diff --git a/src/cfg.rs b/src/cfg.rs index cbb6ceaf..fab383b5 100644 --- a/src/cfg.rs +++ b/src/cfg.rs @@ -30,6 +30,12 @@ pub struct CFGInfo { /// indices. Otherwise, it will be approximate, but should still /// be usable for heuristic purposes. pub approx_loop_depth: Vec, + /// For each block, the next block which is at a lower loop depth, or + /// `Block::invalid` if it is not in a loop. + pub next_outer_loop: Vec, + /// For each block, the previous block which is at a lower loop depth, or + /// `Block::invalid` if it is not in a loop. + pub prev_outer_loop: Vec, } impl CFGInfo { @@ -48,6 +54,8 @@ impl CFGInfo { let mut block_exit = vec![ProgPoint::before(Inst::invalid()); f.num_blocks()]; let mut backedge_in = vec![0; f.num_blocks()]; let mut backedge_out = vec![0; f.num_blocks()]; + let mut next_outer_loop = vec![Block::invalid(); f.num_blocks()]; + let mut prev_outer_loop = vec![Block::invalid(); f.num_blocks()]; for block in 0..f.num_blocks() { let block = Block::new(block); @@ -100,9 +108,11 @@ impl CFGInfo { let mut approx_loop_depth = vec![]; let mut backedge_stack: SmallVec<[usize; 4]> = smallvec![]; let mut cur_depth = 0; + let mut max_depth = 0; for block in 0..f.num_blocks() { if backedge_in[block] > 0 { cur_depth += 1; + max_depth = max_depth.max(cur_depth); backedge_stack.push(backedge_in[block]); } @@ -118,6 +128,28 @@ impl CFGInfo { } } + let mut block_at_depth: SmallVec<[Block; 4]> = + smallvec![Block::invalid(); max_depth as usize]; + for block in 0..f.num_blocks() { + let cur_depth = approx_loop_depth[block]; + for depth in cur_depth..max_depth { + block_at_depth[depth as usize] = Block::new(block); + } + if cur_depth > 0 { + prev_outer_loop[block] = block_at_depth[cur_depth as usize - 1]; + } + } + block_at_depth.fill(Block::invalid()); + for block in (0..f.num_blocks()).rev() { + let cur_depth = approx_loop_depth[block]; + for depth in cur_depth..max_depth { + block_at_depth[depth as usize] = Block::new(block); + } + if cur_depth > 0 { + next_outer_loop[block] = block_at_depth[cur_depth as usize - 1]; + } + } + Ok(CFGInfo { postorder, domtree, @@ -125,6 +157,8 @@ impl CFGInfo { block_entry, block_exit, approx_loop_depth, + next_outer_loop, + prev_outer_loop, }) } diff --git a/src/index.rs b/src/index.rs index df16df0b..10fcef2d 100644 --- a/src/index.rs +++ b/src/index.rs @@ -37,6 +37,23 @@ macro_rules! define_index { self.storage.push(value); idx } + + #[inline(always)] + pub fn get_pair_mut(&mut self, ix1: $ix, ix2: $ix) -> (&mut $elem, &mut $elem) { + if ix1 < ix2 { + let (first, second) = self.storage.split_at_mut(ix1.index() + 1); + ( + &mut first[ix1.index()], + &mut second[ix2.index() - ix1.index() - 1], + ) + } else { + let (first, second) = self.storage.split_at_mut(ix2.index() + 1); + ( + &mut second[ix1.index() - ix2.index() - 1], + &mut first[ix2.index()], + ) + } + } } impl core::ops::Index<$ix> for $storage { diff --git a/src/ion/merge.rs b/src/ion/merge.rs index 09bd2906..af8d66f3 100644 --- a/src/ion/merge.rs +++ b/src/ion/merge.rs @@ -14,14 +14,14 @@ use super::{Env, LiveBundleIndex, SpillSet, SpillSlotIndex, VRegIndex}; use crate::{ - ion::data_structures::{BlockparamOut, CodeRange}, - Function, Inst, OperandConstraint, OperandKind, PReg, ProgPoint, + ion::data_structures::CodeRange, Block, Function, OperandConstraint, OperandKind, PReg, + ProgPoint, }; -use alloc::format; +use alloc::{format, vec::Vec}; use smallvec::smallvec; impl<'a, F: Function> Env<'a, F> { - pub fn merge_bundles(&mut self, from: LiveBundleIndex, to: LiveBundleIndex) -> bool { + fn merge_bundles(&mut self, from: LiveBundleIndex, to: LiveBundleIndex) -> bool { if from == to { // Merge bundle into self -- trivial merge. return true; @@ -74,8 +74,7 @@ impl<'a, F: Function> Env<'a, F> { } }; - // Check for overlap in LiveRanges and for conflicting - // requirements. + // Check for overlap in LiveRanges. let ranges_from = &self.bundles[from].ranges[..]; let ranges_to = &self.bundles[to].ranges[..]; let mut idx_from = 0; @@ -109,18 +108,6 @@ impl<'a, F: Function> Env<'a, F> { } } - // Check for a requirements conflict. - if self.bundles[from].cached_stack() - || self.bundles[from].cached_fixed() - || self.bundles[to].cached_stack() - || self.bundles[to].cached_fixed() - { - if self.merge_bundle_requirements(from, to).is_err() { - trace!(" -> conflicting requirements; aborting merge"); - return false; - } - } - trace!(" -> committing to merge"); // If we reach here, then the bundles do not overlap -- merge @@ -237,9 +224,8 @@ impl<'a, F: Function> Env<'a, F> { true } - pub fn merge_vreg_bundles(&mut self) { - // Create a bundle for every vreg, initially. - trace!("merge_vreg_bundles: creating vreg bundles"); + // Create a bundle for every vreg, initially. + fn create_vreg_bundles(&mut self) { for vreg in 0..self.vregs.len() { let vreg = VRegIndex::new(vreg); if self.vregs[vreg].ranges.is_empty() { @@ -303,12 +289,15 @@ impl<'a, F: Function> Env<'a, F> { }); self.bundles[bundle].spillset = ssidx; } + } - for inst in 0..self.func.num_insts() { - let inst = Inst::new(inst); + /// Merges bundles in the given block. + fn merge_bundles_in_block(&mut self, block: Block) { + let insns = self.func.block_insns(block); - // Attempt to merge Reuse-constraint operand outputs with the - // corresponding inputs. + // Attempt to merge Reuse-constraint operand outputs with the + // corresponding inputs. + for inst in insns.iter() { for op in self.func.inst_operands(inst) { if let OperandConstraint::Reuse(reuse_idx) = op.constraint() { let src_vreg = op.vreg(); @@ -329,25 +318,75 @@ impl<'a, F: Function> Env<'a, F> { } // Attempt to merge blockparams with their inputs. - for i in 0..self.blockparam_outs.len() { - let BlockparamOut { - from_vreg, to_vreg, .. - } = self.blockparam_outs[i]; - trace!( - "trying to merge blockparam v{} with input v{}", - to_vreg.index(), - from_vreg.index() - ); - let to_bundle = self.ranges[self.vregs[to_vreg].ranges[0].index].bundle; - debug_assert!(to_bundle.is_valid()); - let from_bundle = self.ranges[self.vregs[from_vreg].ranges[0].index].bundle; - debug_assert!(from_bundle.is_valid()); - trace!( - " -> from bundle{} to bundle{}", - from_bundle.index(), - to_bundle.index() - ); - self.merge_bundles(from_bundle, to_bundle); + for (i, &succ) in self.func.block_succs(block).iter().enumerate() { + let blockparams_in = self.func.block_params(succ); + let blockparams_out = self.func.branch_blockparams(block, insns.last(), i); + for (&blockparam_in, &blockparam_out) in blockparams_in.iter().zip(blockparams_out) { + let from_vreg = VRegIndex::new(blockparam_out.vreg()); + let to_vreg = VRegIndex::new(blockparam_in.vreg()); + trace!( + "trying to merge blockparam v{} with input v{}", + to_vreg.index(), + from_vreg.index() + ); + let to_bundle = self.ranges[self.vregs[to_vreg].ranges[0].index].bundle; + debug_assert!(to_bundle.is_valid()); + let from_bundle = self.ranges[self.vregs[from_vreg].ranges[0].index].bundle; + debug_assert!(from_bundle.is_valid()); + trace!( + " -> from bundle{} to bundle{}", + from_bundle.index(), + to_bundle.index() + ); + self.merge_bundles(from_bundle, to_bundle); + } + } + } + + /// Get a list of blocks sorted by priority. + /// + /// Each merge eliminates the need for one move instruction in the final + /// program. However a successful merge can cause a future merge to fail + /// due to interference. Therefore we want to prioritize merges that have + /// the highest impact: + /// + /// * We want to eliminate moves in loops that are executed many times. + /// * We want to eliminate moves in split critical edge blocks, since it + /// allows the entire block to be eliminated with jump threading. + /// + /// The heuristic is based on `compareMBBPriority` from LLVM. + fn blocks_by_merge_priority(&mut self) -> Vec { + let loop_depth = |block: Block| self.cfginfo.approx_loop_depth[block.index()]; + let is_split_edge = |block: Block| { + // A split critical edge is an artifical block which only exists for + // the sake of passing blockparams to another block. + self.func.block_insns(block).len() == 1 && self.func.block_succs(block).len() == 1 + }; + let num_connections = + |block: Block| self.func.block_succs(block).len() + self.func.block_preds(block).len(); + + let mut blocks: Vec<_> = (0..self.func.num_blocks()).map(|i| Block::new(i)).collect(); + blocks.sort_by(|&a, &b| { + // Prioritize deeper loops first. + loop_depth(a) + .cmp(&loop_depth(b)) + .reverse() + // Prioritize critical edges to enable jump threading. + .then_with(|| is_split_edge(a).cmp(&is_split_edge(b)).reverse()) + // Prioritize blocks which are more connected in the CFG. + .then_with(|| num_connections(a).cmp(&num_connections(b)).reverse()) + // Otherwise follow the original block order. + }); + blocks + } + + pub fn merge_vreg_bundles(&mut self) { + trace!("merge_vreg_bundles: creating vreg bundles"); + + self.create_vreg_bundles(); + + for block in self.blocks_by_merge_priority() { + self.merge_bundles_in_block(block); } trace!("done merging bundles"); diff --git a/src/ion/mod.rs b/src/ion/mod.rs index 4d32eb8c..f42ae734 100644 --- a/src/ion/mod.rs +++ b/src/ion/mod.rs @@ -38,6 +38,7 @@ use smallvec::smallvec; pub(crate) mod dump; pub(crate) mod moves; pub(crate) mod spill; +pub(crate) mod split; pub(crate) mod stackmap; impl<'a, F: Function> Env<'a, F> { @@ -99,6 +100,7 @@ impl<'a, F: Function> Env<'a, F> { self.build_liveranges(); self.fixup_multi_fixed_vregs(); self.merge_vreg_bundles(); + self.legalize_bundle_requirements(); self.queue_bundles(); if trace_enabled!() { self.dump_state(); diff --git a/src/ion/process.rs b/src/ion/process.rs index 371ab84b..f5d44362 100644 --- a/src/ion/process.rs +++ b/src/ion/process.rs @@ -14,19 +14,17 @@ use super::{ spill_weight_from_constraint, Env, LiveBundleIndex, LiveBundleVec, LiveRangeFlag, - LiveRangeIndex, LiveRangeKey, LiveRangeList, LiveRangeListEntry, PRegIndex, RegTraversalIter, - Requirement, SpillWeight, UseList, VRegIndex, + LiveRangeIndex, LiveRangeKey, PRegIndex, RegTraversalIter, Requirement, SpillWeight, }; use crate::{ ion::data_structures::{ - CodeRange, BUNDLE_MAX_NORMAL_SPILL_WEIGHT, MAX_SPLITS_PER_SPILLSET, - MINIMAL_BUNDLE_SPILL_WEIGHT, MINIMAL_FIXED_BUNDLE_SPILL_WEIGHT, + CodeRange, BUNDLE_MAX_NORMAL_SPILL_WEIGHT, MINIMAL_BUNDLE_SPILL_WEIGHT, + MINIMAL_FIXED_BUNDLE_SPILL_WEIGHT, }, - Allocation, Function, FxHashSet, Inst, InstPosition, OperandConstraint, OperandKind, PReg, - ProgPoint, RegAllocError, + Allocation, Function, Inst, OperandConstraint, OperandKind, PReg, ProgPoint, RegAllocError, }; use core::fmt::Debug; -use smallvec::{smallvec, SmallVec}; +use smallvec::smallvec; #[derive(Clone, Debug, PartialEq, Eq)] pub enum AllocRegResult { @@ -390,603 +388,6 @@ impl<'a, F: Function> Env<'a, F> { } } - pub fn split_and_requeue_bundle( - &mut self, - bundle: LiveBundleIndex, - mut split_at: ProgPoint, - reg_hint: PReg, - // Do we trim the parts around the split and put them in the - // spill bundle? - mut trim_ends_into_spill_bundle: bool, - ) { - self.stats.splits += 1; - trace!( - "split bundle {:?} at {:?} and requeue with reg hint (for first part) {:?}", - bundle, - split_at, - reg_hint, - ); - - // Split `bundle` at `split_at`, creating new LiveRanges and - // bundles (and updating vregs' linked lists appropriately), - // and enqueue the new bundles. - - let spillset = self.bundles[bundle].spillset; - - // Have we reached the maximum split count? If so, fall back - // to a "minimal bundles and spill bundle" setup for this - // bundle. See the doc-comment on - // `split_into_minimal_bundles()` above for more. - if self.spillsets[spillset].splits >= MAX_SPLITS_PER_SPILLSET { - self.split_into_minimal_bundles(bundle, reg_hint); - return; - } - self.spillsets[spillset].splits += 1; - - debug_assert!(!self.bundles[bundle].ranges.is_empty()); - // Split point *at* start is OK; this means we peel off - // exactly one use to create a minimal bundle. - let bundle_start = self.bundles[bundle].ranges.first().unwrap().range.from; - debug_assert!(split_at >= bundle_start); - let bundle_end = self.bundles[bundle].ranges.last().unwrap().range.to; - debug_assert!(split_at < bundle_end); - - // Is the split point *at* the start? If so, peel off the - // first use: set the split point just after it, or just - // before it if it comes after the start of the bundle. - if split_at == bundle_start { - // Find any uses; if none, just chop off one instruction. - let mut first_use = None; - 'outer: for entry in &self.bundles[bundle].ranges { - for u in &self.ranges[entry.index].uses { - first_use = Some(u.pos); - break 'outer; - } - } - trace!(" -> first use loc is {:?}", first_use); - split_at = match first_use { - Some(pos) => { - if pos.inst() == bundle_start.inst() { - ProgPoint::before(pos.inst().next()) - } else { - ProgPoint::before(pos.inst()) - } - } - None => ProgPoint::before( - self.bundles[bundle] - .ranges - .first() - .unwrap() - .range - .from - .inst() - .next(), - ), - }; - trace!( - "split point is at bundle start; advancing to {:?}", - split_at - ); - } else { - // Don't split in the middle of an instruction -- this could - // create impossible moves (we cannot insert a move between an - // instruction's uses and defs). - if split_at.pos() == InstPosition::After { - split_at = split_at.next(); - } - if split_at >= bundle_end { - split_at = split_at.prev().prev(); - } - } - - debug_assert!(split_at > bundle_start && split_at < bundle_end); - - // We need to find which LRs fall on each side of the split, - // which LR we need to split down the middle, then update the - // current bundle, create a new one, and (re)-queue both. - - trace!(" -> LRs: {:?}", self.bundles[bundle].ranges); - - let mut last_lr_in_old_bundle_idx = 0; // last LR-list index in old bundle - let mut first_lr_in_new_bundle_idx = 0; // first LR-list index in new bundle - for (i, entry) in self.bundles[bundle].ranges.iter().enumerate() { - if split_at > entry.range.from { - last_lr_in_old_bundle_idx = i; - first_lr_in_new_bundle_idx = i; - } - if split_at < entry.range.to { - first_lr_in_new_bundle_idx = i; - - // When the bundle contains a fixed constraint, we advance the split point to right - // before the first instruction with a fixed use present. - if self.bundles[bundle].cached_fixed() { - for u in &self.ranges[entry.index].uses { - if u.pos < split_at { - continue; - } - - if matches!(u.operand.constraint(), OperandConstraint::FixedReg { .. }) { - split_at = ProgPoint::before(u.pos.inst()); - - if split_at > entry.range.from { - last_lr_in_old_bundle_idx = i; - } - - trace!(" -> advancing split point to {split_at:?}"); - - trim_ends_into_spill_bundle = false; - - break; - } - } - } - - break; - } - } - - trace!( - " -> last LR in old bundle: LR {:?}", - self.bundles[bundle].ranges[last_lr_in_old_bundle_idx] - ); - trace!( - " -> first LR in new bundle: LR {:?}", - self.bundles[bundle].ranges[first_lr_in_new_bundle_idx] - ); - - // Take the sublist of LRs that will go in the new bundle. - let mut new_lr_list: LiveRangeList = self.bundles[bundle] - .ranges - .iter() - .cloned() - .skip(first_lr_in_new_bundle_idx) - .collect(); - self.bundles[bundle] - .ranges - .truncate(last_lr_in_old_bundle_idx + 1); - self.bundles[bundle].ranges.shrink_to_fit(); - - // If the first entry in `new_lr_list` is a LR that is split - // down the middle, replace it with a new LR and chop off the - // end of the same LR in the original list. - if split_at > new_lr_list[0].range.from { - debug_assert_eq!(last_lr_in_old_bundle_idx, first_lr_in_new_bundle_idx); - let orig_lr = new_lr_list[0].index; - let new_lr = self.ranges.add(CodeRange { - from: split_at, - to: new_lr_list[0].range.to, - }); - self.ranges[new_lr].vreg = self.ranges[orig_lr].vreg; - trace!(" -> splitting LR {:?} into {:?}", orig_lr, new_lr); - let first_use = self.ranges[orig_lr] - .uses - .iter() - .position(|u| u.pos >= split_at) - .unwrap_or(self.ranges[orig_lr].uses.len()); - let rest_uses: UseList = self.ranges[orig_lr] - .uses - .iter() - .cloned() - .skip(first_use) - .collect(); - self.ranges[new_lr].uses = rest_uses; - self.ranges[orig_lr].uses.truncate(first_use); - self.ranges[orig_lr].uses.shrink_to_fit(); - self.recompute_range_properties(orig_lr); - self.recompute_range_properties(new_lr); - new_lr_list[0].index = new_lr; - new_lr_list[0].range = self.ranges[new_lr].range; - self.ranges[orig_lr].range.to = split_at; - self.bundles[bundle].ranges[last_lr_in_old_bundle_idx].range = - self.ranges[orig_lr].range; - - // Perform a lazy split in the VReg data. We just - // append the new LR and its range; we will sort by - // start of range, and fix up range ends, once when we - // iterate over the VReg's ranges after allocation - // completes (this is the only time when order - // matters). - self.vregs[self.ranges[new_lr].vreg] - .ranges - .push(LiveRangeListEntry { - range: self.ranges[new_lr].range, - index: new_lr, - }); - } - - let new_bundle = self.bundles.add(); - trace!(" -> creating new bundle {:?}", new_bundle); - self.bundles[new_bundle].spillset = spillset; - for entry in &new_lr_list { - self.ranges[entry.index].bundle = new_bundle; - } - self.bundles[new_bundle].ranges = new_lr_list; - - if trim_ends_into_spill_bundle { - // Finally, handle moving LRs to the spill bundle when - // appropriate: If the first range in `new_bundle` or last - // range in `bundle` has "empty space" beyond the first or - // last use (respectively), trim it and put an empty LR into - // the spill bundle. (We are careful to treat the "starts at - // def" flag as an implicit first def even if no def-type Use - // is present.) - while let Some(entry) = self.bundles[bundle].ranges.last().cloned() { - let end = entry.range.to; - let vreg = self.ranges[entry.index].vreg; - let last_use = self.ranges[entry.index].uses.last().map(|u| u.pos); - if last_use.is_none() { - let spill = self - .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true) - .unwrap(); - trace!( - " -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}", - bundle, - entry.index, - spill - ); - self.bundles[spill].ranges.push(entry); - self.bundles[bundle].ranges.pop(); - self.ranges[entry.index].bundle = spill; - continue; - } - let last_use = last_use.unwrap(); - let split = ProgPoint::before(last_use.inst().next()); - if split < end { - let spill = self - .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true) - .unwrap(); - self.bundles[bundle].ranges.last_mut().unwrap().range.to = split; - self.ranges[self.bundles[bundle].ranges.last().unwrap().index] - .range - .to = split; - let range = CodeRange { - from: split, - to: end, - }; - let empty_lr = self.ranges.add(range); - self.bundles[spill].ranges.push(LiveRangeListEntry { - range, - index: empty_lr, - }); - self.ranges[empty_lr].bundle = spill; - self.vregs[vreg].ranges.push(LiveRangeListEntry { - range, - index: empty_lr, - }); - trace!( - " -> bundle {:?} range {:?}: last use implies split point {:?}", - bundle, - entry.index, - split - ); - trace!( - " -> moving trailing empty region to new spill bundle {:?} with new LR {:?}", - spill, - empty_lr - ); - } - break; - } - while let Some(entry) = self.bundles[new_bundle].ranges.first().cloned() { - if self.ranges[entry.index].has_flag(LiveRangeFlag::StartsAtDef) { - break; - } - let start = entry.range.from; - let vreg = self.ranges[entry.index].vreg; - let first_use = self.ranges[entry.index].uses.first().map(|u| u.pos); - if first_use.is_none() { - let spill = self - .get_or_create_spill_bundle(new_bundle, /* create_if_absent = */ true) - .unwrap(); - trace!( - " -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}", - new_bundle, - entry.index, - spill - ); - self.bundles[spill].ranges.push(entry); - self.bundles[new_bundle].ranges.drain(..1); - self.ranges[entry.index].bundle = spill; - continue; - } - let first_use = first_use.unwrap(); - let split = ProgPoint::before(first_use.inst()); - if split > start { - let spill = self - .get_or_create_spill_bundle(new_bundle, /* create_if_absent = */ true) - .unwrap(); - self.bundles[new_bundle] - .ranges - .first_mut() - .unwrap() - .range - .from = split; - self.ranges[self.bundles[new_bundle].ranges.first().unwrap().index] - .range - .from = split; - let range = CodeRange { - from: start, - to: split, - }; - let empty_lr = self.ranges.add(range); - self.bundles[spill].ranges.push(LiveRangeListEntry { - range, - index: empty_lr, - }); - self.ranges[empty_lr].bundle = spill; - self.vregs[vreg].ranges.push(LiveRangeListEntry { - range, - index: empty_lr, - }); - trace!( - " -> bundle {:?} range {:?}: first use implies split point {:?}", - bundle, - entry.index, - first_use, - ); - trace!( - " -> moving leading empty region to new spill bundle {:?} with new LR {:?}", - spill, - empty_lr - ); - } - break; - } - } - - if self.bundles[bundle].ranges.len() > 0 { - self.recompute_bundle_properties(bundle); - let prio = self.bundles[bundle].prio; - self.allocation_queue - .insert(bundle, prio as usize, reg_hint); - } - if self.bundles[new_bundle].ranges.len() > 0 { - self.recompute_bundle_properties(new_bundle); - let prio = self.bundles[new_bundle].prio; - self.allocation_queue - .insert(new_bundle, prio as usize, reg_hint); - } - } - - /// Splits the given bundle into minimal bundles per Use, falling - /// back onto the spill bundle. This must work for any bundle no - /// matter how many conflicts. - /// - /// This is meant to solve a quadratic-cost problem that exists - /// with "normal" splitting as implemented above. With that - /// procedure, , splitting a bundle produces two - /// halves. Furthermore, it has cost linear in the length of the - /// bundle, because the resulting half-bundles have their - /// requirements recomputed with a new scan, and because we copy - /// half the use-list over to the tail end sub-bundle. - /// - /// This works fine when a bundle has a handful of splits overall, - /// but not when an input has a systematic pattern of conflicts - /// that will require O(|bundle|) splits (e.g., every Use is - /// constrained to a different fixed register than the last - /// one). In such a case, we get quadratic behavior. - /// - /// This method implements a direct split into minimal bundles - /// along the whole length of the bundle, putting the regions - /// without uses in the spill bundle. We do this once the number - /// of splits in an original bundle (tracked by spillset) reaches - /// a pre-determined limit. - /// - /// This basically approximates what a non-splitting allocator - /// would do: it "spills" the whole bundle to possibly a - /// stackslot, or a second-chance register allocation at best, via - /// the spill bundle; and then does minimal reservations of - /// registers just at uses/defs and moves the "spilled" value - /// into/out of them immediately. - pub fn split_into_minimal_bundles(&mut self, bundle: LiveBundleIndex, reg_hint: PReg) { - let mut removed_lrs: FxHashSet = FxHashSet::default(); - let mut removed_lrs_vregs: FxHashSet = FxHashSet::default(); - let mut new_lrs: SmallVec<[(VRegIndex, LiveRangeIndex); 16]> = smallvec![]; - let mut new_bundles: SmallVec<[LiveBundleIndex; 16]> = smallvec![]; - - let spillset = self.bundles[bundle].spillset; - let spill = self - .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true) - .unwrap(); - - trace!( - "Splitting bundle {:?} into minimal bundles with reg hint {}", - bundle, - reg_hint - ); - - let mut last_lr: Option = None; - let mut last_bundle: Option = None; - let mut last_inst: Option = None; - let mut last_vreg: Option = None; - - let mut spill_uses = UseList::new(); - - for entry in core::mem::take(&mut self.bundles[bundle].ranges) { - let lr_from = entry.range.from; - let lr_to = entry.range.to; - let vreg = self.ranges[entry.index].vreg; - - removed_lrs.insert(entry.index); - removed_lrs_vregs.insert(vreg); - trace!(" -> removing old LR {:?} for vreg {:?}", entry.index, vreg); - - let mut spill_range = entry.range; - let mut spill_starts_def = false; - - let mut last_live_pos = entry.range.from; - for u in core::mem::take(&mut self.ranges[entry.index].uses) { - trace!(" -> use {:?} (last_live_pos {:?})", u, last_live_pos); - - let is_def = u.operand.kind() == OperandKind::Def; - - // If this use has an `any` constraint, eagerly migrate it to the spill range. The - // reasoning here is that in the second-chance allocation for the spill bundle, - // any-constrained uses will be easy to satisfy. Solving those constraints earlier - // could create unnecessary conflicts with existing bundles that need to fit in a - // register, more strict requirements, so we delay them eagerly. - if u.operand.constraint() == OperandConstraint::Any { - trace!(" -> migrating this any-constrained use to the spill range"); - spill_uses.push(u); - - // Remember if we're moving the def of this vreg into the spill range, so that - // we can set the appropriate flags on it later. - spill_starts_def = spill_starts_def || is_def; - - continue; - } - - // If this is a def of the vreg the entry cares about, make sure that the spill - // range starts right before the next instruction so that the value is available. - if is_def { - trace!(" -> moving the spill range forward by one"); - spill_range.from = ProgPoint::before(u.pos.inst().next()); - } - - // If we just created a LR for this inst at the last - // pos, add this use to the same LR. - if Some(u.pos.inst()) == last_inst && Some(vreg) == last_vreg { - self.ranges[last_lr.unwrap()].uses.push(u); - trace!(" -> appended to last LR {:?}", last_lr.unwrap()); - continue; - } - - // The minimal bundle runs through the whole inst - // (up to the Before of the next inst), *unless* - // the original LR was only over the Before (up to - // the After) of this inst. - let to = core::cmp::min(ProgPoint::before(u.pos.inst().next()), lr_to); - - // If the last bundle was at the same inst, add a new - // LR to the same bundle; otherwise, create a LR and a - // new bundle. - if Some(u.pos.inst()) == last_inst { - let cr = CodeRange { from: u.pos, to }; - let lr = self.ranges.add(cr); - new_lrs.push((vreg, lr)); - self.ranges[lr].uses.push(u); - self.ranges[lr].vreg = vreg; - - trace!( - " -> created new LR {:?} but adding to existing bundle {:?}", - lr, - last_bundle.unwrap() - ); - // Edit the previous LR to end mid-inst. - self.bundles[last_bundle.unwrap()] - .ranges - .last_mut() - .unwrap() - .range - .to = u.pos; - self.ranges[last_lr.unwrap()].range.to = u.pos; - // Add this LR to the bundle. - self.bundles[last_bundle.unwrap()] - .ranges - .push(LiveRangeListEntry { - range: cr, - index: lr, - }); - self.ranges[lr].bundle = last_bundle.unwrap(); - last_live_pos = ProgPoint::before(u.pos.inst().next()); - continue; - } - - // Otherwise, create a new LR. - let pos = ProgPoint::before(u.pos.inst()); - let pos = core::cmp::max(lr_from, pos); - let cr = CodeRange { from: pos, to }; - let lr = self.ranges.add(cr); - new_lrs.push((vreg, lr)); - self.ranges[lr].uses.push(u); - self.ranges[lr].vreg = vreg; - - // Create a new bundle that contains only this LR. - let new_bundle = self.bundles.add(); - self.ranges[lr].bundle = new_bundle; - self.bundles[new_bundle].spillset = spillset; - self.bundles[new_bundle].ranges.push(LiveRangeListEntry { - range: cr, - index: lr, - }); - new_bundles.push(new_bundle); - - // If this use was a Def, set the StartsAtDef flag for the new LR. - if is_def { - self.ranges[lr].set_flag(LiveRangeFlag::StartsAtDef); - } - - trace!( - " -> created new LR {:?} range {:?} with new bundle {:?} for this use", - lr, - cr, - new_bundle - ); - - last_live_pos = ProgPoint::before(u.pos.inst().next()); - - last_lr = Some(lr); - last_bundle = Some(new_bundle); - last_inst = Some(u.pos.inst()); - last_vreg = Some(vreg); - } - - if !spill_range.is_empty() { - // Make one entry in the spill bundle that covers the whole range. - // TODO: it might be worth tracking enough state to only create this LR when there is - // open space in the original LR. - let spill_lr = self.ranges.add(spill_range); - self.ranges[spill_lr].vreg = vreg; - self.ranges[spill_lr].bundle = spill; - self.ranges[spill_lr].uses.extend(spill_uses.drain(..)); - new_lrs.push((vreg, spill_lr)); - - if spill_starts_def { - self.ranges[spill_lr].set_flag(LiveRangeFlag::StartsAtDef); - } - - self.bundles[spill].ranges.push(LiveRangeListEntry { - range: spill_range, - index: spill_lr, - }); - self.ranges[spill_lr].bundle = spill; - trace!( - " -> added spill range {:?} in new LR {:?} in spill bundle {:?}", - spill_range, - spill_lr, - spill - ); - } else { - assert!(spill_uses.is_empty()); - } - } - - // Remove all of the removed LRs from respective vregs' lists. - for vreg in removed_lrs_vregs { - self.vregs[vreg] - .ranges - .retain(|entry| !removed_lrs.contains(&entry.index)); - } - - // Add the new LRs to their respective vreg lists. - for (vreg, lr) in new_lrs { - let range = self.ranges[lr].range; - let entry = LiveRangeListEntry { range, index: lr }; - self.vregs[vreg].ranges.push(entry); - } - - // Recompute bundle properties for all new bundles and enqueue - // them. - for bundle in new_bundles { - if self.bundles[bundle].ranges.len() > 0 { - self.recompute_bundle_properties(bundle); - let prio = self.bundles[bundle].prio; - self.allocation_queue - .insert(bundle, prio as usize, reg_hint); - } - } - } - pub fn process_bundle( &mut self, bundle: LiveBundleIndex, @@ -1002,48 +403,26 @@ impl<'a, F: Function> Env<'a, F> { if self.pregs[hint_reg.index()].is_stack { hint_reg = PReg::invalid(); } - trace!("process_bundle: bundle {:?} hint {:?}", bundle, hint_reg,); + trace!("process_bundle: bundle {:?} hint {:?}", bundle, hint_reg); - let req = match self.compute_requirement(bundle) { - Ok(req) => req, - Err(conflict) => { - trace!("conflict!: {:?}", conflict); - // We have to split right away. We'll find a point to - // split that would allow at least the first half of the - // split to be conflict-free. - debug_assert!( - !self.minimal_bundle(bundle), - "Minimal bundle with conflict!" - ); - self.split_and_requeue_bundle( - bundle, - /* split_at_point = */ conflict.suggested_split_point(), - reg_hint, - /* trim_ends_into_spill_bundle = */ - conflict.should_trim_edges_around_split(), - ); - return Ok(()); - } - }; + let req = self + .compute_requirement(bundle) + .expect("invalid bundle requirements"); // If no requirement at all (because no uses), and *if* a // spill bundle is already present, then move the LRs over to // the spill bundle right away. - match req { - Requirement::Any => { - if let Some(spill) = - self.get_or_create_spill_bundle(bundle, /* create_if_absent = */ false) - { - let mut list = - core::mem::replace(&mut self.bundles[bundle].ranges, smallvec![]); - for entry in &list { - self.ranges[entry.index].bundle = spill; - } - self.bundles[spill].ranges.extend(list.drain(..)); - return Ok(()); + if req == Requirement::Any { + if let Some(spill) = + self.get_or_create_spill_bundle(bundle, /* create_if_absent = */ false) + { + let mut list = core::mem::replace(&mut self.bundles[bundle].ranges, smallvec![]); + for entry in &list { + self.ranges[entry.index].bundle = spill; } + self.bundles[spill].ranges.extend(list.drain(..)); + return Ok(()); } - _ => {} } // Try to allocate! @@ -1296,20 +675,7 @@ impl<'a, F: Function> Env<'a, F> { // Adjust `split_at_point` if it is within a deeper loop // than the bundle start -- hoist it to just before the // first loop header it encounters. - let bundle_start_depth = self.cfginfo.approx_loop_depth - [self.cfginfo.insn_block[bundle_start.inst().index()].index()]; - let split_at_depth = self.cfginfo.approx_loop_depth - [self.cfginfo.insn_block[split_at_point.inst().index()].index()]; - if split_at_depth > bundle_start_depth { - for block in (self.cfginfo.insn_block[bundle_start.inst().index()].index() + 1) - ..=self.cfginfo.insn_block[split_at_point.inst().index()].index() - { - if self.cfginfo.approx_loop_depth[block] > bundle_start_depth { - split_at_point = self.cfginfo.block_entry[block]; - break; - } - } - } + split_at_point = self.adjust_split_point_backward(split_at_point, bundle_start); self.split_and_requeue_bundle( bundle, diff --git a/src/ion/requirement.rs b/src/ion/requirement.rs index fc049e20..ac66cb3d 100644 --- a/src/ion/requirement.rs +++ b/src/ion/requirement.rs @@ -157,18 +157,4 @@ impl<'a, F: Function> Env<'a, F> { trace!(" -> final: {:?}", req); Ok(req) } - - pub fn merge_bundle_requirements( - &self, - a: LiveBundleIndex, - b: LiveBundleIndex, - ) -> Result { - let req_a = self - .compute_requirement(a) - .map_err(|_| RequirementConflict)?; - let req_b = self - .compute_requirement(b) - .map_err(|_| RequirementConflict)?; - req_a.merge(req_b) - } } diff --git a/src/ion/split.rs b/src/ion/split.rs new file mode 100644 index 00000000..25da0551 --- /dev/null +++ b/src/ion/split.rs @@ -0,0 +1,779 @@ +/* + * This file was initially derived from the files + * `js/src/jit/BacktrackingAllocator.h` and + * `js/src/jit/BacktrackingAllocator.cpp` in Mozilla Firefox, and was + * originally licensed under the Mozilla Public License 2.0. We + * subsequently relicensed it to Apache-2.0 WITH LLVM-exception (see + * https://github.com/bytecodealliance/regalloc2/issues/7). + * + * Since the initial port, the design has been substantially evolved + * and optimized. + */ + +//! Code related to bundle splitting. + +use super::{ + requirement::Requirement, Env, LiveBundleIndex, LiveBundleVec, LiveRangeFlag, LiveRangeIndex, + LiveRangeListEntry, UseList, VRegIndex, +}; +use crate::{ + ion::data_structures::{CodeRange, MAX_SPLITS_PER_SPILLSET}, + Allocation, Function, FxHashSet, Inst, InstPosition, OperandConstraint, OperandKind, PReg, + ProgPoint, +}; +use core::fmt::Debug; +use smallvec::{smallvec, SmallVec}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum AllocRegResult { + Allocated(Allocation), + Conflict(LiveBundleVec, ProgPoint), + ConflictWithFixed(u32, ProgPoint), + ConflictHighCost, +} + +impl<'a, F: Function> Env<'a, F> { + pub fn split_and_requeue_bundle( + &mut self, + bundle: LiveBundleIndex, + mut split_at: ProgPoint, + reg_hint: PReg, + // Do we trim the parts around the split and put them in the + // spill bundle? + mut trim_ends_into_spill_bundle: bool, + ) { + self.stats.splits += 1; + trace!( + "split bundle {:?} at {:?} and requeue with reg hint (for first part) {:?}", + bundle, + split_at, + reg_hint, + ); + + // Split `bundle` at `split_at`, creating new LiveRanges and + // bundles (and updating vregs' linked lists appropriately), + // and enqueue the new bundles. + + let spillset = self.bundles[bundle].spillset; + + // Have we reached the maximum split count? If so, fall back + // to a "minimal bundles and spill bundle" setup for this + // bundle. See the doc-comment on + // `split_into_minimal_bundles()` above for more. + if self.spillsets[spillset].splits >= MAX_SPLITS_PER_SPILLSET { + self.split_into_minimal_bundles(bundle, reg_hint); + return; + } + self.spillsets[spillset].splits += 1; + + debug_assert!(!self.bundles[bundle].ranges.is_empty()); + // Split point *at* start is OK; this means we peel off + // exactly one use to create a minimal bundle. + let bundle_start = self.bundles[bundle].ranges.first().unwrap().range.from; + debug_assert!(split_at >= bundle_start); + let bundle_end = self.bundles[bundle].ranges.last().unwrap().range.to; + debug_assert!(split_at < bundle_end); + + // Is the split point *at* the start? If so, peel off the + // first use: set the split point just after it, or just + // before it if it comes after the start of the bundle. + if split_at == bundle_start { + // Find any uses; if none, just chop off one instruction. + let mut first_use = None; + 'outer: for entry in &self.bundles[bundle].ranges { + for u in &self.ranges[entry.index].uses { + first_use = Some(u.pos); + break 'outer; + } + } + trace!(" -> first use loc is {:?}", first_use); + split_at = match first_use { + Some(pos) => { + if pos.inst() == bundle_start.inst() { + ProgPoint::before(pos.inst().next()) + } else { + ProgPoint::before(pos.inst()) + } + } + None => ProgPoint::before( + self.bundles[bundle] + .ranges + .first() + .unwrap() + .range + .from + .inst() + .next(), + ), + }; + trace!( + "split point is at bundle start; advancing to {:?}", + split_at + ); + } else { + // Don't split in the middle of an instruction -- this could + // create impossible moves (we cannot insert a move between an + // instruction's uses and defs). + if split_at.pos() == InstPosition::After { + split_at = split_at.next(); + } + if split_at >= bundle_end { + split_at = split_at.prev().prev(); + } + } + + // If this bundle contains a fixed constraint and the first fixed use is + // located after the split point then move the split point forward to + // just before the first fixed use. + // + // This allows the first half of the bundle to be fully unconstrained by + // the fixed use, which may make it allocatable. + if self.bundles[bundle].cached_fixed() { + let first_fixed_use = self.bundles[bundle] + .ranges + .iter() + .flat_map(|entry| self.ranges[entry.index].uses.iter()) + .filter(|u| matches!(u.operand.constraint(), OperandConstraint::FixedReg(_))) + .next() + .unwrap(); + if first_fixed_use.pos > split_at { + let before_fixed = ProgPoint::before(first_fixed_use.pos.inst()); + trace!(" -> advancing split point to first fixed use at {before_fixed:?}"); + split_at = self.adjust_split_point_backward(before_fixed, split_at); + trim_ends_into_spill_bundle = false; + } + } + + let new_bundle = self.bundles.add(); + trace!(" -> creating new bundle {:?}", new_bundle); + self.bundles[new_bundle].spillset = spillset; + self.split_bundle_at(bundle, new_bundle, split_at, true); + + if trim_ends_into_spill_bundle { + // Finally, handle moving LRs to the spill bundle when + // appropriate: If the first range in `new_bundle` or last + // range in `bundle` has "empty space" beyond the first or + // last use (respectively), trim it and put an empty LR into + // the spill bundle. (We are careful to treat the "starts at + // def" flag as an implicit first def even if no def-type Use + // is present.) + self.trim_bundle_tail(bundle); + self.trim_bundle_head(new_bundle); + } + + if self.bundles[bundle].ranges.len() > 0 { + self.recompute_bundle_properties(bundle); + let prio = self.bundles[bundle].prio; + self.allocation_queue + .insert(bundle, prio as usize, reg_hint); + } + if self.bundles[new_bundle].ranges.len() > 0 { + self.recompute_bundle_properties(new_bundle); + let prio = self.bundles[new_bundle].prio; + self.allocation_queue + .insert(new_bundle, prio as usize, reg_hint); + } + } + + /// Core functionality for splitting a bundle into two halves and moving the + /// first or second half into another bundle. + /// + /// Note that this doesn't call `recompute_bundle_properties`, it is the + /// caller's responsibility to do so if necessary. + fn split_bundle_at( + &mut self, + bundle_idx: LiveBundleIndex, + dest_bundle_idx: LiveBundleIndex, + split_at: ProgPoint, + move_second_half: bool, + ) { + let (src_bundle, dest_bundle) = self.bundles.get_pair_mut(bundle_idx, dest_bundle_idx); + let existing_dest_bundle_ranges = dest_bundle.ranges.len(); + + // Splits must happen at the before position of an instruction. + debug_assert_eq!(split_at.pos(), InstPosition::Before); + + // The split point must be within the bundle. + debug_assert!(split_at > src_bundle.ranges[0].range.from); + debug_assert!(split_at < src_bundle.ranges.last().unwrap().range.to); + + // We need to find which LRs fall on each side of the split, + // and which LR we need to split down the middle. + + trace!( + "splitting bundle {:?} at {:?}, moving {} half into bundle {:?}", + bundle_idx, + split_at, + if move_second_half { "second" } else { "first" }, + dest_bundle_idx + ); + trace!(" -> LRs: {:?}", src_bundle.ranges); + + let first_lr_in_second_half = src_bundle + .ranges + .partition_point(|r| r.range.to <= split_at); + let last_lr_in_first_half = + if src_bundle.ranges[first_lr_in_second_half].range.from < split_at { + first_lr_in_second_half + } else { + first_lr_in_second_half - 1 + }; + + trace!( + " -> last LR in first half: LR {:?}", + src_bundle.ranges[last_lr_in_first_half] + ); + trace!( + " -> first LR in second half: LR {:?}", + src_bundle.ranges[first_lr_in_second_half] + ); + + let lr_to_split = src_bundle.ranges[last_lr_in_first_half].clone(); + if move_second_half { + trace!(" -> moving second half into bundle {:?}", dest_bundle_idx); + dest_bundle + .ranges + .extend_from_slice(&src_bundle.ranges[first_lr_in_second_half..]); + src_bundle.ranges.truncate(last_lr_in_first_half + 1); + } else { + trace!(" -> moving first half into bundle {:?}", dest_bundle_idx); + dest_bundle + .ranges + .extend_from_slice(&src_bundle.ranges[..last_lr_in_first_half + 1]); + src_bundle.ranges.drain(..first_lr_in_second_half); + } + src_bundle.ranges.shrink_to_fit(); + + // If both bundles share a LR that is split down the middle, split it + // into 2 separate LRs. + if last_lr_in_first_half == first_lr_in_second_half { + let orig_lr_index = lr_to_split.index; + let new_lr_index = self.ranges.add(CodeRange { + from: split_at, + to: lr_to_split.range.to, + }); + let (orig_lr, new_lr) = self.ranges.get_pair_mut(orig_lr_index, new_lr_index); + orig_lr.range.to = split_at; + new_lr.vreg = orig_lr.vreg; + trace!( + " -> splitting LR {:?} into {:?}", + orig_lr_index, + new_lr_index + ); + + // Transfer all uses after the split point to the new live range. + let split_use = orig_lr.uses.partition_point(|u| u.pos < split_at); + new_lr.uses = UseList::from_slice(&orig_lr.uses[split_use..]); + orig_lr.uses.truncate(split_use); + orig_lr.uses.shrink_to_fit(); + + if move_second_half { + dest_bundle.ranges[existing_dest_bundle_ranges].range.from = split_at; + dest_bundle.ranges[existing_dest_bundle_ranges].index = new_lr_index; + src_bundle.ranges[last_lr_in_first_half].range.to = split_at; + } else { + dest_bundle.ranges[existing_dest_bundle_ranges + last_lr_in_first_half] + .range + .to = split_at; + src_bundle.ranges[0].range.from = split_at; + src_bundle.ranges[0].index = new_lr_index; + new_lr.bundle = bundle_idx; + } + + // Perform a lazy split in the VReg data. We just + // append the new LR and its range; we will sort by + // start of range, and fix up range ends, once when we + // iterate over the VReg's ranges after allocation + // completes (this is the only time when order + // matters). + self.vregs[new_lr.vreg].ranges.push(LiveRangeListEntry { + range: new_lr.range, + index: new_lr_index, + }); + + self.recompute_range_properties(orig_lr_index); + self.recompute_range_properties(new_lr_index); + } + + for entry in &self.bundles[dest_bundle_idx].ranges[existing_dest_bundle_ranges..] { + self.ranges[entry.index].bundle = dest_bundle_idx; + } + } + + /// Trims any "empty space" at the end of the given bundle and moves the + /// live ranges into the spill bundle. + fn trim_bundle_tail(&mut self, bundle: LiveBundleIndex) { + // Select a split point after the last use. + let Some(last_use) = self.bundles[bundle] + .ranges + .iter() + .flat_map(|entry| self.ranges[entry.index].uses.iter()) + .next_back() + else { + return; + }; + let bundle_end = ProgPoint::before( + self.bundles[bundle] + .ranges + .last() + .unwrap() + .range + .to + .next() + .inst(), + ); + let split = ProgPoint::before(last_use.pos.inst().next()); + let split = self.adjust_split_point_forward(split, bundle_end); + + if split != bundle_end { + let spill = self + .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true) + .unwrap(); + trace!( + "trimming tail of bundle {:?} into spill bundle {:?}", + bundle, + spill + ); + self.split_bundle_at(bundle, spill, split, true); + } + } + + /// Trims any "empty space" at the start of the given bundle and moves the + /// live ranges into the spill bundle. + fn trim_bundle_head(&mut self, bundle: LiveBundleIndex) { + // Select a split point before the first use. + let Some(first_use) = self.bundles[bundle] + .ranges + .iter() + .flat_map(|entry| self.ranges[entry.index].uses.iter()) + .next() + else { + return; + }; + let bundle_start = ProgPoint::before( + self.bundles[bundle] + .ranges + .first() + .unwrap() + .range + .from + .inst(), + ); + let split = ProgPoint::before(first_use.pos.inst()); + let split = self.adjust_split_point_backward(split, bundle_start); + + if split != bundle_start { + let spill = self + .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true) + .unwrap(); + trace!( + "trimming head of bundle {:?} into spill bundle {:?}", + bundle, + spill + ); + self.split_bundle_at(bundle, spill, split, false); + } + } + + /// Splits the given bundle into minimal bundles per Use, falling + /// back onto the spill bundle. This must work for any bundle no + /// matter how many conflicts. + /// + /// This is meant to solve a quadratic-cost problem that exists + /// with "normal" splitting as implemented above. With that + /// procedure, , splitting a bundle produces two + /// halves. Furthermore, it has cost linear in the length of the + /// bundle, because the resulting half-bundles have their + /// requirements recomputed with a new scan, and because we copy + /// half the use-list over to the tail end sub-bundle. + /// + /// This works fine when a bundle has a handful of splits overall, + /// but not when an input has a systematic pattern of conflicts + /// that will require O(|bundle|) splits (e.g., every Use is + /// constrained to a different fixed register than the last + /// one). In such a case, we get quadratic behavior. + /// + /// This method implements a direct split into minimal bundles + /// along the whole length of the bundle, putting the regions + /// without uses in the spill bundle. We do this once the number + /// of splits in an original bundle (tracked by spillset) reaches + /// a pre-determined limit. + /// + /// This basically approximates what a non-splitting allocator + /// would do: it "spills" the whole bundle to possibly a + /// stackslot, or a second-chance register allocation at best, via + /// the spill bundle; and then does minimal reservations of + /// registers just at uses/defs and moves the "spilled" value + /// into/out of them immediately. + pub fn split_into_minimal_bundles(&mut self, bundle: LiveBundleIndex, reg_hint: PReg) { + let mut removed_lrs: FxHashSet = FxHashSet::default(); + let mut removed_lrs_vregs: FxHashSet = FxHashSet::default(); + let mut new_lrs: SmallVec<[(VRegIndex, LiveRangeIndex); 16]> = smallvec![]; + let mut new_bundles: SmallVec<[LiveBundleIndex; 16]> = smallvec![]; + + let spillset = self.bundles[bundle].spillset; + let spill = self + .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true) + .unwrap(); + + trace!( + "Splitting bundle {:?} into minimal bundles with reg hint {}", + bundle, + reg_hint + ); + + let mut last_lr: Option = None; + let mut last_bundle: Option = None; + let mut last_inst: Option = None; + let mut last_vreg: Option = None; + + let mut spill_uses = UseList::new(); + + for entry in core::mem::take(&mut self.bundles[bundle].ranges) { + let lr_from = entry.range.from; + let lr_to = entry.range.to; + let vreg = self.ranges[entry.index].vreg; + + removed_lrs.insert(entry.index); + removed_lrs_vregs.insert(vreg); + trace!(" -> removing old LR {:?} for vreg {:?}", entry.index, vreg); + + let mut spill_range = entry.range; + let mut spill_starts_def = false; + + let mut last_live_pos = entry.range.from; + for u in core::mem::take(&mut self.ranges[entry.index].uses) { + trace!(" -> use {:?} (last_live_pos {:?})", u, last_live_pos); + + let is_def = u.operand.kind() == OperandKind::Def; + + // If this use has an `any` constraint, eagerly migrate it to the spill range. The + // reasoning here is that in the second-chance allocation for the spill bundle, + // any-constrained uses will be easy to satisfy. Solving those constraints earlier + // could create unnecessary conflicts with existing bundles that need to fit in a + // register, more strict requirements, so we delay them eagerly. + if u.operand.constraint() == OperandConstraint::Any { + trace!(" -> migrating this any-constrained use to the spill range"); + spill_uses.push(u); + + // Remember if we're moving the def of this vreg into the spill range, so that + // we can set the appropriate flags on it later. + spill_starts_def = spill_starts_def || is_def; + + continue; + } + + // If this is a def of the vreg the entry cares about, make sure that the spill + // range starts right before the next instruction so that the value is available. + if is_def { + trace!(" -> moving the spill range forward by one"); + spill_range.from = ProgPoint::before(u.pos.inst().next()); + } + + // If we just created a LR for this inst at the last + // pos, add this use to the same LR. + if Some(u.pos.inst()) == last_inst && Some(vreg) == last_vreg { + self.ranges[last_lr.unwrap()].uses.push(u); + trace!(" -> appended to last LR {:?}", last_lr.unwrap()); + continue; + } + + // The minimal bundle runs through the whole inst + // (up to the Before of the next inst), *unless* + // the original LR was only over the Before (up to + // the After) of this inst. + let to = core::cmp::min(ProgPoint::before(u.pos.inst().next()), lr_to); + + // If the last bundle was at the same inst, add a new + // LR to the same bundle; otherwise, create a LR and a + // new bundle. + if Some(u.pos.inst()) == last_inst { + let cr = CodeRange { from: u.pos, to }; + let lr = self.ranges.add(cr); + new_lrs.push((vreg, lr)); + self.ranges[lr].uses.push(u); + self.ranges[lr].vreg = vreg; + + trace!( + " -> created new LR {:?} but adding to existing bundle {:?}", + lr, + last_bundle.unwrap() + ); + // Edit the previous LR to end mid-inst. + self.bundles[last_bundle.unwrap()] + .ranges + .last_mut() + .unwrap() + .range + .to = u.pos; + self.ranges[last_lr.unwrap()].range.to = u.pos; + // Add this LR to the bundle. + self.bundles[last_bundle.unwrap()] + .ranges + .push(LiveRangeListEntry { + range: cr, + index: lr, + }); + self.ranges[lr].bundle = last_bundle.unwrap(); + last_live_pos = ProgPoint::before(u.pos.inst().next()); + continue; + } + + // Otherwise, create a new LR. + let pos = ProgPoint::before(u.pos.inst()); + let pos = core::cmp::max(lr_from, pos); + let cr = CodeRange { from: pos, to }; + let lr = self.ranges.add(cr); + new_lrs.push((vreg, lr)); + self.ranges[lr].uses.push(u); + self.ranges[lr].vreg = vreg; + + // Create a new bundle that contains only this LR. + let new_bundle = self.bundles.add(); + self.ranges[lr].bundle = new_bundle; + self.bundles[new_bundle].spillset = spillset; + self.bundles[new_bundle].ranges.push(LiveRangeListEntry { + range: cr, + index: lr, + }); + new_bundles.push(new_bundle); + + // If this use was a Def, set the StartsAtDef flag for the new LR. + if is_def { + self.ranges[lr].set_flag(LiveRangeFlag::StartsAtDef); + } + + trace!( + " -> created new LR {:?} range {:?} with new bundle {:?} for this use", + lr, + cr, + new_bundle + ); + + last_live_pos = ProgPoint::before(u.pos.inst().next()); + + last_lr = Some(lr); + last_bundle = Some(new_bundle); + last_inst = Some(u.pos.inst()); + last_vreg = Some(vreg); + } + + if !spill_range.is_empty() { + // Make one entry in the spill bundle that covers the whole range. + // TODO: it might be worth tracking enough state to only create this LR when there is + // open space in the original LR. + let spill_lr = self.ranges.add(spill_range); + self.ranges[spill_lr].vreg = vreg; + self.ranges[spill_lr].bundle = spill; + self.ranges[spill_lr].uses.extend(spill_uses.drain(..)); + new_lrs.push((vreg, spill_lr)); + + if spill_starts_def { + self.ranges[spill_lr].set_flag(LiveRangeFlag::StartsAtDef); + } + + self.bundles[spill].ranges.push(LiveRangeListEntry { + range: spill_range, + index: spill_lr, + }); + self.ranges[spill_lr].bundle = spill; + trace!( + " -> added spill range {:?} in new LR {:?} in spill bundle {:?}", + spill_range, + spill_lr, + spill + ); + } else { + assert!(spill_uses.is_empty()); + } + } + + // Remove all of the removed LRs from respective vregs' lists. + for vreg in removed_lrs_vregs { + self.vregs[vreg] + .ranges + .retain(|entry| !removed_lrs.contains(&entry.index)); + } + + // Add the new LRs to their respective vreg lists. + for (vreg, lr) in new_lrs { + let range = self.ranges[lr].range; + let entry = LiveRangeListEntry { range, index: lr }; + self.vregs[vreg].ranges.push(entry); + } + + // Recompute bundle properties for all new bundles and enqueue + // them. + for bundle in new_bundles { + if self.bundles[bundle].ranges.len() > 0 { + self.recompute_bundle_properties(bundle); + let prio = self.bundles[bundle].prio; + self.allocation_queue + .insert(bundle, prio as usize, reg_hint); + } + } + } + + /// If the given split point is within a loop, try to move it out of the + /// loop by moving it forward, up to the given limit. + pub fn adjust_split_point_forward(&self, mut split: ProgPoint, limit: ProgPoint) -> ProgPoint { + debug_assert!(split <= limit); + let orig_split = split; + while split != limit { + let block = self.cfginfo.insn_block[split.inst().index()]; + let next_outer_block = self.cfginfo.next_outer_loop[block.index()]; + if next_outer_block.is_invalid() { + break; + } + debug_assert!( + self.cfginfo.approx_loop_depth[next_outer_block.index()] + < self.cfginfo.approx_loop_depth[block.index()] + ); + + let new_split = self.cfginfo.block_entry[next_outer_block.index()]; + if new_split <= limit { + split = new_split; + } else { + break; + } + } + if split != orig_split { + trace!(" -> moving split point out of loop from {orig_split:?} to {split:?}"); + } + split + } + + /// If the given split point is within a loop, try to move it out of the + /// loop by moving it backward, up to the given limit. + pub fn adjust_split_point_backward(&self, mut split: ProgPoint, limit: ProgPoint) -> ProgPoint { + debug_assert!(split >= limit); + let orig_split = split; + while split != limit { + let block = self.cfginfo.insn_block[split.inst().index()]; + let prev_outer_block = self.cfginfo.prev_outer_loop[block.index()]; + if prev_outer_block.is_invalid() { + break; + } + debug_assert!( + self.cfginfo.approx_loop_depth[prev_outer_block.index()] + < self.cfginfo.approx_loop_depth[block.index()] + ); + + // Place the split before the terminator instruction of the block. + let new_split = + ProgPoint::before(self.cfginfo.block_exit[prev_outer_block.index()].inst()); + if new_split >= limit { + split = new_split; + } else { + break; + } + } + if split != orig_split { + trace!(" -> moving split point out of loop from {orig_split:?} to {split:?}"); + } + split + } + + /// When bundles are first created, they may have conflicting requirements, + /// for example two uses with different fixed registers. Additionally, + /// bundle merging may have introduced additional conflict. + /// + /// We handle all requirement conflicts upfront by splitting the bundle into + /// pieces which each have satisfiable requirements. + pub fn legalize_bundle_requirements(&mut self) { + // New bundles may be inserted while we iterate, but those are + // guaranteed to have satisfiable requirements. + for bundle in 0..self.bundles.len() { + self.split_bundle_for_requirements(LiveBundleIndex::new(bundle)); + } + } + + fn split_bundle_for_requirements(&mut self, bundle: LiveBundleIndex) { + // Nothing to do if the bundle has no fixed or stack constraints + if !self.bundles[bundle].cached_fixed() && !self.bundles[bundle].cached_stack() { + return; + } + + // Outer loop for each time we need to split the bundle. + 'outer: loop { + trace!("Checking bundle {bundle:?} for conflicting requirements"); + + // Iterate over the uses in the bundle in reverse order. + let mut last_fixed_reg = None; + let mut last_fixed_stack = None; + let mut last_stack = None; + let mut last_reg = None; + let mut requirement = Requirement::Any; + for u in self.bundles[bundle] + .ranges + .iter() + .flat_map(|entry| self.ranges[entry.index].uses.iter()) + .rev() + { + trace!(" -> use {:?}", u); + + // Check whether the new requirement from the use conflicts with + // existing requirements. + let new_req = self.requirement_from_operand(u.operand); + match new_req.merge(requirement) { + Ok(merged) => { + // No conflict! Keep track of the last seen use of each + // type and continue. + requirement = merged; + trace!(" -> req {requirement:?}"); + match new_req { + Requirement::FixedReg(_) => last_fixed_reg = Some(u.pos), + Requirement::FixedStack(_) => last_fixed_stack = Some(u.pos), + Requirement::Register => last_reg = Some(u.pos), + Requirement::Stack => last_stack = Some(u.pos), + Requirement::Any => {} + } + } + Err(_) => { + trace!(" -> conflict"); + + // Conflict! We need to split the bundle to make it + // allocatable. Find the previous conflicting use. + let potentially_conflicting = match new_req { + Requirement::FixedReg(_) => { + [last_fixed_reg, last_fixed_stack, last_stack] + } + Requirement::FixedStack(_) => { + [last_fixed_reg, last_fixed_stack, last_reg] + } + Requirement::Register => [last_fixed_stack, last_stack, None], + Requirement::Stack => [last_fixed_reg, last_fixed_stack, last_reg], + Requirement::Any => unreachable!(), + }; + let mut conflicting = None; + for pos in potentially_conflicting { + let Some(pos) = pos else { continue }; + match conflicting { + Some(prev) => conflicting = Some(core::cmp::min(prev, pos)), + None => conflicting = Some(pos), + } + } + let conflicting = conflicting.unwrap(); + + // Split the bundle between the 2 conflicting uses, + // while taking loop depth into account. + let start = ProgPoint::before(u.pos.inst().next()); + let end = ProgPoint::before(conflicting.inst()); + let split_at = self.adjust_split_point_backward(end, start); + let new_bundle = self.bundles.add(); + trace!(" -> creating new bundle {:?}", new_bundle); + self.bundles[new_bundle].spillset = self.bundles[bundle].spillset; + self.split_bundle_at(bundle, new_bundle, split_at, true); + + // Now that we've split off a conflicting use, continue + // the scan in the original bundle. + continue 'outer; + } + } + } + + // No conflicts found, we're done! + break; + } + } +}