package com.seaofnodes.simple.codegen; import com.seaofnodes.simple.node.*; import com.seaofnodes.simple.type.Type; import com.seaofnodes.simple.util.Ary; import com.seaofnodes.simple.util.Utils; import java.util.BitSet; import java.util.IdentityHashMap; // Interference Graph abstract public class IFG { // Interference Graph Support // Map from a Basic Block to Live-Out: {a map from a Live Range to a Def} private static final IdentityHashMap> BBOUTS = new IdentityHashMap<>(); static void resetBBLiveOut() { for( IdentityHashMap bbout : BBOUTS.values() ) bbout.clear(); } static final IdentityHashMap TMP = new IdentityHashMap<>(); // Inteference Graph: Array of Bitsets static final Ary IFG = new Ary<>(BitSet.class); static void resetIFG() { for( BitSet bs : IFG ) if( bs!=null ) bs.clear(); } // Set matching bit private static void addIFG( LRG lrg0, LRG lrg1 ) { short x0 = lrg0._lrg, x1 = lrg1._lrg; // Triangulate if( x0 < x1 ) _addIFG(x0,x1); else _addIFG(x1,x0); } // Add lrg1 to lrg0's conflict set private static void _addIFG( short x0, short x1 ) { BitSet ifg = IFG.atX(x0); if( ifg==null ) IFG.setX(x0,ifg = new BitSet()); ifg.set(x1); } private static final Ary WORK = new Ary<>(CFGNode.class); static void pushWork(CFGNode bb) { if( WORK.find(bb) == -1 ) WORK.push(bb); } // ------------------------------------------------------------------------ // Visit all blocks, using the live-out LRGs per-block and doing a backwards // walk over each block. At the end of the walk, push the live-out sets to // prior blocks. Due to loops, no single visitation order suffices. This // problem can be fairly efficiently solved with a proper LIVE calculation // and bitsets. In the interests of simplicity (and assumption of smaller // programs) I am skipping LIVE and directly computing it during the IFG // building. // Start by setting the live-out of the exit block (to empty), and putting // it on a worklist. // - Pull from worklist a block who's live-out has changed // - Walk backwards, adding interferences and computing live. // - At block head, "push" the new live-out to prior blocks. // - - If they pick up new live-outs, put them on the worklist. public static boolean build( int round, RegAlloc alloc ) { // Reset all to empty resetBBLiveOut(); resetIFG(); WORK.clear(); // Last block has nothing live out for( CFGNode bb : alloc._code._cfg ) if( bb.blockHead()) WORK.push( bb ); // Process blocks until no more changes while( !WORK.isEmpty() ) do_block(round,alloc,WORK.pop()); if( alloc.success() ) convert2DAdjacency(alloc); return alloc.success(); } // Walk one block backwards, compute live-in from live-out, and build IFG static void do_block( int round, RegAlloc alloc, CFGNode bb ) { assert bb.blockHead(); IdentityHashMap live_out = BBOUTS.get(bb); TMP.clear(); if( live_out != null ) TMP.putAll(live_out); // Copy bits to temp // A backwards walk over instructions in the basic block for( int inum = bb.nOuts()-1; inum >= 0; inum-- ) { Node n = bb.out(inum); if( n.in(0) != bb ) continue; // In a backwards walk, proj users come before the node itself if( n instanceof MultiNode ) for( Node proj : n.outs() ) if( proj instanceof ProjNode ) do_node(alloc,proj); if( n instanceof MachNode ) do_node(alloc,n); } // The bb head kills register, e.g. a CallEnd and caller-save registers if( bb instanceof MachNode mach ) kills(alloc,mach); // Push live-sets backwards to priors in CFG. if( bb instanceof RegionNode ) for( int i=1; i lrgs = BBOUTS.computeIfAbsent( bb, k -> new IdentityHashMap<>() ); for( LRG lrg : TMP.keySet() ) { Node def = TMP.get(lrg); // Effective def comes from phi input from prior block if( def instanceof PhiNode phi && phi.cfg0()==priorbb ) { assert i!=0; def = phi.in( i ); } // The synthetic exit carries no value on this Phi arm. if( def._type==Type.TOP ) continue; Node def_bb = lrgs.get(lrg); if( def_bb==null ) { lrgs.put(lrg,def); pushWork(bb); } else { // Alive twice with different definitions; self-conflict selfConflict(alloc,def,lrg,def_bb); } } } static void convert2DAdjacency( RegAlloc alloc ) { // Convert the 2-D array of bits (a 1-D array of BitSets) into an // adjacency matrix. int maxlrg = alloc._LRGS.length; for( int i=1; i=0; lrg=ifg.nextSetBit(lrg+1) ) { LRG lrg1 = alloc._LRGS[lrg]; lrg0.addNeighbor(lrg1); lrg1.addNeighbor(lrg0); } } } } // ------------------------------------------------------------------------ // Color the inference graph. // Coloring works by removing "trivial" live ranges from the IFG - those // live ranges with fewer neighbors than available colors (registers). // These are trivial because even if every neighbor takes a unique color, // there's at least one more available to color this live range. // If we hit a clique of non-trivial live ranges, we pick one to be "at // risk" of not-coloring. Good choices include live ranges with a large // area and few hot uses. // Then we reverse and put-back live ranges - picking a color as we go. // If there's no spare color we'll have to spill this at-risk live range. public static boolean color(int round, RegAlloc alloc) { int maxlrg = alloc._LRGS.length, nlrgs=0; for( int i=1; i swork ) swork = sptr; // Walk all neighbors and remove if( lrg._adj != null ) { for( LRG nlrg : lrg._adj ) { // Remove and compress out neighbor if( nlrg.removeCompress(lrg) ) { // Neighbor is just exactly going trivial as 'lrg' is removed from IFG // Find "j" position in the color_stack int jx = swork; while( color_stack[jx] != nlrg ) jx++; // Add trivial neighbor to trivial list. Pull lrg j out of // unknown set, since its now in the trivial set swap(color_stack, swork++, jx); } } } } // Reverse simplify (unstack the color stack), and set colors (registers) for live ranges while( sptr > 0 ) { LRG lrg = color_stack[--sptr]; if( lrg==null ) continue; RegMaskRW rmask = lrg._mask.copy(); // Walk neighbors and remove adjacent colors if( lrg._adj!=null ) { for( LRG nlrg : lrg._adj ) { nlrg.reinsert(lrg); int reg = nlrg._reg; if( reg!= -1 ) // Failed neighbors do not count rmask.clr(reg); // Remove neighbor from my choices } } // At-risk live-range did not color? if( rmask.isEmpty() ) { alloc.fail(lrg); lrg._reg = -1; } else { // Pick first available register short reg = rmask.firstReg(); // Pick a "good" color from the choices. Typically, biased-coloring // removes some over-spilling. if( rmask.size() > 1 ) reg = biasColor(alloc,lrg,reg,rmask); lrg._reg = reg; // Assign the register } } return alloc.success(); } // Pick LRG from color stack private static void pickColor(LRG[] color_stack, int sptr, int swork, RegAlloc alloc) { // Out of trivial colorable, pick an at-risk to pull if( sptr==swork ) swap(color_stack,sptr,pickRisky(color_stack,sptr,alloc)); // When coloring, we'd like to give more choices; so when coloring we'd // like to see the single-def first (since no choices anyway), then // non-split related (so more live ranges get colored), then // split-related last, so they have more colors to bias towards. // Working in reverse, pick first split-related with many regs, then // those with some regs, then single-def. int bidx=sptr; LRG best=color_stack[bidx]; for( int idx = sptr+1; idx < swork; idx++ ) if( betterLRG(best,color_stack[idx]) ) best = color_stack[bidx=idx]; if( bidx != sptr ) swap(color_stack,sptr,bidx); // Pick best at sptr } // Returns true if 'lrg' is better than 'best' private static boolean betterLRG( LRG best, LRG lrg ) { // If single-def varies, keep the not-single-def if( best.size1() != lrg.size1() ) return best.size1(); // If hasSplit varies, keep the hasSplit if( best.hasSplit() != lrg.hasSplit() ) return lrg.hasSplit(); // If both are single register (so both sizes are 1), // return a multi-use over a multi-def if( best.size1() ) return !best._multiUse && lrg._multiUse; // Keep large register count return best.size() < lrg.size(); } // Pick a live range that hasn't already spilled, or has a single-def- // single-use that are not adjacent. private static int pickRisky( LRG[] color_stack, int sptr, RegAlloc alloc ) { int best=sptr; int bestScore = pickRiskyScore(color_stack[best],alloc); for( int i=sptr+1; i bestScore ) { best = i; bestScore = iScore; } } return best; } // Pick a live range to pull, that might not color. // // Picking a live range with a very large span, with defs and uses outside // loops means spilling a relative cheap live range and getting that // register over a large area. // // Picking a live range that is very close to coloring might allow it to // color despite being risky. private static int pickRiskyScore( LRG lrg, RegAlloc alloc ) { // Callee-save live ranges cover the whole function and are cheap to // split. Rank scheduled area recovered against the loop-scaled cost // of the def- and use-side moves. if( lrg._machDef instanceof CalleeSaveNode ) return areaScore(functionArea(lrg._fun,alloc), splitCost((Node)lrg._machDef,(Node)lrg._machUse)); if( lrg._splitDef != null && lrg._splitDef. in(1) instanceof CalleeSaveNode && lrg._splitUse != null && lrg._splitUse.out(0) instanceof ReturnNode ) return areaScore(functionArea(lrg._fun,alloc), splitCost(lrg._splitDef,lrg._splitUse)); // Pick single-def clonables that are not right next to their single-use. // Rank by the area recovered by cloning closer to the use. if( !lrg._multiDef && lrg._machDef!=null && lrg._machUse!=null && lrg._machDef.isClone() ) { Node def = ((Node)lrg._machDef); Node use = ((Node)lrg._machUse); CFGNode cfg = def.cfg0(); int area = cfg != use.cfg0() ? functionArea(lrg._fun,alloc) : cfg._outputs.find(use)-cfg._outputs.find(def)-1; // A single-use function address can move next to its use without // adding an instruction. Prefer that to saving a callee-save // register just to hold the address. Stay in the same block and // require a gap so the clone makes progress. if( def instanceof FunPtrNode && !lrg._multiUse && cfg == use.cfg0() && area > 0 ) return 1000000; if( area > 0 ) return areaScore(area,loopCost(use)); } // Default for ranges without a specific area/cost estimate. return 1000 + (lrg._multiUse ? -100 : 0); } private static int functionArea(FunNode fun, RegAlloc alloc) { int area=0; for( CFGNode bb : alloc._code._cfg ) if( bb.fun()==fun ) area += bb.nOuts(); return area; } private static int areaScore(int area, int cost) { return 1000 + (area<<10)/cost; } private static int splitCost(Node def, Node use) { return loopCost(def) + (use==null ? 0 : loopCost(use)); } private static int loopCost(Node n) { return 1 << (n.cfg0().loopDepth()*3); } private static short biasColor( RegAlloc alloc, LRG lrg, short reg, RegMask mask ) { if( mask.size1() ) return reg; // Check chain of splits up the def-chain. Take first allocated // register, and if it's available in the mask, take it. Node def = lrg._splitDef, use = lrg._splitUse; int tidx=0, cnt=0; while( def != null || use != null ) { if( cnt++ > 10 ) break; if( def != null ) { short bias = biasColor( alloc, def, mask ); if( bias >= 0 ) return bias; // Good bias if( bias == -2 ) def = null; // Kill this side, no more searching else if( (tidx=biasable(def)) == 0 ) def = null; } if( use != null ) { short bias = biasColor( alloc, use, mask ); if( bias >= 0 ) return bias; // Good bias if( bias == -2 ) use = null; // Kill this side, no more searching else if( biasable(use)==0 ) use = null; } if( def != null ) { short bias = biasColorNeighbors( alloc, def, mask ); if( bias >= 0 ) return bias; // Advance def side def = def.in(tidx); if( alloc.lrg(def)==null ) def=null; } if( use != null ) { short bias = biasColorNeighbors( alloc, use, mask ); if( bias >= 0 ) return bias; use = use.out(0); if( biasable(use)==0 ) use=null; } } return mask.firstReg(); } private static int biasable(Node split) { if( split instanceof SplitNode ) return 1; // Yes biasable, advance is slot 1 if( split instanceof PhiNode phi ) return phi.region() instanceof LoopNode ? 2 : 1; // Prefer the backedge in loops if( !(split instanceof MachNode mach) ) return 0; // Not biasable return mach.twoAddress(); // Only biasable if 2-addr } // 3-way return: // - good bias reg, take it & exit // - this path is cutoff; do not search here anymore // - advance this side private static short biasColor( RegAlloc alloc, Node split, RegMask mask ) { short bias = alloc.lrg(split)._reg; if( bias != -1 ) { if( mask.test(bias) ) return bias; // Good bias else return -2; // Kill this side } else return -1; // Advance this side } // Check if we can match "split" color, or else trim "mask" to colors // "split" might get. private static short biasColorNeighbors( RegAlloc alloc, Node split, RegMask mask ) { LRG slrg = alloc.lrg(split); if( slrg._adj == null ) return -1; // No trimming // Can I limit my own choices to valid neighbor choices? for( LRG alrg : slrg._adj ) { short reg = alrg._reg; if( reg == -1 && alrg._mask.size1() ) reg = alrg._mask.firstReg(); if( reg != -1 ) { mask.clr(reg); if( mask.size1() ) return mask.firstReg(); } } return -1; // No obvious color choice, but mask got trimmed } private static void swap( LRG[] ary, int x, int y ) { LRG tmp = ary[x]; ary[x] = ary[y]; ary[y] = tmp; } }