diff options
Diffstat (limited to 'chapter-sccp/src/main/java/com/seaofnodes/simple/codegen')
20 files changed, 4206 insertions, 0 deletions
diff --git a/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/BuildLRG.java b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/BuildLRG.java new file mode 100644 index 00000000..2319714b --- /dev/null +++ b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/BuildLRG.java @@ -0,0 +1,95 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.Type; +import com.seaofnodes.simple.type.TypeMem; + +abstract public class BuildLRG { + // Compute live ranges in a single forwards pass. Every def is a new live + // range except live ranges are joined at Phi. Also, gather the + // intersection of register masks in each live range. Example: "free + // always zero" register forces register zero, and calling conventions + // typically force registers. + + // Sets FAILED to the set of hard-conflicts (means no need for an IFG, + // since it will not color, just split the conflicted ranges now). Returns + // true if no hard-conflicts, although we still might not color. + public static boolean run(int round, RegAlloc alloc) { + for( Node bb : alloc._code._cfg ) + for( Node n : bb.outs() ) { + // A synthetic never-exit value has no runtime definition. + if( n._type==Type.TOP ) continue; + if( n instanceof PhiNode phi && !(phi._type instanceof TypeMem) ) { + // All real Phi inputs end up with the same LRG. + // Pass 1: find any pre-existing LRG, to avoid make-then-Union a LRG + LRG lrg = alloc.lrg(phi); + if( lrg == null ) + for( int i=1; i<phi.nIns(); i++ ) + if( (lrg = alloc.lrg(phi.in(i))) != null ) + break; + // If none, make one. + if( lrg==null ) lrg = alloc.newLRG(n); + if( phi instanceof MachNode ) defLRG(alloc,n); + // Pass 2: everybody uses the same LRG + lrg=alloc.union(lrg,phi); + for( int i=phi instanceof ParmNode ? 2 : 1; i<n.nIns(); i++ ) + // TOP keeps the type merge neutral, but needs no register. + if( n.in(i)._type!=Type.TOP ) + lrg=alloc.union(lrg,n.in(i)); + if( lrg._mask!=null && lrg._mask.isEmpty() ) + alloc.fail(lrg); + + } else if( n instanceof MachNode mach ) { + // Attempt to commute ops to keep live ranges compatible. + if( mach.commutes() && n.nOuts()==1 ) { + int uidx = n.out(0)._inputs.find(n); + RegMask mask1 = n.in (1) instanceof MachNode machx ? machx.outregmap() : alloc.lrg(n.in(1))._mask; + RegMask mask2 = n.in (2) instanceof MachNode machx ? machx.outregmap() : alloc.lrg(n.in(2))._mask; + RegMask masko = n.out(0) instanceof MachNode machx ? machx.regmap(uidx): mach.outregmap(); + if( masko!=null && !mask1.overlap(masko) && mask2.overlap(masko) ) + n.swap12(); + } + + // Define live range + defLRG(alloc,n); + + // Now, look in the opposite direction. How are incoming + // LRGs affected by this node: For all uses, make live lrgs + for( int i=1; i<n.nIns(); i++ ) + if( n.in(i)!=null ) { + LRG lrg2 = alloc.lrg(n.in(i)); + if( lrg2 != null ) { // Anti-dep or other, no LRG + RegMask use_mask = mach.regmap(i); // use_mask is also null for anti-dep + if( use_mask!=null && !lrg2.machUse(mach,(short)i,use_mask.size1()).and(use_mask) ) + alloc.fail(lrg2); // Empty register mask, must split + } + } + + } + + // MultiNodes have projections which set registers + if( n instanceof MultiNode && !(n instanceof CFGNode) ) + for( Node proj : n.outs() ) + if( proj instanceof MachNode ) + defLRG(alloc,proj); + } + + // Collect live ranges + alloc.unify(); + + return alloc.success(); + } + + private static void defLRG( RegAlloc alloc, Node n ) { + MachNode mach = (MachNode)n; + RegMask def_mask = mach.outregmap(); + if( def_mask == null ) return; + LRG lrg = mach.twoAddress() == 0 + ? alloc.newLRG(n) // Define a new LRG for N + : alloc.lrg2(n,mach.twoAddress()); // Use the matching 2-adr input + // Record mask and mach + if( !lrg.machDef(mach,def_mask.size1()).and(def_mask) ) + alloc.fail(lrg); // Empty register mask, must split + } + +} diff --git a/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/Coalesce.java b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/Coalesce.java new file mode 100644 index 00000000..689ceceb --- /dev/null +++ b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/Coalesce.java @@ -0,0 +1,105 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.util.Ary; +import com.seaofnodes.simple.util.Utils; +import java.util.BitSet; +import java.util.IdentityHashMap; + +// Coalesce copies +abstract public class Coalesce { + + public static boolean coalesce( int round, RegAlloc alloc ) { + + // Walk all the splits, looking for coalesce chances + boolean progress = false; + for( CFGNode bb : alloc._code._cfg ) { + outer: + for( int j=0; j<bb.nOuts(); j++ ) { + Node n = bb.out(j); + if( !(n instanceof SplitNode) ) continue; + LRG v1 = alloc.lrg(n); + LRG v2 = alloc.lrg(n.in(1)); + if( v1 != v2 ) { + LRG ov1 = v1, ov2 = v2; + // Get the smaller neighbor count in v1 + if( v1.nadj() > v2.nadj() ) + { v1 = v2; v2 = alloc.lrg(n); } + int v2len = v2._adj==null ? 0 : v2._adj._len; + // See that they do not conflict (coalescing would make a self-conflict) + if( v1._adj!=null && v2._adj!=null && v1._mask.overlap(v2._mask) ) { + // Walk the smaller neighbors, and add to the larger. + // Check for direct conflicts along the way. + for( LRG v3 : v1._adj ) { + if( v3==v2 ) { + // Direct conflict, skip this LRG pair + v2._adj.setLen(v2len); + continue outer; + } + // Add missing LRGs from V1 to V2 + if( v2._adj.find(v3)== -1 ) + v2._adj.push(v3); + } + } + // Most constrained mask + RegMask mask = v1._mask==v2._mask ? v1._mask : v1._mask.copy().and(v2._mask); + // Check for capacity + if( v2.nadj() >= mask.size() ) { + // Fails capacity, will not be trivial colorable + if( v2._adj!=null ) v2._adj.setLen(v2len); + continue; + } + + // Union lrgs. + progress = true; + + // Since `n` is dying, remove from the LRGS BEFORE doing + // the union; this will let the union code keep the + // matching bits from the other side, preserving some + // info for biased coloring + if( ov1. _machDef==n ) ov1. _machDef = null; + if( ov1._splitDef==n ) ov1._splitDef = null; + if( ov2. _machUse==n ) ov2. _machUse = null; + if( ov2._splitUse==n ) ov2._splitUse = null; + // Keep larger adjacency list. + Ary<LRG> larger = v2._adj; + v2._adj = null; + + // Union lrgs. Swaps again, based on small lrg + LRG vWin = v1.union(v2); + vWin._adj = larger; + LRG vLose = v1.leader() ? v2 : v1; + + // LRGs below v2len were always in v2, but maybe not in v1. + // LRGs above v2len are in v1 but were not in v2. + + // Walk v2's neighbors, replacing vLose with vWin. + if( larger != null ) + for( LRG vn : larger ) { + int idx = vn._adj.find(vLose); + if( idx != -1 ) + vn._adj.del(idx); + if( vn._adj.find(vWin) == -1 ) + vn._adj.add(vWin); + } + } else { + + // Since `n` is dying, remove from the LRGS + if( v1. _machDef==n ) v1. _machDef=null; + if( v1. _machUse==n ) v1. _machUse=null; + if( v1._splitDef==n ) v1._splitDef=null; + if( v1._splitUse==n ) v1._splitUse=null; + } + + // Remove junk split + n.removeSplit(); + j--; + } + } + + if( progress ) + alloc.unify(); + + return true; + } +} diff --git a/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/CodeGen.java b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/CodeGen.java new file mode 100644 index 00000000..71b9e495 --- /dev/null +++ b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/CodeGen.java @@ -0,0 +1,393 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.graph.GraphObserver; + +import com.seaofnodes.simple.IterPeeps; +import com.seaofnodes.simple.Parser; +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.print.*; +import com.seaofnodes.simple.type.*; +import com.seaofnodes.simple.util.Ary; +import com.seaofnodes.simple.util.SB; +import com.seaofnodes.simple.util.Utils; +import java.io.IOException; +import java.util.*; + +@SuppressWarnings("unchecked") +public class CodeGen { + public GraphObserver<Node> _obs; + public static final String PORTS = "com.seaofnodes.simple.node.cpus"; + // Last created CodeGen as a global; used all over to avoid passing about a + // "context". + public static CodeGen CODE; + + public enum Phase { + Parse, // Build ideal IR from source + Iter, // Pessimistic peepholes + Opto, // Optimistic SCCP within each function + IterPostOpto, // Peepholes and inlining using SCCP results + FinalizeIR, // Finish function cleanup and layouts + TypeCheck, // Check the surviving ideal graph + LoopTree, // Discover loops and connect nonreturning paths + Select, // Select machine instructions + Schedule, // Global code motion + LocalSched, // Order instructions within blocks + RegAlloc, // Assign registers and stack slots + Encoding, // Encode instructions + Export; // Link in memory or write ELF + + } + public Phase _phase; + + // Run through target, skipping phases already completed. + public CodeGen driver(Phase target) { return driver(target,null,null); } + public CodeGen driver(Phase target, String cpu, String callingConv) { return driver(target,cpu,callingConv,0); } + public CodeGen driver(Phase target, String cpu, String callingConv, int dump) { + if( target.ordinal()>=Phase.RegAlloc.ordinal() && cpu==null && _mach==null ) + throw new IllegalArgumentException("Machine code requires a CPU target"); + if( target==Phase.Export ) throw new IllegalArgumentException("Use exportELF with an output file or null for memory linking"); + step(target, Phase.Parse, this::parse, dump); + step(target, Phase.Iter, this::iter, dump); + step(target, Phase.Opto, this::opto, dump); + step(target, Phase.IterPostOpto, this::iterPostOpto, dump); + step(target, Phase.FinalizeIR, this::finalizeIR, dump); + step(target, Phase.TypeCheck, this::typeCheck, dump); + step(target, Phase.LoopTree, this::loopTree, dump); + step(target, Phase.Select, () -> instSelect(cpu,callingConv), dump); + step(target, Phase.Schedule, this::GCM, dump); + step(target, Phase.LocalSched, this::localSched, dump); + step(target, Phase.RegAlloc, this::regAlloc, dump); + step(target, Phase.Encoding, this::encode, dump); + return this; + } + + private void step(Phase target, Phase next, Runnable work, int dump) { + if( next.ordinal()>target.ordinal() || (_phase!=null && _phase.ordinal()>=next.ordinal()) ) return; + Phase prior = _phase; + work.run(); + if( _phase!=prior && (dump & (1<<_phase.ordinal()))!=0 ) { + if( (dump & (1<<30))!=0 ) System.err.println("After "+_phase+":"); + System.err.println(this); + } + } + + public final long[] _times = new long[Phase.values().length]; + private CodeGen phase(Phase next, boolean ordered, Runnable work) { + assert ordered : "Cannot run "+next+" after "+_phase; + _phase = next; + long t0 = System.currentTimeMillis(); + try { work.run(); } + finally { _times[next.ordinal()] = System.currentTimeMillis()-t0; } + if( _obs!=null ) _obs.phase(next.name()); + return this; + } + + + // --------------------------- + // Compilation source code + public final String _src; + // Compile-time known initial argument type + public final TypeInteger _arg; + + // --------------------------- + public CodeGen( String src ) { this(src, TypeInteger.BOT, 123L, true ); } + public CodeGen( String src, TypeInteger arg) { this(src, arg==null ? TypeInteger.BOT : arg, 123L, true ); } + + public CodeGen( String src, TypeInteger arg, long workListSeed, boolean reset ) { + CODE = this; + if( reset ) Type.reset(); + _main = makeFun(TypeFunPtr.MAIN); + _phase = null; + _callingConv = null; + _start = new StartNode(arg); + _stop = new StopNode(src); + _src = src; + _arg = arg; + _iter = new IterPeeps(workListSeed); + P = new Parser(this,arg); + } + + + // Run all the phases through final ELF emission + public CodeGen driver( String cpu, String callingConv, String obj ) throws IOException { + return driver(Phase.Encoding,cpu,callingConv).exportELF(obj); + } + + + // --------------------------- + /** + * A counter, for unique node id generation. Starting with value 1, to + * avoid bugs confusing node ID 0 with uninitialized values. + */ + public int _uid = 1; + public int UID() { return _uid; } + public int getUID() { + return _uid++; + } + + // Next available memory alias number + private int _alias = 2; // 0 is for control, 1 for memory + public int getALIAS() { return _alias++; } + + + // Inlining can insert CFG edges and invalidate cached depth ordering. + private char _iDepthVersion; + public char iDepthVersion() { return _iDepthVersion; } + public void invalidateIDepthCaches() { + assert _iDepthVersion < Character.MAX_VALUE : "Dominator cache version exceeds 65535"; + ++_iDepthVersion; + } + + // Popular visit bitset, declared here, so it gets reused all over + public final BitSet _visit = new BitSet(); + public BitSet visit() { assert _visit.isEmpty(); return _visit; } + + // Start and stop; end points of the generated IR + public StartNode _start; + public StopNode _stop; + + // Global Value Numbering. Hash over opcode and inputs; hits in this table + // are structurally equal. + public final HashMap<Node,Node> _gvn = new HashMap<>(); + + // Source of unique function indices + private int _fidx =0; + public TypeFunPtr makeFun( TypeFunPtr fun ) { + int fidx = _fidx++; + assert fidx<64; // TODO: need a larger FIDX space + return fun.makeFrom(fidx); + } + // Signature for MAIN + public TypeFunPtr _main; + public void setMain(FunNode main) { + _main = main.sig(); + link(main); + } + // Reverse from a constant function pointer to the IR function being called. + // Error to call with a non-constant TFP + public FunNode link( TypeFunPtr tfp ) { return link(tfp.fidx()); } + // Read-only diagnostic lookup; preserve stale entries. + public FunNode _link(TypeFunPtr tfp) { return _link(tfp.fidx()); } + public FunNode _link(int fidx) { return _linker.atX(fidx); } + // Return the FunNode from a fidx, cleaning up dead entries. + public FunNode link( int fidx ) { + FunNode fun =_linker.atX(fidx); + if( fun!=null && fun.isDead() ) { + _linker.setX(fidx,null); fun = null; } + return fun; + } + + // Insert linker mapping from constant function signature to the function + // being called. + public void link(FunNode fun) { + int fidx = fun.sig().fidx(); + _linker.setX(fidx,fun); + } + + // "Linker" mapping from constant TypeFunPtrs to heads of function. These + // TFPs all have exact single fidxs and their return is wiped to BOTTOM (so + // the return is not part of the match). + final Ary<FunNode> _linker = new Ary<>(FunNode.class); + + // Parser object + public final Parser P; + + // Parse ASCII text into Sea-of-Nodes IR + public CodeGen parse() { + return phase(Phase.Parse, _phase==null, () -> { + P.parse(); + }); + } + + + // --------------------------- + // Iterator peepholes. + public final IterPeeps _iter; + // Stop tracking deps while assertin + public boolean _midAssert; + // Statistics on peepholes + public int _iter_cnt, _iter_nop_cnt; + // Stat gathering + public void iterCnt() { if( !_midAssert ) _iter_cnt++; } + public void iterNop() { if( !_midAssert ) _iter_nop_cnt++; } + + // Pessimistic peepholes reach a fixed point after parsing. + public CodeGen iter() { return phase(Phase.Iter, _phase==Phase.Parse, () -> _iter.iterate(this)); } + + // SCCP computes types and queues changed nodes; Iter applies their rewrites. + public CodeGen opto() { return phase(Phase.Opto, _phase==Phase.Iter, () -> Opto.opto(this)); } + public CodeGen iterPostOpto() { return phase(Phase.IterPostOpto, _phase==Phase.Opto, () -> _iter.iterate(this)); } + + public CodeGen finalizeIR() { return phase(Phase.FinalizeIR, _phase==Phase.IterPostOpto, () -> FinalizeIR.run(this)); } + public <N extends Node> N add( N n ) { return _iter.add(n); } + public void addAll( Ary<Node> ary ) { _iter.addAll(ary); } + + // --------------------------- + // Last check for bad programs + public CodeGen typeCheck() { return phase(Phase.TypeCheck, _phase==Phase.FinalizeIR, () -> TypeCheck.run(this)); } + + // --------------------------- + // Build the loop tree; break never-exit loops + public CodeGen loopTree() { + return phase(Phase.LoopTree, _phase==Phase.TypeCheck, () -> { + // Build the loop tree, fix never-exit loops + _start.buildLoopTree(_start,_stop); + }); + } + + // --------------------------- + // Code generation CPU target + public Machine _mach; + // Chosen calling convention (usually either Win64 or SystemV) + public String _callingConv; + // Callee save registers + public RegMask _callerSave; + + // All returns have the following inputs: + // 0 - ctrl + // 1 - memory + // 2 - varies with returning GPR,FPR + // 3 - RPC + // 4+- Caller save registers + public RegMask[] _retMasks; + // Return Program Counter + public RegMask _rpcMask; + + // Convert to target hardware nodes + public CodeGen instSelect( String cpu, String callingConv ) { return instSelect(cpu,callingConv,PORTS); } + public CodeGen instSelect( String cpu, String callingConv, String base ) { + if( cpu==null ) return this; + return phase(Phase.Select, _phase==Phase.LoopTree, + () -> new InstSelect(this).run(cpu,callingConv,base)); + } + + + // --------------------------- + // Control Flow Graph in Reverse Post Order. + public Ary<CFGNode> _cfg = new Ary<>(CFGNode.class); + + // Global schedule (code motion) nodes + public CodeGen GCM() { + return phase(Phase.Schedule, (_phase==Phase.LoopTree || _phase==Phase.Select), () -> { + GlobalCodeMotion.buildCFG(this); + + }); + } + + // --------------------------- + // Local (basic block) scheduler phase, a classic list scheduler + public CodeGen localSched() { + return phase(Phase.LocalSched, _phase==Phase.Schedule, () -> ListScheduler.sched(this)); + } + + + // --------------------------- + // Register Allocation + public RegAlloc _regAlloc; + public CodeGen regAlloc() { + return phase(Phase.RegAlloc, _phase==Phase.LocalSched, () -> { + _regAlloc = new RegAlloc(this); + _regAlloc.regAlloc(); + }); + } + + // Human readable register name + public String reg(Node n) { return reg(n,null); } + public String reg(Node n, FunNode fun) { + if( _phase.ordinal() >= Phase.RegAlloc.ordinal() ) { + String s = _regAlloc.reg(n,fun); + if( s!=null ) return s; + } + return "N"+ n._nid; + } + + + // --------------------------- + // Encoding + public Encoding _encoding; // Encoding object + public void preEncode() { } // overridden by alternative ports + public CodeGen encode() { + return phase(Phase.Encoding, _phase==Phase.RegAlloc, () -> { + _encoding = new Encoding(this); + preEncode(); + _encoding.encode(); + }); + } + + // --------------------------- + // Exporting to external formats + public CodeGen exportELF(String fname) throws IOException { + assert _phase==Phase.Encoding; + _phase=Phase.Export; + long t0=System.currentTimeMillis(); + try { + if( fname == null ) new LinkMem(this).link(); // In memory patching + else new ElfFile(this).export(fname); // External ELF file + } finally { _times[Phase.Export.ordinal()]=System.currentTimeMillis()-t0; } + if( _obs!=null ) _obs.phase(Phase.Export.name()); + return this; + } + + // --------------------------- + public boolean _asmLittle=true; + SB asm(SB sb) { return ASMPrinter.print(sb,this); } + public String asm() { return asm(new SB()).toString(); } + + + // Testing shortcuts + public Node ctrl() { return _stop.ret().ctrl(); } + public Node expr() { return _stop.ret().expr(); } + public String print() { return _stop.print(); } + + // Debugging helper + @Override public String toString() { + return _phase.ordinal() > Phase.Schedule.ordinal() + ? IRPrinter._prettyPrint( this ) + : IRPrinter.prettyPrint(this); + } + + // Debugging helper + public Node f(int idx) { return _stop.find(idx); } + + + String printCFG() { + if( _cfg==null ) return "no CFG"; + SB sb = new SB(); + for( CFGNode cfg : _cfg ) { + sb.fix(8,""+(int)cfg._idepth); + IRPrinter.printLine( cfg, sb ); + } + return sb.toString(); + } + + public static void print_as_hex(Encoding enc) { + for (byte b : enc._bits.toByteArray()) { + System.out.print(String.format("%02X", b)); + } + System.out.println(); + } + + public void print_as_hex() { + for (byte b : _encoding._bits.toByteArray()) { + System.out.print(String.format("%02X", b)); + } + System.out.println(); + } + + //// Debug purposes for now +// public CodeGen printENCODING() { +// ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); +// +// for(Node bb : CodeGen.CODE._cfg) { +// for(Node n: bb.outs()) { +// if(n instanceof MachNode) { +// ((MachNode) n).encoding(E); +// } +// } +// } +// +// print_as_hex(outputStream); +// // Get the raw bytes from the output stream +// return this; +// } +} diff --git a/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/ElfExpr.java b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/ElfExpr.java new file mode 100644 index 00000000..9d9b8a1c --- /dev/null +++ b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/ElfExpr.java @@ -0,0 +1,87 @@ +package com.seaofnodes.simple.codegen; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class ElfExpr { + public static void main(String[] args) { + try (FileOutputStream fos = new FileOutputStream("hello.o")) { + ByteBuffer buffer = ByteBuffer.allocate(512); + buffer.order(ByteOrder.LITTLE_ENDIAN); + + // === ELF HEADER (64 bytes) === + buffer.put(new byte[]{ + 0x7F, 'E', 'L', 'F', // ELF Magic + 2, 1, 1, 0, // 64-bit, little-endian, version + 0, 0, 0, 0, 0, 0, 0, // ABI padding + 2, 0, // Type: Executable (ET_EXEC) + 0x3E, 0x00, // Machine: x86-64 + 1, 0, 0, 0, // ELF version + 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Entry point + 0, 0, 0, 0, 0, 0, 0, 0, // Program header offset + 64, 0, 0, 0, 0, 0, 0, 0, // Section header offset + 0, 0, 0, 0, // Flags + 64, 0, // ELF header size + 0, 0, // Program header entry size + 0, 0, // Number of program header entries + 40, 0, // Section header entry size + 2, 0, // Number of section headers + 1, 0 // Section header string table index + }); + + // === .TEXT SECTION (Executable Code) === + byte[] textSection = { + // sys_write (write(1, message, 13)) + (byte) 0x48, (byte) 0xC7, (byte) 0xC0, 0x01, 0x00, 0x00, 0x00, // mov rax, 1 + (byte) 0x48, (byte) 0xC7, (byte) 0xC7, 0x01, 0x00, 0x00, 0x00, // mov rdi, 1 + (byte) 0x48, (byte) 0xBE, (byte)0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // mov rsi, message + (byte) 0x48, (byte) 0xC7, (byte) 0xC2, 0x0D, 0x00, 0x00, 0x00, // mov rdx, 13 + (byte) 0x0F, (byte) 0x05, // syscall + + // sys_exit (exit(0)) + (byte) 0x48, (byte) 0xC7, (byte) 0xC0, 0x3C, 0x00, 0x00, 0x00, // mov rax, 60 + (byte) 0x48, (byte) 0x31, (byte) 0xFF, // xor rdi, rdi + (byte) 0x0F, (byte) 0x05 // syscall + }; + + // Append section data + buffer.position(256); + buffer.put(textSection); + + // === STRING MESSAGE (.data Section) === + byte[] message = "Hello, World!\n".getBytes(); + buffer.position(400); + buffer.put(message); + + // === SECTION HEADERS === + buffer.position(64); + + // Null Section (required as first entry) + buffer.put(new byte[40]); + + // .text Section + buffer.putInt(1); // Offset in section string table (".text") + buffer.putInt(1); // Type: SHT_PROGBITS + buffer.putLong(6); // Flags: Executable | Allocatable + buffer.putLong(0); // Virtual address + buffer.putLong(256); // Offset to section data + buffer.putLong(textSection.length); // Section size + buffer.putInt(0); // Link + buffer.putInt(0); // Info + buffer.putLong(4); // Address alignment + buffer.putLong(0); // No extra entries + + // === SECTION HEADER STRING TABLE === + buffer.position(500); + buffer.put(".text".getBytes()); + buffer.put((byte) 0x00); // Null terminator + + fos.write(buffer.array()); + System.out.println("ELF object file 'hello.o' created."); + } catch (IOException e) { + e.printStackTrace(); + } + } +} diff --git a/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/ElfFile.java b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/ElfFile.java new file mode 100644 index 00000000..b6267600 --- /dev/null +++ b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/ElfFile.java @@ -0,0 +1,382 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.*; +import com.seaofnodes.simple.codegen.Encoding.BAOS; +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.*; +import com.seaofnodes.simple.util.Ary; +import java.io.*; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.HashMap; +import java.util.Map; + +public class ElfFile { + + final CodeGen _code; + Ary<Section> _sections = new Ary<>(Section.class); + + ElfFile( CodeGen code ) { _code = code; } + + public static int writeCString(DataSection strtab, String name) { + // place name in the string table + int pos = strtab._contents.size(); + byte[] name_bytes = name.getBytes(); + strtab._contents.write(name_bytes, 0, name_bytes.length); + // the strings are null terminated + strtab._contents.write(0); + return pos; + } + + public static void write4( ByteArrayOutputStream bits, int op ) { + bits.write(op ); + bits.write(op>> 8); + bits.write(op>>16); + bits.write(op>>24); + } + public static void write8( ByteArrayOutputStream bits, long i64 ) { + write4(bits, (int) i64 ); + write4(bits, (int)(i64>>32)); + } + + public static final int SYMBOL_SIZE = 4+1+1+2+8+8; + + public static final int SYM_BIND_LOCAL = 0; + public static final int SYM_BIND_GLOBAL = 1; + public static final int SYM_BIND_WEAK = 2; + + public static final int SYM_TYPE_NOTYPE = 0; + public static final int SYM_TYPE_OBJECT = 1; + public static final int SYM_TYPE_FUNC = 2; + public static final int SYM_TYPE_SECTION = 3; + + public static class Symbol { + String _name; + int _name_pos; + + // index in the symbol table + int _index; + + // which section is it in + int _parent; + + // depends on the symbol type + // TODO + int _value, _size; + + // top 4bits are the "bind", bottom 4 are "type" + int _info; + + // bind 1011 + // 00001111 + // 10110000 + + Symbol(String name, int parent, int bind, int type) { + _name = name; + _parent = parent; + _info = (bind << 4) | (type & 0xF); + } + + void writeHeader(ByteBuffer out) { + out.putInt(_name_pos); // name + out.put((byte) _info); // info + // default visibility + out.put((byte) 0); // other + out.putShort((short) _parent); // shndx + out.putLong(_value); // value + out.putLong(_size); // size + } + } + + public static final int SHF_WRITE = 1; + public static final int SHF_ALLOC = 2; + public static final int SHF_EXECINSTR = 4; + public static final int SHF_STRINGS = 32; + public static final int SHF_INFO_LINK = 64; + + public static final int SECTION_HDR_SIZE = 64; + abstract public static class Section { + String _name; + int _index; + + int _type, _flags; + int _link, _info; + + // where's the name in the string table + int _name_pos; + int _file_offset; + + Section(String name, int type) { + _name = name; + _type = type; + } + + void writeHeader(ByteBuffer out) { + out.putInt(_name_pos); // name + out.putInt(_type); // type + out.putLong(_flags); // flags + out.putLong(0); // addr + out.putLong(_file_offset); // offset + out.putLong(size()); // size + out.putInt(_link); // link + out.putInt(_info); // info + out.putLong(16); // addralign + if (_type == 2) { + out.putLong(SYMBOL_SIZE);// entsize + } else if (_type == 4) { + out.putLong(24); // entsize + } else { + out.putLong(0); // entsize + } + } + + abstract void write(ByteBuffer out); + abstract int size(); + } + + public static class SymbolSection extends Section { + Ary<Symbol> _symbols = new Ary<>(Symbol.class); + Ary<Symbol> _loc = new Ary<>(Symbol.class); + + SymbolSection(String name, int type) { + super(name, type); + } + + @Override + void writeHeader(ByteBuffer out) { + // points to string table section + _link = 1; + + // number of non-local symbols + _info = _loc.len() + 1; + + super.writeHeader(out); + } + + void push(Symbol s) { + if ((s._info >> 4) != SYM_BIND_LOCAL) { + _symbols.push(s); + } else { + _loc.push(s); + } + } + + @Override + void write(ByteBuffer out) { + // Index 0 both designates the first entry in the table and serves as the undefined symbol index + for( int i = 0; i < SYMBOL_SIZE/4; i++ ) { + out.putInt(0); + } + // index is already set + for( Symbol s : _loc ) { + s.writeHeader(out); + } + for( Symbol s : _symbols ) { + s.writeHeader(out); + } + } + + @Override + int size() { + return (1 + _symbols.len() + _loc.len()) * SYMBOL_SIZE; + } + } + + public static class DataSection extends Section { + ByteArrayOutputStream _contents; + + DataSection(String name, int type) { + super(name, type); + _contents = new ByteArrayOutputStream(); + } + + DataSection(String name, int type, ByteArrayOutputStream contents) { + super(name, type); + _contents = contents; + } + + @Override + void write(ByteBuffer out) { + out.put(_contents.toByteArray()); + } + + @Override + int size() { + return _contents.size(); + } + } + + private void pushSection(Section s) { + _sections.push(s); + // ELF is effectively "1-based", due to the NULL section + s._index = _sections._len; + } + + /* creates function and stores where it starts*/ + private final HashMap<TypeFunPtr,Symbol> _funcs = new HashMap<>(); + private void encodeFunctions(SymbolSection symbols, DataSection text) { + for( int i=0; i<_code._cfg._len; i++ ) { + if( !(_code._cfg.at(i) instanceof FunNode fun) ) continue; + // skip until the function ends + while( !(_code._cfg.at(i) instanceof ReturnNode ret) ) + i++; + int end = _code._encoding._opStart[ret._nid] + _code._encoding._opLen[ret._nid]; + + if( fun._name==null ) continue; // Anonymous addresses are relocated within .text. + Symbol func = new Symbol(fun._name, text._index, SYM_BIND_GLOBAL, SYM_TYPE_FUNC); + func._value = _code._encoding._opStart[fun._nid]; + func._size = end - func._value; + symbols.push(func); + _funcs.put(fun.sig(), func); + } + } + + public void export(String fname) throws IOException { + DataSection strtab = new DataSection(".strtab", 3 /* SHT_SYMTAB */); + // first byte is reserved for an empty string + strtab._contents.write(0); + pushSection(strtab); + + // place all the symbols + SymbolSection symbols = new SymbolSection(".symtab", 2 /* SHT_STRTAB */); + pushSection(symbols); + + // zero flag by default + // we've already constructed this entire section in the encoding phase + DataSection text = new DataSection(".text", 1 /* SHT_PROGBITS */, _code._encoding._bits); + text._flags = SHF_ALLOC | SHF_WRITE | SHF_EXECINSTR; + pushSection(text); + + // Build and write constant pool + BAOS cpool = new BAOS(); + Encoding enc = _code._encoding; + enc.writeConstantPool(cpool,false); + DataSection rdata = new DataSection(".rodata", 1 /* SHT_PROGBITS */, cpool); + rdata._flags = SHF_ALLOC; + pushSection(rdata); + + // populate function symbols + encodeFunctions(symbols, text); + + int idx = 1; + for( Section s : _sections ) { + Symbol sym = new Symbol(s._name, idx++, SYM_BIND_LOCAL, SYM_TYPE_SECTION); + // we can reuse the same name pos from the actual section + sym._name_pos = s._name_pos; + sym._size = s.size(); + symbols.push(sym); + } + + // calculate local index + int num = 1; + for( Symbol s : symbols._loc ) + s._index = num++; + // extra space for .rela.text + int start_global = num+1; // Add one to skip the final .rela.text local symbol + for( Symbol a: symbols._symbols ) + a._index = start_global++; + + // create .text relocations + DataSection text_rela = new DataSection(".rela.text", 4 /* SHT_RELA */); + for( Node n : enc._externals.keySet()) { + int nid = n._nid; + String extern = enc._externals.get(n); + + Symbol sym = new Symbol(extern, 0, SYM_BIND_GLOBAL, SYM_TYPE_NOTYPE); + sym._index = start_global++; + symbols.push(sym); + + int offset = enc._opStart[nid] + enc._opLen[nid] - 4; + + // u64 offset + write8(text_rela._contents, offset); + // u64 info + write8(text_rela._contents, ((long)sym._index << 32L) | 4L /* PLT32 */); + // i64 addend + write8(text_rela._contents, -4); + } + + // Write relocations for the constant pool + for( Encoding.Relo relo : enc._bigCons.values() ) { + Symbol glob = new Symbol("CPOOL$"+start_global, rdata._index, SYM_BIND_GLOBAL, SYM_TYPE_FUNC); + glob._value = relo._target; + glob._size = relo._t.size(); + glob._index = start_global++; + symbols.push(glob); + write8(text_rela._contents, relo._opStart+relo._off); + write8(text_rela._contents, ((long)glob._index << 32L) | relo._elf ); + write8(text_rela._contents, -4); + } + + text_rela._flags = SHF_INFO_LINK; + text_rela._link = 2; + text_rela._info = text._index; + pushSection(text_rela); + + // Final .rela.text symbol + Symbol sym = new Symbol(text_rela._name, num++, SYM_BIND_LOCAL, SYM_TYPE_SECTION); + sym._name_pos = text_rela._name_pos; + sym._size = text_rela.size(); + symbols.push(sym); + + // populate string table + for( Section s : _sections ) { s._name_pos = writeCString(strtab, s._name); } + for( Symbol s : symbols._symbols ) { s._name_pos = writeCString(strtab, s._name); } + for( Symbol s : symbols._loc ) { s._name_pos = writeCString(strtab, s._name); } + + int size = 64; // ELF header + // size of all section data + for( Section s : _sections ) { + s._file_offset = size; + size += s.size(); + } + // headers go at the very end + int shdr_offset = size; + // there's a "null" section at the start of the section headers + size += SECTION_HDR_SIZE*(1 + _sections.len()); + + // System.out.printf("Hello %d, %d, %s\n", size, shdr_offset); + ByteBuffer out = ByteBuffer.allocate(size); + out.order(ByteOrder.LITTLE_ENDIAN); + + // ELF header + out.put(new byte[]{ 0x7F, 'E', 'L', 'F', 2, 1, 1, 0 }); + out.put(new byte[8]); + out.putShort((short)1 /* ET_REL */); // type + out.putShort((short)62 /* AMD64 */); // machine + out.putInt(1); // version + out.putLong(0); // entry + out.putLong(0); // phoff + out.putLong(shdr_offset); // shoff + out.putInt(0); // flags + out.putShort((short)64); // ehsize + out.putShort((short)0); // phentsize + out.putShort((short)0); // phnum + out.putShort((short)64); // shentsize + out.putShort((short) (1 + _sections.len())); // shnum + out.putShort((short)1); // shstrndx + + // raw section data + for( Section s : _sections ) { + s.write(out); + } + + // "null" section + for( int i = 0; i < 16; i++ ) { + out.putInt(0); + } + + for( Section s : _sections ) { + s.writeHeader(out); + } + assert out.position() == size; + + File file = new File(fname); + if( file.getParentFile()!=null ) + file.getParentFile().mkdirs(); + BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file)); + bos.write(out.array()); + bos.close(); + } + +} diff --git a/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/Encoding.java b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/Encoding.java new file mode 100644 index 00000000..b5a8e148 --- /dev/null +++ b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/Encoding.java @@ -0,0 +1,539 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.print.IRPrinter; +import com.seaofnodes.simple.type.*; +import com.seaofnodes.simple.util.Ary; +import com.seaofnodes.simple.util.SB; +import com.seaofnodes.simple.util.Utils; +import java.io.ByteArrayOutputStream; +import java.util.*; + + +/** + * Instruction encodings + * <p> + * This class holds the encoding bits, plus the relocation information needed + * to move the code to a non-zero offset, or write to an external file (ELF). + * <p> + * There are also a bunch of generic utilities for managing bits and bytes + * common in all encodings. + */ +public class Encoding implements com.seaofnodes.isa.CodeSink { + + // Top-level program graph structure + public final CodeGen _code; + + // Instruction bytes. The registers are encoded already. Relocatable data + // is in a fixed format depending on the kind of relocation. + + // - RIP-relative offsets into the same chunk are just encoded with their + // check relative offsets; no relocation info is required. + + // - Fixed address targets in the same encoding will be correct for a zero + // base; moving the entire encoding requires adding the new base address. + + // - RIP-relative to external chunks have a zero offset; the matching + // relocation info will be used to patch the correct value. + + public static class BAOS extends ByteArrayOutputStream { + public byte[] buf() { return buf; } + void set( byte[] buf0, int count0 ) { buf=buf0; count=count0; } + }; + final public BAOS _bits = new BAOS(); + + public int [] _opStart; // Start of opcodes, by _nid + public byte[] _opLen; // Length of opcodes, by _nid + + // Function headers now padded when printing + public boolean _padFunHeads; + + // Big Constant relocation info. + public static class Relo { + public final Node _op; + public final Type _t; // Constant type + public final int _align; // Log alignment, captured for diagnostic display + public final int _structSize; // Capture layout during encoding, not printing + public final byte _off; // Offset from start of opcode + public final byte _elf; // ELF relocation type, e.g. 2/PC32 + public int _target; // Where constant is finally placed + public int _opStart; // Opcode start + Relo( Node op, Type t, byte off, byte elf ) { + _op=op; _t=t; _off=off; _elf=elf; + _align = t.alignment(); + _structSize = t instanceof TypeStruct ts ? ts.size() : 0; + } + } + public final HashMap<Node,Relo> _bigCons = new HashMap<>(); + + Encoding( CodeGen code ) { _code = code; } + + // Shortcut to the defining register + public short reg(Node n) { + return _code._regAlloc.regnum(n); + } + + public int read1(int idx) { + byte[] buf = _bits.buf(); + return (buf[idx]&0xFF); + } + public int read2(int idx) { + byte[] buf = _bits.buf(); + return (buf[idx]&0xFF) | (buf[idx+1]&0xFF) <<8; + } + public int read4(int idx) { + byte[] buf = _bits.buf(); + return (buf[idx]&0xFF) | (buf[idx+1]&0xFF) <<8 | (buf[idx+2]&0xFF)<<16 | (buf[idx+3]&0xFF)<<24; + } + public long read8(int idx) { + return (read4(idx) & 0xFFFFFFFFL) | read4(idx+4); + } + + // This buffer is invalid/moving until after all encodings are written + public byte[] bits() { return _bits.buf(); } + + // 4 byte little-endian write + public void patch4( int idx, int val ) { + byte[] buf = _bits.buf(); + buf[idx ] = (byte)(val ); + buf[idx+1] = (byte)(val>> 8); + buf[idx+2] = (byte)(val>>16); + buf[idx+3] = (byte)(val>>24); + } + + static void padN(int n, BAOS bits) { + while( (bits.size()+n-1 & -n) > bits.size() ) + bits.write(0); + } + + // Convenience for writing log-N + static void addN( int log, Type t, BAOS bits ) { + long x = t instanceof TypeInteger ti + ? ti.value() + : log==3 + ? Double.doubleToRawLongBits( ((TypeFloat)t).value()) + : Float.floatToRawIntBits((float)((TypeFloat)t).value()); + addN(log,x,bits); + } + static void addN( int log, long x, BAOS bits ) { + for( int i=0; i < 1<<log; i++ ) { + bits.write((int)x); + x >>= 8; + } + } + + public Encoding add1( int op ) { addN(0,op,_bits); return this; } + public Encoding add2( int op ) { addN(1,op,_bits); return this; } + public Encoding add4( int op ) { addN(2,op,_bits); return this; } + public Encoding add8(long op ) { addN(3,op,_bits); return this; } + + + // Nodes need "relocation" patching; things done after code is placed. + // Record src and dst Nodes. + private final HashMap<Node,CFGNode> _internals = new HashMap<>(); + // Source is a Call, destination in the Fun. + public Encoding relo( CallNode call ) { + _internals.put(call,_code.link(call.tfp())); + return this; + } + public Encoding relo( ConstantNode con ) { + TypeFunPtr tfp = (TypeFunPtr)con._con; + _internals.put(con,_code.link(tfp)); + return this; + } + public void jump( CFGNode jmp, CFGNode dst ) { + _internals.put(jmp,dst.uctrlSkipEmpty()); + } + + + final HashMap<Node,String> _externals = new HashMap<>(); + public Encoding external( Node n, String extern ) { + _externals.put(n,extern); + return this; + } + + // Store t as a 32/64 bit constant in the code space; generate RIP-relative + // addressing to load it + public void largeConstant( Node relo, Type t, int off, int elf ) { + assert t.isConstant(); + assert (byte)off == off; + assert (byte)elf == elf; + _bigCons.put(relo,new Relo(relo,t,(byte)off,(byte)elf)); + } + + // -------------------------------------------------- + void encode() { + // Basic block layout: negate branches to keep blocks in-order; insert + // unconditional jumps. Attempt to keep backwards branches taken, + // forwards not-taken (this is the default prediction on most + // hardware). Layout is still Reverse Post Order but with more + // restrictions. + basicBlockLayout(); + + // Write encoding bits in order into a big byte array. + // Record opcode start and length. + writeEncodings(); + + // Short-form RIP-relative support: replace long encodings with short + // encodings and compact the code, changing all the offsets. + compactShortForm(); + + // Patch RIP-relative and local encodings now. + patchLocalRelocations(); + } + + // -------------------------------------------------- + // Basic block layout: negate branches to keep blocks in-order; insert + // unconditional jumps. Attempt to keep backwards branches taken, forwards + // not-taken (this is the default prediction on most hardware). Layout is + // still Reverse Post Order but with more restrictions. + private void basicBlockLayout() { + IdentityHashMap<LoopNode,Ary<CFGNode>> rpos = new IdentityHashMap<>(); + Ary<CFGNode> rpo = new Ary<>(CFGNode.class); + rpos.put(_code._start.loop(),rpo); + BitSet visit = _code.visit(); + rpo.add(_code._stop); + FunNode main = _code.link(_code._main); + for( Node n : _code._start._outputs ) + if( n instanceof FunNode fun && fun != main ) { + int x = rpo._len; + _rpo_cfg(fun, visit, rpos ); + assert rpo.at(x) instanceof ReturnNode; + } + // Do any main function last... so it lands at offset 0 in the RPO + if( main!=null ) + _rpo_cfg(main, visit, rpos ); + rpo.add(_code._start); + + // Reverse in-place + for( int i=0; i< rpo.size()>>1; i++ ) + rpo.swap(i,rpo.size()-1-i); + visit.clear(); + _code._cfg = rpo; // Save the new ordering + } + + + private void _rpo_cfg(CFGNode bb, BitSet visit, IdentityHashMap<LoopNode,Ary<CFGNode>> rpos ) { + if( bb==null || visit.get(bb._nid) ) return; + visit.set(bb._nid); + CFGNode next = bb.uctrl(); + + // Loops run an inner "rpo_cfg", then append the entire loop body in + // place. This keeps loop bodies completely contained, although if + // Some Day Later we have real profile data we ought to arrange the + // branch orderings based on frequency. + if( bb instanceof LoopNode loop ) { + Ary<CFGNode> body = new Ary<>(CFGNode.class); // Private RPO for the loop + rpos.put(loop.loop(),body); // Find it via loop tree + _rpo_cfg(next,visit,rpos); // RPO the loop + body.add(loop); // Include loop last (first in RPO) + Ary<CFGNode> outer = rpos.get(loop.cfg(1).loop()); + outer.addAll(body); // Append to original CFG + return; + } + + // IfNodes visit 2 sides, and may choose to reorder them in the RPO + if( bb instanceof IfNode iff ) { + // Pick out the T/F projections + CProjNode t = iff.cproj(0); + CProjNode f = iff.cproj(1); + // Invert the branch or not + if( shouldInvert(t,f,iff.loopDepth()) ) { + iff.negate(); + t.invert(); + f.invert(); + CProjNode tmp=f; f=t; t=tmp; // Swap t/f + } + + // Whichever side is visited first becomes last in the RPO. With + // no loops, visit the False side last (so True side first) so that + // when the False RPO visit returns, the IF is immediately next. + // When the RPO is reversed, the fall-through path will always be + // following the IF. + _rpo_cfg(t,visit,rpos); // True side first + next = f; // False side last + } + + // If the *next* BB has already been visited, and we are not already a + // jump, we may need an unconditional forwards jump here + Ary<CFGNode> rpo = rpos.get(bb.loop()); + if( next!=null && visit.get(next._nid) && !(bb instanceof IfNode) ) { + boolean needJump = next instanceof LoopNode + // Empty blocks from an IF will invert the IF and backwards + // branch to the loop head. If not empty, or not an IF + // will need a jump. + ? (bb.nOuts()>1 || !(bb.cfg0() instanceof IfNode) ) + // Forwards jump. If all the blocks, in RPO order, to our + // target are empty, we will fall in and not need a jump. + : !backwardsEmptyScan(rpo,next); + if( needJump ) { + CFGNode jmp = _code._mach.jump(); + jmp._ltree = bb._ltree; + jmp.setDefX(0,bb); + next.setDef(next._inputs.find(bb),jmp); + rpo.add(jmp); + } + } + + _rpo_cfg(next,visit,rpos); + rpo.add(bb); + } + + // Should this test be inverted? + private static boolean shouldInvert(CFGNode t, CFGNode f, int bld) { + int tld = t.loopDepth(), fld = f.loopDepth(); + // These next two are symmetric and can happen in any order; if `tld < + // bld` is true, the `fld < bld` must be false, or else both directions + // exit the loop... and the IF test would not be in the loop. + + // true to exit a loop usually (and false falls into Yet Another Loop + // Block), invert if false is taking an empty backwards branch. + if( tld < bld ) return forwardsEmptyScan(f,bld); + // false to exit a loop, normally invert to true (except empty back) + if( fld < bld ) return !forwardsEmptyScan(t,bld); + + // Not exiting the loop; staying at this depth (or going deeper loop). + + // Jump/true to an empty backwards block + if( forwardsEmptyScan(t,bld) ) return false; + if( forwardsEmptyScan(f,bld) ) return true ; + + // Fall/false into a full block, Jump/true to an empty block. + if( f.nOuts()>1 && t.nOuts()==1 ) return false; + if( t.nOuts()>1 && f.nOuts()==1 ) return true ; + // Jump to a merge point, assuming other things are jumping there as well + if( f.out(0) instanceof RegionNode ) return true ; + if( t.out(0) instanceof RegionNode ) return false; + // Everything else equal, use pre-order + return t._pre > f._pre; + } + + // Do we continue forwards from 'c' to the Loop back edge, without doing + // anything else? Then this branch can directly jump to the loop head. + private static boolean forwardsEmptyScan( CFGNode c, int bld ) { + if( c.nOuts()!=1 || c.loopDepth()!=bld ) return false; + return c.uctrl() instanceof RegionNode cfg && + ((cfg instanceof LoopNode && cfg._ltree==c._ltree) || forwardsEmptyScan(cfg,bld)); + } + + // Is the CFG from "next" to the end empty? This means jumping to "next" + // will naturally fall into the end. + private static boolean backwardsEmptyScan(Ary<CFGNode> rpo, CFGNode next) { + for( int i=rpo._len-1; rpo.at(i)!=next; i-- ) + if( rpo.at(i).nOuts()!=1 ) + return false; + return true; + } + + // -------------------------------------------------- + // Write encoding bits in order into a big byte array. + // Record opcode start and length. + public FunNode _fun; // Currently encoding function + private void writeEncodings() { + _opStart= new int [_code.UID()]; + _opLen = new byte[_code.UID()]; + for( CFGNode bb : _code._cfg ) { + if( !(bb instanceof MachNode mach0) ) + _opStart[bb._nid] = _bits.size(); + else if( bb instanceof FunNode fun ) { + _fun = fun; // Currently encoding function + _opStart[bb._nid] = _bits.size(); + mach0.encoding( this ); + _opLen[bb._nid] = (byte) (_bits.size() - _opStart[bb._nid]); + } + for( Node n : bb._outputs ) { + if( n instanceof MachNode mach && !(n instanceof FunNode) ) { + _opStart[n._nid] = _bits.size(); + mach.encoding( this ); + _opLen[n._nid] = (byte) (_bits.size() - _opStart[n._nid]); + } + } + } + } + + // -------------------------------------------------- + // Short-form RIP-relative support: replace short encodings with long + // encodings and expand the code, changing all the offsets. + private void compactShortForm() { + int len = _code._cfg._len; + int[] oldStarts = new int[len]; + for( int i=0; i<len; i++ ) + oldStarts[i] = _opStart[_code._cfg.at(i)._nid]; + + // TODO: Rewrite this algo to use the small "_jmps" list of just the + // jumps instead of walking all blocks. + // TODO2: Allow short encodings on non-jumps. This algo doesn't + // really care jumps vs other rip-relative things. + + // Check the size of short jumps, adjust the block starts for a whole + // pass over all blocks. If some jumps go long, they might stretch the + // distance for other jumps, so another pass is needed. + int slide = -1; + while( slide != 0) { // While no fails + slide = 0; + for( int i=0; i<len; i++ ) { + CFGNode bb = _code._cfg.at(i); + _opStart[bb._nid] += slide; + // Slide down all other (non-CFG) ops in the block + for( Node n : bb._outputs ) + if( n instanceof MachNode && !(n instanceof CFGNode) ) + _opStart[n._nid] += slide; + if( bb instanceof RIPRelSize riprel ) { + CFGNode target = (bb instanceof IfNode iff ? iff.cproj(0) : (CFGNode)bb.out(0)).uctrlSkipEmpty(); + // Delta is from opStart to opStart. X86 at least counts + // the delta from the opEnd, but we don't have the end until + // we decide the size - so the encSize has to deal + int delta = _opStart[target._nid] - _opStart[bb._nid]; + byte opLen = riprel.encSize(delta); + // Recorded size is smaller than the current size? + if( _opLen[bb._nid] < opLen ) { + // Start sliding the code down; record slide amount and new size + slide += opLen - _opLen[bb._nid]; + _opLen[bb._nid] = opLen; + } + } + } + } + + + // Go again, inserting padding on function headers. Since no + // short-jumps span function headers, the padding will not make any + // short jumps fail. + for( int i=0; i<len; i++ ) { + CFGNode bb = _code._cfg.at(i); + // Functions pad to align 16 + if( bb instanceof FunNode ) { + int newStart = _opStart[bb._nid]+slide; + slide += (newStart+15 & -16)-newStart; + } + _opStart[bb._nid] += slide; + for( Node n : bb._outputs ) + if( n instanceof MachNode && !(n instanceof CFGNode) ) + _opStart[n._nid] += slide; + } + _padFunHeads = true; + + // Copy/slide the bits to make space for all the longer branches + int grow = _opStart[_code._cfg.at(len-1)._nid] - oldStarts[len-1]; + if( grow > 0 ) { // If no short-form ops, nothing to do here + int end = _bits.size(); + byte[] bits = new byte[end+grow]; + for( int i=len-1; i>=0; i-- ) { + int start = oldStarts[i]; + if( start==0 && i>1 ) continue; + int oldStart = oldStarts[i]; + int newStart = _opStart[_code._cfg.at(i)._nid]; + System.arraycopy(_bits.buf(),oldStart,bits,newStart,end-start); + end = start; + } + _bits.set(bits,bits.length); + } + + } + + + // -------------------------------------------------- + // Patch local encodings now + void patchLocalRelocations() { + // Walk the local code-address relocations + for( Node src : _internals.keySet() ) { + int start = _opStart[src._nid]; + Node dst = _internals.get(src); + int target = _opStart[dst._nid]; + ((RIPRelSize)src).patch(this, start, _opLen[src._nid], target - start); + } + } + + // -------------------------------------------------- + // Write the constant pool into the BAOS and optionally patch locally + void writeConstantPool( BAOS bits, boolean patch ) { + padN(16,bits); + + // radix sort the big constants by alignment + Ary<Relo>[] raligns = new Ary[5]; + for( Node op : _bigCons.keySet() ) { + Relo relo = _bigCons.get(op); + int align = relo._t.alignment(); + Ary<Relo> relos = raligns[align]==null ? (raligns[align]=new Ary<>(Relo.class)) : raligns[align]; + relos.add(relo); + } + + + // Types can be used more than once; collapse the dups + HashMap<Type,Integer> targets = new HashMap<>(); + + // By alignment + for( int align = 4; align >= 0; align-- ) { + Ary<Relo> relos = raligns[align]; + if( relos == null ) continue; + for( Relo relo : relos ) { + // Map from relo to constant start and patch + Integer target = targets.get(relo._t); + if( target==null ) { + targets.put(relo._t,target = bits.size()); + // Write constant into constant pool + switch( relo._t ) { + case TypeTuple tt -> cpool(align,bits,tt); + case TypeStruct ts -> cpool(bits,ts); + // Simple primitive (e.g. larger int, float) + default -> addN(align,relo._t,bits); + } + } + // Record target address and opcode start + relo._target = target; + relo._opStart= _opStart[relo._op._nid]; + // Go ahead and locally patch in-memory + if( patch ) + ((RIPRelSize)relo._op).patch(this, relo._opStart, _opLen[relo._op._nid], relo._target - relo._opStart); + } + } + } + + // Constant tuples put all entries at same alignment + private void cpool(int align, BAOS bits, TypeTuple tt) { + for( Type tx : tt._types ) + addN(align,tx,bits); + } + + // Structs use internal field layout + private void cpool( BAOS bits, TypeStruct ts ) { + // Field order by offset + int[] layout = ts.layout(); + int off=0; // offset in the struct + for( int fn=0; fn<ts._fields.length; fn++ ) { + Field f = ts._fields[layout[fn]]; + int foff = ts. offset(layout[fn]); + // Pad up to field + while( off < foff ) { bits.write(0); off++; }; + // Constant array fields are special + if( f._fname=="[]" ) { // Must be a constant array + ((TypeConAry)f._t).write(bits); + off += ((TypeConAry)f._t).len(); + } else { + int log = f._t.log_size(); + addN(log,f._t,bits); + off += 1<<log; + } + } + } + + + // A series of libc/external calls that Simple can link against in a JIT. + // Since no runtime in the JVM process, using magic numbers for the CPU + // emulators to pick up on. + public static int SENTINEL_CALLOC = -4; + public static int SENTINEL_WRITE = -8; + + void patchGlobalRelocations() { + for( Node src : _externals.keySet() ) { + int start = _opStart[src._nid]; + String dst = _externals.get(src); + int target = switch( dst ) { + case "calloc" -> SENTINEL_CALLOC; + case "write" -> SENTINEL_WRITE ; + default -> throw Utils.TODO(); + }; + ((RIPRelSize)src).patch(this, start, _opLen[src._nid], target - start); + } + } +} diff --git a/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/FinalizeIR.java b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/FinalizeIR.java new file mode 100644 index 00000000..14cf56ba --- /dev/null +++ b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/FinalizeIR.java @@ -0,0 +1,94 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.Parser; +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.*; +import java.util.Arrays; + +// Settle layouts and detach the analyzed call graph before the backend. +abstract class FinalizeIR { + static void run(CodeGen code) { + // SCCP kept conservative entries. Rewrites have now relinked the + // surviving calls, so remove uncalled bodies just as before SCCP. + int progress = -1; + while( progress != code._start.nOuts() ) { + progress = code._start.nOuts(); + FunNode main = code.link(code._main); + for( int i=0; i<code._start.nOuts(); i++ ) { + Node use = code._start.out(i); + if( use instanceof FunNode fun && fun != main && + fun.nIns()==2 && fun.in(1)==code._start && !fun.hasPtr() && + (main!=null || fun._name==null || fun._name.startsWith("sys.")) ) { + code.add(fun).setDef(1,Parser.XCTRL); + code.addAll(fun._outputs); + i--; + } + } + code._iter.iterate(code); + } + fieldSizes(code); + code._iter.iterate(code); + removeCallGraph(code); + Arrays.sort(code._stop._inputs._es,0,code._stop.nIns(),(x,y) -> x._nid - y._nid); + } + + // Freeze field sizes; do struct layouts; convert field offsets into + // constants. + private static void fieldSizes( CodeGen code ) { + for( int i=0; i<code._start.nOuts(); i++ ) { + Node use = code._start.out(i); + if( use instanceof ConFldOffNode off ) { + TypeMemPtr tmp = (TypeMemPtr) Parser.TYPES.get(off._name); + off.subsume( off.asOffset(tmp._obj) ); + i--; + } + } + } + + // The remaining passes assume all calls are unlinked; i.e. we are throwing + // away the Call Graph here. Functions only reachable from internal calls + // need to be re-hooked to stop/start less they go dead. + private static void removeCallGraph(CodeGen code) { + code._start.walk( x -> { + // Unlink all existing (conservative) linkages. + if( x instanceof CallNode call ) { + for( Node y : call._outputs ) { + // Keep function alive and hooked to Start so it gets + // codegen'd. Function is callable by this call via + // TFP, although perhaps not by any other means. + if( y instanceof FunNode fun && !(fun.in(1) instanceof StartNode) ) { + ParmNode rpc = fun.rpc(); + if( rpc==null ) // Ensure valid RPC + rpc = makeRPC(fun); + // Insert a hook to Start and Stop + fun.insertDef(1,code._start); + for( Node use : fun._outputs ) + if( use instanceof ParmNode parm ) + parm.insertDef(1,ConstantNode.make(parm._type).peephole()); + code._stop.addDef(fun.ret()); + } + } + call.unlink_all(); + } + return null; + } ); + + } + + // Make a valid Parm RPC. + // First make sure a valid RPC; single-call into multi-fun will have each + // function having a single call site, so the RPC becomes a constant and + // folds - but since multiple targets, the Call never inlines. Recreate a + // valid RPC so codegen (and Eval2) understands the calling convention. + private static ParmNode makeRPC( FunNode fun ) { + ParmNode rpc = new ParmNode("$rpc",0,fun.ret().rpc()._type,fun); + for( int i=1; i<fun.nIns(); i++ ) { + CallEndNode cend = ((CallNode)fun.in(i)).cend(); + rpc.addDef(ConstantNode.make(cend._rpc).peephole()); + } + fun.ret().setDef(3,rpc.init()); + return rpc; + } + + +} diff --git a/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/GlobalCodeMotion.java b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/GlobalCodeMotion.java new file mode 100644 index 00000000..da45db4f --- /dev/null +++ b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/GlobalCodeMotion.java @@ -0,0 +1,311 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.IterPeeps.WorkList; +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.*; +import com.seaofnodes.simple.util.Ary; +import com.seaofnodes.simple.util.Utils; +import java.util.*; + +public abstract class GlobalCodeMotion { + + // Arrange that the existing isCFG() Nodes form a valid CFG. The + // Node.use(0) is always a block tail (either IfNode or head of the + // following block). There are no unreachable infinite loops. + public static void buildCFG( CodeGen code ) { + Ary<CFGNode> rpo = new Ary<>(CFGNode.class); + _rpo_cfg(null, code._start, code.visit(), rpo); + // Reverse in-place + for( int i=0; i< rpo.size()>>1; i++ ) + rpo.swap(i,rpo.size()-1-i); + // Set global CFG + code._cfg = rpo; + + schedEarly(code); + + // Break up shared global constants by functions + breakUpGlobalConstants(code._start); + + code._visit.clear(); + schedLate (code); + } + + // Post-Order of CFG + private static void _rpo_cfg(CFGNode def, Node use, BitSet visit, Ary<CFGNode> rpo) { + if( !(use instanceof CFGNode cfg) || visit.get(cfg._nid) ) + return; // Been there, done that + if( def instanceof ReturnNode && use instanceof CallEndNode ) + return; + assert !( def instanceof CallNode && use instanceof FunNode ); // All calls unwired now + visit.set(cfg._nid); + for( Node useuse : cfg._outputs ) + _rpo_cfg(cfg,useuse,visit,rpo); + rpo.add(cfg); + } + + // After early scheduling, every global chain member has Start in slot 0. + // Give each function one private copy of the entire dependency graph. + private static void breakUpGlobalConstants( StartNode start ) { + var globals = new IdentityHashMap<Node,Boolean>(); + var cons = new ArrayList<Node>(); + for( Node con : start._outputs ) + if( con!=null && !(con instanceof CFGNode) && + (con.isConst() || (con instanceof MachNode mach && mach.outregmap()!=null)) ) { + globals.put(con,true); + cons.add(con.keep()); // Preserve original inputs while rewiring users. + } + + var copies = new IdentityHashMap<FunNode,IdentityHashMap<Node,Node>>(); + for( Node con : cons ) + for( Node use : con._outputs.asAry() ) { + if( use==null || globals.containsKey(use) ) continue; + FunNode fun = useFun(use); + if( fun==null ) continue; // Global metadata has no function owner. + var local = copies.computeIfAbsent(fun,f -> new IdentityHashMap<>()); + Node copy = cloneGlobal(con,fun,globals,local); + for( int i=0; i<use.nIns(); i++ ) + if( use.in(i)==con ) + use.setDef(i,copy); + } + for( Node con : cons ) con.unkill(); + } + + private static Node cloneGlobal(Node con, FunNode fun, IdentityHashMap<Node,Boolean> globals, + IdentityHashMap<Node,Node> copies) { + Node copy = copies.get(con); + if( copy!=null ) return copy; + copies.put(con,copy=con.copyEmpty()); + copy.addDef(fun); + for( int i=1; i<con.nIns(); i++ ) { + Node def = con.in(i); + copy.addDef(globals.containsKey(def) ? cloneGlobal(def,fun,globals,copies) : def); + } + return copy; + } + + private static FunNode useFun(Node use) { + if( use instanceof ReturnNode ret ) return ret.fun(); + if( use instanceof ParmNode parm ) return parm.fun(); + CFGNode cfg = use.cfg0(); + while( cfg!=null && !(cfg instanceof FunNode) ) + cfg = cfg.idom(); + return (FunNode)cfg; + } + + + // ------------------------------------------------------------------------ + // Visit all nodes in CFG Reverse Post-Order, essentially defs before uses + // (except at loops). Since defs are visited first - and hoisted as early + // as possible, when we come to a use we place it just after its deepest + // input. + private static void schedEarly(CodeGen code) { + // Reverse Post-Order on CFG + for( CFGNode cfg : code._cfg ) { + cfg.loopDepth(); + for( Node n : cfg._inputs ) + _schedEarly(n,code ); + // In dead infinite loops, entire code blocks may be unreachable + // from below. Reach down from the CFG to their Phis so their + // inputs are scheduled too. + if( cfg instanceof RegionNode ) + for( Node phi : cfg._outputs ) + if( phi instanceof PhiNode ) + _schedEarly(phi,code ); + } + } + + private static void _schedEarly(Node n, CodeGen code) { + if( n==null || code._visit.get(n._nid) ) return; // Been there, done that + assert !(n instanceof CFGNode); + code._visit.set(n._nid); + // Schedule inputs first, except Phis: following their backedges would + // enter a data cycle before its control has been scheduled. + for( Node def : n._inputs ) + if( def!=null && !(def instanceof PhiNode) ) + _schedEarly(def,code); + + // An existing edge 0 already supplies control (or a Phi/Proj binding). + if( n.in(0)==null ) { + // Schedule at deepest input + CFGNode early = code._start; // Maximally early, lowest idepth + for( int i=1; i<n.nIns(); i++ ) + if( n.in(i)!=null && n.in(i).cfg0().idepth() > early.idepth() ) + early = n.in(i).cfg0(); // Latest/deepest input + n.setDef(0,early); + } + } + + // ------------------------------------------------------------------------ + private static void schedLate( CodeGen code ) { + CFGNode[] late = new CFGNode[code.UID()]; + Node[] ns = new Node[code.UID()]; + // Record Load NIDs at all their CFG block choices, then check against + // Store block choices to force a Load above an anti-dependent Store. + int[] anti = new int[code.UID()]; + // Breadth-first scheduling + breadth(code._stop,ns,late,anti); + + // Copy the best placement choice into the control slot + for( int i=0; i<late.length; i++ ) + if( ns[i] != null && !(ns[i] instanceof ProjNode) ) + ns[i].setDef(0,late[i]); + } + + private static void breadth(Node stop, Node[] ns, CFGNode[] late, int[] anti) { + // Things on the worklist have some (but perhaps not all) uses done. + WorkList<Node> work = new WorkList<>(); + work.push(stop); + Node n; + outer: + while( (n = work.pop()) != null ) { + assert late[n._nid]==null; // No double visit + // These I know the late schedule of, and need to set early for loops + if( n instanceof CFGNode cfg ) late[n._nid] = cfg.blockHead() ? cfg : cfg.cfg(0); + else if( n instanceof PhiNode phi ) late[n._nid] = phi.region(); + else if( n instanceof ProjNode && n.in(0) instanceof CFGNode cfg ) late[n._nid] = cfg; + else { + + // All uses done? + for( Node use : n._outputs ) + if( use!=null && late[use._nid]==null ) + continue outer; // Nope, await all uses done + + // Loads need their memory inputs' uses also done + if( n instanceof MemOpNode ld && ld._isLoad ) + for( Node memuse : ld.antiDeps() ) + if( late[memuse._nid]==null ) + continue outer; + + // All uses done, schedule + _doSchedLate(n,ns,late,anti); + } + + // A use just finished; reconsider its inputs and waiting loads, + // even when the shared memory input was already scheduled. + for( Node def : n._inputs ) { + if( def==null ) continue; + if( late[def._nid]==null ) work.push(def); + wakeLoads(def,late,work,new BitSet()); + } + if( n instanceof LoopNode loop ) + for( Node phi : loop._outputs ) + if( phi instanceof PhiNode && late[phi._nid]==null ) + work.push(phi); + } + } + + private static void _doSchedLate(Node n, Node[] ns, CFGNode[] late, int[] anti) { + // Walk uses, gathering the LCA (Least Common Ancestor) of uses + CFGNode early = n.in(0) instanceof CFGNode cfg ? cfg : n.in(0).cfg0(); + assert early != null; + CFGNode lca = null; + for( Node use : n._outputs ) + if( use != null ) + lca = use_block(n,use, late).domLCA(lca,null); + + // Loads may need anti-dependencies, raising their LCA + if( n instanceof MemOpNode load && load._isLoad ) + lca = find_anti_dep(lca,load,early,late,anti); + + + // Walk up from the LCA to the early, looking for best place. This is + // the lowest execution frequency, approximated by least loop depth and + // deepest control flow. + CFGNode best = lca; + lca = lca.idom(); // Already found best for starting LCA + for( ; lca != early.idom(); lca = lca.idom() ) + if( better(lca,best) ) + best = lca; + assert !(best instanceof IfNode); + ns [n._nid] = n; + late[n._nid] = best; + } + + // Block of use. Normally from late[] schedule, except for Phis, which go + // to the matching Region input. + private static CFGNode use_block(Node n, Node use, CFGNode[] late) { + if( use instanceof ParmNode ) return late[use._nid]; + if( !(use instanceof PhiNode phi) ) + return late[use._nid]; + CFGNode found=null; + for( int i=1; i<phi.nIns(); i++ ) + if( phi.in(i)==n ) + found = phi.region().cfg(i).domLCA(found,null); // Can be more than one matching input. + + assert found!=null; + return found; + } + + + // Least loop depth first, then largest idepth + private static boolean better( CFGNode lca, CFGNode best ) { + return lca.loopDepth() < best.loopDepth() || + lca instanceof NeverNode || + lca.idepth() > best.idepth() || + best instanceof IfNode; + } + + // An allocation may consume a load's memory through a partial aggregate. + private static void wakeLoads(Node def, CFGNode[] late, WorkList<Node> work, BitSet visit) { + if( visit.get(def._nid) ) return; + visit.set(def._nid); + for( Node out : def._outputs ) + if( out instanceof MemOpNode ld && ld._isLoad && late[ld._nid]==null ) work.push(ld); + if( def instanceof MemMergeNode ) + for( int i=1; i<def.nIns(); i++ ) + if( def.in(i)!=null ) wakeLoads(def.in(i),late,work,visit); + } + + private static CFGNode find_anti_dep(CFGNode lca, MemOpNode load, CFGNode early, CFGNode[] late, int[] anti) { + // We could skip final-field loads here. + // Walk LCA->early, flagging Load's block location choices + for( CFGNode cfg=lca; early!=null && cfg!=early.idom(); cfg = cfg.idom() ) + anti[cfg._nid] = load._nid; + // Walk load->mem uses, looking for Stores causing an anti-dep + for( Node mem : load.antiDeps() ) { + switch( mem ) { + case MemOpNode st: + assert late[st._nid]!=null; + lca = anti_dep(load,late[st._nid],st.cfg0(),lca,st,anti); + break; + case CallNode st: + assert late[st._nid]!=null; + lca = anti_dep(load,late[st._nid],st.cfg0(),lca,st,anti); + break; + case PhiNode phi: + // Inputs carrying different memory might overwrite this alias. + // Raise the LCA for those paths; unchanged memory does not + // clobber the load. No scheduling edge to the Phi. + for( int i=1; i<phi.nIns(); i++ ) + if( phi.in(i)!=load.mem() ) + lca = anti_dep(load,phi.region().cfg(i),load.mem().cfg0(),lca,null,anti); + break; + case NewNode st: + assert late[st._nid]!=null; + lca = anti_dep(load,late[st._nid],st.cfg0(),lca,st,anti); + break; + default: throw Utils.TODO(); + } + } + return lca; + } + + // + private static CFGNode anti_dep( MemOpNode load, CFGNode stblk, CFGNode defblk, CFGNode lca, Node st, int[] anti ) { + // A conditional writer can precede the load without dominating its late block. + // Walk back to the first overlap with the load's placement range. + // Another Phi input need not be dominated by this memory definition. + // Include its depth, and retain the sentinel for an entry's null idom. + for( ; stblk != defblk.idom() && stblk.idepth() >= defblk.idepth(); stblk = stblk.idom() ) { + // Store and Load overlap, need anti-dependence + if( anti[stblk._nid]==load._nid ) { + lca = stblk.domLCA(lca,null); // Raise Loads LCA + if( lca == stblk && st != null && !(st instanceof CallNode) && st._inputs.find(load) == -1 ) // And if something moved, + st.addDef(load); // Add anti-dep as well; Calls already end their block. + return lca; // Cap this stores' anti-dep to here + } + } + return lca; + } + +} diff --git a/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/IFG.java b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/IFG.java new file mode 100644 index 00000000..bcd1957d --- /dev/null +++ b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/IFG.java @@ -0,0 +1,539 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.Type; +import com.seaofnodes.simple.util.Ary; +import com.seaofnodes.simple.util.Utils; +import java.util.BitSet; +import java.util.IdentityHashMap; + +// Interference Graph +abstract public class IFG { + // Interference Graph Support + + // Map from a Basic Block to Live-Out: {a map from a Live Range to a Def} + private static final IdentityHashMap<CFGNode,IdentityHashMap<LRG,Node>> BBOUTS = new IdentityHashMap<>(); + static void resetBBLiveOut() { + for( IdentityHashMap<LRG,Node> bbout : BBOUTS.values() ) + bbout.clear(); + } + static final IdentityHashMap<LRG,Node> TMP = new IdentityHashMap<>(); + + // Inteference Graph: Array of Bitsets + static final Ary<BitSet> IFG = new Ary<>(BitSet.class); + static void resetIFG() { + for( BitSet bs : IFG ) + if( bs!=null ) bs.clear(); + } + + // Set matching bit + private static void addIFG( LRG lrg0, LRG lrg1 ) { + short x0 = lrg0._lrg, x1 = lrg1._lrg; + // Triangulate + if( x0 < x1 ) _addIFG(x0,x1); + else _addIFG(x1,x0); + } + // Add lrg1 to lrg0's conflict set + private static void _addIFG( short x0, short x1 ) { + BitSet ifg = IFG.atX(x0); + if( ifg==null ) + IFG.setX(x0,ifg = new BitSet()); + ifg.set(x1); + } + + + private static final Ary<CFGNode> WORK = new Ary<>(CFGNode.class); + static void pushWork(CFGNode bb) { + if( WORK.find(bb) == -1 ) + WORK.push(bb); + } + + // ------------------------------------------------------------------------ + // Visit all blocks, using the live-out LRGs per-block and doing a backwards + // walk over each block. At the end of the walk, push the live-out sets to + // prior blocks. Due to loops, no single visitation order suffices. This + // problem can be fairly efficiently solved with a proper LIVE calculation + // and bitsets. In the interests of simplicity (and assumption of smaller + // programs) I am skipping LIVE and directly computing it during the IFG + // building. + + // Start by setting the live-out of the exit block (to empty), and putting + // it on a worklist. + + // - Pull from worklist a block who's live-out has changed + // - Walk backwards, adding interferences and computing live. + // - At block head, "push" the new live-out to prior blocks. + // - - If they pick up new live-outs, put them on the worklist. + + public static boolean build( int round, RegAlloc alloc ) { + // Reset all to empty + resetBBLiveOut(); + resetIFG(); + + // Last block has nothing live out + assert WORK.isEmpty(); + for( CFGNode bb : alloc._code._cfg ) + if( bb.blockHead()) + WORK.push( bb ); + + // Process blocks until no more changes + while( !WORK.isEmpty() ) + do_block(round,alloc,WORK.pop()); + + if( alloc.success() ) + convert2DAdjacency(alloc); + return alloc.success(); + } + + // Walk one block backwards, compute live-in from live-out, and build IFG + static void do_block( int round, RegAlloc alloc, CFGNode bb ) { + assert bb.blockHead(); + IdentityHashMap<LRG,Node> live_out = BBOUTS.get(bb); + TMP.clear(); + if( live_out != null ) + TMP.putAll(live_out); // Copy bits to temp + + // A backwards walk over instructions in the basic block + for( int inum = bb.nOuts()-1; inum >= 0; inum-- ) { + Node n = bb.out(inum); + if( n.in(0) != bb ) continue; + // In a backwards walk, proj users come before the node itself + if( n instanceof MultiNode ) + for( Node proj : n.outs() ) + if( proj instanceof ProjNode ) + do_node(alloc,proj); + if( n instanceof MachNode ) + do_node(alloc,n); + } + + // The bb head kills register, e.g. a CallEnd and caller-save registers + if( bb instanceof MachNode mach ) + kills(alloc,mach); + + // Push live-sets backwards to priors in CFG. + if( bb instanceof RegionNode ) + for( int i=1; i<bb.nIns(); i++ ) + mergeLiveOut(alloc,bb,i); + else + mergeLiveOut(alloc,bb,0); + } + + private static void do_node(RegAlloc alloc, Node n) { + // Defining means killing live LRG + LRG lrg = alloc.lrg(n); + if( lrg != null ) { + // Check for def-side self-conflict live ranges. These must split, + // and only happens during the first round a particular LRG splits. + selfConflict(alloc,n,lrg); + TMP.remove(lrg); // Kill def + } + + // Phis use and define the same live range, i.e. these LRGs already + // marked conflicted, no need to mark again + if( n instanceof PhiNode ) + return; + + // Kill any killed registers; milli-code routines like New can kill + // will not being a CFG. + if( n instanceof MachNode m ) + kills(alloc,m); + + // Interfere n with all live + if( lrg!=null ) { + // Interfere n with all live + for( LRG tlrg : TMP.keySet() ) { + assert tlrg.leader(); + // Skip self + if( lrg != tlrg && + // And register sets overlap + lrg._mask.overlap(tlrg._mask) ) + // Then tlrg and lrg interfere. If lrg needs the single + // last tlrg register at some point, either tlrg or lrg + // must fail. If *n* (a subset of lrg) needs the single + // last tlrg register then only tlrg must fail. + if( ((MachNode)n).outregmap().size1() ) { + if( !tlrg.clr(lrg._mask.firstReg()) ) // Clear bit, no interference + alloc.fail(tlrg); // Clearing drives mask to empty + } else addIFG(lrg,tlrg); // Add interference + } + } + + + // Check for self-conflict live ranges. These must split, and only + // happens during the first round when a particular LRG splits. + // Also record use-side spills for biased coloring. + // Then make all inputs live. + for( int i=1; i<n.nIns(); i++ ) { + Node def = n.in(i); + if( def==null ) continue; + LRG lrg1 = alloc.lrg(def); + if( lrg1==null ) continue; // Anti-dependence, etc, no register + + // Check use-side for self-conflict; if so we MUST split + selfConflict(alloc,def,lrg1); + + // Look for a must-use single register conflicting with some other must-def. + if( n instanceof MachNode m ) { + RegMask ni_mask = m.regmap(i); + if( ni_mask!=null && ni_mask.size1() ) { // Must-use single register + // Search all current live + for( LRG tlrg : TMP.keySet() ) { + assert tlrg.leader(); + Node live = TMP.get(tlrg); + if( live != def && live instanceof MachNode lmach && lmach.outregmap().overlap(ni_mask) ) { + // Deny the register, since it absolutely must be used here + if( !tlrg.clr(ni_mask.firstReg()) ) + // Then direct reg-reg conflict between use here (at n.in(i)) and def (of tlrg) there. + // Fail the older live range, it must move its register. + alloc.fail( tlrg ); + } + } + } + } + + // All inputs live - except in self conflicts, where we keep the prior def alive + // until it goes. + TMP.put(lrg1,def); + } + } + + // Single-register defines *must* have their register; other live ranges + // *must* avoid this register, so instead of interfering we remove the + // single register from the other rmask. + private static void kills( RegAlloc alloc, MachNode m ) { + RegMask killMask = m.killmap(); + if( killMask==null ) return; + // Kill registers with all live + for( LRG tlrg : TMP.keySet() ) { + assert tlrg.leader(); + // Always, tlrg cannot use kills + if( tlrg._mask.overlap(killMask) ) { + if( !tlrg.sub(killMask) ) alloc.fail(tlrg); + } + } + } + + // Check for self-conflict live ranges. These must split, and only happens + // during the first round a particular LRG splits. + private static void selfConflict(RegAlloc alloc, Node n, LRG lrg) { + selfConflict(alloc,n,lrg,TMP.get(lrg)); + } + private static void selfConflict(RegAlloc alloc, Node n, LRG lrg, Node prior) { + if( prior!=null && prior != n ) { + lrg.selfConflict(prior); + lrg.selfConflict(n); + alloc.fail(lrg); // 2 unrelated values live at once same live range; self-conflict + } + } + + // Merge TMP into bb's live-out set; if changes put bb on WORK + private static void mergeLiveOut( RegAlloc alloc, CFGNode priorbb, int i ) { + CFGNode bb = priorbb.cfg(i); + if( bb == null ) return; // Start has no prior + while( !bb.blockHead() ) bb = bb.cfg0(); + + // Lazy get live-out set for bb + IdentityHashMap<LRG, Node> lrgs = BBOUTS.computeIfAbsent( bb, k -> new IdentityHashMap<>() ); + + for( LRG lrg : TMP.keySet() ) { + Node def = TMP.get(lrg); + // Effective def comes from phi input from prior block + if( def instanceof PhiNode phi && phi.cfg0()==priorbb ) { + assert i!=0; + def = phi.in( i ); + } + // The synthetic exit carries no value on this Phi arm. + if( def._type==Type.TOP ) continue; + Node def_bb = lrgs.get(lrg); + if( def_bb==null ) { + lrgs.put(lrg,def); + pushWork(bb); + } else { + // Alive twice with different definitions; self-conflict + selfConflict(alloc,def,lrg,def_bb); + } + } + } + + + static void convert2DAdjacency( RegAlloc alloc ) { + // Convert the 2-D array of bits (a 1-D array of BitSets) into an + // adjacency matrix. + int maxlrg = alloc._LRGS.length; + for( int i=1; i<maxlrg; i++ ) { + BitSet ifg = IFG.atX(i); + if( ifg != null ) { + LRG lrg0 = alloc._LRGS[i]; + for( int lrg = ifg.nextSetBit(0); lrg>=0; lrg=ifg.nextSetBit(lrg+1) ) { + LRG lrg1 = alloc._LRGS[lrg]; + lrg0.addNeighbor(lrg1); + lrg1.addNeighbor(lrg0); + } + } + } + } + + + // ------------------------------------------------------------------------ + // Color the inference graph. + + // Coloring works by removing "trivial" live ranges from the IFG - those + // live ranges with fewer neighbors than available colors (registers). + // These are trivial because even if every neighbor takes a unique color, + // there's at least one more available to color this live range. + + // If we hit a clique of non-trivial live ranges, we pick one to be "at + // risk" of not-coloring. Good choices include live ranges with a large + // area and few hot uses. + + // Then we reverse and put-back live ranges - picking a color as we go. + // If there's no spare color we'll have to spill this at-risk live range. + + public static boolean color(int round, RegAlloc alloc) { + int maxlrg = alloc._LRGS.length, nlrgs=0; + for( int i=1; i<maxlrg; i++ ) + if( alloc._LRGS[i] != null ) + nlrgs++; + + // Simplify + + // Walk all the LRGS looking for some trivial ones to start with. + // During this pass all LRGS are broken into 3 disjoint sets: + // - trivial, removed from IFG; color_stack[0 to sptr] + // - trivial, not (yet) removed from IFG; color_stack[sptr to swork] + // - unknown; color_stack[work to maxlrg] + + // Gather all not-unified (not-null); separate trivial and non-trivial set. + int sptr = 0, swork = 0; + LRG[] color_stack = new LRG[nlrgs]; + for( int i=0, j=0; i<maxlrg; i++ ) { + LRG lrg = alloc._LRGS[i]; + if( lrg==null ) continue; // Unified lrgs are null here + color_stack[j++] = lrg; + if( lrg.lowDegree() ) swap(color_stack, swork++, j-1); + } + + // Pull all lrgs from IFG, in trivial order if possible + while( sptr < color_stack.length ) { + // Swap best color up front + pickColor(color_stack,sptr,swork); + + // Pick a trivial lrg, and (temporarily) remove from the IFG. + LRG lrg = color_stack[sptr++]; + // If sptr was swork, then pulled an at-risk lrg + if( sptr > swork ) + swork = sptr; + + // Walk all neighbors and remove + if( lrg._adj != null ) { + for( LRG nlrg : lrg._adj ) { + // Remove and compress out neighbor + if( nlrg.removeCompress(lrg) ) { + // Neighbor is just exactly going trivial as 'lrg' is removed from IFG + // Find "j" position in the color_stack + int jx = swork; while( color_stack[jx] != nlrg ) jx++; + // Add trivial neighbor to trivial list. Pull lrg j out of + // unknown set, since its now in the trivial set + swap(color_stack, swork++, jx); + } + } + } + } + + // Reverse simplify (unstack the color stack), and set colors (registers) for live ranges + while( sptr > 0 ) { + LRG lrg = color_stack[--sptr]; + if( lrg==null ) continue; + RegMaskRW rmask = lrg._mask.copy(); + // Walk neighbors and remove adjacent colors + if( lrg._adj!=null ) { + for( LRG nlrg : lrg._adj ) { + nlrg.reinsert(lrg); + int reg = nlrg._reg; + if( reg!= -1 ) // Failed neighbors do not count + rmask.clr(reg); // Remove neighbor from my choices + } + } + // At-risk live-range did not color? + if( rmask.isEmpty() ) { + alloc.fail(lrg); + lrg._reg = -1; + } else { + // Pick first available register + short reg = rmask.firstReg(); + // Pick a "good" color from the choices. Typically, biased-coloring + // removes some over-spilling. + if( rmask.size() > 1 ) reg = biasColor(alloc,lrg,reg,rmask); + lrg._reg = reg; // Assign the register + } + } + + return alloc.success(); + } + + // Pick LRG from color stack + private static void pickColor(LRG[] color_stack, int sptr, int swork) { + // Out of trivial colorable, pick an at-risk to pull + if( sptr==swork ) + swap(color_stack,sptr,pickRisky(color_stack,sptr)); + // When coloring, we'd like to give more choices; so when coloring we'd + // like to see the single-def first (since no choices anyway), then + // non-split related (so more live ranges get colored), then + // split-related last, so they have more colors to bias towards. + + // Working in reverse, pick first split-related with many regs, then + // those with some regs, then single-def. + int bidx=sptr; + LRG best=color_stack[bidx]; + for( int idx = sptr+1; idx < swork; idx++ ) + if( betterLRG(best,color_stack[idx]) ) + best = color_stack[bidx=idx]; + if( bidx != sptr ) + swap(color_stack,sptr,bidx); // Pick best at sptr + } + + private static boolean betterLRG( LRG best, LRG lrg ) { + // If single-def varies, keep the not-single-def + if( best.size1() != lrg.size1() ) + return best.size1(); + // If hasSplit varies, keep the hasSplit + if( best.hasSplit() != lrg.hasSplit() ) + return lrg.hasSplit(); + // Keep large register count + return best.size() < lrg.size(); + } + + // Pick a live range that hasn't already spilled, or has a single-def- + // single-use that are not adjacent. + private static int pickRisky( LRG[] color_stack, int sptr ) { + int best=sptr; + int bestScore = pickRiskyScore(color_stack[best]); + for( int i=sptr+1; i<color_stack.length; i++ ) { + if( bestScore == 1000000 ) return best; // Already max score + int iScore = pickRiskyScore(color_stack[i]); + if( iScore > bestScore ) + { best = i; bestScore = iScore; } + } + return best; + } + + // Pick a live range to pull, that might not color. + // + // Picking a live range with a very large span, with defs and uses outside + // loops means spilling a relative cheap live range and getting that + // register over a large area. + // + // Picking a live range that is very close to coloring might allow it to + // color despite being risky. + private static int pickRiskyScore( LRG lrg ) { + // Pick single-def clonables that are not right next to their single-use. + // Failing to color these will clone them closer to their uses. + if( !lrg._multiDef && lrg._machDef!=null && lrg._machUse!=null && lrg._machDef.isClone() ) { + Node def = (Node)lrg._machDef; + Node use = (Node)lrg._machUse; + CFGNode cfg = def.cfg0(); + if( lrg._multiUse || cfg != use.cfg0() || // Many uses or different blocks OR + // Same block, but not close + cfg._outputs.find(def)+1 < cfg._outputs.find(use) ) + return 1000000; + } + + // Always pick callee-save registers as being very large area recovered + // and very cheap to spill. + if( lrg._machDef instanceof CalleeSaveNode ) + return 1000000-2-lrg._mask.firstReg(); + if( lrg._splitDef != null && lrg._splitDef.in(1) instanceof CalleeSaveNode && + lrg._splitUse != null && lrg._splitUse.out(0) instanceof ReturnNode ) + return 1000000-1; + + // TODO: cost/benefit model. Perhaps counting loop-depth (freq) of def/use for cost + // and "area" for benefit + return 1000; + } + + private static short biasColor( RegAlloc alloc, LRG lrg, short reg, RegMask mask ) { + if( mask.size1() ) return reg; + // Check chain of splits up the def-chain. Take first allocated + // register, and if it's available in the mask, take it. + Node def = lrg._splitDef, use = lrg._splitUse; + int tidx=0, cnt=0; + + while( def != null || use != null ) { + if( cnt++ > 10 ) break; + + if( def != null ) { + short bias = biasColor( alloc, def, mask ); + if( bias >= 0 ) return bias; // Good bias + if( bias == -2 ) def = null; // Kill this side, no more searching + else if( (tidx=biasable(def)) == 0 ) def = null; + } + + if( use != null ) { + short bias = biasColor( alloc, use, mask ); + if( bias >= 0 ) return bias; // Good bias + if( bias == -2 ) use = null; // Kill this side, no more searching + else if( biasable(use)==0 ) use = null; + } + + if( def != null ) { + short bias = biasColorNeighbors( alloc, def, mask ); + if( bias >= 0 ) return bias; + // Advance def side + def = def.in(tidx); + if( alloc.lrg(def)==null ) def=null; + } + + if( use != null ) { + short bias = biasColorNeighbors( alloc, use, mask ); + if( bias >= 0 ) return bias; + use = use.out(0); + if( biasable(use)==0 ) use=null; + } + + } + return mask.firstReg(); + } + + private static int biasable(Node split) { + if( split instanceof SplitNode ) return 1; // Yes biasable, advance is slot 1 + if( split instanceof PhiNode phi ) return phi.region() instanceof LoopNode ? 2 : 1; // Prefer the backedge in loops + if( !(split instanceof MachNode mach) ) return 0; // Not biasable + return mach.twoAddress(); // Only biasable if 2-addr + } + + // 3-way return: + // - good bias reg, take it & exit + // - this path is cutoff; do not search here anymore + // - advance this side + private static short biasColor( RegAlloc alloc, Node split, RegMask mask ) { + short bias = alloc.lrg(split)._reg; + if( bias != -1 ) { + if( mask.test(bias) ) return bias; // Good bias + else return -2; // Kill this side + } else return -1; // Advance this side + } + + // Check if we can match "split" color, or else trim "mask" to colors + // "split" might get. + private static short biasColorNeighbors( RegAlloc alloc, Node split, RegMask mask ) { + LRG slrg = alloc.lrg(split); + if( slrg._adj == null ) return -1; // No trimming + + // Can I limit my own choices to valid neighbor choices? + for( LRG alrg : slrg._adj ) { + short reg = alrg._reg; + if( reg == -1 && alrg._mask.size1() ) + reg = alrg._mask.firstReg(); + if( reg != -1 ) { + mask.clr(reg); + if( mask.size1() ) + return mask.firstReg(); + } + } + return -1; // No obvious color choice, but mask got trimmed + } + + private static void swap( LRG[] ary, int x, int y ) { + LRG tmp = ary[x]; ary[x] = ary[y]; ary[y] = tmp; + } +} diff --git a/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/InstSelect.java b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/InstSelect.java new file mode 100644 index 00000000..e8e87785 --- /dev/null +++ b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/InstSelect.java @@ -0,0 +1,90 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; +import java.util.BitSet; +import java.util.IdentityHashMap; + +// Replace ideal nodes with machine nodes, then rebuild the copied graph's uses. +class InstSelect { + private final CodeGen _code; + InstSelect(CodeGen code) { _code=code; } + + @SuppressWarnings("unchecked") + void run(String cpu, String callingConv, String base) { + _code._callingConv = callingConv; + + // Look for CPU in fixed named place: + // com.seaofnodes.simple.node.cpus."cpu"."cpu.class" + String clzFile = base+"."+cpu+"."+cpu; + try { _code._mach = ((Class<Machine>) Class.forName( clzFile )).getDeclaredConstructor(new Class[]{CodeGen.class}).newInstance(_code); } + catch( Exception e ) { throw new RuntimeException(e); } + + // Build global copies of common register masks. + long callerSave = _code._mach.callerSave(); + long neverSave = _code._mach. neverSave(); + int maxReg = Math.min(64,_code._mach.regs().length); + assert maxReg>=64 || (-1L << maxReg & callerSave)==0; // No stack slots in callerSave + _code._callerSave = new RegMask(callerSave); + + // Build a Return RegMask array. All returns have the following inputs: + // 0 - ctrl + // 1 - memory + // 2 - varies with returning GPR,FPR + // 3 - RPC + // 4+- Caller save registers + _code._retMasks = new RegMask[(maxReg-_code._callerSave.size()-Long.bitCount( neverSave ))+4]; + for( int reg=0, i=4; reg<maxReg; reg++ ) + if( !_code._callerSave.test(reg) && ((1L<<reg)&neverSave)==0 ) + _code._retMasks[i++] = new RegMask(reg); + _code._rpcMask = new RegMask(_code._mach.rpc()); + _code._retMasks[3] = _code._rpcMask; + + // Convert to machine ops + _code._uid = 1; // All new machine nodes reset numbering + var map = new IdentityHashMap<Node,Node>(); + _instSelect( _code._stop, map ); + _code._stop = ( StopNode)map.get(_code._stop ); + StartNode start = (StartNode)map.get(_code._start); + _code._start = start==null ? new StartNode(_code._start) : start; + _instOuts(_code._stop,_code.visit()); + _code._visit.clear(); + } + + private Node _instSelect( Node n, IdentityHashMap<Node,Node> map ) { + if( n==null ) return null; + Node x = map.get(n); + if( x !=null ) return x; // Been there, done that + + // If n is a MachNode already, then its part of a multi-node expansion. + // It does not need instruction selection (already selected!) + // but it does need its inputs walked. + if( n instanceof MachNode ) { + for( int i=0; i < n.nIns(); i++ ) + n._inputs.set(i, _instSelect(n.in(i),map) ); + return n; + } + + // Produce a machine node from n; map it to flag as done so stops cycles. + map.put(n, x=_code._mach.instSelect(n) ); + // Stop retains the callable body. Its lifetime edge is not a machine operand. + if( n instanceof FunPtrNode ) x._inputs.pop(); + // Walk machine op and replace inputs with mapped inputs + for( int i=0; i < x.nIns(); i++ ) + x._inputs.set(i, _instSelect(x.in(i),map) ); + if( x instanceof MachNode mach ) + mach.postSelect(_code); // Post selection action + + return x; + } + + private void _instOuts( Node n, BitSet visit ) { + if( visit.get(n._nid) ) return; + visit.set(n._nid); + for( Node in : n._inputs ) + if( in!=null ) { + in._outputs.push(n); + _instOuts(in,visit); + } + } + +} diff --git a/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/LRG.java b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/LRG.java new file mode 100644 index 00000000..0c36ae2b --- /dev/null +++ b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/LRG.java @@ -0,0 +1,235 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.util.Ary; +import com.seaofnodes.simple.util.SB; +import com.seaofnodes.simple.util.Utils; +import java.util.IdentityHashMap; + +/** A Live Range + * <p> + * A live range is a set of nodes and edges which must get the same register. + * Live ranges form an interconnected web with almost no limits on their + * shape. Live ranges also gather the set of register constraints from all + * their parts. + * <p> + * Most of the fields in this class are for spill heuristics, but there are a + * few key ones that define what a LRG is. LRGs have a unique dense integer + * `_lrg` number which names the LRG. New `_lrg` numbers come from the + * `RegAlloc._lrg_num` counter. LRGs can be unioned together - + * (<a href="https://en.wikipedia.org/wiki/Disjoint-set_data_structure">this is the Union-Find algorithm</a>) + * - and when this happens the lower numbered `_lrg` wins. Unioning only + * happens during `BuildLRG` and happens because either a `Phi` is forcing all + * its inputs and outputs into the same register, or because of a 2-address + * instruction. LRGs have matching `union` and `find` calls, and a set + * `_leader` field. + */ +public class LRG { + + // Dense live range numbers + final short _lrg; + + // U-F leader; null if leader + LRG _leader; + + // Choosen register + short _reg; + + // Count of single-register defs and uses + short _1regDefCnt, _1regUseCnt; + // Count of all defs, uses. Mostly interested 1 vs many + boolean _multiDef, _multiUse; + + // A sample MachNode def in the live range + public MachNode _machDef, _machUse; + short _uidx; // _machUse input + + // Some splits used in biased coloring + MachConcreteNode _splitDef, _splitUse; + + // All the self-conflicting defs for this live range + IdentityHashMap<Node,String> _selfConflicts; + + // AND of all masks involved; null if none have been applied yet + RegMask _mask; + + // Adjacent Live Range neighbors. Only valid during coloring + Ary<LRG> _adj; + + public int nadj() { return _adj==null ? 0 : _adj._len; } + + public RegMask useMask() { return _machUse==null ? null : _machUse.regmap(_uidx); } + + void addNeighbor(LRG lrg) { + if( _adj==null ) _adj = new Ary<>(LRG.class); + _adj.push(lrg); + } + + // Remove and compress neighbor list; store ex-neighbor past + // end for unwinding Simplify. True if exactly going low-degree + boolean removeCompress( LRG lrg ) { + _adj.swap(_adj.find(lrg),_adj._len-1); + return _adj._len-- == _mask.size(); + } + + void reinsert( LRG lrg ) { + assert _adj._es[_adj._len]==lrg; + _adj._len++; + } + + // More registers than neighbors + boolean lowDegree() { return nadj() < _mask.size(); } + + LRG( short lrg ) { _lrg = lrg; _reg = -1; } + + boolean leader() { return _leader == null; } + + // Fast-path Find from the Union-Find algorithm + LRG find() { + if( _leader==null ) return this; // I am the leader + if( _leader._leader==null ) // I point to the leader + return _leader; + return _rollup(); + } + // Slow-path rollup of U-F + private LRG _rollup() { + LRG ldr = _leader._leader; + // Roll-up + while( ldr._leader!=null ) ldr = ldr._leader; + LRG l2 = this; + while( l2 != ldr ) { + LRG l3 = l2._leader; + l2._leader = ldr; + l2 = l3; + } + return ldr; + } + + // Union `this` and `lrg`, keeping the lower numbered _lrg. + // Includes a number of fast-path cutouts. + LRG union( LRG lrg ) { + assert leader(); + if( lrg==null ) return this; + lrg = lrg.find(); + if( lrg==this ) return this; + return _lrg < lrg._lrg ? _union(lrg) : lrg._union(this); + } + // Union `this` and `lrg`, folding together all stats. + private LRG _union( LRG lrg ) { + // Set U-F leader + lrg._leader = this; + // Fold together stats + if( _machDef==null ) { + _machDef = lrg._machDef; + } else if( lrg._machDef!=null ) { + if( _machDef != lrg._machDef ) _multiDef=true; + if( _1regDefCnt==0 ) + _machDef = lrg._machDef; + else if( _machDef==lrg._machDef ) + _1regDefCnt--; + } + _1regDefCnt += lrg._1regDefCnt; + _multiDef |= lrg._multiDef; + + if( _machUse==null ) { + _machUse = lrg._machUse; + _uidx = lrg._uidx; + } else if( lrg._machUse!=null ) { + if( _machUse != lrg._machUse ) _multiUse=true; + if( _1regUseCnt==0 ) { + _machUse = lrg._machUse; + _uidx = lrg._uidx; + } + else if( _machUse==lrg._machUse ) + _1regUseCnt--; + } + _1regUseCnt += lrg._1regUseCnt; + _multiUse |= lrg._multiUse; + + // Fold deepest Split + _splitDef = deepSplit(_splitDef,lrg._splitDef); + _splitUse = deepSplit(_splitUse,lrg._splitUse); + + // Fold together masks + RegMask mask = _mask.and(lrg._mask); + if( mask==null ) + mask = _mask.copy().and(lrg._mask); + _mask = mask; + return this; + } + + private static MachConcreteNode deepSplit( MachConcreteNode s0, MachConcreteNode s1 ) { + return s0==null || (s1!=null && s0.cfg0().loopDepth() < s1.cfg0().loopDepth()) ? s1 : s0; + } + + // Record any Mach def for spilling heuristics + LRG machDef( MachNode def, boolean size1 ) { + if( _machDef!=null && _machDef!=def ) + _multiDef = true; + if( _machDef==null || size1 ) + _machDef = def; + if( size1 ) + _1regDefCnt++; + if( def instanceof SplitNode split ) + _splitDef = deepSplit(_splitDef,split); + return this; + } + + // Record any Mach use for spilling heuristics + LRG machUse( MachNode use, short uidx, boolean size1 ) { + if( _machUse!=null && _machUse!=use ) + _multiUse = true; + if( _machUse==null || size1 ) + { _machUse = use; _uidx = uidx; } + if( size1 ) + _1regUseCnt++; + if( use instanceof SplitNode split ) + _splitUse = deepSplit(_splitUse,split); + return this; + } + + boolean hasSplit() { return _splitDef != null || _splitUse != null; } + short size() { return _mask.size(); } + boolean size1() { return _mask.size1(); } + + + void selfConflict( Node def ) { + if( _selfConflicts == null ) _selfConflicts = new IdentityHashMap<>(); + _selfConflicts.put(def,""); + } + + + // Record intersection of all register masks. + // True if still has registers + boolean and( RegMask mask ) { + RegMask mask2 = mask.and(_mask); + if( mask2==null ) + mask2 = _mask.copy().and(mask); + _mask = mask2; + return !_mask.isEmpty(); + } + // Remove this singular register + // True if still has registers + boolean clr( int reg ) { + if( _mask.clr(reg) ) return true; + _mask = _mask.copy(); // Need a mutable copy + return _mask.clr(reg); + } + // Subtract mask (AND complement) + // True if still has registers + boolean sub( RegMask mask ) { + RegMask mask2 = _mask.sub(mask); + if( mask2==null ) + mask2 = _mask.copy().sub(mask); + _mask = mask2; + return !_mask.isEmpty(); + } + + @Override public String toString() { return toString(new SB()).toString(); } + + public SB toString( SB sb ) { + sb.p("V").p(_lrg); + if( _mask!=null ) _mask.toString(sb); + return sb; + } +} diff --git a/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/LinkMem.java b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/LinkMem.java new file mode 100644 index 00000000..2415203d --- /dev/null +++ b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/LinkMem.java @@ -0,0 +1,23 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.Node; +import com.seaofnodes.simple.util.Utils; + +// In-memory linking +public class LinkMem { + final CodeGen _code; + LinkMem( CodeGen code ) { _code = code; } + + public CodeGen link() { + Encoding enc = _code._encoding; + + // Patch external calls internally + enc.patchGlobalRelocations(); + + // Write any large constants into a constant pool; they + // are accessed by RIP-relative addressing. + enc.writeConstantPool(enc._bits,true); + + return _code; + } +} diff --git a/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/ListScheduler.java b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/ListScheduler.java new file mode 100644 index 00000000..ddf5a679 --- /dev/null +++ b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/ListScheduler.java @@ -0,0 +1,261 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.IterPeeps.WorkList; +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.*; +import com.seaofnodes.simple.util.Ary; +import com.seaofnodes.simple.util.Utils; +import java.util.*; + +public abstract class ListScheduler { + + // eXtra per-node stuff for scheduling. + // Nodes are all block-local, and projections fold into their parent. + private static final IdentityHashMap<Node,XSched> XS = new IdentityHashMap<>(); + private static class XSched { + + Node _n; // Node this extra is for + int _bcnt; // Not-ready not-scheduled inputs + int _rcnt; // Ready not-scheduled inputs + boolean _ruse; // Node IS a "remote use", uses a other-block value + boolean _rdef; // Node IS a "remote def" of a value used in other block + boolean _single; // Defines a single register, likely to conflict + + static final Ary<XSched> FREE = new Ary<>(XSched.class); + static void alloc(CFGNode bb, Node n) { + XSched x = FREE.pop(); + XS.put(n, (x==null ? new XSched() : x).init(bb,n)); + } + void free() { _n = null; FREE.push(this); } + + static XSched get(Node n) { return n==null ? null : XS.get(n instanceof ProjNode && !(n.in(0) instanceof CallEndNode) ? n.in(0) : n); } + + private XSched init(CFGNode bb, Node n) { + _n = n; + _bcnt = 0; + _ruse = _rdef = _single = false; + + // Count block-locals and remote inputs + if( !(n instanceof PhiNode) ) + for( Node def : n._inputs ) + if( def != null && def != bb ) { + if( def.cfg0()==bb ) _bcnt++; // Raise block-local input count + else if( !def.isMem() ) _ruse = true; // Remote register user + } + // Count remote outputs and allowed registers + if( n instanceof MultiNode ) + for( Node use : n._outputs ) + computeSingleRDef(bb,use); + else + computeSingleRDef(bb,n); + + if( n instanceof NewNode || n instanceof CallNode ) + bb.fun()._hasCalls = true; + return this; + } + + private void computeSingleRDef(CFGNode bb, Node n) { + // See if this is 2-input, and that input is single-def + if( n instanceof MachNode mach && mach.twoAddress() != 0 ) { + XSched xs = XS.get(n.in(mach.twoAddress())); + if( xs != null ) + _single = xs._single; + } + // Visit all outputs + RegMaskRW rmask = !_single && n instanceof MachNode mach && mach.outregmap() != null ? mach.outregmap().copy() : null; + for( Node use : n._outputs ) { + // Remote use, so this is a remote def + if( use!=null && use.cfg0()!=bb ) _rdef = true; + // And mask between n's def mask and all n's users masks' + if( use instanceof MachNode mach && rmask != null ) + for( int i=1; i<use.nIns(); i++ ) + if( use.in(i)==n ) + rmask.and(mach.regmap(i)); + } + // Defines in a single register + _single |= rmask!=null && rmask.size1(); + } + + // If _bcnt==0, declare ready; move user bcnts into rcnts. + boolean isReady() { + if( _bcnt > 0 || _rcnt > 0 ) return false; + if( _n instanceof MultiNode ) + for( Node use : _n._outputs ) + if( use instanceof ProjNode ) readyUp(use); + // anti-dep from a multireg directly + else _readyUp(use); + else + readyUp(_n); + return true; + } + private static void readyUp(Node n) { + for( Node use : n._outputs ) + _readyUp(use); + } + private static void _readyUp(Node use) { + XSched xs = get(use); + if( xs!=null && !(use instanceof PhiNode) ) + { assert xs._bcnt>0; xs._bcnt--; xs._rcnt++; } + } + + boolean decIsReady() { + assert _rcnt > 0; + _rcnt--; + return isReady(); + } + } + + + // List schedule every block + public static void sched( CodeGen code ) { + XS.clear(); + for( CFGNode bb : code._cfg ) + if( bb.blockHead() ) + local(bb,false); + } + + // Classic list scheduler + private static void local( CFGNode bb, boolean trace ) { + assert XS.isEmpty(); + int len = bb.nOuts(); + + // Count block-locals, remote inputs and outputs + for( int i=0; i<len; i++ ) + XSched.alloc(bb,bb.out(i)); + + // Nodes are ready if they are not used by other nodes in this block. + // Move ready up front. + int ready = 0; + for( int i=0; i<len; i++ ) + if( XSched.get(bb.out(i)).isReady() ) // No block-local inputs? + bb._outputs.swap(ready++,i); // Move into ready set + assert len==0 || ready > 0; + + // Classic list scheduler. Behind sched is scheduled. Between sched + // and ready have zero counts and are ready to schedule. Ahead of + // ready is undefined. + // + // As nodes are scheduled, the nodes they use decrement their bcnts[]. Once + // that gets to 0, all the nodes using it are scheduled, so it's ready. + int sched = 0; + while( sched < len ) { + assert sched < ready; + int pick = best(bb._outputs,sched,ready,trace); // Pick best + Node best = bb._outputs.swap(pick,sched++); // Swap best to head of ready and move into the scheduled set + // Lower ready count of users, and since projections are ready as + // soon as their multi is ready, lower those now too. + for( Node use : best._outputs ) + if( use instanceof ProjNode ) + for( Node useuse : use._outputs ) + ready = ready(bb,useuse,ready); + else + ready = ready(bb,use,ready); + } + + // Reset for next block + for( XSched x : XS.values() ) + x.free(); + XS.clear(); + } + + private static int ready(CFGNode bb, Node use, int ready) { + if( use!=null && use.in(0)==bb && !(use instanceof PhiNode) && XSched.get(use).decIsReady() ) + bb._outputs.set(ready++, use); // Became ready, move into ready set + //bb._outputs.swap(ready++,use); // Became ready, move into ready set + return ready; + } + + + // Pick best between sched and ready. + private static int best( Ary<Node> blk, int sched, int ready, boolean trace ) { + int pick = sched; + Node p = blk.at(pick); + int score = score(p); + if( trace ) { System.out.printf("%4d N%d ",score,p._nid); System.out.println(p); } + for( int i=sched+1; i<ready; i++ ) { + Node n = blk.at(i); + int nscore = score(n); + if( trace ) { System.out.printf("%4d N%d ",nscore,n._nid); System.out.println(n); } + if( nscore > score ) + { score = nscore; pick = i; p = n; } + } + if( trace ) { System.out.printf("Pick: N%d %s\n\n",p._nid,p); } + return pick; + } + + // Highest score wins. Max at 1000, min at 10, except specials. + private static final int[] CNT = new int[3]; + static int score( Node n ) { + if( n instanceof ProjNode ) return 1001; // Pinned at block entry + if( n instanceof CProjNode ) return 1001; // Pinned at block entry + if( n instanceof PhiNode ) return 1000; + if( n instanceof CFGNode ) return 1; // Pinned at block exit + + int score = 500; + // Ideal nodes ignore register pressure scheduling + if( !(n instanceof MachNode) ) + return score; + + // Register pressure local scheduling: avoid overlapping specialized + // registers usages. + + // Subtract 10 (delay) if this op forces a live range to exist that + // cannot be resolved immediately; i.e., live count goes up. Scale to + // 20 if multi-def. + + // Subtract 100 if this op forces a single-def live range to exist + // which might conflict on the same register with other live ranges. + // Defines a single register based on def & uses, and the output is not + // ready. Scale to 200 for multi-def. + CNT[1]=CNT[2]=0; + XSched xn = XSched.get(n); + // If defining a remote value, just generically stall alot. Value is + // used in a later block, can we delay until the end of this block? + if( xn._rdef ) score = 200; // If defining a remote value, just generically stall alot + boolean flags = false; + if( n instanceof MultiNode ) { + for( Node use : n._outputs ) + flags |= singleUseNotReady( use, xn._single ); + } else + flags |= singleUseNotReady( n, xn._single ); + score += -10 * Math.min( CNT[1], 2 ); + score += -100 * Math.min( CNT[2], 2 ); + if( flags ) return 10; + + // Add 10 if this op ends a single-def live range, add 20 for 2 or more. + // Scale to 100,200 if the single-def is also single-register. + CNT[1]=CNT[2]=0; + for( int i=1; i<n.nIns(); i++ ) { + XSched xd = XSched.get( n.in(i) ); + if( xd != null && n.in(i).nOuts()==1 ) + CNT[xd._single || ((MachNode)n).regmap(i)!=null && ((MachNode)n).regmap(i).size1() ? 2 : 1]++; + } + score += 10 * Math.min( CNT[1], 2 ); + score += 100 * Math.min( CNT[2], 2 ); + + // Nothing special + assert 10 <= score && score <= 990; + return score; + } + + // Single use, and that use is not ready to fire once this one does. If + // true, stalling 'n' might reduce 'n's lifetime. Stall by 1 (2 if single) + // per not-ready. + + // If CFG user in this block, it must be last, so definitely stall + // single-def (which stalls flags on x86,arm until the jmp). + private static boolean singleUseNotReady( Node n, boolean single ) { + // If n can rematerialize, assume allocator will split into private uses + // as needed. No impact on local scheduling. + if( n.nOuts()>1 && n instanceof MachNode mach && mach.isClone() ) + return false; + for( Node use : n.outs() ) { + XSched xu = XSched.get(use); + if( xu != null && xu._n instanceof CFGNode ) return true; + if( xu != null && xu._bcnt > 0 ) + CNT[single ? 2 : 1]++; // Since bcnt>0 or CFG, stall until user is more ready + } + return false; + } + +} diff --git a/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/Machine.java b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/Machine.java new file mode 100644 index 00000000..a95faefc --- /dev/null +++ b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/Machine.java @@ -0,0 +1,108 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.TypeFunPtr; +import com.seaofnodes.simple.type.TypeInteger; + +abstract public class Machine { + // Human readable machine name. Something like "x86-64" or "arm" or "risc5" + public abstract String name(); + // Default opcode size for printing; 4 bytes for most 32-bit risc chips + public int defaultOpSize() { return 4; } + // Human-readable name for a register number, e.g. "RAX" or "R0" + public abstract String[] regs(); + // Create a split op; any register to any register, including stack slots + public abstract SplitNode split( String kind, byte round, LRG lrg); + // List of caller-save registers, as a 64bit mask + public abstract long callerSave(); + // List of never-save registers, e.g. RSP or a ZERO register if you have one + public abstract long neverSave(); + // Call Argument Mask. Passed in the function signature and argument + // number (2-based; 0 is for control and 1 for memory). Also passed in a 0 + // for the function itself, or for *outgoing* calls, the maximum stack slot + // given to the incoming function arguments (stack slots reserved for + // incoming arguments). + public abstract RegMask callArgMask(TypeFunPtr tfp, int arg, int maxArgSlot); + // Return register mask, based on signature (GPR vs FPR) + public abstract RegMask retMask(TypeFunPtr tfp); + // Return PC register + public abstract int rpc(); + // Return a MachNode unconditional branch + public abstract CFGNode jump(); + // Instruction select from ideal nodes + public abstract Node instSelect( Node n ); + + /** Stack Slot Numbering + +RA gives all registers a number, starting at 0. Stack slots continue this +numbering going up from the last register number. Common register numbers are +0-15 for GPRs, 16-31 for FPRs, 32 for flags, and stack slots starting at +register#33 going up. These numbers are machine-specific, YMMV, etc; e.g. RPC +is only in stack slot 0 on X86; other cpus start with the rpc in a register +which may spill into any generic spill slot. + +For ease of reading and printing, stack slots start again at slot#0 - but +during RA they are actually biased by CPU.MAX_REG. + +Stack Layout during Reg Alloc: + + Slot Value Description + ---- ------- ----------- +MAXREG+ R+1 argR maxArgSlot(calleR tfp), can be 0 + ... +MAXREG+ 1 arg0 + ---- -- Caller ------ +MAXREG+ 0 RPC +MAXREG+ R+E+N PAD optional alignment + +MAXREG+ + Spills Caller Frame Space; N spills +MAXREG+ R+E+1 Spills + +MAXREG+ R+E argE maxArgSlot(calleE tfp), can be 0 + ... +MAXREG+ R+2 arg0 + ---- -- Callee ------- + ... + +Slots defined by maxArgSlot do not have to contain the actual passed argument; +they can be "shadow slots". If they exist, they are property of the frame / +FunNode where the TFP comes from. + +For a given calleR, there are generally many calleE's and they all share the +low-range stack slots. + +Example:, Win64, main(int arg) { calloc(1,size); } + + Post-Warp: SP+56 shadow-main-arg +Pre-alloc Post-Alloc SP+48 RPC +--- ------- ---------- ------------- +40 spill#3 SP+40 SP+40 spill#3 +39 spill#2 SP+32 "" +38 spill#1 SP+24 "" +37 spill#0 SP+16 "" spill#0 +36 shadow-calloc#size SP+ 8 SP+ 8 shadow-calloc#size +35 shadow-calloc#1 SP+ 0 SP+ 0 shadow-calloc#1 +--- --- FRAME BASE ---------- ---- FRAME BASE ------- +34 shadow-main-arg SP+56 +33 RPC SP+48 + + + */ + + + // Maximum stack slot (or 0) for the args in this TFP. This will include + // shadow slots if defined in the ABI, even if all arguments are passed in + // registers. + public abstract short maxArgSlot(TypeFunPtr tfp); + + // Only the declared C result width describes the physical return register. + // Simple functions already return canonical full-width integer values. + protected static TypeInteger cReturn(ProjNode prj) { + if( prj._idx==2 && prj.in(0) instanceof CallEndNode cend && + cend.call().fptr() instanceof ExternNode ext && + ((TypeFunPtr)ext._con).ret() instanceof TypeInteger ret && ret.log_size()<3 ) + return ret; + return null; + } + +} diff --git a/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/Opto.java b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/Opto.java new file mode 100644 index 00000000..e6334414 --- /dev/null +++ b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/Opto.java @@ -0,0 +1,53 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.*; +import com.seaofnodes.simple.util.Ary; + +// Sparse conditional constant propagation within each function. +abstract public class Opto { + + public static void opto(CodeGen code) { + Ary<Type> oldTypes = new Ary<>(Type.class); + code._start.walk(n -> { + oldTypes.setX(n._nid,n._type); + if( n instanceof CallEndNode cend ) cend.freezeResult(); + n._type = Type.TOP; + code.add(n); + // Calls retain their signature and whole-memory effects, but do + // not pass optimistic values between functions in this chapter. + if( n instanceof CallNode call ) call.unlink_all(); + return null; + }); + // Keep each function's unknown-caller Start input. Its parameters + // receive their declared types, independently of the actual calls. + // The next chapter removes those inputs and discovers call edges. + Node n; + while( (n = code._iter._work.pop()) != null ) { + if( n.isDead() ) continue; + Type old = n._type, next = n.compute(); + if( old == next ) continue; + assert old.isa(next); // Optimistic types only fall. + assert next.isa(oldTypes.at(n._nid)) : n.getClass().getSimpleName()+"#"+n._nid+" old="+oldTypes.at(n._nid)+" next="+next; + n._type = next; + code.addAll(n._outputs); + n.moveDepsToWorklist(code._iter); + assert check(code); + } + code._start.walk(x -> { + assert x.compute() == x._type; + assert x._type.isa(oldTypes.at(x._nid)); + code.add(x); // Run the rewrites enabled by the new types. + return null; + }); + } + + private static boolean check(CodeGen code) { + code._start.walk(n -> { + Type next = n.compute(); + assert next == n._type || (n._type.isa(next) && code._iter._work.on(n)); + return null; + }); + return true; + } +} diff --git a/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/RIPRelSize.java b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/RIPRelSize.java new file mode 100644 index 00000000..777182dc --- /dev/null +++ b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/RIPRelSize.java @@ -0,0 +1,14 @@ +package com.seaofnodes.simple.codegen; + +// This Node rip-relative encodings has sizes that vary by delta +public interface RIPRelSize { + // delta is the distance from the opcode *start* to the target start. Each + // hardware target adjusts as needed, X86 measures from the opcode *end* + // and small jumps are 2 bytes, so they'll need to subtract 2. + byte encSize(int delta); + + // Patch full encoding in. Used for both short and long forms. delta + // again is from the opcode *start*. + void patch( Encoding enc, int opStart, int opLen, int delta ); + +} diff --git a/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/RegAlloc.java b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/RegAlloc.java new file mode 100644 index 00000000..565fa6ba --- /dev/null +++ b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/RegAlloc.java @@ -0,0 +1,704 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.util.Ary; +import com.seaofnodes.simple.util.Utils; +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.Type; +import java.util.Arrays; +import java.util.BitSet; +import java.util.IdentityHashMap; + +/** + * "Briggs/Chaitin/Click". + * Graph coloring. + * <p> + * Every def and every use have a bit-set of allowed registers. + * Fully general to bizarre chips. + * Multiple outputs fully supported, e.g. add-with-carry or combined div/rem or "xor rax,rax" setting both rax and flags. + * <p> + * Intel 2-address accumulator-style ops fully supported. + * Op-to-stack-spill supported. + * All addressing modes supported. + * Register pairs could be supported with some grief. + * <p> + * Splitting instead of spilling (simpler implementation). + * Stack slots are "just another register" (tiny stack frames, simpler implementation). + * Stack/unstack during coloring with a marvelous trick (simpler implementation). + * <p> + * Both bitset and adjacency list formats for the interference graph; one of + * the few times it's faster to change data structures mid-flight rather than + * just wrap one of the two. + * <p> + * Liveness computation and interference graph built in the same one pass (one + * fewer passes per round of coloring). + * <p> + * Single-register def or use live ranges deny neighbors their required + * register and thus do not interfere, vs interfering and denying the color + * and coloring time. Basically all function calls do this, but many oddball + * registers also, e.g. older div/mod/mul ops. (5x smaller IFG, the only + * O(n^2) part of this operation). + */ + +public class RegAlloc { + // Main Coloring Algorithm: + // Repeat until colored: + // Build Live Ranges (LRGs) + // - Intersect allowed registers + // If hard conflicts (LRGs with no allowed registers) + // - Pre-split conflicted LRGs, and repeat + // Build Interference Graph (IFG) + // - Self conflicts split now + // Color (remove trivial first then any until empty; reverse assign colors + // - If color fails: + // - - Split uncolorable LRGs + + // Top-level program graph structure + final CodeGen _code; + + public int _spills, _spillScaled; + + // ----------------------- + // Live ranges with self-conflicts or no allowed registers + private final IdentityHashMap<LRG,String> _failed = new IdentityHashMap<>(); + // Loop Phis get one chance to color after only cold self-conflict splits. + // If the same Phi self-conflicts again, split its hot def/backedge also. + private final IdentityHashMap<PhiNode,String> _deferredLoopSelf = new IdentityHashMap<>(); + void fail( LRG lrg ) { + assert lrg.leader(); + _failed.put(lrg,""); + } + boolean success() { return _failed.isEmpty(); } + + + // ----------------------- + // Map from Nodes to Live Ranges + private final IdentityHashMap<Node,LRG> _lrgs = new IdentityHashMap<>(); + short _lrg_num; + + // Define a new LRG, and assign n + LRG newLRG( Node n ) { + LRG lrg = lrg(n); + if( lrg!=null ) return lrg; + lrg = new LRG(_lrg_num++); + LRG old = _lrgs.put(n,lrg); assert old==null; + return lrg; + } + + // LRG for n + LRG lrg( Node n ) { + LRG lrg = _lrgs.get(n); + if( lrg==null ) return null; + LRG lrg2 = lrg.find(); + if( lrg != lrg2 ) + _lrgs.put(n,lrg2); + return lrg2; + } + + // Find LRG for n.in(idx), and also map n to it + LRG lrg2( Node n, int idx ) { + LRG lrg = lrg(n.in(idx)); + return union(lrg,n); + } + + // Union any lrg for n with lrg and map to the union + LRG union( LRG lrg, Node n ) { + LRG lrgn = _lrgs.get(n); + LRG lrg3 = lrg.union(lrgn); + _lrgs.put(n,lrg3); + return lrg3; + } + + // Force all unified to roll up; collect live ranges + LRG[] _LRGS; + void unify() { + Ary<LRG> lrgs = new Ary<>(LRG.class); + for( Node n : _lrgs.keySet() ) { + LRG lrg = lrg(n); + lrgs.setX(lrg._lrg,lrg); + } + _LRGS = lrgs.asAry(); + // Remove unified lrgs from failed set also + for( LRG lrg : _failed.keySet().toArray(new LRG[0]) ) { + if( !lrg.leader() ) { + LRG lrg2 = lrg.find(); + _failed.remove(lrg); + _failed.put(lrg2,""); + } + } + } + + + // Diagnostic lookup: do not compress leaders or rewrite the node mapping. + private LRG _lrg(Node n) { + LRG lrg = _lrgs.get(n); + if( lrg != null ) + while( lrg._leader != null ) lrg = lrg._leader; + return lrg; + } + + public short regnum( Node n ) { + LRG lrg = _lrg(n); + return lrg==null ? -1 : lrg._reg; + } + + // Printable register number for node n + String reg( Node n ) { return reg(n,null); } + String reg( Node n, FunNode fun ) { + LRG lrg = _lrg(n); + if( lrg==null ) return null; + // No register yet, use LRG + if( lrg._reg == -1 ) return "V"+lrg._lrg; + // Chosen machine register unless stack-slot and past RA + String[] regs = _code._mach.regs(); + if( lrg._reg < regs.length || _code._phase.ordinal() <= CodeGen.Phase.RegAlloc.ordinal() || fun==null ) + return RegMask.reg(regs,lrg._reg); + // Stack-slot past RA uses the frame layout logic + return "[rsp+"+fun.computeStackOffset(_code,lrg._reg)+"]"; + } + + // ----------------------- + RegAlloc( CodeGen code ) { _code = code; } + + public void regAlloc() { + // Insert callee-save registers + String[] regs = _code._mach.regs(); + long neverSave= _code._mach.neverSave(); + for( CFGNode bb : _code._cfg ) + if( bb instanceof FunNode fun ) { + ReturnNode ret = fun.ret(); + int len = Math.min(regs.length,64); + for( int reg=0; reg<len; reg++ ) + if( !_code._callerSave.test(reg) && ((1L<<reg)&neverSave)==0 ) { + ret.addDef(new CalleeSaveNode(fun,reg,regs[reg])); + assert ret.regmap(ret.nIns()-1).firstReg()==reg; + } + } + // Cache reg masks for New and Call + for( CFGNode bb : _code._cfg ) { + if( bb instanceof CallEndMach cend ) cend.cacheRegs(_code); + for( Node n : bb._outputs ) + if( n instanceof NewNode nnn ) nnn.cacheRegs(_code); + } + + // Optional diagnostic for values accidentally shared across functions. + //assert verifyFunctionLocalEdges(); + + // Top driver: repeated rounds of coloring and splitting. + byte round=0; + while( !graphColor(round) ) { + split(round); + if( round >= 7 ) // Really expect to be done soon + throw new IllegalStateException("Register allocation made no progress after eight rounds"); + round++; + } + postColor(); // Remove no-op spills + } + + boolean verifyFunctionLocalEdges() { + for( CFGNode bb : _code._cfg ) { + if( bb instanceof StartNode ) continue; + FunNode fun = bb.fun(); + for( Node use : bb._outputs ) + if( use instanceof MachNode mach && !(use instanceof ParmNode) ) + for( int i=1; i<use.nIns(); i++ ) { + Node def = use.in(i); + if( def==null || mach.regmap(i)==null || def.cfg0()==null || def.cfg0() instanceof StartNode ) continue; + assert def.cfg0().fun()==fun : "Cross-function register edge: "+def.uniqueName()+" -> "+use.uniqueName()+" input "+i; + } + } + return true; + } + + private boolean graphColor(byte round) { + _failed.clear(); + _lrgs.clear(); + _lrg_num = 1; + _LRGS=null; + + return + // Build Live Ranges + BuildLRG.run(round,this) && // if no hard register conflicts + // Build Interference Graph + IFG.build(round,this) && // If no self conflicts or uncolorable + // Conservative coalesce copies + Coalesce.coalesce(round,this) && + // Color attempt + IFG.color(round,this); // If colorable + } + + // ----------------------- + // Split conflicted live ranges. + void split(byte round) { + + // In C2, all splits are handling in one pass over the program. Here, + // in the name of clarity, we'll handle each failing live range + // independently... which generally requires a full pass over the + // program for each failing live range. i.e., might be a lot of + // passes. + + // Sort, to avoid non-deterministic HashMap ordering + LRG[] splits = _failed.keySet().toArray(new LRG[0]); + Arrays.sort(splits, (x,y) -> x._lrg - y._lrg ); + for( LRG lrg : splits ) + split(round,lrg); + } + + // Split this live range, top level heuristic + boolean split( byte round, LRG lrg ) { + assert lrg.leader(); // Already rolled up + + if( lrg._selfConflicts != null ) + return splitSelfConflict(round,lrg); + + // Register mask when empty; split around defs and uses with limited + // register masks. + if( lrg._mask.isEmpty() && (!lrg._multiDef || lrg._1regUseCnt==1) ) { + if( lrg._1regDefCnt <= 1 && + lrg._1regUseCnt <= 1 && + (lrg._1regDefCnt + lrg._1regUseCnt) > 0 && + // A fixed-register clone with flexible uses must split at the uses. + !(lrg._1regDefCnt==1 && lrg._machDef.isClone() && lrg._1regUseCnt==0) ) + return splitEmptyMaskSimple(round,lrg); + // Repeated single-reg uses from a single def. Special for archs + // with more fixed regs. + if( !lrg._multiDef && lrg._1regDefCnt <= 1 && lrg._1regUseCnt > 2 ) + if( splitEmptyMaskByUse(round,lrg) ) + return true; + } + + // Generic split-by-loop depth. + return splitByLoop(round,lrg); + } + + // Split live range with an empty mask. Specifically forces splits at + // single-register defs or uses and not elsewhere. + boolean splitEmptyMaskSimple( byte round, LRG lrg ) { + // Live range has a single-def single-register, and/or a single-use + // single-register. Split after the def and before the use. Does not + // require a full pass. + + // Split just after def + if( lrg._1regDefCnt==1 && !lrg._machDef.isClone() ) + // An earlier copy is reusable only if no intervening clobber. + // Being in the same block alone is insufficient. Example: + // alloc + // V1/rax - forced by alloc + // alloc + // V2/rax - kills prior RAX + // st4 [V1],len - No good, must split around + insertAfterAndReplace(makeSplit("def/empty1",round,lrg),(Node)lrg._machDef,false); + // Split just before use + if( lrg._1regUseCnt==1 ) + insertBefore((Node)lrg._machUse,lrg._uidx,"use/empty1",round,lrg); + return true; + } + + // Popular single-def value: share a split among uses with compatible masks. + boolean splitEmptyMaskByUse( byte round, LRG lrg ) { + Node def = (Node)lrg._machDef; + // Snapshot distinct users: rewriting an input changes def's output list, + // and a call can use the same value in more than one argument. + Ary<Node> uses = new Ary<>(Node.class); + for( Node use : def._outputs ) + if( uses.find(use)==-1 ) uses.push(use); + + Ary<RegMask> rclass = new Ary<>(RegMask.class); + int ncalls=0; + for( Node use : uses ) + if( use instanceof MachNode mach ) { + // Sharing copies across several calls usually loses to their kills. + if( use instanceof CallNode && ++ncalls > 1 ) return false; + for( int i=1; i<use.nIns(); i++ ) + if( use.in(i)==def && mach.regmap(i)!=null ) + putIntoRegClass(rclass,mach.regmap(i)); + } + if( rclass._len <= 1 ) return false; + + for( RegMask rmask : rclass ) { + Node split = makeSplit(def,"popular",round,lrg,rmask); + split.insertAfter(def); + if( split.nIns()>1 ) split.setDef(1,def); + for( Node use : uses ) + if( use instanceof MachNode mach ) + for( int i=1; i<use.nIns(); i++ ) { + RegMask mask = mach.regmap(i); + if( use.in(i)==def && mask!=null && mask.overlap(rmask) ) { + assert rmask.and(mask)==rmask; + use.setDefOrdered(i,split); + } + } + } + return true; + } + + // Narrowing preserves all earlier users of a class; new classes are disjoint. + // Each use contains its first overlapping class, so one pass suffices. + private static void putIntoRegClass( Ary<RegMask> rclass, RegMask rmask ) { + for( int i=0; i<rclass._len; i++ ) { + RegMask omask = rclass.at(i); + if( !omask.overlap(rmask) ) continue; + if( omask.and(rmask)!=omask ) + rclass.set(i,new RegMask(omask.copy().and(rmask))); + return; + } + rclass.push(rmask); + } + + + // Self conflicts require Phis (or two-address). + // Insert a split after every def. + boolean splitSelfConflict( byte round, LRG lrg ) { + // Sort conflict set, so we're deterministic + Node[] conflicts = lrg._selfConflicts.keySet().toArray(new Node[0]); + Arrays.sort(conflicts, (x,y) -> x._nid - y._nid ); + + // First encounter with a loop Phi: try only cold splits this round. + // Remember the Phi across rounds so a persistent self-conflict forces + // the hot def/backedge splits on the next encounter. + IdentityHashMap<PhiNode,String> deferred = new IdentityHashMap<>(); + for( Node def : conflicts ) { + if( def instanceof PhiNode phi && phi.region() instanceof LoopNode ) + deferLoopSelf(phi,deferred); + for( Node use : def._outputs ) + if( use instanceof PhiNode phi && phi.region() instanceof LoopNode && + phi._inputs.find(def)==2 ) + deferLoopSelf(phi,deferred); + } + + // For all conflicts + for( Node def : conflicts ) { + assert lrg(def)==lrg; // Might be conflict use-side + // Split before each use that extends the live range; i.e. is a + // Phi or two-address + for( int i=0; i<def._outputs._len; i++ ) { + Node use = def.out(i); + if( (use instanceof PhiNode phi && + !(deferred.containsKey(phi) && phi.region() instanceof LoopNode && phi._inputs.find(def)==2) && + !(phi.region() instanceof LoopNode loop && phi.in(2)==def && def.cfg0().idepth() > loop.idepth() ) ) || + (use instanceof MachNode mach && mach.twoAddress()!=0 && use.in(mach.twoAddress())==def) ) + insertBefore( use, use._inputs.find(def), "use/self/use",round,lrg ); + } + // Split after the Phi and before its entry. Also isolate inputs + // shared with another Phi: otherwise the shared input keeps the + // conflicting live ranges joined, even after splitting the entry. + if( def instanceof PhiNode phi && !(def instanceof ParmNode) ) { + if( !deferred.containsKey(phi) ) { + SplitNode split = makeSplit("def/self",round,lrg); + insertAfterAndReplace(split,def,false); + if( split.nOuts()==0 ) + split.killOrdered(); + } + for( int i=1; i<(deferred.containsKey(phi) ? 2 : phi.nIns()); i++ ) + if( i==1 || sharedPhiInput(phi,i) ) + insertBefore(phi,i,"use/self/phi",round,lrg); + } + // Split before two-address ops which extend the live range + if( def instanceof MachNode mach && mach.twoAddress()!= 0 ) + insertBefore(def,mach.twoAddress(),"use/self/two",round,lrg); + } + return true; + } + + private void deferLoopSelf(PhiNode phi, IdentityHashMap<PhiNode,String> deferred) { + if( !_deferredLoopSelf.containsKey(phi) ) { + _deferredLoopSelf.put(phi,""); + deferred.put(phi,""); + } + } + + + private static boolean sharedPhiInput(PhiNode phi, int i) { + for( Node use : phi.in(i)._outputs ) + if( use instanceof PhiNode && use!=phi ) return true; + return false; + } + + // Generic: split around the outermost loop with non-split def/uses. This + // covers both self-conflicts (once we split deep enough) and register + // pressure issues. + boolean splitByLoop( byte round, LRG lrg ) { + findAllLRG(lrg); + + // Find min loop depth for all non-split defs and uses. + long ld = (-1L<<32) | 9999; + for( Node n : _ns ) { + if( lrg(n)==lrg ) // This is a LRG def + ld = ldepth(ld,n,n.cfg0()); + // PhiNodes check all CFG inputs + if( n instanceof PhiNode phi ) { + for( int i=1; i<n.nIns(); i++ ) + if( phi.in(i)._type!=Type.TOP ) + ld = ldepth(ld, phi.in(i), phi.region().cfg(i)); + } else { + // Others check uses + for( int i=1; i<n.nIns(); i++ ) + if( lrgSame(n.in(i),lrg) ) // This is a LRG use + ld = ldepth(ld,n,n.cfg0()); + } + } + int min = (int)ld; + int max = (int)(ld>>32); + + + // If the minLoopDepth is less than the maxLoopDepth: for-all defs and + // uses, if at minLoopDepth or lower, split after def and before use. + for( Node n : _ns ) { + if( n instanceof SplitNode split && !split.isNonStackSplit() && min!=max ) continue; // Ordinary splits already reach stack storage. + if( n.isDead() ) continue; // Some Cloneable went dead by other spill changes + // If this is a 2-address commutable op (e.g. AddX86, MulX86) and the rhs has only a single user, + // commute the inputs... which chops the LHS live ranges' upper bound to just the RHS. + if( n instanceof MachNode mach && lrg(n)==lrg && mach.twoAddress()==1 && mach.commutes() && n.in(2).nOuts()==1 ) + n.swap12(); + + if( lrg(n)==lrg && // This is a LRG def + // At loop boundary, or splitting in inner loop + (min==max || n.cfg0().loopDepth() <= min) ) { + // Cloneable constants will be cloned at uses, not after def + if( !(n instanceof MachNode mach && mach.isClone()) && + // Single user is already a split adjacent + !(n.nOuts()==1 && n.out(0) instanceof SplitNode split && sameBlockNoClobber(split) ) ) + // Split after def in min loop nest + insertAfterAndReplace( makeSplit("def/loop",round,lrg), n,true); + } + + // PhiNodes check all CFG inputs + if( n instanceof PhiNode phi && !(n instanceof ParmNode)) { + for( int i=1; i<n.nIns(); i++ ) + // No split in front of a split + if( !(n.in(i) instanceof SplitNode) && + // splitting in inner loop or at loop border + (min==max || phi.region().cfg(i).loopDepth() <= min) && + // and not around the backedge of a loop (bad place to force a split, hard to remove) + !(phi.region() instanceof LoopNode && i==2 && (phi.in(i) instanceof PhiNode pp && pp.region()==phi.region())) ) + // Split before phi-use in prior block + insertBefore(phi,i, "use/loop/phi",round,lrg); + + } else { + // Others check uses + for( int i=1; i<n.nIns(); i++ ) { + // This is a LRG use + // splitting in inner loop or at loop border + if( lrgSame( n.in( i ), lrg ) && + (min == max || (n.in(i) instanceof MachNode mach && mach.isClone()) || n.cfg0().loopDepth() <= min) ) + // Split before in this block + insertBefore( n, i, "use/loop/use", round,lrg, false ); + } + } + } + return true; + } + + private boolean lrgSame(Node x, LRG lrg) { + LRG xlrg = lrg(x); + while( xlrg==null && x instanceof SplitNode ) + xlrg=lrg(x = x.in(1)); + return xlrg==lrg; + } + + private static long ldepth( long ld, Node n, CFGNode cfg ) { + // Ordinary splits already reach stack storage; register-only transfers do not. + if( n instanceof SplitNode split && !split.isNonStackSplit() ) return ld; + // Collect min/max loop depth + int min = (int)ld; + int max = (int)(ld>>32); + int d = cfg.loopDepth(); + // if n will lower the min loop and is in the tail end of the loop + // header, splitting "around" the loop will not help. Treat n as being + // in the loop. + if( d < min ) { + if( cfg.uctrl() instanceof LoopNode loop && loop.entry()==cfg ) { + for( int i=cfg.nOuts()-2; i>=0; i-- ) { + Node out = cfg.out(i); + if( n==out ) + { d = loop.loopDepth(); break; } // Treat n as being "in the loop" + if( !((out instanceof MachNode mach && mach.isClone()) || out instanceof SplitNode ) ) + break; // Treat b as "normal", out of loop + } + } + } + + // lower min, raise max, and re-fold + min = Math.min(min,d); + max = Math.max(max,d); + return ((long)max<<32) | min; + } + + // Find all members of a live range, both defs and uses + private final Ary<Node> _ns = new Ary<>(Node.class); + void findAllLRG( LRG lrg ) { + _ns.clear(); + int wd = 0; + if( lrg._machDef!=null ) _ns.push((Node)lrg._machDef); + if( lrg._machUse!=null ) _ns.push((Node)lrg._machUse); + while( wd < _ns._len ) { + Node n = _ns.at(wd++); + if( lrg(n)!=lrg ) continue; + for( Node def : n._inputs ) + if( lrg(def)==lrg && _ns.find(def)== -1 ) + _ns.push(def); + for( Node use : n._outputs ) + if( _ns.find(use)== -1 ) + _ns.push(use); + } + for( Node n : _ns ) assert !n.isDead(); + } + + void insertBefore(Node n, int i, String kind, byte round, LRG lrg, boolean skip) { + Node def = n.in(i); + // Phi splitting may visit the synthetic never-exit arm. + if( def._type==Type.TOP ) return; + // Effective block for use + CFGNode cfg = n instanceof PhiNode phi ? phi.region().cfg(i) : n.cfg0(); + // Use-side RegMask, if available + RegMask umask = n instanceof MachNode mach ? mach.regmap(i) : null; + // Def is a split ? + if( skip && def instanceof SplitNode ) { + boolean singleReg = umask!=null && umask.size1(); + // Same block, multiple registers, split is only used by n, + // assume this is good enough and do not split again. + if( cfg==def.cfg0() && def.nOuts()==1 && !singleReg ) + return; + } + makeSplit(def,kind,round,lrg,umask).insertBefore(n, i); + // Skip split-of-split same block + if( skip && def instanceof SplitNode && cfg==def.cfg0() ) + n.in(i).setDefOrdered(1,def.in(1)); + } + void insertBefore(Node n, int i, String kind, byte round, LRG lrg) { + insertBefore(n,i,kind,round,lrg,true); + } + + // Replace uses of `def` with `split`, and insert `split` immediately after + // `def` in the basic block. + public void insertAfterAndReplace( Node split, Node def, boolean must ) { + split.insertAfter(def); + if( split.nIns()>1 ) split.setDef(1,def); + for( int j=def.nOuts()-1; j>=0; j-- ) { + Node use = def.out(j); + if( use==split ) continue; // Skip self + // Can we avoid a split of a split? 'this' split is used by + // another split in the same block. + if( !must && use instanceof SplitNode split2 && sameBlockNoClobber(split2) ) + continue; + int idx = use._inputs.find(def); + use.setDefOrdered(idx,split); + if( j < def.nOuts() ) j++; + } + } + + private Node makeSplit( Node def, String kind, byte round, LRG lrg, RegMask umask ) { + // Clone simple constants if possible + Node split = def instanceof MachNode mach && mach.isClone() && (umask==null || mach.outregmap().overlap(umask)) + ? mach.copy() + : _code._mach.split(kind,round,lrg); + _lrgs.put(split,lrg); + return split; + } + private SplitNode makeSplit( String kind, byte round, LRG lrg ) { + SplitNode split = _code._mach.split(kind,round,lrg); + _lrgs.put(split,lrg); + return split; + } + + + // ----------------------- + // POST PASS: Remove empty spills that biased-coloring made + private void postColor() { + int maxReg = -1; + for( CFGNode bb : _code._cfg ) { // For all ops + if( bb instanceof FunNode ) + maxReg = -1; // Reset for new function + // Compute frame size, based on arguments and largest reg seen + if( bb instanceof ReturnNode ret ) + ret.fun().computeFrameAdjust(_code,maxReg); + // Raise frame size by max stack args passed, even if ignored + if( bb instanceof CallEndMach cend ) + maxReg = Math.max(maxReg,cend._xslot); + + for( int j=0; j<bb.nOuts(); j++ ) { + Node n = bb.out(j); + if( lrg(n)!=null ) + maxReg = Math.max(maxReg,lrg(n)._reg+1); + // Raise frame size by max stack args passed to New + if( n instanceof NewNode nnn ) + maxReg = Math.max(maxReg,nnn._xslot); + + if( !(n instanceof SplitNode ) ) continue; + int defreg = lrg(n )._reg; + int usereg = lrg(n.in(1))._reg; + // Attempt to bypass split + if( defreg != usereg && splitBypass(bb,j,n,defreg) ) + usereg = lrg(n.in(1))._reg; + + // Same register, or the next instruction can use the source directly. + if( defreg == usereg || splitForward(bb,j,n,usereg) ) { + n.removeSplit(); + j--; + continue; + } + // Split is kept; count against split score + _spills++; + _spillScaled += (1<<bb.loopDepth()*3); + assert _spillScaled >= 0; + } + } + } + + // A sole, adjacent use can read the source register directly. Nothing can + // clobber it in between; only the operand mask and two-address tie remain. + private boolean splitForward( CFGNode bb, int j, Node split, int reg ) { + if( split.nOuts()!=1 || j+1==bb.nOuts() ) return false; + Node use = split.out(0); + if( use!=bb.out(j+1) || use instanceof PhiNode || use instanceof CFGNode || + !(use instanceof MachNode mach) ) return false; + int i = use._inputs.find(split); + RegMask mask = mach.regmap(i); + return mask!=null && mask.test(reg) && mach.twoAddress()!=i; + } + + private boolean splitBypass( CFGNode bb, int j, Node lo, int defreg ) { + // Attempt to bypass split + Node hi = lo.in(1); + while( true ) { + if( !(hi instanceof SplitNode && lo.cfg0() == hi.cfg0()) ) + return false; + if( lrg(hi.in(1))._reg==defreg ) + break; + hi = hi.in(1); + } + // Walk backward from the later copy, including instructions without results. + for( int idx = j-1; bb.out(idx) != hi; idx--) { + Node n = bb.out(idx); + if( clobbers(n,defreg) ) + return false; // Clobbered + } + lo.setDefOrdered(1,hi.in(1)); + return true; + } + + + private boolean clobbers(Node n, int reg) { + LRG lrg = lrg(n); + return lrg!=null && (lrg._reg==reg || lrg._reg==-1 && lrg._mask.size1() && lrg._mask.firstReg()==reg) || + n instanceof MachNode mach && mach.killmap()!=null && mach.killmap().test(reg); + } + + public boolean sameBlockNoClobber( SplitNode split ) { + Node def = split.in(1); + CFGNode cfg = def.cfg0(); + if( cfg != split.cfg0() ) return false; // Not same block + // Get multinode head + Node def0 = def instanceof ProjNode ? def.in(0) : def; + int defreg = lrg(def)._reg; + if( defreg == -1 ) defreg = lrg(def)._mask.firstReg(); + if( defreg == -1 ) return false; // no allowed registers -> clobbered + for( int idx = cfg._outputs.find(split) -1; idx >= 0; idx-- ) { + Node n = cfg.out(idx); + if( n==def0 ) return true; // No clobbers + if( lrg(n) == lrg(def) ) return false; // Self conflict + if( clobbers(n,defreg) ) + return false; // Clobbered + } + throw Utils.TODO(); + } +} diff --git a/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/RegMask.java b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/RegMask.java new file mode 100644 index 00000000..303bba9e --- /dev/null +++ b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/RegMask.java @@ -0,0 +1,122 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.util.SB; +import com.seaofnodes.simple.util.Utils; + +/** RegMask + * A "register mask" - 1 bit set for each allowed register. In addition, + * "stack slot" registers may be allowed, effectively making the set infinite. + * <p> + * For smaller and simpler machines it suffices to make such masks an i64 or + * i128 (64- or 128-bit integers), and this presentation is by far the better + * way to go... if all register allocations can fit in this bit limitation. + * The allocator will need bits for stack-based parameters and for splits + * which cannot get a register. For a 32-register machine like the X86, add 1 + * for flags - gives 33 registers. Using a Java `long` has 64 bits, leaving + * 31 for spills and stack passing. This is adequate for nearly all + * allocations; only the largest allocations will run this out. However, if + * we move to a chip with 64 registers we'll immediately run out, and need at + * least a 128 bit mask. Since you cannot *return* a 128 bit value directly + * in Java, Simple will pick up a `RegMask` class object. +*/ +public class RegMask { + + long _bits0, _bits1; + + private static final RegMask EMPTY = new RegMask(0L); + public static final RegMask FULL = new RegMask(-1L); + + public RegMask(int bit) { + if( bit < 64 ) _bits0 = 1L<<bit; + else _bits1 = 1L<<(bit-64); + } + public RegMask(long bits ) { _bits0 = bits; } + public RegMask(long bits0, long bits1 ) { _bits0 = bits0; _bits1 = bits1; } + public RegMask(RegMask r) { _bits0 = r._bits0; _bits1 = r._bits1; } + private RegMask() { _bits0 = _bits1 = 0; } + + // AND, with copy-on-write if changing + RegMask and( RegMask mask ) { + if( mask==null ) return this; + long bits0 = _bits0 & mask._bits0; + long bits1 = _bits1 & mask._bits1; + if( bits0==_bits0 && bits1==_bits1 ) return this; + if( bits0==mask._bits0 && bits1==mask._bits1 ) return mask; + if( bits0==0 && bits1==0 ) return EMPTY; + // Update-in-place a mutable mask, or make a defensive copy + return null; + } + RegMask sub( RegMask mask ) { + if( mask==null ) return this; + long bits0 = _bits0 & ~mask._bits0; + long bits1 = _bits1 & ~mask._bits1; + if( bits0==_bits0 && bits1==_bits1 ) return this; + if( bits0==mask._bits0 && bits1==mask._bits1 ) return mask; + if( bits0==0 && bits1==0 ) return EMPTY; + // Update-in-place a mutable mask, or make a defensive copy + return null; + } + + // Fails if bit is set, because this is immutable + public boolean clr( int reg ) { + if( reg < 64 ) return ((_bits0 >> reg)&1)==0; + if( reg < 128 ) return ((_bits1 >> reg)&1)==0; + return true; + } + + + public short firstReg() { + if( isEmpty() ) return -1; + return (short)(_bits0 != 0 + ? Long.numberOfTrailingZeros(_bits0) + : Long.numberOfTrailingZeros(_bits1)+64); + } + public short nextReg(short reg) { + if( reg < 63 ) { + long bits0 = _bits0 & (-1L << (reg+1)); + if( bits0!=0 ) return (short)Long.numberOfTrailingZeros(bits0); + } + long bits1 = reg < 64 ? _bits1 : reg < 127 ? _bits1 & (-1L << (reg-63)) : 0; + return bits1==0 ? -1 : (short)(64+Long.numberOfTrailingZeros(bits1)); + } + + boolean isEmpty() { return _bits0==0 && _bits1==0; } + + public boolean test( int reg ) { + return ((reg<64 ? (_bits0 >> reg) : (_bits1 >> (reg-64))) & 1) != 0; + } + + // checks if the 2 masks have at least 1 bit in common + public boolean overlap( RegMask mask ) { return (_bits0 & mask._bits0)!=0 || (_bits1 & mask._bits1)!=0; } + + // Defensive writable copy + public RegMaskRW copy() { return new RegMaskRW( _bits0, _bits1 ); } + + // Has exactly 1 bit set + public boolean size1() { + return !isEmpty() && (((_bits0 & -_bits0)==_bits0 && _bits1==0) || + ((_bits1 & -_bits1)==_bits1 && _bits0==0)); + } + + // Cardinality + public short size() { return (short)(Long.bitCount(_bits0)+Long.bitCount(_bits1)); } + + @Override public String toString() { return toString(new SB()).toString(); } + public SB toString(SB sb) { + Machine mach = CodeGen.CODE._mach; + if( _bits0==0 && _bits1==0 ) return sb.p("[]"); + sb.p("["); + String[] regs = mach.regs(); + for( int i=0; i<64; i++ ) + if( ((_bits0 >> i)&1) != 0 ) + sb.p(reg(regs,i)).p(","); + for( int i=0; i<64; i++ ) + if( ((_bits1 >> i)&1) != 0 ) + sb.p(reg(regs,i+64)).p(","); + return sb.unchar().p("]"); + } + + public static String reg(String[] regs, int i) { + return i<regs.length ? regs[i] : "[stk#"+i+"]"; + } +} diff --git a/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/RegMaskRW.java b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/RegMaskRW.java new file mode 100644 index 00000000..1da9a3b1 --- /dev/null +++ b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/RegMaskRW.java @@ -0,0 +1,36 @@ +package com.seaofnodes.simple.codegen; + +// Mutable regmask(writable) +public class RegMaskRW extends RegMask { + public RegMaskRW(long x, long y) { super(x,y); } + // clears bit at position r. Returns true if the mask is still not empty. + public boolean clr(int r) { + if( r < 64 ) _bits0 &= ~(1L<<(r )); + else _bits1 &= ~(1L<<(r-64)); + return _bits0!=0 || _bits1!=0; + } + // sets bit at position r. + public void set(int r) { + if( r < 64 ) _bits0 |= (1L<<(r )); + else _bits1 |= (1L<<(r-64)); + } + @Override RegMaskRW and( RegMask mask ) { + if( mask==null ) return this; + _bits0 &= mask._bits0; + _bits1 &= mask._bits1; + return this; + } + @Override RegMaskRW sub( RegMask mask ) { + if( mask==null ) return this; + _bits0 &= ~mask._bits0; + _bits1 &= ~mask._bits1; + return this; + } + public RegMaskRW or( RegMask mask ) { + if( mask!=null ) { + _bits0 |= mask._bits0; + _bits1 |= mask._bits1; + } + return this; + } +} diff --git a/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/TypeCheck.java b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/TypeCheck.java new file mode 100644 index 00000000..0997fd2f --- /dev/null +++ b/chapter-sccp/src/main/java/com/seaofnodes/simple/codegen/TypeCheck.java @@ -0,0 +1,15 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.Parser; + +abstract class TypeCheck { + static void run(CodeGen code) { + Parser.ParseException err = code._stop.walk(n -> { + Parser.ParseException msg = n.err(); + if( msg != null && code._obs != null ) code._obs.error(n, msg.getMessage()); + return msg; + }); + if( err != null ) + throw err; + } +} |