diff options
Diffstat (limited to 'chapter29/src/main/java/com/seaofnodes/simple/codegen')
32 files changed, 8222 insertions, 0 deletions
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/BuildLRG.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/BuildLRG.java new file mode 100644 index 00000000..0b314978 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/BuildLRG.java @@ -0,0 +1,101 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.Type; +import com.seaofnodes.simple.type.TypeMem; + +abstract public class BuildLRG { + // Compute live ranges in a single forwards pass. Every def is a new live + // range except live ranges are joined at Phi. Also, gather the + // intersection of register masks in each live range. Example: "free + // always zero" register forces register zero, and calling conventions + // typically force registers. + + // Sets FAILED to the set of hard-conflicts (means no need for an IFG, + // since it will not color, just split the conflicted ranges now). Returns + // true if no hard-conflicts, although we still might not color. + public static boolean run(int round, RegAlloc alloc) { + FunNode fun = null; + for( Node bb : alloc._code._cfg ) { + if( bb instanceof FunNode fun2 ) + fun = fun2; + for( Node n : bb.outs() ) { + // A synthetic never-exit value has no runtime definition. + if( n._type==Type.TOP ) continue; + if( n instanceof PhiNode phi && !(phi._type instanceof TypeMem) ) { + if( phi instanceof MachNode mach && mach.outregmap()==null ) + continue; + // All real Phi inputs end up with the same LRG. + // Pass 1: find any pre-existing LRG, to avoid make-then-Union a LRG + LRG lrg = alloc.lrg(phi); + if( lrg == null ) + for( int i=1; i<phi.nIns(); i++ ) + if( (lrg = alloc.lrg(phi.in(i))) != null ) + break; + // If none, make one. + if( lrg==null ) lrg = alloc.newLRG(n,fun); + if( phi instanceof MachNode ) defLRG(alloc,n,fun); + // Pass 2: everybody uses the same LRG + lrg=alloc.union(lrg,phi); + for( int i=phi instanceof ParmNode ? 2 : 1; i<n.nIns(); i++ ) + // TOP keeps the type merge neutral, but needs no register. + if( n.in(i)._type!=Type.TOP ) + lrg=alloc.union(lrg,n.in(i)); + if( lrg._mask!=null && lrg._mask.isEmpty() ) + alloc.fail(lrg); + + } else if( n instanceof MachNode mach ) { + // Attempt to commute ops to keep live ranges compatible. + if( mach.commutes() && n.nOuts()==1 ) { + int uidx = n.out(0)._inputs.find(n); + RegMask mask1 = n.in (1) instanceof MachNode machx ? machx.outregmap() : alloc.lrg(n.in(1))._mask; + RegMask mask2 = n.in (2) instanceof MachNode machx ? machx.outregmap() : alloc.lrg(n.in(2))._mask; + RegMask masko = n.out(0) instanceof MachNode machx ? machx.regmap(uidx): mach.outregmap(); + if( masko!=null && !mask1.overlap(masko) && mask2.overlap(masko) ) + n.swap12(); + } + + // Define live range + defLRG(alloc,n,fun); + + // Now, look in the opposite direction. How are incoming + // LRGs affected by this node: For all uses, make live lrgs + for( int i=1; i<n.nIns(); i++ ) + if( n.in(i)!=null ) { + LRG lrg2 = alloc.lrg(n.in(i)); + if( lrg2 != null ) { // Anti-dep or other, no LRG + RegMask use_mask = mach.regmap(i); // use_mask is also null for anti-dep + if( use_mask!=null && !lrg2.machUse(mach,(short)i,use_mask.size1()).and(use_mask) ) + alloc.fail(lrg2); // Empty register mask, must split + } + } + + } + + // MultiNodes have projections which set registers + if( n instanceof MultiNode && !(n instanceof CFGNode) ) + for( Node proj : n.outs() ) + if( proj instanceof MachNode ) + defLRG(alloc,proj,fun); + } + } + + // Collect live ranges + alloc.unify(); + + return alloc.success(); + } + + private static void defLRG( RegAlloc alloc, Node n, FunNode fun ) { + MachNode mach = (MachNode)n; + RegMask def_mask = mach.outregmap(); + if( def_mask == null ) return; + LRG lrg = mach.twoAddress() == 0 + ? alloc.newLRG(n,fun) // Define a new LRG for N + : alloc.lrg2(n,mach.twoAddress()); // Use the matching 2-adr input + // Record mask and mach + if( !lrg.machDef(mach,def_mask.size1()).and(def_mask) ) + alloc.fail(lrg); // Empty register mask, must split + } + +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/ClassInitDependencies.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/ClassInitDependencies.java new file mode 100644 index 00000000..3547b86c --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/ClassInitDependencies.java @@ -0,0 +1,238 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.parser.Lexer; +import com.seaofnodes.simple.parser.Parser; +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.*; +import com.seaofnodes.simple.util.*; +import java.util.*; + +// Search for Class Initialization Cycles. + +// Done after Opto, so dead code does not make a cycle. + +// Build the inits-graph, a call-graph plus CLInits plus class parents. + +// (1) Scan all functions f, building _edges[f][]: the set of <clinit> fcns +// from CompUnits from CLInitNodes, and the set of fcn call targets in f and +// the class parent. These are all reported as fidxs (ints). +// +// (2) Walk the _edges graph using Tarjans SCC. Reject cycles containing more +// than one <clinit>. +// +// (3) Report on <clinit> cycles, if any +// +public class ClassInitDependencies { + private final CodeGen _code; + // Function FIDX -> callees, touched initializers, and parent initializer. + public final int[][] _edgess; + private final int[] _pres; + private final int[] _lows; + private final int[] _stk ; + private final int[] _spos; // Stack position; FIDX -> stack slot + 1, or 0 when absent. + private int _sp; + private int _pre; + + + // Build once; tests can inspect the graph independently of cycle checking. + public ClassInitDependencies( CodeGen code ) { + _code = code; + assert code._phase.ordinal() >= CodeGen.Phase.IterPostOpto.ordinal(); + _pres = new int[code._linker._len]; + _lows = new int[code._linker._len]; + _stk = new int[code._linker._len]; + _spos = new int[code._linker._len]; + + // + BitSet visit = code.visit(); // Visit set per-fcn + Ary<CFGNode> work = new Ary<>(CFGNode.class); // Worklist per-fcn + IntHashMap iset = new IntHashMap(0); // dedup hash set per-fcn + _edgess = new int[code._linker._len][]; + + // For all functions, build the _edges array + for( FunNode fun : code._linker ) + if( fun!=null && !fun.isDead() ) + // Visit function and collect direct class touches and callees + _edgess[fun.sig().fidx()] = walk(code,fun,work,iset,visit,null); + + // FIDX zero represents escapees - an unknown callee, including a + // callback supplied by another compilation. Its code can call any + // exposed function. Keep this edge in saved summaries so later + // applications contribute their callbacks too. + + // Filter escapees by dead and not a class (class initializers are + // reached through bodies, not directly callable function pointers). + int[] escaped = ((TypeTuple)code._stop._type).mem()._escFs; + AryInt callees = new AryInt(); + // An imported Stop can still carry FULL (only local bodies have edges) + // so limit by existing FIDXs. + int limit = code._linker._len; + for( int fidx = XInt.next(escaped,0,limit); fidx >= 0; fidx = XInt.next(escaped,fidx,limit) ) { + FunNode fun = code.link(fidx); // Discard dead bodies and external C functions. + if( fun!=null && !fun.isClz() ) callees.push(fidx); + } + _edgess[0] = callees.isEmpty() ? null : callees.asAry(); + visit.clear(); + } + + // Return the outgoing init edge set for this function + private static int[] walk( CodeGen code, FunNode fun, Ary<CFGNode> work, IntHashMap iset, BitSet visit, + Ary<Lexer> sites) { + if( fun.summary()!=null ) return fun.summary()._classDependencyFIDXs; + // Walk function body, looking for outgoing init edges + assert iset .isEmpty(); // int hash set for de-dup + assert visit.isEmpty(); // needs a visit set, because loops + assert work .isEmpty(); // needs a worklist, because DAG, parallel explore paths + + // Children require their parents to init first; + // add child->parent edge + if( fun.isClz() && fun._compunit._par != null ) + iset.put(code.link(fun._compunit._par).sig().fidx(),1); + + work.push(fun); + while( !work.isEmpty() ) { + CFGNode cfg = work.pop(); + if( visit.get(cfg._nid) ) continue; // Been there, done that + visit.set(cfg._nid); + assert !cfg._type.isHigh(); + // Do not walk off end of function + if( cfg instanceof ReturnNode ) continue; + + // CLInit adds a class init requirement to this function + // (plus all parent classes) + if( cfg instanceof CLInitNode init ) { + FunNode clz = code.link(init.cu()); // <clinit> referred to + iset.put(clz.sig().fidx(),1); // Add to edge set + if( sites!=null && sites.atX(clz.sig().fidx())==null ) sites.setX(clz.sig().fidx(),init._loc); + } + + // Calls union all their fptr fidxs into the callee set + if( cfg instanceof CallNode call && call.fptr()._type instanceof TypeFunPtr fp ) { + if( XInt.isHigh(fp.fidxs()) ) { + iset.put(0,1); + if( sites!=null && sites.atX(0)==null ) sites.setX(0,call._loc); + } + // Union call targets into the callee set + else for( int dstx = XInt.next(fp.fidxs(),0); dstx>=0; dstx=XInt.next(fp.fidxs(),dstx) ) { + iset.put(dstx,1); + if( sites!=null && sites.atX(dstx)==null ) sites.setX(dstx,call._loc); + } + } + + // Forward recursion to walk the whole function + for( Node use : cfg._outputs ) { + // Do not walk into another function + if( !(use instanceof CFGNode cfg1) || use instanceof FunNode || use instanceof StopNode ) continue; + // Expect control flow edge is sensible + assert (use instanceof RegionNode r && r._inputs.find(cfg)>0) || use.in(0)==cfg; + if( !visit.get(cfg1._nid) ) // Small speed opt + work.add(cfg1); + } + } + + visit.clear(); + int[] edges = iset.isEmpty() ? null : iset.keySet(); + iset.clear(); + return edges; + } + + // Return an error string, or null if no cycle contains multiple clinits. + public String check() { + Arrays.fill(_pres,0); + // A prior check may have stopped with a cycle still on the stack. + while( _sp>0 ) _spos[_stk[--_sp]] = 0; + _pre = 1; + + int idx = -1; + for( int fidx = 0; fidx < _edgess.length; fidx++ ) + if( _pres[fidx] == 0 && // Already visited + (idx = scc(fidx)) != -1 ) // And found a cycle + return report(idx); // Report cycle + return null; + } + + // Every internal edge of an SCC belongs to a cycle. Print all of them: + // stack order is not a path, and enumerating all cycle paths can explode. + // This bookkeeping and formatting are only needed on an error. + private String report(int idx) { + BitSet members = new BitSet(); + for( int i=idx; i<_sp; i++ ) members.set(_stk[i]); + SB sb = new SB().p("Cyclic class initialization:"); + // Recover one surviving source site per edge only after finding an + // error. A representative touch is enough; runtime first-touch order + // need not be statically known. Lexer snapshots retain the source CU + // even when the node has been inlined into a different function. + Ary<CFGNode> work = new Ary<>(CFGNode.class); + IntHashMap iset = new IntHashMap(0); + BitSet visit = new BitSet(); + Ary<Lexer> sites = new Ary<>(Lexer.class); + for( int i=idx; i<_sp; i++ ) { + int src = _stk[i]; + FunNode fun = _code.link(src); + Arrays.fill(sites._es,null); + sites.clear(); + if( fun!=null ) walk(_code,fun,work,iset,visit,sites); + if( _edgess[src]!=null ) + for( int dst : _edgess[src] ) + if( members.get(dst) ) { + sb.p("\n ").p(name(src)).p(" -> ").p(name(dst)); + Lexer loc = sites.atX(dst); + if( loc!=null && loc._cu!=null ) + sb.p('\n').p(SourceError.format(loc._cu,loc.pos(),"cyclic class initialization")); + else if( fun!=null && fun.isClz() && dst!=0 && fun._compunit._par==_code.link(dst)._compunit ) + sb.p("\n parent initialization requirement"); + } + } + return sb.toString(); + } + + private String name(int fidx) { + if( fidx==0 ) return "unknown call"; + FunNode fun = _code.link(fidx); + String owner=fun._compunit._cname; + if( fun.isClz() ) return owner; + String label=fun.label(); + return (label.startsWith(owner+".") ? label : owner+"."+label)+" [fidx "+fidx+"]"; + } + + // Tarjan SCC + int scc( int fidx ) { + // Assign pre-order number + _lows[fidx] = _pres[fidx] = _pre++; + _spos[fidx] = _sp+1; + _stk[_sp++] = fidx; + // walk outbound edges + if( _edgess[fidx]!=null ) { + for( int edge : _edgess[fidx] ) { + if( edge >= _pres.length ) continue; // Out of bounds happens when calling C fcns + if( _pres[edge] == 0 ) { // Not visited + int idx = scc(edge); + if( idx != -1 ) return idx; // Found a cycle + _lows[fidx] = Math.min(_lows[fidx],_lows[edge]); + } else if( _edgess[edge]!=null && _spos[edge]!=0 ) { + _lows[fidx] = Math.min(_lows[fidx],_pres[edge]); + } + } + } + // Found a cycle + if( _lows[fidx] == _pres[fidx] ) { + int idx = _spos[fidx]-1; + if( hasClzCycle(idx) ) + return idx; // Found a class cycle + // Pop stack, including fidx, should be a really short loop + while( _sp>idx ) _spos[_stk[--_sp]] = 0; + } + return -1; // No cycle + } + + private boolean hasClzCycle(int idx) { + int clinits = 0; + for( int i=idx; i<_sp; i++ ) { + FunNode fun = _code.link(_stk[i]); + if( fun!=null && fun.isClz() && ++clinits > 1 ) + return true; + } + return false; + } + +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/Coalesce.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Coalesce.java new file mode 100644 index 00000000..689ceceb --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Coalesce.java @@ -0,0 +1,105 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.util.Ary; +import com.seaofnodes.simple.util.Utils; +import java.util.BitSet; +import java.util.IdentityHashMap; + +// Coalesce copies +abstract public class Coalesce { + + public static boolean coalesce( int round, RegAlloc alloc ) { + + // Walk all the splits, looking for coalesce chances + boolean progress = false; + for( CFGNode bb : alloc._code._cfg ) { + outer: + for( int j=0; j<bb.nOuts(); j++ ) { + Node n = bb.out(j); + if( !(n instanceof SplitNode) ) continue; + LRG v1 = alloc.lrg(n); + LRG v2 = alloc.lrg(n.in(1)); + if( v1 != v2 ) { + LRG ov1 = v1, ov2 = v2; + // Get the smaller neighbor count in v1 + if( v1.nadj() > v2.nadj() ) + { v1 = v2; v2 = alloc.lrg(n); } + int v2len = v2._adj==null ? 0 : v2._adj._len; + // See that they do not conflict (coalescing would make a self-conflict) + if( v1._adj!=null && v2._adj!=null && v1._mask.overlap(v2._mask) ) { + // Walk the smaller neighbors, and add to the larger. + // Check for direct conflicts along the way. + for( LRG v3 : v1._adj ) { + if( v3==v2 ) { + // Direct conflict, skip this LRG pair + v2._adj.setLen(v2len); + continue outer; + } + // Add missing LRGs from V1 to V2 + if( v2._adj.find(v3)== -1 ) + v2._adj.push(v3); + } + } + // Most constrained mask + RegMask mask = v1._mask==v2._mask ? v1._mask : v1._mask.copy().and(v2._mask); + // Check for capacity + if( v2.nadj() >= mask.size() ) { + // Fails capacity, will not be trivial colorable + if( v2._adj!=null ) v2._adj.setLen(v2len); + continue; + } + + // Union lrgs. + progress = true; + + // Since `n` is dying, remove from the LRGS BEFORE doing + // the union; this will let the union code keep the + // matching bits from the other side, preserving some + // info for biased coloring + if( ov1. _machDef==n ) ov1. _machDef = null; + if( ov1._splitDef==n ) ov1._splitDef = null; + if( ov2. _machUse==n ) ov2. _machUse = null; + if( ov2._splitUse==n ) ov2._splitUse = null; + // Keep larger adjacency list. + Ary<LRG> larger = v2._adj; + v2._adj = null; + + // Union lrgs. Swaps again, based on small lrg + LRG vWin = v1.union(v2); + vWin._adj = larger; + LRG vLose = v1.leader() ? v2 : v1; + + // LRGs below v2len were always in v2, but maybe not in v1. + // LRGs above v2len are in v1 but were not in v2. + + // Walk v2's neighbors, replacing vLose with vWin. + if( larger != null ) + for( LRG vn : larger ) { + int idx = vn._adj.find(vLose); + if( idx != -1 ) + vn._adj.del(idx); + if( vn._adj.find(vWin) == -1 ) + vn._adj.add(vWin); + } + } else { + + // Since `n` is dying, remove from the LRGS + if( v1. _machDef==n ) v1. _machDef=null; + if( v1. _machUse==n ) v1. _machUse=null; + if( v1._splitDef==n ) v1._splitDef=null; + if( v1._splitUse==n ) v1._splitUse=null; + } + + // Remove junk split + n.removeSplit(); + j--; + } + } + + if( progress ) + alloc.unify(); + + return true; + } +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/CodeGen.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/CodeGen.java new file mode 100644 index 00000000..bdb510b4 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/CodeGen.java @@ -0,0 +1,889 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.graph.GraphObserver; +import com.seaofnodes.simple.parser.Parser; +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.print.*; +import com.seaofnodes.simple.type.*; +import com.seaofnodes.simple.util.*; +import java.io.File; +import java.util.*; + + +// Compile Driver for a Compilation Unit +@SuppressWarnings("unchecked") +public class CodeGen { + + // Last created CodeGen as a global; used all over to avoid passing about a "context". + public static CodeGen CODE; + + // Location of built-in CPU ports + public static final String PORTS = "com.seaofnodes.simple.node.cpus"; + + // Compile phases + public enum Phase { + Parse, // Parse ASCII text into Sea-of-Nodes IR + Iter, // Pessimistic peepholes after parse + Opto, // SCCP, numeric modes, seed the peephole worklist + IterPostOpto, // Peepholes and inlining using SCCP results + FinalizeIR, // Field layout, offset cleanup, function/CU roots + TypeCheck, // Last check for bad programs + Serialize, // Save ideal IR, including opaque class initialization + Lower, // Expand checked class initialization to ordinary IR + LoopTree, // Build a loop tree; break infinite loops + Unlink, // Unlink call sites before machine code generation + Select, // Convert to target hardware nodes + Schedule, // Global schedule (code motion) nodes + LocalSched, // Local schedule + RegAlloc, // Register allocation + Encoding, // Encoding + Export, // Export + LastPhase // After last phase + } + public Phase _phase; + // Null for normal compilation; owned by this compilation, not a global viewer. + public GraphObserver<Node> _obs; + + /** True when tests deliberately randomize Iter worklist order. */ + public static boolean iterSeedOverridden() { return System.getProperty("simple.iter.seed") != null; } + + // Expensive assert sampling: + // -Dsimple.assert.expensive=0 disables these checks + // -Dsimple.assert.expensive=1 checks every step + // -Dsimple.assert.expensive=N checks once every 1<<N steps + private static final int EXPENSIVE_ASSERT_LOG = Integer.getInteger("simple.assert.expensive",8); + public static boolean expensiveAssert() { return EXPENSIVE_ASSERT_LOG > 0; } + public static boolean expensiveAssert( int trip ) { + if( EXPENSIVE_ASSERT_LOG <= 0 ) return false; + if( EXPENSIVE_ASSERT_LOG == 1 ) return true; + int shift = Math.min(EXPENSIVE_ASSERT_LOG,30); + return (trip & ((1<<shift)-1)) == 0; + } + + // --------------------------- + // Module Source Root + public final String _modDir; + // Module Build Root + public final String _buildDir; + // Search path from CWD for `.o` files containing external symbols and types. + // This can contain archives, zips and other .o file containers. + public final Ary<String> _externPaths; + + // Dotted path from module root to file containing the source, sans ".smp". + // e.g. module root: "sys", nested source name "sys.io". + public final String _srcName; + + // Only available for tests, otherwise source comes from the CompUnit + private final String _src; + + // Current Working Directory; default module base + private final String _cwd; + + // Compilation Units used in this compile; typically one per source/object + // file, although you can nest structs and get more than one. These are all + // required to compile, and are expected to have their source timestamps + // ahead of their obj file. Some of these may entirely be dead or inlined + // in this CompUnit and thus not needed in any linking stage. All the + // alive ones (inlined away or not) participate in class init cycle checks. + public HashMap<String,CompUnit> _compunits; + + // Test shortcut for only one compilation unit + public CompUnit compunit() { + assert _compunits.size()==1; + for( CompUnit cu : _compunits.values() ) + return cu; + assert false : "no compilation unit"; + return null; + } + + // Frozen after ParseAll: public initializers and function-valued fields. + public boolean publicFIDX(int fidx) { + return fidx >= GlobalBits.RESERVED && XInt.bit(publicFIDXs(),fidx); + } + + private void gatherPublicFields(TypeStruct clz, boolean classFields) { + boolean hasCtor = clz.field("<ctor>") != null; + for( Field fld : clz._fields ) { + // With a user constructor, class:N.N is the private <init>. + if( fld._fname.charAt(0)=='_' || fld._t==Type.TOP || + (hasCtor && clz.hiddenInit(fld)) ) continue; + if( classFields ) _publicAliases = XInt.make(_publicAliases,fld._alias); + if( fld._t instanceof TypeFunPtr tfp ) + _publicFIDXs = XInt.meet(_publicFIDXs,tfp._fidxs); + } + } + + public static boolean hasUserConstructor(TypeStruct self) { + Type t = Parser.TYPES.get(Parser.addClzPrefix(self._name)); + return t instanceof TypeStruct clz && clz.field("<ctor>") != null; + } + + // A forward reference has a placeholder class, not a definition yet. + private static boolean resolvedClass(TypeStruct clz) { + Type t = Parser.TYPES.get(clz._name.substring(Parser.CLZ.length())); + return !(t instanceof TypeStruct ts) || !ts._fref; + } + + private int[] _publicFIDXs, _publicAliases; + + /** + * Collect every possible public escape - after ParseAll is done - in case + * some code calls a generic function. Example: + * var apply = = { {int->int} f, int x -> f(x); }; // Generic apply() call + * The CallEnd from `f(x)` has to use generic TypeMem.BOT, which + * immediately escapes *everything* including known private functions. + * As weak as this is, its still sharper than "everything". + */ + void freezePublicInterface() { + assert _publicFIDXs==null && _publicAliases==null; + _publicFIDXs = _publicAliases = XInt.EMPTY; + for( Type t : Parser.TYPES.values() ) + if( t instanceof TypeStruct clz && + Parser.startsClzPrefix(clz._name) && + Parser.isPublicClassName(clz._name) && + resolvedClass(clz) ) { + // Gather function pointers from public fields of both class + // and instance objects. + gatherPublicFields(clz,true); + Type self = Parser.TYPES.get(clz._name.substring(Parser.CLZ.length())); + if( self instanceof TypeStruct ts ) gatherPublicFields(ts,false); + } + // Limit broad function types to known FIDXs, excluding reserved bits. + // Public class initializers are roots without needing a field reference. + int[] fidxs = XInt.EMPTY; + for( int fidx = GlobalBits.RESERVED; fidx<_linker._len; fidx++ ) { + FunNode fun = _linker.atX(fidx); + if( XInt.bit(_publicFIDXs,fidx) || (fun!=null && fun.isClz() && fun.isPublic()) ) + fidxs = XInt.make(fidxs,fidx); + } + _publicFIDXs = fidxs; + } + + public int[] publicFIDXs() { assert _publicFIDXs !=null; return _publicFIDXs; } + public int[] publicAliases(){ assert _publicAliases!=null; return _publicAliases; } + public boolean publicInterfaceFrozen() { return _publicFIDXs != null; } + + // --------------------------- + + // Very common nodes, cached here + public final ConstantNode ZERO; + public final XCtrlNode XCTRL; + + public Node constant( Type t ) { + assert _phase==null || _phase.ordinal() <= Phase.Lower.ordinal(); + return ConstantNode.make(t).peephole(); + } + public ConstantNode con( Type t ) { return (ConstantNode)constant(t); } + public ConstantNode con( long con ) { return con==0 ? ZERO : con(TypeInteger.constant(con)); } + + + // --------------------------- + // Test setup; no module nor file with specific argument + public CodeGen( String src ) { this(src, 126L, true ); } + + // Test setup; no module nor file; can alter seed & argument; can re-run same CodeGen + public CodeGen( String src, Type arg ) { + this(null,null,null,null, src, 126L, true, arg); + } + // Test setup; no module nor file; can alter seed & argument; can re-run same CodeGen + public CodeGen( String src, long workListSeed, boolean resetTypes ) { + this(null,null,null,null, src, workListSeed, resetTypes, TypeInteger.BOT); + } + + // Generic CodeGen, including full module setup + public CodeGen( String modDir, String buildDir, Ary<String> externPaths, + String srcName, String src, long workListSeed, boolean resetTypes, Type arg ) { + // Public singleton to avoid passing about this state to a huge count + // of places. Probably becomes a TLS at some point. + CODE = this; + if( resetTypes ) Type.reset(); // Reset to recover from failed test + + _cwd = System.getProperty("user.dir")+"/"; + _modDir = modDir == null ? _cwd : modDir; + _buildDir = buildDir == null ? _cwd : buildDir; + _externPaths = externPaths; + _srcName = srcName; + _callingConv = null; // Calling convention + // Source code from test strings, not files + _src = src; + // All the compilation units + _compunits = new HashMap<>(); + _phase = null; + // Start GVN table + _gvn = new HashMap<>(); + // Allow whole-suite worklist-order sweeps without rewriting the many + // explicit seeds used by tests and command-line compilation. + String iterSeed = System.getProperty("simple.iter.seed"); + _iter = new IterPeeps(iterSeed == null ? workListSeed : Long.parseLong(iterSeed)); + // End points of graph + _stop = new StopNode().init(); + _start = new StartNode(null,_stop,arg).init(); + ZERO = con(TypeInteger.ZERO).keep(); + XCTRL = new XCtrlNode().peephole().keep(); + P = new Parser(this); + } + + + // Run requested phases. File export must request Serialize before Lower; + // requesting Export up front does that automatically. Encoding alone does + // not retain reloadable ideal IR. Choose combined in-memory output before + // Encoding; otherwise saved IR requests separate object encodings. + + // No code emission, just IR generation + public CodeGen driver( Phase phase ) { return driver(phase,null,null,false,false,0); } + // No object file writing, but code generation for a specific cpu/os pair (allows emulation) + public CodeGen driver( Phase phase, String cpu, String callingConv ) { + if( phase.ordinal() < Phase.Export.ordinal() ) + // No code outputted + return driver(phase,cpu,callingConv,false,false,0); + if( _srcName == null ) + throw new RuntimeException("No source filename provided, so do not know how to name the obj file"); + boolean emitEntrySymbol = !_srcName.contains("."); + return driver( phase, cpu, callingConv, false, emitEntrySymbol, 0 ); + } + // Write an object file for a specific cpu/os pair + + public CodeGen driver( String cpu, String callingConv, boolean inMemory, boolean emitEntrySymbol ) { return driver(Phase.Export,cpu,callingConv,inMemory,emitEntrySymbol,0); } + + // The Generic driver + public CodeGen driver( Phase target, String cpu, String callingConv, boolean inMemory, boolean emitEntrySymbol, int dump ) { + // Parse a Simple file, and for mentioned external symbols either load + // from .o or recursively parse + step(target, Phase.Parse, this::parse, dump); + // Run peepholes to a fixed point + step(target, Phase.Iter, this::iter, dump); + // Run SCCP (and resolve int/flt expressions) + step(target, Phase.Opto, this::opto, dump); + // Run peepholes to a fixed point again, on the better types. + step(target, Phase.IterPostOpto, this::iterPostOpto, dump); + // Field layouts; field symbols become constant offsets + step(target, Phase.FinalizeIR, this::finalizeIR, dump); + // Final error check - bad programs fail here + step(target, Phase.TypeCheck, this::typeCheck, dump); + // Save reloadable ideal IR for a later seperate compilation + step(target, Phase.Serialize, () -> Serialize.forTarget(this,target,inMemory), dump); + // Eval2 executes ideal operations, including CLInit, directly. Only + // machine compilation expands them before scheduling. + step(target, Phase.Lower, () -> { if( cpu!=null ) lower(); }, dump); + // Compute the loop-tree, used for approximating exec frequencies for + // scheduling and register allocation + step(target, Phase.LoopTree, this::loopTree, dump); + // Detach call-graph edges; ideal execution retains unloaded summaries + // as dispatch targets and schedules their bodies on first use. + step(target, Phase.Unlink, () -> unlink(cpu!=null), dump); + // Instruction selection + step(target, Phase.Select, () -> instSelect(cpu,callingConv), dump); + // Global Code Motion + step(target, Phase.Schedule, this::GCM, dump); + // Local (basic block only) scheduling + step(target, Phase.LocalSched, this::localSched, dump); + // Register allocation + step(target, Phase.RegAlloc, this::regAlloc, dump); + // Encode machine instructions into bits + step(target, Phase.Encoding, () -> encode(inMemory), dump); + // Export bits (and serialized IR) into ELF files + step(target, Phase.Export, () -> exportELF(inMemory,emitEntrySymbol), dump); + return this; + } + + // The driver names the order; optional phases decide locally whether to run. + private void step(Phase target, Phase next, Runnable work, int dump) { + if( next.ordinal() > target.ordinal() || (_phase!=null && _phase.ordinal()>=next.ordinal()) ) return; + Phase prior = _phase; + work.run(); + if( _phase!=prior ) dump(dump); + } + + // All phase entry points share ordering checks, timing, and viewer notification. + // Record elapsed time even on failure, but report completion only on success. + CodeGen phase(Phase next, boolean ordered, Runnable work) { + assert ordered : "Cannot run "+next+" after "+_phase; + // Serialize reads the checked graph; other phases can change it. + if( next!=Phase.Serialize ) _classInitDependencies = null; + _phase = next; + long t0 = System.currentTimeMillis(); + work.run(); // Do The Phase + _times[next.ordinal()] = System.currentTimeMillis() - t0; + if( _obs!=null ) _obs.phase(next.name()); // Record for viewer + return this; + } + + public String entryClinitName() { + return Parser.addClzPrefix(_srcName==null ? "Test" : _srcName.replace('/','.'))+".<clinit>"; + } + + // Verbose printing during compilation + private void dump(int dump) { + int p2 = _phase.ordinal(); + if( (dump & (1<<p2)) == 0 ) return; + if( (dump & (1<<30)) != 0 ) + System.err.println("After "+_phase+":"); + System.err.println(_phase.ordinal() >= Phase.LocalSched.ordinal() + ? asm(new SB()).toString() + : IRPrinter.prettyPrint(this)); + } + + + // Record times by phase + public final long[] _times = new long[Phase.LastPhase.ordinal()]; + + // --------------------------- + /** + * A counter, for unique node id generation. Starting with value 1, to + * avoid bugs confusing node ID 0 with uninitialized values. + */ + public int _uid = 1; + public int UID() { return _uid; } + public int getUID() { + return _uid++; + } + + + // These next fields all 2-way map some *global* program feature to a + // local dense index, suitable for packing in BitSets. The mapping takes a + // global value - always a {source file/class} name, and an order number + // counting up per feature in the same file. The dense local index can be + // different in different compilations. + + // The global info allows loading the same remote class info from different + // paths and aligning them. Example: Compiling A loads pre-compiled B and + // C, both of which load pre-compiled D. The 2 different D loads unify via + // this global info. + + // These mappings are all trivial identities when compiling one file; they + // only become complex when loading separately compiled code. + + // Compute local function index (FIDX) from global function info. This is + // called *in order* during parsing, and that order is part of the global + // unique mapping + public final GlobalBits _aliases = new GlobalBits(); + public int alias(String clz) { return _aliases.next(clz); } + + // C storage has one global alias per symbol, including across object files. + public int externAlias(String name) { return _aliases.next("C:"+name,GlobalBits.RESERVED); } + + // External data addresses supplied by an in-memory linker (image offsets). + public final HashMap<String,Integer> _externDataAddresses = new HashMap<>(); + + // Compute local function index (FIDX) from global function info. This is + // called *in order* during parsing, and that order is part of the global + // unique mapping + public final GlobalBits _fidxs = new GlobalBits(); + // Return local index for specific global file & order. Used to find fidxs for e.g. FREF class <init> fcns + public int fidx( String clz, int order ) { return _fidxs.next(clz, order); } + // New fidx in the current file + public int fidx( String clz ) { return fidx(clz, -1); } + + // Compute local RPC index from global RPC info, one per call + public final GlobalBits _rpcs = new GlobalBits(); + public int rpc( String clz ) { return _rpcs.next(clz); } + + + // Inlining can insert CFG edges and invalidate cached depth ordering. + private char _iDepthVersion; + public char iDepthVersion() { return _iDepthVersion; } + public void invalidateIDepthCaches() { + assert _iDepthVersion < Character.MAX_VALUE : "Dominator cache version exceeds 65535"; + ++_iDepthVersion; + } + + // Popular visit bitset, declared here, so it gets reused all over + public final BitSet _visit = new BitSet(); + public BitSet visit() { assert _visit.isEmpty(); return _visit; } + + // Start and Stop; end points of the generated IR + public StartNode _start; + public StopNode _stop; + // Iter creates this shared analysis result. Start's unused backedge slot + // roots it until Unlink; serialized Stops retain its escape facts instead. + public GlobalEscapeNode _globalEscapes; + + // Global Value Numbering. Hash over opcode and inputs; hits in this table + // are structurally equal. + public final HashMap<Node,Node> _gvn; + + // Reverse from a constant function pointer to the IR function being called. + // Error to call with a non-constant TFP + public FunNode link( TypeFunPtr tfp ) { return link(tfp.fidx()); } + // Read-only linker lookup, suitable for printers and debugger display. + // In particular, do not lazily remove a partially dead function. + public FunNode _link(int fidx) { return _linker.atX(fidx); } + public FunNode _link(TypeFunPtr tfp) { return _link(tfp.fidx()); } + // Return the FunNode from a fidx + public FunNode link( int fidx ) { + FunNode fun =_linker.atX(fidx); + if( fun!=null && fun.isDead() ) { + _linker.setX(fidx,null); fun = null; } + return fun; + } + // Return the <clinit> for a CompUnit + public FunNode link( CompUnit cu ) { + for( FunNode fun : _linker ) + if( fun!=null && fun._compunit == cu && fun.isClz() ) + return fun; + return null; + } + + // Insert linker mapping from constant function signature to the function + // being called. + public void link(FunNode fun) { + int fidx = fun.sig().fidx(); + _linker.setX(fidx,fun); + add(_globalEscapes); // A new or replacement summary may expose escapes. + } + + // "Linker" mapping from constant TypeFunPtrs to heads of function. These + // TFPs all have exact single fidxs and their return is wiped to BOTTOM (so + // the return is not part of the match). + public final Ary<FunNode> _linker = new Ary<>(FunNode.class); + + // Extern function declarations; input is the FIDX assigned to the name. + final HashMap<Integer,String> _externFunc = new HashMap<>(); + public void externFunc(int fidx, String ex) { + assert !_externFunc.containsKey(fidx); + _externFunc.put(fidx,ex); + } + public String externFunc(int fidx) { return _externFunc.get(fidx); } + + public String funcName(int fidx ) { + FunNode fun = _link(fidx); + if( fun!=null ) return fun._name; + return externFunc(fidx); + } + + // Select which source CompUnit's object this encoding pass emits. + CompUnit _emitUnit; + boolean emits(CompUnit cu) { return cu!=null && cu._src!=null && (_emitUnit==null || _emitUnit==cu); } + boolean emits(FunNode fun) { return emits(fun._compunit); } + final LinkedHashMap<CompUnit,Encoding> _encodings = new LinkedHashMap<>(); + + + // --------------------------- + // Parser object + public final Ary<CallNode> _calls = new Ary<>(CallNode.class); + + public final Parser P; + + // Parse ASCII text into Sea-of-Nodes IR + public CodeGen parse() { return phase(Phase.Parse, _phase==null, () -> ParseAll.parse(this,_src)); } + + + // --------------------------- + // Iterator peepholes. + public final IterPeeps _iter; + // Stop tracking deps while assertin + public boolean _midAssert; + // Statistics on peepholes + public int _iter_cnt, _iter_nop_cnt; + // Stat gathering + public void iterCnt() { if( !_midAssert ) _iter_cnt++; } + public void iterNop() { if( !_midAssert ) _iter_nop_cnt++; } + + // Pessimistic peepholes after parsing. This lifts loaded and parsed + // types before the optimistic Opto pass resets types and lets them fall. + public CodeGen iter() { return phase(Phase.Iter, _phase==Phase.Parse, () -> Iter.run(this)); } + + // SCCP and numeric-mode resolution; leave cleanup work queued for Iter. + public CodeGen opto() { return phase(Phase.Opto, _phase==Phase.Iter, () -> Opto.opto(this)); } + + public CodeGen iterPostOpto() { return phase(Phase.IterPostOpto, _phase==Phase.Opto, () -> _iter.iterate(this)); } + + // Materialize an inline candidate at an empty worklist. Loading queues + // the new body and affected callers before CallEnd considers inlining. + public void loadForInline(CompUnit cu) { + assert _phase==Phase.IterPostOpto && _iter._work.isEmpty(); + ParseAll.loadCompUnit(this,cu); + } + + // TypeCheck saves its successful analysis for Serialize. Loading bodies + // or entering another graph-changing phase discards it. + ClassInitDependencies _classInitDependencies; + + public CodeGen finalizeIR() { return phase(Phase.FinalizeIR, _phase==Phase.IterPostOpto, () -> FinalizeIR.run(this)); } + + // Cross-phase `keep()`s belong here! We have a few nodes which we can + // lose all uses of, but later find them and want to re-add output edges. + // If we don't `keep()` them, they'll dead-code-elimnate. + + // These are mostly around adding and removal functions from other + // compilation units. E.g. a StartCU, when all his functions are exactly + // called with no escapes and no default input. Those functions will still + // go out in their own ELF, and will later hook on to the StartCU during + // code emission. Between e.g. Opto and code emission the StartCU has no + // direct users - but we're going to want the StartCU at elf-writer time! + + // The Parser and `idealize()` `keep()`s remain locally balanced. + + // We make a few collections of these cross-phase `keep()`s. + + // A few more cross-phase example lifetimes: + + // Until FinalizeIR: Opto and import loading retain StartCU's memory + // projection while unknown-caller edges disappear and may be rebuilt. + // CompUnit._start.proj(1) finds it; Opto.linkStart uses it for the memory + // Parm, found by scanning fun._outputs. FinalizeIR first holds the StartCU + // itself, then releases these projections and queues ordinary dead cleanup. + + // Until Unlink: FinalizeIR retains StartCU even if it has no users left. + // CompUnit._start is the non-graph reference used by Serialize and Unlink. + // Unlink.run/forEval install scheduling edges before releasing these holds. + // A Java reference alone does not protect a node from graph deletion. + // + // Replacement is the exception to those release points: ParseAll must + // release our old wrapper/projection holds before subsuming the wrapper. + // It registers the replacement projection for the next FinalizeIR call. + private final Ary<Node> _keptUntilFinalize = new Ary<>(Node.class); + private final Ary<Node> _keptUntilUnlink = new Ary<>(Node.class); + + void keepUntilFinalize(Node n) { keepOnce(_keptUntilFinalize,n); } + void keepUntilUnlink (Node n) { keepOnce(_keptUntilUnlink ,n); } + private static void keepOnce(Ary<Node> kept, Node n) { + if( kept.find(n)==-1 ) kept.add(n.keep()); + } + + // FinalizeIR has retained the wrappers; unused projections may now die. + void releaseFinalizedNodes() { + for( Node n : _keptUntilFinalize ) add(n.unkeep()); + _keptUntilFinalize.clear(); + } + + // Unlink has installed the scheduling edges. No optimization is needed. + void releaseUnlinkedNodes() { + for( Node n : _keptUntilUnlink ) n.unkeep(); + _keptUntilUnlink.clear(); + } + + // Only release our holds. A caller's temporary keep is a separate owner. + void releaseReplacedNode(Node n) { + releaseReplacedNode(_keptUntilFinalize,n); + releaseReplacedNode(_keptUntilUnlink,n); + } + private static void releaseReplacedNode(Ary<Node> kept, Node n) { + int idx = kept.find(n); + if( idx!=-1 ) kept.del(idx).unkeep(); + } + + // Root a surviving function without claiming that it has unknown callers. + public void keepInCompUnit(FunNode fun) { + StopCUNode stop = fun._compunit._stop; + if( stop._inputs.find(fun.ret()) == -1 ) + add(stop).addDef(fun.ret()); + } + public <N extends Node> N add( N n ) { return _iter.add(n); } + public void addAll( Ary<Node> ary ) { _iter.addAll(ary); } + public void addAll( Node n ) { _iter.add(n); _iter.addAll(n._inputs); } + + // --------------------------- + // Last check for bad programs + public CodeGen typeCheck() { return phase(Phase.TypeCheck, _phase==Phase.FinalizeIR, () -> TypeCheck.run(this)); } + + // --------------------------- + // TypeCheck queued surviving CLInits during its existing error walk. + public CodeGen lower() { + return phase(Phase.Lower, (_phase==Phase.TypeCheck || _phase==Phase.Serialize), () -> _iter.iterate(this)); + } + + // --------------------------- + // Build the loop tree; break never-exit loops + public CodeGen loopTree() { + return phase(Phase.LoopTree, (_phase==Phase.Lower || _phase==Phase.TypeCheck || _phase==Phase.Serialize), + () -> _start.buildLoopTree(_linker,_stop)); + } + + // --------------------------- + public BAOS _serial; + final LinkedHashMap<CompUnit,BAOS> _serials = new LinkedHashMap<>(); + public void serialize() { + phase(Phase.Serialize, _phase==Phase.TypeCheck, () -> Serialize.serialize(this)); + } + + // --------------------------- + // Code generation CPU target + public Machine _mach; + // Chosen calling convention (usually either Win64 or SystemV) + public String _callingConv; + public String _cCallingConv; + // Callee save registers + public RegMask _callerSave; + public RegMask _cCallerSave; + + // All returns have the following inputs: + // 0 - ctrl + // 1 - memory + // 2 - varies with returning GPR,FPR + // 3 - RPC + // 4+- Caller save registers + public RegMask[] _retMasks; + // Return Program Counter + public RegMask _rpcMask; + + // Convert to target hardware nodes + public CodeGen instSelect( String cpu, String callingConv ) { return instSelect(cpu,callingConv,PORTS); } + public CodeGen instSelect( String cpu, String callingConv, String base ) { + return InstSelect.select(this,cpu,callingConv,base); + } + + // --------------------------- + public void unlink() { + unlink(true); + } + private void unlink(boolean removeImports) { + phase(Phase.Unlink, _phase==Phase.LoopTree, () -> { + _start.setDef(2,null); // Releases the compiler-only escape analysis. + if( removeImports ) Unlink.run(this,true); + else Unlink.forEval(this,_start,_stop); + }); + } + + // --------------------------- + // Control Flow Graph in Reverse Post Order. + public Ary<CFGNode> _cfg = new Ary<>(CFGNode.class); + + // Global schedule (code motion) nodes + public CodeGen GCM() { + return phase(Phase.Schedule, (_phase==Phase.Unlink || _phase==Phase.Select), () -> GlobalCodeMotion.buildCFG(this)); + } + + // --------------------------- + // Local (basic block) scheduler phase, a classic list scheduler + public CodeGen localSched() { return phase(Phase.LocalSched, _phase==Phase.Schedule, () -> ListScheduler.sched(this)); } + + + // --------------------------- + // Register Allocation + public RegAlloc _regAlloc; + public CodeGen regAlloc() { + return phase(Phase.RegAlloc, _phase==Phase.LocalSched, () -> { + _regAlloc = new RegAlloc(this); + _regAlloc.regAlloc(); + }); + } + + // Human-readable register name + public String reg(Node n) { + if( _phase.ordinal() >= Phase.RegAlloc.ordinal() ) { + String s = _regAlloc.reg(n); + if( s!=null ) return s; + } + return "N"+ n._nid; + } + + + // --------------------------- + // Encoding + public Encoding _encoding; + public CodeGen encode() { return encode(false); } + private CodeGen encode(boolean inMemory) { return phase(Phase.Encoding, _phase==Phase.RegAlloc, () -> { + // Saved ideal IR requests one object per unit, including when the + // driver stopped at Serialize and resumed later. In-memory linking + // explicitly requests one combined image instead. + if( inMemory || _serials.isEmpty() ) { _encoding = new Encoding(this).encode(); return; } + for( CompUnit cu : _serials.keySet() ) { + _emitUnit = cu; + Encoding enc = new Encoding(this).encode(); + _encodings.put(cu,enc); + if( cu._fname.equals(_srcName==null ? "Test" : _srcName) ) _encoding = enc; + } + _emitUnit = null; + }); } + + // --------------------------- + // Exporting to external formats + ElfWriter _elf; + public CodeGen exportELF( boolean inMemory, boolean emitEntrySymbol ) { + return phase(Phase.Export, _phase==Phase.Encoding, () -> { + if( inMemory ) { + assert _encodings.isEmpty() : "In-memory output must be selected before Encoding"; + new LinkMem(this).link(_encoding); + } + else { + if( _serials.isEmpty() ) + throw new IllegalStateException("ELF export needs ideal IR: request Export from the driver, or Serialize before Lower"); + assert !_encodings.isEmpty() : "File output must be selected before Encoding"; + Encoding entry = _encoding; + BAOS serial = _serial; + try { + for( var e : _encodings.entrySet() ) { + _emitUnit = e.getKey(); + _encoding = e.getValue(); + _serial = _serials.get(_emitUnit); + _elf = new ElfWriter(this).export(emitEntrySymbol && _emitUnit._fname.equals(_srcName)); + } + // Mutually dependent units built together must not become + // stale merely because their files were written in sequence. + long time = System.currentTimeMillis(); + for( CompUnit cu : _encodings.keySet() ) { + File obj = new File(_buildDir+"/"+cu._fname+".o"); + if( !obj.setLastModified(time) ) throw new IllegalStateException("Cannot timestamp "+obj); + } + } finally { _emitUnit=null; _encoding=entry; _serial=serial; } + } + }); + } + + + // --------------------------- + + // Search a external path list, each path is recursively searched for .o + // files, which are partially loaded and searched for public symbols. + + // This is a state machine which lazily searches down the set of search + // paths until the requested module is class is found. + + // Map from external Strings to either partially read Simple ElfFile or ExternNode + public final HashMap<String,Object> _externSymbols = new HashMap<>(); + + // State machine elements; the outermost element is _externPaths + private int _extPathIdx; // Search index into the extern path list + private final Ary<File> _files = new Ary<>(File.class); // Files in the current extern path being searched + private final HashSet<File> _seenFiles = new HashSet<>(); + private int _extFileIdx; // Index into _files + + public ElfReader findExternalSimple( String name ) { + return findExternalSymbol(name) instanceof ElfReader elf ? elf : null; + } + + public ExternNode findExternal( String name ) { + return findExternalSymbol(name) instanceof ExternNode extern ? extern : null; + } + + // Search the search-path for the name; return an ElfReader if found in a + // Simple-made ELF; return an ExternNode if from another ELF, or null if + // not found. + private Object findExternalSymbol( String name ) { + // State Machine! + + while( true ) { + // Check for an immediate hit + switch( _externSymbols.get(name) ) { + + case ExternNode extern: + // Name maps to an ExternNode + return extern; + + case ElfReader elf: + return elf; + + case null: + // Name is unknown. Advance the file-system search, pulling + // out new ELF files from the search directory list and loading + // their published symbols. + + // The last directory was mined out, so get the next search + // directory and find all ELF-like files. + if( _extFileIdx >= _files._len ) { + if( _externPaths==null || _extPathIdx >= _externPaths._len ) + return null; // A true miss in the whole search path + loadExtPath(); // Load next batch file of files + break; + } + // Load and parse an ELF header + ElfReader elf = getElf(_files.at(_extFileIdx++)); + if( elf == null ) break; // Not an ELF + // Load public symbols from the ELF + elf.loadPublicSymbols(); + if( elf._strs==null ) break; // No .simple section with strings + // Put public symbol in global table; first names shadow all others + // Map from symbol string to ELFReader; most lookups + // will miss and no need to unpack ElfReader types + _externSymbols.putIfAbsent(elf._strs[1],elf); + + // Publish definitions only; referenced classes belong to other objects. + for( String fname : elf._units ) + _externSymbols.putIfAbsent(Parser.addClzPrefix(fname.replace('/','.')),elf); + // A -L object can name dependencies alongside itself. They + // need metadata discovery even when no directory was supplied. + for( String dep : elf._deps ) { + File file = elf.dependencyFile(dep); + if( file.isFile() ) addExtFile(file); + } + break; + + default: throw Utils.TODO("should not reach here"); + } + } + } + + // Load all the .o files in the next external path. + private void loadExtPath() { + _files.clear(); + _extFileIdx = 0; + String extpath = _externPaths.at(_extPathIdx++); + File f = new File(extpath); + if( !f.isAbsolute() ) + f = new File(_cwd+extpath); + fillExtFiles(f); + } + + // Recursive search (TODO: gzip, archives) and gather all .o files. + private void fillExtFiles(File dir) { + if( dir.isDirectory() ) + for( File f : dir.listFiles() ) + fillExtFiles(f); + else if( dir.getName().endsWith(".o") ) + addExtFile(dir); + } + + private void addExtFile(File file) { + file=file.toPath().toAbsolutePath().normalize().toFile(); + if( _seenFiles.add(file) ) _files.add(file); + } + + // Map from object file names to ElfReaders + public final HashMap<String,ElfReader> _elfs = new HashMap<>(); + public ElfReader getElf(File f) { + ElfReader elf = _elfs.get(f.getPath()); + if( elf != null ) + return elf; + _elfs.put(f.getPath(), elf = ElfReader.load(f,null)); + return elf; + } + + // --------------------------- + public boolean _asmLittle=true; + public String asm() { return asm(new SB()).toString(); } + SB asm(SB sb) { + ASMPrinter.print(sb,this); + return sb; + } + + + // Testing shortcuts + public Node ctrl() { return compunit()._stop.ret().ctrl(); } + public Node expr() { return compunit()._stop.ret().expr(); } + public String print() { return compunit()._stop.print(); } + + // Debugging helper + @Override public String toString() { + if( _stop == null ) + return "No StopNode"; + return IRPrinter.prettyPrint(this); + } + + // Debugging helper + public Node f(int idx) { + for( CompUnit cu : _compunits.values() ) { + Node n = (cu._stop == null ? _stop : cu._stop).find(idx); + if( n != null ) return n; + } + return null; + } + + + String printCFG() { + if( _cfg==null ) return "no CFG"; + SB sb = new SB(); + for( CFGNode cfg : _cfg ) { + sb.fix(8,""+(int)cfg._idepth); + IRPrinter.printLine( cfg, sb ); + } + return sb.toString(); + } + + public static void print_as_hex(Encoding enc) { + for (byte b : enc._bits.toByteArray()) { + System.out.print(String.format("%02X", b)); + } + System.out.println(); + } +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/CompUnit.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/CompUnit.java new file mode 100644 index 00000000..2b091b3c --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/CompUnit.java @@ -0,0 +1,170 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.parser.Parser; +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.Type; +import com.seaofnodes.simple.type.TypeStruct; +import com.seaofnodes.simple.util.*; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; +import static com.seaofnodes.simple.codegen.CodeGen.CODE; + +// A compilation-unit, which is an external reference and source and object +// files. These are mirrored over a file system tree (so form a tree), both +// the source tree and build/object tree. +public class CompUnit { + public final CompUnit _par; // Parent, e.g. A/B + final String _fname; // Full slashed file name starting from module root, e.g. A/B/C + final public String _cname; // Full dotted class name starting from module root, e.g. A.B.C + final String _name; // Base name, e.g. C + final ExternNode _ext; // Found in external libs; e.g. libc + final File _smp; // Simple source file, e.g. module_root/A/B/C.smp + public final File _obj; // ELF output file, e.g. build/A/B/C.o + final public String _src; // Source code, from file or test case + // Original source path from an imported diagnostic location. Does not + // affect discovery/rebuild decisions, which use _smp and _src. + public String _sourceFile; + + public String sourceFile() { + return _smp!=null ? _smp.getAbsolutePath() : _sourceFile; + } + + public TypeStruct _clz; // The one TypeStruct being published + + // _deps records units required for types, code, or class references, + // without requiring their initializers to run. These discovery/rebuild + // dependencies may form a cycle. + Ary<CompUnit> _deps; + + // Per-Compilation-Unit Start/StopNodes, keeping alive all exported Nodes. + // Null means no code loaded (yet). + public StartCUNode _start; + public StopCUNode _stop ; + + + // Discovered source file. Source is loaded if required to parse, and null + // otherwise. + CompUnit( CompUnit par, String fname, String cname, String name ) throws IOException { + _par = par; + _fname= fname; + _cname= cname; + _name = name; + _ext = null; + _smp = new File( CODE. _modDir + "/" + fname + ".smp"); + _obj = new File( CODE._buildDir + "/" + fname + ".o" ); + + boolean mustCompile = _smp.exists() && // Has source to compile + ( fname.equals(CODE._srcName) || // Explicit input is a compilation request + !_obj.exists() || // No object file + _smp.lastModified() > _obj.lastModified() || // Out-of-date with source? + !current(_obj,new HashSet<>()) ); // Out-of-date anywhere in its inputs? + _src = mustCompile ? new String(Files.readAllBytes(_smp.toPath())) : null; + } + + // Test case. No source file. Source code passed in as a String. Fake + // class name. + CompUnit( String fname, String src ) { + _par = null; + _name = fname; + _fname= fname; + _cname= fname; + _ext = null; + _smp = null; + _obj = null; + _src = src; + } + + // External CompUnit, pulled in from an external library like libc + CompUnit( ExternNode ext, String symbol ) { + _par = null; + _name = symbol; + _fname= _name; + _cname= null; + _ext = ext; + _smp = null; + _obj = null; + _src = null; + } + + // External Simple object file, or namespace parent, found on an external search path. + CompUnit( File obj, CompUnit par, String fname, String cname, String name ) { + _par = par; + _name = name; + _fname= fname; + _cname= cname; + _ext = null; + _smp = null; + _obj = obj; + _src = null; + } + + // Check source-backed inputs before importing any saved types or IR. + // A dependency's source can be newer while its object is unchanged. + // Use a fresh visited set for each root: encountering a cycle is safe, but + // caching a provisional "current" answer from inside it would not be. + private static boolean current(File obj, HashSet<File> visit) { + if( obj==null || !obj.isFile() ) return false; + if( !visit.add(obj.getAbsoluteFile()) ) return true; + ElfReader elf = CODE.getElf(obj); + if( elf==null ) return false; + elf.loadPublicSymbols(); + if( elf._units==null ) return false; + long time = obj.lastModified(); + for( String unit : elf._units ) { + // Never import another copy of a unit explicitly being compiled. + CompUnit cu = CODE._compunits.get(unit); + if( unit.equals(CODE._srcName) || (cu!=null && cu._src!=null) ) return false; + File src = new File(CODE._modDir+"/"+unit+".smp"); + if( !src.isFile() || src.lastModified()>time ) return false; + } + for( String dep : elf._deps ) { + if( Arrays.asList(elf._units).contains(dep) ) continue; // Owned by this object. + File file = dependencyObj(dep,true); + if( file==null || !file.isFile() || file.lastModified()>time ) return false; + // Object-only libraries are supplied as a coherent build. Their + // input versions cannot be inferred from installation timestamps. + if( file.equals(new File(CODE._buildDir+"/"+dep+".o")) && !current(file,visit) ) return false; + } + return true; + } + + // Resolve dependencies the same way for freshness checks and actual loads. + static File dependencyObj(String fname, boolean local) { + File obj = new File(CODE._buildDir+"/"+fname+".o"); + if( local && (obj.isFile() || new File(CODE._modDir+"/"+fname+".smp").isFile()) ) return obj; + ElfReader elf = CODE.findExternalSimple(Parser.addClzPrefix(fname.replace('/','.'))); + return elf==null ? null : elf._file; + } + + // Add a discovered dependency, for later writing out in an ELF file, ".simple" section + void addDep(CompUnit dep) { + if( _deps == null ) _deps = new Ary<>(CompUnit.class); + _deps.add(dep); + } + + public void addFun(CodeGen code, FunNode fun) { + assert fun.in(1)==_start; + _stop.addDef(fun.ret()); + code.add(_stop); + _stop.addDep(fun); + } + + void setStart(CodeGen code) { + _stop = new StopCUNode().init(); + _start = new StartCUNode(code._start,_stop, Type.BOTTOM, _fname).init(); + code._stop.addDef(_stop); + } + + @Override public String toString() { + String x = "{"+_cname; + if( _smp!=null && _smp.exists()) x += ",hasSMP"; + if( _obj!=null && _obj.exists()) x += ",hasObj"; + if( _src!=null ) x += ",hasSrc"; + if( _ext != null ) x += ",isExt"; + x += "}"; + return x; + } +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/ElfReader.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/ElfReader.java new file mode 100644 index 00000000..1dced3a8 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/ElfReader.java @@ -0,0 +1,245 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.Node; +import com.seaofnodes.simple.type.TypeMemPtr; +import com.seaofnodes.simple.type.TypeStruct; +import com.seaofnodes.simple.util.Ary; +import com.seaofnodes.simple.util.BAOS; +import com.seaofnodes.simple.util.Utils; +import java.io.*; +import java.nio.file.Files; +import java.util.HashMap; + +public class ElfReader { + + public final File _file; + public final byte[] _buf; + int _x; // Cursor into buffer + // ELF Header + public final ElfHeader _header; + // All sections + public final Section[] _sections; + // Simple section, if available + public final Section _simple; + // Byte array in/out stream - used as a reader + public final BAOS _bais; + // Compilation unit; written into FunNodes + public final CompUnit _compunit; + + // Strings from the Simple section, either small public set or whole set + public String[] _strs; + // Dependent file names + public String[] _deps; + // Compilation units defined by this object, independent of init analysis. + public String[] _units; + // Only published symbol this file + public TypeStruct _clz; + // Canonical named types, distinct from optimized same-named graph states. + public TypeStruct[] _pubTypes; + // Nodes read in this Elf + public Ary<Node> _nodes; + Serialize.TypeRead _types; + FunctionSummary[] _functions; + + // Load an Elf file (mostly lazily) + public static ElfReader load( File f, CompUnit cu ) { + try { return new ElfReader(f,cu); } + catch( IOException ioe ) { return null; } + } + + // Load the Simple section from the ELF file + private ElfReader( File f, CompUnit cu ) throws IOException { + _file = f; + _compunit = cu; + _buf = Files.readAllBytes(f.toPath()); + _header = new ElfHeader(); + // No attempt is made to parse the program header table. + _sections = new Section[_header._shnum]; + for( int i=0; i<_header._shnum; i++ ) + _sections[i] = new Section(i); + + // Load section names + Section secstr = _sections[_header._shstrndx]; + for( Section sec : _sections ) + if( sec._type != 0 ) + sec._name = secstr.str(sec._noff); + + Section simple = null; + for( Section sec : _sections ) + if( ".simple".equals(sec._name) ) + simple = sec; + _simple = simple; + _bais = _simple==null ? null : new BAOS(_buf, (int)_simple._offset); + } + + // For Serialize self-test + ElfReader( BAOS bais ) { + _file = null; + _compunit = null; + _buf = bais.buf(); + _header = null; + _sections = null; + _simple = null; + _bais = bais; + } + + // Load the public symbols and dependent objs, but not the whole IR + void loadPublicSymbols() { + // Check already loaded and cached + if( _strs == null && _simple != null ) + // Read only the published public strings, for speed purposes when + // scanning a large count of ELF files but getting hits in only a + // few of them. i.e. optimizing (prematurely?) for the traditional + // case of linking a small file against a large library + Serialize.read_public_strs(this); + } + + File dependencyFile(String unit) { + File root = _file; + for( String part : _units[0].split("/") ) root=root.getParentFile(); + return new File(root,unit+".o"); + } + + // Loads the entire IR + public TypeStruct loadSimple(CodeGen code) { + code._classInitDependencies = null; // Loaded bodies can change init edges. + Serialize.readAll(code, this, code._aliases, code._fidxs, code._rpcs); + return _clz; + } + + // Loads strings, mappings, and types, but not Nodes. Used after the + // public-string scan finds a matching Simple ELF, so parsing can resolve + // imported field/array aliases before the full graph is linked in later. + public TypeStruct loadPublicTypes(CodeGen code) { + Serialize.readTypes(code, this, code._aliases, code._fidxs, code._rpcs); + return _clz; + } + + int u1() { return _buf[_x++]&0xFF; } + int magic() { // Big endian + return (u1() << 24) | (u1() << 16) | (u1() << 8) | u1(); + } + int u2() { return u1() | (u1()<<8); } // Little endian + int u4() { // Little endian + return u1() | (u1() << 8) | (u1() << 16) | (u1() << 24); + } + long u8() { return u4() | ((long)u4()<<32); } + void skip(int skip) { _x += skip; } + + + public class ElfHeader { + final int _magic; + final boolean _format32; // Format32 or format64 + final byte _osabi; + final boolean _little; // True if little endian, false if big endian + final char _type; + final char _machine; + final long _entry; // Process entry point + final long _phoff; // Program header table offset, from file start + final long _shoff; // Section header table offset, from file start + final int _flags; // Flags + final int _phnum; // Number of program header entries + final int _shentsize; + final int _shnum; // Number of section header entries + final int _shstrndx; // Section header index contains section names + + + ElfHeader() throws IOException { + _x = 0; + if( (_magic = magic()) != 0x7F454C46 ) bad("magic number",_magic); + byte EI_CLASS = (byte)u1(); + if( EI_CLASS != 1 && EI_CLASS != 2 ) bad("EI_CLASS",EI_CLASS); + _format32 = (EI_CLASS==1); + byte EI_DATA = (byte)u1(); + if( EI_DATA != 1 && EI_DATA != 2 ) bad("EI_DATA",EI_DATA); + _little = EI_DATA == 1; + if( u1() != 1 ) bad("version",1); + if( (_osabi=(byte)u1()) != 0 ) bad("EI_VERSION",_osabi); + byte abiversion = (byte)u1(); + if( abiversion != 0 ) bad("EI_ABIVERSION",abiversion); + skip(7); // Pad + + // Endian matters now + if( !_little ) throw new IOException("TODO: Big endian"); + if( (_type = (char)u2()) != 1 ) bad("ET_REL",_type); + if( (_machine = (char)u2()) != 0x3E ) bad("AMD x86-64",_machine); + int ver = u4(); + if( ver != 1 ) bad("Version",ver); + // Format32 matters now + if( _format32 ) bad("Format",1); + _entry = u8(); // Process entry point + _phoff = u8(); // Program header table offset + _shoff = u8(); // Section header table offset + _flags = u4(); // Flags + int ehsize = u2(); + if( ehsize != 0x40 ) bad("EHSIZE",ehsize); + int phentsize = u2(); + if( phentsize != 0x38 && phentsize != 0 ) bad("PHENTSIZE",phentsize); + _phnum = u2(); + _shentsize = u2(); + if( _shentsize != 0x40 ) bad("SHENTSIZE",_shentsize); + _shnum = u2(); + // String section + _shstrndx = u2(); + assert _x == ehsize; + } + } + + public class Section { + final int _noff; // Offset in string section for name + final int _type; // Section type + final long _flags; + final long _addr; // Offset in virtual mem, if loaded + final long _offset; // Offset in file + final long _size; // Size in bytes + final int _link; // Associated section index, varies + final int _info; // Varies + final byte _align; // Log2 alignment + final int _esize; // Element size, if non-zero + String _name; + + Section(int sidx) throws IOException { + _x = (int)(_header._shoff + (sidx * _header._shentsize)); + _noff = u4(); + _type = u4(); + _flags = u8(); + _addr = u8(); + _offset= u8(); + _size = u8(); + _link = u4(); + _info = u4(); + long align = u8(); + if( align==0 ) align=1; + _align = (byte)Long.numberOfTrailingZeros(align); + if( (1L << _align ) != align ) + throw new IOException("Bad section alignment: "+align); + _esize = (int)u8(); + } + + String str(int off) { + assert _type == 3/*SHT_STRTAB*/; // String table section header + int start = (int)(_offset+off); + int x = start; + while( _buf[x++] !=0 ) ; + return new String(_buf,start,x-start-1); + } + } + + private static void bad(String msg, int val) throws IOException { + throw new IOException("Bad "+msg+": "+val); + } + + + + //public TypeMemPtr lookup( String name ) { + // if( _simple == null ) return null; + // BAOS baos = new BAOS(_buf, (int)_simple._offset); + // Ary<String> strs = Serialize.read_strs(baos); + // + // + // + // Ary<Node> nodes = Serialize.read_types(baos); + // + // throw Utils.TODO("Elf load symbol "+name); + //} +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/ElfWriter.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/ElfWriter.java new file mode 100644 index 00000000..fcf5c287 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/ElfWriter.java @@ -0,0 +1,468 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.TypeFunPtr; +import com.seaofnodes.simple.type.TypeStruct; +import com.seaofnodes.simple.util.Ary; +import com.seaofnodes.simple.util.BAOS; +import java.io.*; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.HashMap; + +public class ElfWriter { + + final CodeGen _code; + Ary<Section> _sections; + StringSection _strtab; + + ElfWriter( CodeGen code ) { + _code = code; + } + + // ------------------------------------------------------------------------ + abstract public class Section { + static final int SHT_NULL = 0; + static final int SHT_PROGBITS= 1; + static final int SHT_SYMTAB = 2; + static final int SHT_STRTAB = 3; + static final int SHT_RELA = 4; + static final int SHT_SIMPLE = 10; // Simple section + + final String _name; + final int _type, _entsize, _index, _name_pos, _flags; + int _link, _info; + int _file_offset; + + Section(String name, int type, int entsize, int flags) { + _name = name; + _type = type; + _entsize = entsize; + _flags = flags; + _index = _sections._len; + _sections.push(this); + _name_pos = name.isEmpty() + ? 0 // Null section + : _strtab == null + ? 1 // StringSection + // All the other sections + : _strtab.writeCString(name); + } + + void writeHeader(ByteBuffer out) { + out.putInt(_name_pos); // name + out.putInt(_type); // type + out.putLong(_flags); // flags + out.putLong(0); // addr + out.putLong(_file_offset); // offset + out.putLong(size()); // size + out.putInt(_link); // link + out.putInt(_info); // info + out.putLong(16); // addralign + out.putLong(_entsize); // Fixed-size element size + } + + abstract void write(ByteBuffer out); + abstract int size(); + } + + // ------------------------------------------------------------------------ + // Generic data sections; carries about a BAOS for bits + public class DataSection extends Section { + static final int SHF_WRITE = 1; + static final int SHF_ALLOC = 2; + static final int SHF_EXECINSTR = 4; + static final int SHF_INFO_LINK = 64; + + final BAOS _contents; + + DataSection( String name, int type, int flags, int entsize ) { this(name, type, flags, entsize, new BAOS() ); } + DataSection( String name, int type, BAOS contents, int flags ) { this(name, type, flags, 0, contents ); } + private DataSection( String name, int type, int flags, int entsize, BAOS contents ) { + super(name, type, entsize, flags); + _contents = contents; + } + + @Override void write(ByteBuffer out) { out.put(_contents.toByteArray()); } + @Override int size() { return _contents.size(); } + public void write2( int op ) { + _contents.write(op ); + _contents.write(op>> 8); + } + public void write4( int op ) { + _contents.write(op ); + _contents.write(op>> 8); + _contents.write(op>>16); + _contents.write(op>>24); + } + public void write8( long i64 ) { + write4( (int) i64 ); + write4( (int)(i64>>32)); + } + + } + + // ------------------------------------------------------------------------ + // Section to collect and de-dup strings + public class StringSection extends DataSection { + final HashMap<String,Integer> _dedup = new HashMap<>(); + + StringSection() { + super(".strtab",SHT_STRTAB, 0, 0 ); + int emptyoff = writeCString(""); // Always empty string at offset 0 + assert emptyoff == 0; + int namepos = writeCString(".strtab"); + assert namepos == _name_pos; + } + + public int writeCString(String name) { + if( name==null ) return 0; // Anonymous functions have no name + Integer ii = _dedup.get(name); + if( ii != null ) return ii; + // place name in the string table + int pos = _contents.size(); + _dedup.put(name,pos); + byte[] name_bytes = name.getBytes(); + _contents.write(name_bytes); + // the strings are null terminated + _contents.write(0); + return pos; + } + } + + // ------------------------------------------------------------------------ + // The relocations; 3 words per relocation + public class ReloSection extends DataSection { + ReloSection( int info ) { + this(".rela.text",info); + } + ReloSection(String name,int info) { + super(name, SHT_RELA, SHF_INFO_LINK, 24/*entsize*/); + _link = 2; + _info = info; + } + + void writeReloOffPlus(int offset, int symidx, byte relo) { + writeRelo(offset,symidx,relo,-4); + } + void writeRelo(int offset,int symidx,byte relo,long addend) { + assert symidx > 0; + write8(offset); + write8(((long)symidx << 32L) | relo); + write8(addend); + } + } + + + // ------------------------------------------------------------------------ + // Symbols section; emit the symbols into the data section directly + public class SymbolSection extends DataSection { + public static final int SYMBOL_SIZE = 4+1+1+2+8+8; + final BAOS _globals; + final int _nlocals; + + SymbolSection(int nlocals) { + super(".symtab", SHT_SYMTAB, 0/*flags*/, SYMBOL_SIZE); + _globals = new BAOS(); + _nlocals = nlocals; + } + + // Global symbol index + int gidx() { return _nlocals+(_globals.size()/SYMBOL_SIZE); } + + public int symbol( String name, int shndx, int bind, int type, long value, long size ) { + int gidx = gidx(); // Global index before I write to _globals + BAOS baos = bind==SYM_BIND_LOCAL ? _contents : _globals; + write4(baos,_strtab.writeCString(name)); + int info = (bind << 4) | (type & 0xF); + baos.write(info); + baos.write(0); // other/visibility + write2(baos,shndx); // section index + write8(baos,value); + write8(baos,size); + return bind==SYM_BIND_LOCAL ? -99 : gidx; + } + + public int symbol( String name, int shndx, int bind, int type ) { + return symbol(name,shndx,bind,type,0,0); + } + + // creates function and stores where it starts. + // Filters to just this class.obj file. + void encodeFunctions(StopNode stop, Encoding enc, int text_idx, boolean emitEntrySymbol) { + for( Node stopcu : stop._inputs ) { + for( Node ret : stopcu._inputs ) { + FunNode fun = ((ReturnNode)ret).fun(); + if( !_code.emits(fun) ) + continue; + int end = enc.opStart(fun.ret()) + enc.opLen(fun.ret()); + long value = enc.opStart(fun); + long size = end - value; + // Stable FIDX names are exported separately below. + if( fun._name != null && !fun._name.equals(_code._fidxs.symbol(fun.sig().fidx())) ) { + // The C runtime owns `main` and adapts argc/argv into + // a normal Simple array. Keep the Simple function on + // its native ABI under a private entry symbol. + String name = emitEntrySymbol && "main".equals(fun._name) + ? "simple_main" + : fun._name; + symbol(name, text_idx, SYM_BIND_GLOBAL, SYM_TYPE_FUNC, value, size); + } + } + } + } + + // Insert a default-main for top-level compilations, if the top-level + // does not declare a main. + void symbolEntryClinit(int text_idx, Encoding enc) { + FunNode clinit = null; + String entryName = _code.entryClinitName(); + for( FunNode fun : _code._linker ) + if( fun != null && _code.emits(fun) ) { + if( "main".equals(fun._name) ) + return; // encodeFunctions exported this as simple_main + if( entryName.equals(fun._name) ) + clinit = fun; // Keep top-level entry, if we don't find an explicit main + } + // No top-level main, so the top-level clinit becomes a default main + if( clinit != null ) { + assert enc.opStart(clinit) == 0; + symbol("simple_main",text_idx, SYM_BIND_GLOBAL, SYM_TYPE_FUNC, enc.opStart(clinit), 0); + return; + } + throw new IllegalStateException("Missing entry <clinit> "+entryName); + } + + static void write2( BAOS bits, int op ) { + bits.write(op ); + bits.write(op>> 8); + } + static void write4( BAOS bits, int op ) { + bits.write(op ); + bits.write(op>> 8); + bits.write(op>>16); + bits.write(op>>24); + } + static void write8( BAOS bits, long i64 ) { + write4(bits, (int) i64 ); + write4(bits, (int)(i64>>32)); + } + + @Override + void writeHeader(ByteBuffer out) { + // points to string table section + _link = _strtab._index; + _info = _nlocals; + super.writeHeader(out); + } + + @Override void write(ByteBuffer out) { + out.put(_contents.toByteArray()); + out.put(_globals .toByteArray()); + } + @Override int size() { + return _contents.size() + _globals.size(); + } + } + + public static final int SYM_BIND_LOCAL = 0; + public static final int SYM_BIND_GLOBAL = 1; + public static final int SYM_BIND_WEAK = 2; + + public static final int SYM_TYPE_NOTYPE = 0; + public static final int SYM_TYPE_OBJECT = 1; + public static final int SYM_TYPE_FUNC = 2; + public static final int SYM_TYPE_SECTION = 3; + + public static final int SECTION_HDR_SIZE = 64; + + // ------------------------------------------------------------------------ + public class SimpleSection extends DataSection { + SimpleSection(BAOS serial) { super(".simple", SHT_SIMPLE, serial, 0); } + } + + + // ------------------------------------------------------------------------ + + // Native code can live above 4G, outside our four-byte function pointers. + // Give address-taken natives a low entry which tail-jumps through the PLT. + // A weak, stable name makes references from separate objects agree on one + // entry. Do not patch references straight to the local copy: that would + // bypass the linker's choice and break pointer equality across objects. + private int nativeEntry(String name, HashMap<String,Integer> entries, + DataSection text, SymbolSection symbols, ReloSection relos) { + Integer old = entries.get(name); + if( old!=null ) return old; + int offset = text.size(); + text._contents.write(0xE9); // jmp rel32; preserve arguments and return address + text.write4(0); + int target = symbols.symbol(name,0,SYM_BIND_GLOBAL,SYM_TYPE_FUNC); + relos.writeReloOffPlus(offset+1,target,(byte)4/*PLT32*/); + int entry = symbols.symbol("__simple_native$"+name,text._index, + SYM_BIND_WEAK,SYM_TYPE_FUNC,offset,5); + entries.put(name,entry); + return entry; + } + + public ElfWriter export(boolean emitEntrySymbol) { + if( _code._serial==null ) + throw new IllegalStateException("ELF export needs ideal IR: request Export from the driver, or Serialize before Lower"); + // Sections are created in the order they are emitted. + _sections = new Ary<>(Section.class); + + // Null section + Section nullSection = new DataSection("",Section.SHT_NULL,0,0); + + // String section next; written out early and available to record/dedup + // strings for all remaining sections + _strtab = new StringSection(); + + // place all the symbols + int nlocals = 1/*null*/+9/*number of sections*/+1/*Simple section*/; + SymbolSection symbols = new SymbolSection(nlocals); + + // we've already constructed these entire sections in the encoding phase + Encoding enc = _code._encoding; + // Append ELF-only thunks to a copy. Export must not change instruction + // offsets or the image used by the in-memory linker and emulators. + BAOS textBits = new BAOS(enc._bits.toByteArray(),enc._bits.size()); + DataSection text = new DataSection(".text" , Section.SHT_PROGBITS, textBits, DataSection.SHF_ALLOC | DataSection.SHF_WRITE | DataSection.SHF_EXECINSTR ); + // Constant pool + DataSection rodata = new DataSection(".rodata", Section.SHT_PROGBITS, enc._cpool, DataSection.SHF_ALLOC ); + // Static/global pool + DataSection rwdata = new DataSection(".data" , Section.SHT_PROGBITS, enc._sdata, DataSection.SHF_ALLOC | DataSection.SHF_WRITE ); + + + // populate function symbols + symbols.encodeFunctions(_code._stop, _code._encoding, text._index, emitEntrySymbol); + // Export the source compilation unit <clinit> as the C runtime "main". + // Encoding keeps this function at text offset zero. + if( emitEntrySymbol ) + symbols.symbolEntryClinit(text._index,enc); + + // create external .text relocations + ReloSection relocations = new ReloSection(text._index); + HashMap<String,Integer> nativeEntries = new HashMap<>(); + + for( Node n : enc._externals.keySet()) { + String extern = enc._externals.get(n); + boolean nativeAddress = n instanceof FunPtrNode && + _code.externFunc(((TypeFunPtr)n._type).fidx())!=null; + int symidx = nativeAddress + ? nativeEntry(extern,nativeEntries,text,symbols,relocations) + : symbols.symbol(extern,0,SYM_BIND_GLOBAL,SYM_TYPE_FUNC); + int offset = enc.opStart(n) + enc.opLen(n) - 4; + // Address materialization and static fields name the same thunk; + // direct calls still go straight to the native symbol's PLT entry. + relocations.writeReloOffPlus(offset,symidx,(byte)(nativeAddress ? 2/*PC32*/ : 4/*PLT32*/)); + } + + // Native ELF output is AMD64. Data addresses use PC32, not PLT calls. + for( Node n : enc._externData.keySet() ) { + String extern = enc._externData.get(n); + int symidx = symbols.symbol(extern, 0, SYM_BIND_GLOBAL, SYM_TYPE_OBJECT); + int offset = enc.opStart(n) + enc.opLen(n) - 4; + relocations.writeReloOffPlus(offset, symidx, (byte)2/*PC32*/); + } + + // Stable function identities are also addressable from data, including + // anonymous functions which have no source-level exported name. + HashMap<String,Integer> dataSymbols=new HashMap<>(); + for( FunNode fun : _code._linker ) + if( fun!=null && !fun.isDead() && _code.emits(fun) && enc.opStart(fun)>=0 ) { + String name=_code._fidxs.symbol(fun.sig().fidx()); + dataSymbols.put(name,symbols.symbol(name,text._index,SYM_BIND_GLOBAL,SYM_TYPE_FUNC,enc.opStart(fun),0)); + } + for( StaticData.ObjectData obj : enc._data.objects.values() ) { + int section=obj.external ? 0 : obj.readOnly ? rodata._index : rwdata._index; + dataSymbols.put(obj.symbol,symbols.symbol(obj.symbol,section,SYM_BIND_GLOBAL,SYM_TYPE_OBJECT, + obj.external ? 0 : obj.offset,obj.external ? 0 : obj.size)); + } + for( Encoding.Relo relo : enc._bigCons.values() ) { + StaticData.ObjectData obj=enc._data.object(relo._t); + relocations.writeReloOffPlus(relo._opStart+relo._off,dataSymbols.get(obj.symbol),relo._elf); + } + ReloSection roRelos=new ReloSection(".rela.rodata",rodata._index); + ReloSection rwRelos=new ReloSection(".rela.data",rwdata._index); + for( StaticData.Relocation relo : enc._data.relocations ) { + String name=enc._data.targetSymbol(relo.value()); + String nativeName=relo.value() instanceof TypeFunPtr fp ? _code.externFunc(fp.fidx()) : null; + Integer sym=nativeName==null ? dataSymbols.get(name) + : nativeEntry(nativeName,nativeEntries,text,symbols,relocations); + if( sym==null ) { + sym=symbols.symbol(name,0,SYM_BIND_GLOBAL,SYM_TYPE_NOTYPE); + dataSymbols.put(name,sym); + } + // AMD64 absolute 32/64-bit addresses; linker rejects overflow. + byte kind=(byte)(relo.logSize()==2 ? 10 : 1); + (relo.object().readOnly ? roRelos : rwRelos).writeRelo(relo.object().offset+relo.offset(),sym,kind,0); + } + + // Create a "Simple" section for Simple types and ir + SimpleSection simple = new SimpleSection(_code._serial); + + // Write the local section header symbols + assert nlocals == 1+_sections._len; + symbols.symbol("",0,0,0); // Null symbol at index 0 + for( Section s : _sections ) + symbols.symbol(s._name, s._index, SYM_BIND_LOCAL, SYM_TYPE_SECTION, 0, s.size()); + + int size = 64; // ELF header size + // size of all section data + for( Section s : _sections ) { + s._file_offset = size; + size += s.size(); + } + // headers go at the very end + int shdr_offset = size; + size += SECTION_HDR_SIZE*_sections.len(); + + ByteBuffer out = ByteBuffer.allocate(size); + out.order(ByteOrder.LITTLE_ENDIAN); + + // ELF header + out.put(new byte[]{ 0x7F, 'E', 'L', 'F', 2, 1, 1, 0 }); + out.put(new byte[8]); + out.putShort((short)1 /* ET_REL */); // type + out.putShort((short)62 /* AMD64 */); // machine + out.putInt(1); // version + out.putLong(0); // entry + out.putLong(0); // phoff + out.putLong(shdr_offset); // shoff + out.putInt(0); // flags + out.putShort((short)64); // ehsize + out.putShort((short)0); // phentsize + out.putShort((short)0); // phnum + out.putShort((short)64); // shentsize + out.putShort((short) _sections.len()); // shnum + out.putShort((short) _strtab._index); // shstrndx + + // raw section data + for( Section s : _sections ) + s.write(out); + + // Section headers + for( Section s : _sections ) + s.writeHeader(out); + + assert out.position() == size; + + String objName = _code._buildDir + "/" + (_code._emitUnit==null ? _code._srcName : _code._emitUnit._fname) + ".o"; + File file = new File(objName); + if( file.getParentFile()!=null ) + file.getParentFile().mkdirs(); + try { + BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file)); + bos.write(out.array()); + bos.close(); + } catch( IOException ioe ) { + // Caller does not want to deal, but it is a failed compilation in any case + throw new RuntimeException(ioe); + } + // Reset for next ELF + _sections = null; + _strtab = null; + return this; + } + +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/Encoding.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Encoding.java new file mode 100644 index 00000000..8b868218 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Encoding.java @@ -0,0 +1,570 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.parser.Parser; +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.print.IRPrinter; +import com.seaofnodes.simple.type.*; +import com.seaofnodes.simple.util.*; +import java.util.*; + + +/** + * Instruction encodings + * <p> + * This class holds the encoding bits, plus the relocation information needed + * to move the code to a non-zero offset, or write to an external file (ELF). + * <p> + * There are also a bunch of generic utilities for managing bits and bytes + * common in all encodings. + */ +public class Encoding implements com.seaofnodes.isa.CodeSink { + + // Top-level program graph structure + public final CodeGen _code; + + // CFG Nodes in RPO order, per-encoding. + public Ary<CFGNode> _cfg; + + // Instruction bytes. The registers are encoded already. Relocatable data + // is in a fixed format depending on the kind of relocation. + + // - RIP-relative offsets into the same chunk are just encoded with their + // chunk relative offsets; no relocation info is required. + + // - Fixed address targets in the same encoding will be correct for a zero + // base; moving the entire encoding requires adding the new base address. + + // - RIP-relative to external chunks have a zero offset; the matching + // relocation info will be used to patch the correct value. + + public final BAOS _bits = new BAOS(); // Instructions / encodings + public final BAOS _cpool = new BAOS(); // Constant r/o pool + public final BAOS _sdata = new BAOS(); // Static r/w pool + public final HashMap<Node,Relo> _bigCons = new HashMap<>(); + public final StaticData _data; + public final HashMap<String,Integer> _externalAddresses = new HashMap<>(); + + // Big Constant relocation info. + public static class Relo { + public final Node _op; + public final Type _t; // Constant type + public final int _align; // Log alignment, captured for diagnostic display + public final int _structSize; // Capture layout during encoding, not printing + public final byte _off; // Offset from start of opcode + public final byte _elf; // ELF relocation type, e.g. 2/PC32 + public int _target; // Where constant is finally placed + public int _opStart; // Opcode start + Relo( Node op, Type t, byte off, byte elf ) { + _op=op; _t=t; _off=off; _elf=elf; + _align = t.alignment(); + _structSize = t instanceof TypeStruct ts ? ts.size() : 0; + _readOnly = readOnly(t); + } + public boolean readOnly() { return _readOnly; } + private final boolean _readOnly; + private static boolean readOnly(Type t) { + // Class initializers still write their fields, even when every + // final value is constant. Their storage must remain writable. + return !(t instanceof TypeStruct ts) || + (!Parser.startsClzPrefix(ts._name) && ts.isConstant()); + } + } + + Encoding( CodeGen code ) { + _code = code; + _data = new StaticData(this); + } + + // Shortcut to the defining register + public short reg(Node n) { + return _code._regAlloc.regnum(n); + } + + public int read1(int idx) { + byte[] buf = _bits.buf(); + return (buf[idx]&0xFF); + } + public int read2(int idx) { + byte[] buf = _bits.buf(); + return (buf[idx]&0xFF) | (buf[idx+1]&0xFF) <<8; + } + public int read4(int idx) { + byte[] buf = _bits.buf(); + return (buf[idx]&0xFF) | (buf[idx+1]&0xFF) <<8 | (buf[idx+2]&0xFF)<<16 | (buf[idx+3]&0xFF)<<24; + } + public long read8(int idx) { + return (read4(idx) & 0xFFFFFFFFL) | read4(idx+4); + } + + // This buffer is invalid/moving until after all encodings are written + public byte[] bits() { return _bits.buf(); } + + // 4 byte little-endian write + public void patch4( int idx, int val ) { + byte[] buf = _bits.buf(); + buf[idx ] = (byte)(val ); + buf[idx+1] = (byte)(val>> 8); + buf[idx+2] = (byte)(val>>16); + buf[idx+3] = (byte)(val>>24); + } + + static void padN(int n, BAOS bits) { + while( (bits.size()+n-1 & -n) > bits.size() ) + bits.write(0); + } + + // Convenience for writing log-N + static void addN( int log, long x, BAOS bits ) { + for( int i=0; i < 1<<log; i++ ) { + bits.write((int)x); + x >>= 8; + } + } + + public Encoding add1( int op ) { addN(0,op,_bits); return this; } + public Encoding add2( int op ) { addN(1,op,_bits); return this; } + public Encoding add4( int op ) { addN(2,op,_bits); return this; } + public Encoding add8(long op ) { addN(3,op,_bits); return this; } + + private final static int SENTINEL = -1; + private final IntHashMap _opStart = new IntHashMap(SENTINEL); + private final IntHashMap _opLen = new IntHashMap(SENTINEL); + + + // Read op start for n + public int opStart(Node n) { return _opStart.get(n._nid); } + // Read op length for n + public byte opLen (Node n) { return (byte)_opLen.get(n._nid); } + + // Set op start for n + public void opStart(Node n, int off) { + assert off != SENTINEL; + _opStart.put(n._nid,off); + } + + // Set op start for n + public void opStartAdd(Node n, int off) { + if( off==0 ) return; + int old = _opStart.get(n._nid); + assert old != SENTINEL && old+off != SENTINEL; + _opStart.put(n._nid,old+off); + } + + // Set op length for n + public void opLen(Node n, byte len) { + assert len != SENTINEL; + _opLen.put(n._nid,len); + } + + // Nodes need "relocation" patching; things done after code is placed. + // Record src and dst Nodes. + private final HashMap<Node,CFGNode> _internals = new HashMap<>(); + // Source is a Call, destination in the Fun. + public Encoding relo( CallNode call ) { + FunNode fun = _code.link(call.tfp()); + if( fun != null && !_code.emits(fun) ) + _externals.put(call,fun._name); + else + _internals.put(call,fun); + return this; + } + // Code address as a constant + public Encoding relo( FunPtrNode fptr ) { + TypeFunPtr tfp = (TypeFunPtr)fptr._type; + // Native functions have no local FunNode, but are still callable. + // Do not treat their addresses as identities of deleted functions. + String nativeName = _code.externFunc(tfp.fidx()); + if( nativeName!=null ) return external(fptr,nativeName); + FunNode fun = _code.link(tfp); + if( fun != null && !_code.emits(fun) ) + _externals.put(fptr,fun._name); + else + _internals.put(fptr,fun); + return this; + } + // Local jump + public void jump( CFGNode jmp, CFGNode dst ) { + _internals.put(jmp,dst.uctrlSkipEmpty()); + } + + // External references only located by Strings a link-time + public final HashMap<Node,String> _externals = new HashMap<>(); + public Encoding external( Node n, String extern ) { + _externals.put(n,extern); + return this; + } + + public final HashMap<Node,String> _externData = new HashMap<>(); + public void externalData(Node n, String name) { _externData.put(n,name); } + + // Store t as a 32/64 bit constant in the code space; generate RIP-relative + // addressing to load it. Type is stored in either the .rodata or .data. + public void largeConstant( Node relo, Type t, int off, int elf ) { + // TODO: Any-old struct in the cpool + assert t.isConstant() || (t instanceof TypeStruct ts && Parser.startsClzPrefix(ts._name)); + assert (byte)off == off; + assert (byte)elf == elf; + _bigCons.put(relo,new Relo(relo,t,(byte)off,(byte)elf)); + } + + // -------------------------------------------------- + Encoding encode( ) { + // Basic block layout: negate branches to keep blocks in-order; insert + // unconditional jumps. Attempt to keep backwards branches taken, + // forwards not-taken (this is the default prediction on most + // hardware). Layout is still Reverse Post Order but with more + // restrictions. + basicBlockLayout( ); + + // Write encoding bits in order into a big byte array. + // Record opcode start and length. + writeEncodings(); + + // Short-form RIP-relative support: replace long encodings with short + // encodings and compact the code, changing all the offsets. + compactShortForm(); + + // Place pool objects and record relocations within their contents. + _data.write(); + + // Patch RIP-relative and local encodings now. + patchLocalRelocations(); + + return this; + } + + // -------------------------------------------------- + // Basic block layout: negate branches to keep blocks in-order; insert + // unconditional jumps. Attempt to keep backwards branches taken, forwards + // not-taken (this is the default prediction on most hardware). Layout is + // still Reverse Post Order but with more restrictions. + private void basicBlockLayout( ) { + IdentityHashMap<LoopNode,Ary<CFGNode>> rpos = new IdentityHashMap<>(); + _cfg = new Ary<>(CFGNode.class); + rpos.put(_code._start.loop(),_cfg); + BitSet visit = _code.visit(); + + // Do them all except the <clinit>s + Ary<FunNode> clinits = new Ary<>(FunNode.class); + FunNode entry = null; + String entryName = _code.entryClinitName(); + for( FunNode fun : _code._linker ) { + if( fun != null && !fun.isDead() && _code.emits(fun) ) { + if( fun.isClz() ) { + if( fun._name.equals(entryName) ) + entry = fun; + else + clinits.add(fun); + } else { + int x = _cfg._len; + _rpo_cfg(fun,visit,rpos); + assert _cfg.at(x) instanceof ReturnNode; + } + } + } + // Now the <clinit>s last, so when reversed they become first. The + // program entry <clinit> is last of all, so it lands at offset zero. + for( FunNode clinit : clinits ) { + int x = _cfg._len; + _rpo_cfg(clinit,visit,rpos); + assert _cfg.at(x) instanceof ReturnNode; + } + if( entry != null ) { + int x = _cfg._len; + _rpo_cfg(entry,visit,rpos); + assert _cfg.at(x) instanceof ReturnNode; + } + + + // Reverse in-place + for( int i=0; i< _cfg.size()>>1; i++ ) + _cfg.swap(i,_cfg.size()-1-i); + visit.clear(); + } + + + private void _rpo_cfg(CFGNode bb, BitSet visit, IdentityHashMap<LoopNode,Ary<CFGNode>> rpos ) { + if( bb==null || visit.get(bb._nid) ) return; + visit.set(bb._nid); + CFGNode next = bb.uctrl(); + + // Loops run an inner "rpo_cfg", then append the entire loop body in + // place. This keeps loop bodies completely contained, although if + // Some Day Later we have real profile data we ought to arrange the + // branch orderings based on frequency. + if( bb instanceof LoopNode loop ) { + Ary<CFGNode> body = new Ary<>(CFGNode.class); // Private RPO for the loop + rpos.put(loop.loop(),body); // Find it via loop tree + _rpo_cfg(next,visit,rpos); // RPO the loop + body.add(loop); // Include loop last (first in RPO) + Ary<CFGNode> outer = rpos.get(loop.cfg(1).loop()); + outer.addAll(body); // Append to original CFG + return; + } + + // IfNodes visit 2 sides, and may choose to reorder them in the RPO + if( bb instanceof IfNode iff ) { + // Pick out the T/F projections + CProjNode t = iff.cproj(0); + CProjNode f = iff.cproj(1); + // Invert the branch or not + if( shouldInvert(t,f,iff.loopDepth()) ) { + iff.negate(); + t.invert(); + f.invert(); + CProjNode tmp=f; f=t; t=tmp; // Swap t/f + } + + // Whichever side is visited first becomes last in the RPO. With + // no loops, visit the False side last (so True side first) so that + // when the False RPO visit returns, the IF is immediately next. + // When the RPO is reversed, the fall-through path will always be + // following the IF. + _rpo_cfg(t,visit,rpos); // True side first + next = f; // False side last + } + + // If the *next* BB has already been visited, and we are not already a + // jump, we may need an unconditional forwards jump here + Ary<CFGNode> rpo = rpos.get(bb.loop()); + if( next!=null && visit.get(next._nid) && !(bb instanceof IfNode) ) { + boolean needJump = next instanceof LoopNode + // Empty blocks from an IF will invert the IF and backwards + // branch to the loop head. If not empty, or not an IF + // will need a jump. + ? (bb.nOuts()>1 || !(bb.cfg0() instanceof IfNode) ) + // Forwards jump. If all the blocks, in RPO order, to our + // target are empty, we will fall in and not need a jump. + : !backwardsEmptyScan(rpo,next); + if( needJump ) { + CFGNode jmp = _code._mach.jump(); + jmp._ltree = bb._ltree; + jmp.setDefX(0,bb); + next.setDef(next._inputs.find(bb),jmp); + rpo.add(jmp); + } + } + + _rpo_cfg(next,visit,rpos); + rpo.add(bb); + } + + // Should this test be inverted? + private static boolean shouldInvert(CFGNode t, CFGNode f, int bld) { + int tld = t.loopDepth(), fld = f.loopDepth(); + // These next two are symmetric and can happen in any order; if `tld < + // bld` is true, then `fld < bld` must be false, or else both directions + // exit the loop... and the IF test would not be in the loop. + + // true to exit a loop usually (and false falls into Yet Another Loop + // Block), invert if false is taking an empty backwards branch. + if( tld < bld ) return forwardsEmptyScan(f,bld); + // false to exit a loop, normally invert to true (except empty back) + if( fld < bld ) return !forwardsEmptyScan(t,bld); + + // Not exiting the loop; staying at this depth (or going deeper loop). + + // Jump/true to an empty backwards block + if( forwardsEmptyScan(t,bld) ) return false; + if( forwardsEmptyScan(f,bld) ) return true ; + + // Fall/false into a full block, Jump/true to an empty block. + if( f.nOuts()>1 && t.nOuts()==1 ) return false; + if( t.nOuts()>1 && f.nOuts()==1 ) return true ; + // Jump to a merge point, assuming other things are jumping there as well + if( f.out(0) instanceof RegionNode ) return true ; + if( t.out(0) instanceof RegionNode ) return false; + // Everything else equal, use pre-order + return t._pre > f._pre; + } + + // Do we continue forwards from 'c' to the Loop back edge, without doing + // anything else? Then this branch can directly jump to the loop head. + private static boolean forwardsEmptyScan( CFGNode c, int bld ) { + if( c.nOuts()!=1 || c.loopDepth()!=bld ) return false; + return c.uctrl() instanceof RegionNode cfg && + ((cfg instanceof LoopNode && cfg._ltree==c._ltree) || forwardsEmptyScan(cfg,bld)); + } + + // Is the CFG from "next" to the end empty? This means jumping to "next" + // will naturally fall into the end. + private static boolean backwardsEmptyScan(Ary<CFGNode> rpo, CFGNode next) { + for( int i=rpo._len-1; rpo.at(i)!=next; i-- ) + if( rpo.at(i).nOuts()!=1 ) + return false; + return true; + } + + // -------------------------------------------------- + // Write encoding bits in order into a big byte array. + // Record opcode start and length. + + // Current function is used by the spill-op encodings to query the stack + // frame layout and get the spill offsets. + private void writeEncodings() { + for( CFGNode bb : _cfg ) { + if( !(bb instanceof MachNode mach0) ) + opStart(bb, _bits.size()); + else if( bb instanceof FunNode ) { + opStart(bb, _bits.size()); + mach0.encoding( this ); + opLen(bb, (byte) (_bits.size() - opStart(bb))); + } + for( Node n : bb._outputs ) { + if( n instanceof MachNode mach && !(n instanceof FunNode) && + // FunPtrs are encoding in the block of their control, not + // the block of the ReturnNode input - which is also a CFG. + !(n instanceof FunPtrNode && bb instanceof ReturnNode) ) { + opStart(n, _bits.size()); + mach.encoding( this ); + opLen(n, (byte) (_bits.size() - opStart(n))); + } + } + } + } + + // -------------------------------------------------- + // Short-form RIP-relative support: replace short encodings with long + // encodings and expand the code, changing all the offsets. + private void compactShortForm() { + int len = _cfg._len; + int[] oldStarts = new int[len]; + for( int i=0; i<len; i++ ) + oldStarts[i] = opStart(_cfg.at(i)); + + // TODO: Rewrite this algo to use the small "_jmps" list of just the + // jumps instead of walking all blocks. + // TODO2: Allow short encodings on non-jumps. This algo doesn't + // really care jumps vs other rip-relative things. + + // Check the size of short jumps, adjust the block starts for a whole + // pass over all blocks. If some jumps go long, they might stretch the + // distance for other jumps, so another pass is needed. + int slide = -1; + while( slide != 0) { // While no fails + slide = 0; + for( int i=0; i<len; i++ ) { + CFGNode bb = _cfg.at(i); + opStartAdd(bb, slide); + // Slide down all other (non-CFG) ops in the block + for( Node n : bb._outputs ) + if( n instanceof MachNode && !(n instanceof CFGNode) && + // Match writeEncodings(): FunPtrs are encoded in the + // block of their control, not again at the Return. + !(n instanceof FunPtrNode && bb instanceof ReturnNode) ) + opStartAdd(n, slide); + if( bb instanceof RIPRelSize riprel ) { + CFGNode target = (bb instanceof IfNode iff ? iff.cproj(0) : (CFGNode)bb.out(0)).uctrlSkipEmpty(); + // Delta is from opStart to opStart. X86 at least counts + // the delta from the opEnd, but we don't have the end until + // we decide the size - so the encSize has to deal + int delta = opStart(target) - opStart(bb); + byte opLen = riprel.encSize(delta); + // Recorded size is smaller than the current size? + if( opLen(bb) < opLen ) { + // Start sliding the code down; record slide amount and new size + slide += opLen - opLen(bb); + opLen(bb, opLen); + } + } + } + } + + + // Go again, inserting padding on function headers. Since no + // short-jumps span function headers, the padding will not make any + // short jumps fail. + for( int i=0; i<len; i++ ) { + CFGNode bb = _cfg.at(i); + // Functions pad to align 16 + if( bb instanceof FunNode ) { + int newStart = opStart(bb)+slide; + slide += (newStart+15 & -16)-newStart; + } + opStartAdd(bb, slide); + for( Node n : bb._outputs ) + if( n instanceof MachNode && !(n instanceof CFGNode) && + !(n instanceof FunPtrNode && bb instanceof ReturnNode) ) + opStartAdd(n, slide); + } + + // Copy/slide the bits to make space for all the longer branches + int grow = opStart(_cfg.at(len-1)) - oldStarts[len-1]; + if( grow > 0 ) { // If no short-form ops, nothing to do here + int end = _bits.size(); + byte[] bits = new byte[end+grow]; + for( int i=len-1; i>=0; i-- ) { + int start = oldStarts[i]; + if( start==0 && i>1 ) continue; + int oldStart = oldStarts[i]; + int newStart = opStart(_cfg.at(i)); + System.arraycopy(_bits.buf(),oldStart,bits,newStart,end-start); + end = start; + } + _bits.set(bits,bits.length); + } + + } + + + // -------------------------------------------------- + // Patch local encodings now + void patchLocalRelocations() { + // Walk the local code-address relocations + for( Node src : _internals.keySet() ) { + int start = opStart(src); + Node dst = _internals.get(src); + // If function is entirely dead, only the function pointer remains + // and, it can only be used to test against zero or equals to + // another function pointer... i.e., there Is No Code Here. + int target = dst == null ? start : opStart(dst); + ((RIPRelSize)src).patch(this, start, opLen(src), target - start); + } + } + + // Emit a single scalar as bits + static void addN( int log, Type t, BAOS bits ) { + long x = switch( t ) { + case TypeInteger ti -> ti.value(); + case TypeFloat tf -> log==3 + ? Double.doubleToRawLongBits( tf.value()) + : Float.floatToRawIntBits((float)tf.value()); + case TypeFunPtr tfp -> 0; // These need to be relocated + case TypeMemPtr tmp -> 0; // These need to be relocated + default -> { if( t==Type.NIL ) yield 0; else throw Utils.TODO(); } + }; + addN(log,x,bits); + } + + // A series of libc/external calls that Simple can link against in a JIT. + // Since no runtime in the JVM process, using magic numbers for the CPU + // emulators to pick up on. + public static int SENTINEL_CALLOC = -4; + public static int SENTINEL_WRITE = -8; + + int externalAddress(String name) { + Integer address=_externalAddresses.get(name); + if( address!=null ) return address; + return switch(name) { + case "calloc" -> SENTINEL_CALLOC; + case "write" -> SENTINEL_WRITE; + default -> throw new IllegalArgumentException("Unresolved external function '"+name+"'"); + }; + } + + void patchGlobalRelocations() { + for( Node src : _externData.keySet() ) { + String name = _externData.get(src); + Integer target = _code._externDataAddresses.get(name); + if( target==null ) throw new IllegalArgumentException("Unresolved C data symbol '"+name+"'"); + int start = opStart(src); + ((RIPRelSize)src).patch(this,start,opLen(src),target-start); + } + for( Node src : _externals.keySet() ) { + int start = opStart(src); + String dst = _externals.get(src); + int target = externalAddress(dst); + ((RIPRelSize)src).patch(this, start, opLen(src), target - start); + } + } +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/FinalizeIR.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/FinalizeIR.java new file mode 100644 index 00000000..69e0104d --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/FinalizeIR.java @@ -0,0 +1,44 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; +import java.util.Arrays; + +// Comnpute field layouts only after SCCP and its peepholes converge. +// Keep surviving functions rooted without introducing unknown callers. +abstract class FinalizeIR { + static void run(CodeGen code) { + // Compute field layouts and replace symbolic field offsets by constants. + for( int i=0; i<code._start.nOuts(); i++ ) { + Node use = code._start.out(i); + if( use instanceof ConFldOffNode off ) { + Node con = off.asOffset(); + assert con != null : off._ts._name+"."+off._fname; + off.subsume(con); + i--; + } + } + code._iter.iterate(code); // Propagate field offsets and clean up. + + // Retain surviving functions for emission without adding unknown callers. + for( FunNode fun : code._linker ) + if( fun != null ) { + // A folded static function field can leave a detached body + // with internal cycles. It has no callers; later phases must + // not revive it merely because its nodes are still allocated. + if( fun.isDead() || fun._type.isHigh() ) { + code._linker.set(fun.sig().fidx(),null); + continue; + } + code.keepInCompUnit(fun); + } + // Hand off the wrapper lifetime before releasing its memory projection. + for( CompUnit cu : code._compunits.values() ) + code.keepUntilUnlink(cu._start); + code.releaseFinalizedNodes(); + // Stable print order: initializers first, then other Returns by NID. + for( Node stop : code._stop._inputs ) + Arrays.sort(stop._inputs._es,0,stop.nIns(), + (x,y) -> ((((ReturnNode)x).fun().isClz() ? -1 : x._nid) - + (((ReturnNode)y).fun().isClz() ? -1 : y._nid) )); + } +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/FunctionSummary.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/FunctionSummary.java new file mode 100644 index 00000000..6cd4035d --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/FunctionSummary.java @@ -0,0 +1,28 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.*; + +// The small, eagerly readable part of a function's serialized implementation. +record FunctionSummary(TypeFunPtr sig, TypeTuple state, String name, int size, int[] classDependencyFIDXs) { + void install(CodeGen code, CompUnit cu) { + FunNode fun = new FunNode(null,sig,name,cu,null,cu._start); + fun._approxUIDs=size; + fun.init(); + // First user stays in output slot zero until the summary is removed. + SummaryNode body = new SummaryNode(state,classDependencyFIDXs,fun); + ParmNode rpc = new ParmNode("$rpc",0,TypeRPC.BOT,fun,code.con(TypeRPC.BOT)).init(); + ParmNode mem = new ParmNode("$mem",1,TypeMem.BOT,fun,cu._start.proj(1)).init(); + body.addDef(mem); + for( int i = 0; i<sig.nargs(); i++ ) + body.addDef(new ParmNode("arg"+i,i+2,sig.arg(i),fun,code.con(sig.arg(i))).init()); + body.init(); + Node ctrl = new CProjNode(body,0,"ctrl").init(); + Node memory = new ProjNode(body,1,"mem").init(); + Node value = new ProjNode(body,2,"value").init(); + ReturnNode ret = new ReturnNode(ctrl,memory,value,rpc,fun).init(); + fun.setRet(ret); + cu._stop.addDef(ret); + code.link(fun); + } +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/GlobalBits.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/GlobalBits.java new file mode 100644 index 00000000..bc9c9d39 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/GlobalBits.java @@ -0,0 +1,177 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.util.*; +import java.util.HashMap; +import java.util.Arrays; +import java.util.Objects; + + +// This utility 2-way maps some *global* program feature to a local dense +// index, suitable for packing in BitSets. The mapping takes a global value - +// always a {source file/class} name and an order number counting up per +// feature in the same file. The dense local index can be different in +// different compilations. + +// The global info allows loading the same remote class info from different +// paths and aligning them. Example: Compiling A loads pre-compiled B and C, +// both of which load pre-compiled D. The 2 different D loads unify via this +// global info. + +// These mappings are all trivial identities when compiling one file; they +// only become complex when loading seperately compiled code. + +public class GlobalBits { + public static final int RESERVED = 2; + private final HashMap<String,int[]> _clzOrder2Local = new HashMap<>(); + private final Ary<String> _local2Clz = new Ary<>(String.class); + private final AryInt _local2Order = new AryInt(); + // Local dense index + private int _local; + + GlobalBits() { + _local = RESERVED; + // Reserved null-class bits are global identities and are never + // remapped. + int[] xs = new int[RESERVED]; + xs[0] = RESERVED; // Per-class next order counter + for( int i=0; i<RESERVED; i++ ) { + if( i > 0 ) xs[i] = i; + _local2Clz .setX(i,null); + _local2Order.setX(i,i); + } + _clzOrder2Local.put(null,xs); + } + + // Get a next local index for this clz + int next( String clz ) { return next(clz,-1); } + + // Given a clz and order, return an existing local index or make a new + // mapping and return that. + int next(String clz, int order ) { + // Get the per-clz Order->Local mapping + int[] xs = _clzOrder2Local.get(clz); + if( xs == null ) + xs = new int[]{RESERVED,1}; // Make, if it does not exist + // Caller does not have an order in mind, just take the next one stored + // in xs[0] + if( order == -1 ) + order = xs[0]++; + // Order exists from prior compilation, but not recorded locally + else if( order >= xs[0] ) + xs[0] = order+1; + // Grow as needed + while( xs.length <= order ) + xs = Arrays.copyOf(xs,xs.length*2); + // Have a local mapping already? + if( xs[order] != 0 ) + return xs[order]; + int local = _local++; + bind(local,clz,order); + return local; // The dense local index + } + + // Bind a specific dense local index to {clz,order}. Used by both fresh + // allocation and deserialization, where local indexes must not compact + // across holes because XInt bitsets are serialized against this space. + private void bind( int local, String clz, int order ) { + if( _local <= local ) + _local = local+1; + _local2Clz .setX(local,clz ); + _local2Order.setX(local,order); + + // Inline-only fidx holes are represented as {null,0}. They need the + // reverse mapping above, but no forward map because xs[0] is the + // per-class next-order counter. + if( clz==null && order==0 ) + return; + + int[] xs = _clzOrder2Local.get(clz); + if( xs == null ) + xs = new int[]{RESERVED,1}; + if( order >= xs[0] ) + xs[0] = order+1; + while( xs.length <= order ) + xs = Arrays.copyOf(xs,xs.length*2); + _clzOrder2Local.put(clz,xs); + xs[order] = local; + } + + // A file-local function that will inline, and never write to disk + public int nextInline( ) { + int local = _local++; + _local2Clz .setX(local,null); + _local2Order.setX(local,0); + return local; + } + // Get a next local index for the same clz as this old index + public int next( int old ) { return next(_local2Clz.at(old)); } + + // Map a file-local dense index to this compilation unit's local dense index. + public int map( GlobalBits fileBits, int fileLocal ) { + if( fileLocal >= fileBits._local2Clz.size() ) + throw new ArrayIndexOutOfBoundsException(""+fileLocal+" >= "+fileBits._local2Clz.size()); + return findOrNext(fileBits._local2Clz.at(fileLocal), fileBits._local2Order.at(fileLocal)); + } + + // Map a global {clz,order} to a local one; keep any existing mapping or + // make a new one as needed. + private int findOrNext(String clz, int order ) { + // Reserved null-class identities are not represented in xs[order], + // because xs[0] is the next-order counter. + if( clz==null && order < _local2Clz.size() && _local2Order.at(order)==order && _local2Clz.at(order)==null ) + return order; + int[] xs = _clzOrder2Local.get(clz); + if( xs != null && order < xs.length ) { + int local = xs[order]; + if( local < _local2Clz.size() && _local2Order.at(local)==order && Objects.equals(_local2Clz.at(local),clz) ) + return local; + } + return next(clz,order); + } + + // Stable native symbol identity, independent of this compilation's dense IDs. + String owner(int local) { return _local2Clz.at(local); } + String symbol(int local) { + return _local2Clz.at(local)+"$F$"+_local2Order.at(local); + } + + // Write out to BAOS enough bits to unwind the local back to global index + void packed( BAOS baos, HashMap<String,Integer> strs ) { + // Unwind a local index to the clz. + baos.packed2(_local2Clz.size()); + for( int i=RESERVED; i<_local2Clz.size(); i++ ) { + String clz = _local2Clz.at(i); + baos.packed2(clz==null ? 0 : strs.get(clz)); + baos.packed2( _local2Order.at(i) ); + } + } + + void gather( HashMap<String,Integer> strs ) { + for( int i=RESERVED; i<_local2Clz.size(); i++ ) + Serialize.gather(strs,_local2Clz.at(i)); + } + + // Read in the BAIS bits needed to make a file-local -> global mapping + static GlobalBits packed( BAOS bais, String[] strs ) { + var gb = new GlobalBits(); + int n = bais.packed2(); + for( int i=RESERVED; i<n; i++ ) { + int sx = bais.packed2(); + String clz = strs[sx]; + if( clz.isEmpty() ) clz = null; + int order = bais.packed2(); + gb.bind(i,clz,order); + } + return gb; + } + + @Override public String toString() { + SB sb = new SB().p('['); + for( int i=0; i<_local2Clz.size(); i++ ) { + if( i > 0 ) sb.p(','); + sb.p(i).p(" <-> ").p(_local2Clz.at(i)).p('#').p(_local2Order.at(i)); + } + return sb.p(']').toString(); + } + +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/GlobalCodeMotion.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/GlobalCodeMotion.java new file mode 100644 index 00000000..86cb2e98 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/GlobalCodeMotion.java @@ -0,0 +1,356 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.codegen.IterPeeps.WorkList; +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.*; +import com.seaofnodes.simple.util.Ary; +import com.seaofnodes.simple.util.AryInt; +import com.seaofnodes.simple.util.Utils; +import java.util.*; + +public abstract class GlobalCodeMotion { + + // Ideal execution needs a schedule, but neither CLInit lowering nor + // interprocedural optimization. Each loaded object is prepared once. + public static Ary<CFGNode> forEval(CodeGen code, StartNode start, StopNode stop) { + Ary<FunNode> funs = Unlink.forEval(code,start,stop); + start.buildLoopTree(funs,stop); + return buildCFG(code,start,stop); + } + + // Arrange that the existing isCFG() Nodes form a valid CFG. The + // Node.use(0) is always a block tail (either IfNode or head of the + // following block). There are no unreachable infinite loops. + public static void buildCFG( CodeGen code ) { + code._cfg = buildCFG(code,code._start,code._stop); + } + + // Schedule one graph, including an object loaded while Eval2 is running. + // Other graphs (and their active evaluation frames) are left untouched. + public static Ary<CFGNode> buildCFG(CodeGen code, StartNode start, StopNode stop) { + Ary<CFGNode> rpo = new Ary<>(CFGNode.class); + _rpo_cfg(null, start, code.visit(), rpo); + // Reverse in-place + for( int i=0; i< rpo.size()>>1; i++ ) + rpo.swap(i,rpo.size()-1-i); + schedEarly(code,start,rpo); + + // Break up shared global constants by functions + breakUpGlobalConstants(start); + + code._visit.clear(); + schedLate(code,stop); + return rpo; + } + + // Post-Order of CFG + private static void _rpo_cfg(CFGNode def, Node use, BitSet visit, Ary<CFGNode> rpo) { + if( !(use instanceof CFGNode cfg) || visit.get(cfg._nid) ) + return; // Been there, done that + // A shared dead-control node can also reach an unloaded function's + // Return. Its wrapper is a dispatch target, not executable code. + if( cfg instanceof FunNode fun && fun.summary()!=null || + cfg instanceof ReturnNode ret && ret.fun().summary()!=null ) return; + assert !( def instanceof ReturnNode && use instanceof CallEndNode ); // All calls unwired now + assert !( def instanceof CallNode && use instanceof FunNode ); // All calls unwired now + visit.set(cfg._nid); + for( Node useuse : cfg._outputs ) + _rpo_cfg(cfg,useuse,visit,rpo); + rpo.add(cfg); + } + + // After early scheduling, every global chain member has Start in slot 0. + // Give each function one private copy of the entire dependency graph. + private static void breakUpGlobalConstants( StartNode start ) { + var globals = new IdentityHashMap<Node,Boolean>(); + var cons = new ArrayList<Node>(); + for( Node con : start._outputs ) + if( con!=null && !(con instanceof CFGNode) && + (con.isConst() || con instanceof MachNode mach && mach.outregmap()!=null) ) { + globals.put(con,true); + cons.add(con.keep()); // Preserve original inputs while rewiring users. + } + + var copies = new IdentityHashMap<FunNode,IdentityHashMap<Node,Node>>(); + for( Node con : cons ) + for( Node use : con._outputs.asAry() ) { + if( use==null || globals.containsKey(use) ) continue; + FunNode fun = useFun(use); + if( fun==null || fun.summary()!=null ) continue; // Metadata has no executable owner. + assert CodeGen.CODE._mach==null || fun.isFromSource(); // Ideal evaluation includes imported bodies. + var local = copies.computeIfAbsent(fun,f -> new IdentityHashMap<>()); + Node copy = cloneGlobal(con,fun,globals,local); + for( int i=0; i<use.nIns(); i++ ) + if( use.in(i)==con ) + use.setDef(i,copy); + } + for( Node con : cons ) con.unkill(); + } + + private static Node cloneGlobal(Node con, FunNode fun, IdentityHashMap<Node,Boolean> globals, + IdentityHashMap<Node,Node> copies) { + Node copy = copies.get(con); + if( copy!=null ) return copy; + copies.put(con,copy=con.copyEmpty()); + copy.addDef(fun); + for( int i=1; i<con.nIns(); i++ ) { + Node def = con.in(i); + copy.addDef(globals.containsKey(def) ? cloneGlobal(def,fun,globals,copies) : def); + } + return copy; + } + + private static FunNode useFun(Node use) { + if( use instanceof ReturnNode ret ) return ret.fun(); + if( use instanceof ParmNode parm ) return parm.fun(); + CFGNode cfg = use.cfg0(); + while( cfg!=null && !(cfg instanceof FunNode) ) + cfg = cfg.idom(); + return (FunNode)cfg; + } + + + // ------------------------------------------------------------------------ + // Visit all nodes in CFG Reverse Post-Order, essentially defs before uses + // (except at loops). Since defs are visited first - and hoisted as early + // as possible, when we come to a use we place it just after its deepest + // input. + private static void schedEarly(CodeGen code, StartNode start, Ary<CFGNode> cfgs) { + // Reverse Post-Order on CFG + for( CFGNode cfg : cfgs ) { + cfg.loopDepth(); + for( Node n : cfg._inputs ) + _schedEarly(n,code,start ); + // In dead infinite loops, entire code blocks may be unreachable + // from below. Reach down from the CFG to their Phis so their + // inputs are scheduled too. + if( cfg instanceof RegionNode ) + for( Node phi : cfg._outputs ) + if( phi instanceof PhiNode ) + _schedEarly(phi,code,start ); + } + } + + private static void _schedEarly(Node n, CodeGen code, StartNode start) { + if( n==null || code._visit.get(n._nid) ) return; // Been there, done that + assert !(n instanceof CFGNode) : n.getClass().getSimpleName()+"#"+n._nid+" outside scheduled CFG"; + code._visit.set(n._nid); + // Schedule inputs first, except Phis: following their backedges would + // enter a data cycle before its control has been scheduled. + for( Node def : n._inputs ) + if( def!=null && !(def instanceof PhiNode) ) + _schedEarly(def,code,start); + + // An existing edge 0 already supplies control (or a Phi/Proj binding). + if( n.in(0)==null ) { + // Schedule at deepest input + CFGNode early = start; // Maximally early, lowest idepth + for( int i=1; i<n.nIns(); i++ ) + if( n.in(i)!=null && n.in(i).cfg0().idepth() > early.idepth() ) + early = n.in(i).cfg0(); // Latest/deepest input + n.setDef(0,early); + } + } + + // ------------------------------------------------------------------------ + // Scheduling is not recursive. Retain scratch capacity between objects, + // but release graph references and reset only the entries we used. + private static CFGNode[] LATE = new CFGNode[1]; + // Load NIDs at their CFG block choices, checked against Store blocks. + private static int[] ANTI = new int[1]; + private static final AryInt TOUCHED = new AryInt(); + private static final Ary<Node> PLACED = new Ary<>(Node.class); + private static final Ary<Node> ANTIDEPS = new Ary<>(Node.class); + private static final WorkList<Node> WORK = new WorkList<>(); + + private static void schedLate( CodeGen code, StopNode stop ) { + assert TOUCHED.isEmpty() && PLACED.isEmpty() && WORK.isEmpty(); + if( code.UID() > LATE.length ) { + int len = Math.max(code.UID(),LATE.length<<1); + LATE = new CFGNode[len]; + ANTI = new int[len]; + } + WORK.clear(); + // Breadth-first scheduling + // Early scheduling has released the shared visit bits. + breadth(stop,PLACED,LATE,ANTI,ANTIDEPS,code.visit()); + + // Copy the best placement choice into the control slot + for( Node n : PLACED ) + if( !(n instanceof ProjNode) ) + n.setDef(0,LATE[n._nid]); + for( int i=0; i<TOUCHED.len(); i++ ) + LATE[TOUCHED.at(i)] = null; + TOUCHED.clear(); + clear(PLACED); + clear(ANTIDEPS); + } + + // Ary.clear retains references; static scratch must release them as well. + private static void clear(Ary<Node> nodes) { + Arrays.fill(nodes._es,0,nodes._len,null); + nodes.clear(); + } + + private static Ary<Node> antiDeps(MemOpNode load, Ary<Node> antideps) { + clear(antideps); + return load.antiDeps(antideps); + } + + private static void breadth(Node stop, Ary<Node> placed, CFGNode[] late, int[] anti, Ary<Node> antideps, BitSet visit) { + // Things on the worklist have some (but perhaps not all) uses done. + WorkList<Node> work = WORK; + work.push(stop); + Node n; + outer: + while( (n = work.pop()) != null ) { + assert late[n._nid]==null; // No double visit + // These I know the late schedule of, and need to set early for loops + if( n instanceof CFGNode cfg ) late[n._nid] = cfg.blockHead() ? cfg : cfg.cfg(0); + else if( n instanceof PhiNode phi ) late[n._nid] = phi.region(); + else if( n instanceof ProjNode && n.in(0) instanceof CFGNode cfg ) late[n._nid] = cfg; + else { + + // All uses done? + for( Node use : n._outputs ) + if( use!=null && late[use._nid]==null ) + continue outer; // Nope, await all uses done + + // Loads need their memory inputs' uses also done + if( n instanceof MemOpNode ld && ld._isLoad ) + for( Node memuse : antiDeps(ld,antideps) ) + if( late[memuse._nid]==null ) + continue outer; + + // All uses done, schedule + _doSchedLate(n,placed,late,anti,antideps); + } + TOUCHED.push(n._nid); + + // A use just finished; reconsider its inputs and waiting loads, + // even when the shared memory input was already scheduled. + for( Node def : n._inputs ) { + if( def==null ) continue; + if( late[def._nid]==null ) work.push(def); + wakeLoads(def,late,work,visit); + } + visit.clear(); + if( n instanceof LoopNode loop ) + for( Node phi : loop._outputs ) + if( phi instanceof PhiNode && late[phi._nid]==null ) + work.push(phi); + } + } + + private static void _doSchedLate(Node n, Ary<Node> placed, CFGNode[] late, int[] anti, Ary<Node> antideps) { + // Walk uses, gathering the LCA (Least Common Ancestor) of uses + CFGNode early = n.in(0) instanceof CFGNode cfg ? cfg : n.in(0).cfg0(); + assert early != null; + CFGNode lca = null; + for( Node use : n._outputs ) + if( use != null ) + lca = use_block(n,use, late).domLCA(lca,null); + + // Loads may need anti-dependencies, raising their LCA + if( n instanceof MemOpNode load && load._isLoad ) + lca = find_anti_dep(lca,load,early,late,anti, antideps); + + // Walk up from the LCA to the early, looking for best place. This is + // the lowest execution frequency, approximated by least loop depth and + // deepest control flow. + CFGNode best = lca; + lca = lca.idom(); // Already found best for starting LCA + for( ; lca != early.idom(); lca = lca.idom() ) + if( better(lca,best) ) + best = lca; + assert !(best instanceof IfNode); + placed.push(n); + late[n._nid] = best; + } + + // Block of use. Normally from late[] schedule, except for Phis, which go + // to the matching Region input. + private static CFGNode use_block(Node n, Node use, CFGNode[] late) { + if( use instanceof ParmNode parm ) + return late[use._nid]; + if( !(use instanceof PhiNode phi) ) + return late[use._nid]; + CFGNode found=null; + for( int i=1; i<phi.nIns(); i++ ) + if( phi.in(i)==n ) + found = phi.region().cfg(i).domLCA(found,null); // Can be more than one matching input. + + assert found!=null; + return found; + } + + + // Least loop depth first, then largest idepth + private static boolean better( CFGNode lca, CFGNode best ) { + return lca.loopDepth() < best.loopDepth() || + lca instanceof NeverNode || + lca.idepth() > best.idepth() || + best instanceof IfNode; + } + + // A writer may consume a load's memory through an aggregate. + private static void wakeLoads(Node def, CFGNode[] late, WorkList<Node> work, BitSet visit) { + if( visit.get(def._nid) ) return; + visit.set(def._nid); + for( Node out : def._outputs ) + if( out instanceof MemOpNode ld && ld._isLoad && late[ld._nid]==null ) work.push(ld); + if( def instanceof MemMergeNode ) + for( int i=1; i<def.nIns(); i++ ) + if( def.in(i)!=null ) wakeLoads(def.in(i),late,work,visit); + } + + private static CFGNode find_anti_dep(CFGNode lca, MemOpNode load, CFGNode early, CFGNode[] late, int[] anti, Ary<Node> antideps) { + // We could skip final-field loads here. + // Walk LCA->early, flagging Load's block location choices + CFGNode marked = lca; + for( CFGNode cfg = marked; cfg!=early.idom(); cfg = cfg.idom() ) + anti[cfg._nid] = load._nid; + // Walk load->mem uses, looking for Stores causing an anti-dep + for( Node mem : antiDeps(load,antideps) ) { + if( mem instanceof PhiNode phi ) { + // Inputs carrying different memory might overwrite this alias. + // Raise the LCA for those paths; the unchanged-memory paths + // do not clobber the load. No scheduling edge to the Phi. + for( int i=1; i<phi.nIns(); i++ ) + if( phi.in(i)!=load.mem() ) + lca = anti_dep(load,phi.region().cfg(i),load.mem().cfg0(),lca,null,anti); + } else { + // Hunt for clobbering stores + assert mem instanceof MemOpNode || mem instanceof EscapeNode || + mem instanceof CallNode || mem instanceof CLInitNode; + assert late[mem._nid]!=null; + lca = anti_dep(load,late[mem._nid],mem.cfg0(),lca,mem,anti); + } + } + // This load's marks are no longer needed; leave scratch clean even + // when the next compilation reuses these node IDs. + for( CFGNode cfg = marked; cfg!=early.idom(); cfg = cfg.idom() ) + anti[cfg._nid] = 0; + return lca; + } + + // + private static CFGNode anti_dep( MemOpNode load, CFGNode stblk, CFGNode defblk, CFGNode lca, Node st, int[] anti ) { + // A conditional writer can precede the load without dominating its late block. + // Walk back to the first overlap with the load's placement range. + // A different Phi input need not be dominated by this memory's + // definition. Stop at its depth rather than looking for that ancestor. + // Retain the sentinel too: a function entry has a null idom. + for( ; stblk != defblk.idom() && stblk.idepth() >= defblk.idepth(); stblk = stblk.idom() ) { + // Store and Load overlap, need anti-dependence + if( anti[stblk._nid]==load._nid ) { + lca = stblk.domLCA(lca,null); // Raise Loads LCA + if( lca == stblk && st != null && !(st instanceof CallNode) && st._inputs.find(load) == -1 ) // And if something moved, + st.addDef(load); // Add anti-dep as well (calls already end their block) + return lca; // Cap this stores' anti-dep to here + } + } + return lca; + } + +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/IFG.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/IFG.java new file mode 100644 index 00000000..d15e66da --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/IFG.java @@ -0,0 +1,586 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.Type; +import com.seaofnodes.simple.util.Ary; +import com.seaofnodes.simple.util.Utils; +import java.util.BitSet; +import java.util.IdentityHashMap; + +// Interference Graph +abstract public class IFG { + // Interference Graph Support + + // Map from a Basic Block to Live-Out: {a map from a Live Range to a Def} + private static final IdentityHashMap<CFGNode,IdentityHashMap<LRG,Node>> BBOUTS = new IdentityHashMap<>(); + static void resetBBLiveOut() { + for( IdentityHashMap<LRG,Node> bbout : BBOUTS.values() ) + bbout.clear(); + } + static final IdentityHashMap<LRG,Node> TMP = new IdentityHashMap<>(); + + // Inteference Graph: Array of Bitsets + static final Ary<BitSet> IFG = new Ary<>(BitSet.class); + static void resetIFG() { + for( BitSet bs : IFG ) + if( bs!=null ) bs.clear(); + } + + // Set matching bit + private static void addIFG( LRG lrg0, LRG lrg1 ) { + short x0 = lrg0._lrg, x1 = lrg1._lrg; + // Triangulate + if( x0 < x1 ) _addIFG(x0,x1); + else _addIFG(x1,x0); + } + // Add lrg1 to lrg0's conflict set + private static void _addIFG( short x0, short x1 ) { + BitSet ifg = IFG.atX(x0); + if( ifg==null ) + IFG.setX(x0,ifg = new BitSet()); + ifg.set(x1); + } + + + private static final Ary<CFGNode> WORK = new Ary<>(CFGNode.class); + static void pushWork(CFGNode bb) { + if( WORK.find(bb) == -1 ) + WORK.push(bb); + } + + // ------------------------------------------------------------------------ + // Visit all blocks, using the live-out LRGs per-block and doing a backwards + // walk over each block. At the end of the walk, push the live-out sets to + // prior blocks. Due to loops, no single visitation order suffices. This + // problem can be fairly efficiently solved with a proper LIVE calculation + // and bitsets. In the interests of simplicity (and assumption of smaller + // programs) I am skipping LIVE and directly computing it during the IFG + // building. + + // Start by setting the live-out of the exit block (to empty), and putting + // it on a worklist. + + // - Pull from worklist a block who's live-out has changed + // - Walk backwards, adding interferences and computing live. + // - At block head, "push" the new live-out to prior blocks. + // - - If they pick up new live-outs, put them on the worklist. + + public static boolean build( int round, RegAlloc alloc ) { + // Reset all to empty + resetBBLiveOut(); + resetIFG(); + WORK.clear(); + + // Last block has nothing live out + for( CFGNode bb : alloc._code._cfg ) + if( bb.blockHead()) + WORK.push( bb ); + + // Process blocks until no more changes + while( !WORK.isEmpty() ) + do_block(round,alloc,WORK.pop()); + + if( alloc.success() ) + convert2DAdjacency(alloc); + return alloc.success(); + } + + // Walk one block backwards, compute live-in from live-out, and build IFG + static void do_block( int round, RegAlloc alloc, CFGNode bb ) { + assert bb.blockHead(); + IdentityHashMap<LRG,Node> live_out = BBOUTS.get(bb); + TMP.clear(); + if( live_out != null ) + TMP.putAll(live_out); // Copy bits to temp + + // A backwards walk over instructions in the basic block + for( int inum = bb.nOuts()-1; inum >= 0; inum-- ) { + Node n = bb.out(inum); + if( n.in(0) != bb ) continue; + // In a backwards walk, proj users come before the node itself + if( n instanceof MultiNode ) + for( Node proj : n.outs() ) + if( proj instanceof ProjNode ) + do_node(alloc,proj); + if( n instanceof MachNode ) + do_node(alloc,n); + } + + // The bb head kills register, e.g. a CallEnd and caller-save registers + if( bb instanceof MachNode mach ) + kills(alloc,mach); + + // Push live-sets backwards to priors in CFG. + if( bb instanceof RegionNode ) + for( int i=1; i<bb.nIns(); i++ ) + mergeLiveOut(alloc,bb,i); + else + mergeLiveOut(alloc,bb,0); + } + + private static void do_node(RegAlloc alloc, Node n) { + // Defining means killing live LRG + LRG lrg = alloc.lrg(n); + if( lrg != null ) { + // Check for def-side self-conflict live ranges. These must split, + // and only happens during the first round a particular LRG splits. + selfConflict(alloc,n,lrg); + TMP.remove(lrg); // Kill def + } + + // Phis use and define the same live range, i.e. these LRGs already + // marked conflicted, no need to mark again + if( n instanceof PhiNode ) + return; + + // Kill any killed registers; milli-code routines like New can kill + // will not being a CFG. + if( n instanceof MachNode m ) + kills(alloc,m); + + // Interfere n with all live + if( lrg!=null ) { + // Interfere n with all live + for( LRG tlrg : TMP.keySet() ) { + assert tlrg.leader(); + // Skip self + if( lrg != tlrg && + // And register sets overlap + lrg._mask.overlap(tlrg._mask) ) + // Then tlrg and lrg interfere. If lrg needs the single + // last tlrg register at some point, either tlrg or lrg + // must fail. If *n* (a subset of lrg) needs the single + // last tlrg register then only tlrg must fail. + if( ((MachNode)n).outregmap().size1() ) { + if( !tlrg.clr(lrg._mask.firstReg()) ) // Clear bit, no interference + alloc.fail(tlrg); // Clearing drives mask to empty + } else addIFG(lrg,tlrg); // Add interference + } + } + + + // Check for self-conflict live ranges. These must split, and only + // happens during the first round when a particular LRG splits. + // Also record use-side spills for biased coloring. + // Then make all inputs live. + for( int i=1; i<n.nIns(); i++ ) { + Node def = n.in(i); + if( def==null ) continue; + LRG lrg1 = alloc.lrg(def); + if( lrg1==null ) continue; // Anti-dependence, etc, no register + + // Check use-side for self-conflict; if so we MUST split + selfConflict(alloc,def,lrg1); + + // All operands are live together, regardless of their input order. + TMP.put(lrg1,def); + } + + // Check fixed-register uses after gathering every operand. An indirect + // call's target comes after its arguments, but must also avoid their + // fixed registers (and may need to move from its incoming register). + for( int i=1; i<n.nIns(); i++ ) { + Node def = n.in(i); + if( def==null || alloc.lrg(def)==null ) continue; + // Look for a must-use single register conflicting with some other must-def. + if( n instanceof MachNode m ) { + RegMask ni_mask = m.regmap(i); + if( ni_mask!=null && ni_mask.size1() ) { // Must-use single register + // Search all current live + for( LRG tlrg : TMP.keySet() ) { + assert tlrg.leader(); + Node live = TMP.get(tlrg); + if( live != def && live instanceof MachNode lmach && lmach.outregmap().overlap(ni_mask) ) { + // Deny the register, since it absolutely must be used here + if( !tlrg.clr(ni_mask.firstReg()) ) + // Then direct reg-reg conflict between use here (at n.in(i)) and def (of tlrg) there. + // Fail the older live range, it must move its register. + alloc.fail( tlrg ); + } + } + } + } + } + } + + // Single-register defines *must* have their register; other live ranges + // *must* avoid this register, so instead of interfering we remove the + // single register from the other rmask. + private static void kills( RegAlloc alloc, MachNode m ) { + RegMask killMask = m.killmap(); + if( killMask==null ) return; + // Kill registers with all live + for( LRG tlrg : TMP.keySet() ) { + assert tlrg.leader(); + // Always, tlrg cannot use kills + if( tlrg._mask.overlap(killMask) ) { + Node live = TMP.get(tlrg); + // Disallow clone-ables from living across killed registers. + // Fail them and let splitting re-clone closer to the target use. + if( live instanceof MachNode mach && mach.isClone() ) + alloc.fail(tlrg); + else if( !tlrg.sub(killMask) ) + alloc.fail(tlrg); + } + } + } + + // Check for self-conflict live ranges. These must split, and only happens + // during the first round a particular LRG splits. + private static void selfConflict(RegAlloc alloc, Node n, LRG lrg) { + selfConflict(alloc,n,lrg,TMP.get(lrg)); + } + private static void selfConflict(RegAlloc alloc, Node n, LRG lrg, Node prior) { + if( prior!=null && prior != n ) { + lrg.selfConflict(prior); + lrg.selfConflict(n); + alloc.fail(lrg); // 2 unrelated values live at once same live range; self-conflict + } + } + + // Merge TMP into bb's live-out set; if changes put bb on WORK + private static void mergeLiveOut( RegAlloc alloc, CFGNode priorbb, int i ) { + CFGNode bb = priorbb.cfg(i); + if( bb == null ) return; // Start has no prior + while( !bb.blockHead() ) bb = bb.cfg0(); + + // Lazy get live-out set for bb + IdentityHashMap<LRG, Node> lrgs = BBOUTS.computeIfAbsent( bb, k -> new IdentityHashMap<>() ); + + for( LRG lrg : TMP.keySet() ) { + Node def = TMP.get(lrg); + // Effective def comes from phi input from prior block + if( def instanceof PhiNode phi && phi.cfg0()==priorbb ) { + assert i!=0; + def = phi.in( i ); + } + // The synthetic exit carries no value on this Phi arm. + if( def._type==Type.TOP ) continue; + Node def_bb = lrgs.get(lrg); + if( def_bb==null ) { + lrgs.put(lrg,def); + pushWork(bb); + } else { + // Alive twice with different definitions; self-conflict + selfConflict(alloc,def,lrg,def_bb); + } + } + } + + + static void convert2DAdjacency( RegAlloc alloc ) { + // Convert the 2-D array of bits (a 1-D array of BitSets) into an + // adjacency matrix. + int maxlrg = alloc._LRGS.length; + for( int i=1; i<maxlrg; i++ ) { + BitSet ifg = IFG.atX(i); + if( ifg != null ) { + LRG lrg0 = alloc._LRGS[i]; + for( int lrg = ifg.nextSetBit(0); lrg>=0; lrg=ifg.nextSetBit(lrg+1) ) { + LRG lrg1 = alloc._LRGS[lrg]; + lrg0.addNeighbor(lrg1); + lrg1.addNeighbor(lrg0); + } + } + } + } + + + // ------------------------------------------------------------------------ + // Color the inference graph. + + // Coloring works by removing "trivial" live ranges from the IFG - those + // live ranges with fewer neighbors than available colors (registers). + // These are trivial because even if every neighbor takes a unique color, + // there's at least one more available to color this live range. + + // If we hit a clique of non-trivial live ranges, we pick one to be "at + // risk" of not-coloring. Good choices include live ranges with a large + // area and few hot uses. + + // Then we reverse and put-back live ranges - picking a color as we go. + // If there's no spare color we'll have to spill this at-risk live range. + + public static boolean color(int round, RegAlloc alloc) { + int maxlrg = alloc._LRGS.length, nlrgs=0; + for( int i=1; i<maxlrg; i++ ) + if( alloc._LRGS[i] != null ) + nlrgs++; + + // Simplify + + // Walk all the LRGS looking for some trivial ones to start with. + // During this pass all LRGS are broken into 3 disjoint sets: + // - trivial, removed from IFG; color_stack[0 to sptr] + // - trivial, not (yet) removed from IFG; color_stack[sptr to swork] + // - unknown; color_stack[work to maxlrg] + + // Gather all not-unified (not-null); separate trivial and non-trivial set. + int sptr = 0, swork = 0; + LRG[] color_stack = new LRG[nlrgs]; + for( int i=0, j=0; i<maxlrg; i++ ) { + LRG lrg = alloc._LRGS[i]; + if( lrg==null ) continue; // Unified lrgs are null here + color_stack[j++] = lrg; + if( lrg.lowDegree() ) swap(color_stack, swork++, j-1); + } + + // Pull all lrgs from IFG, in trivial order if possible + while( sptr < color_stack.length ) { + // Swap best color up front + pickColor(color_stack,sptr,swork,alloc); + + // Pick a trivial lrg, and (temporarily) remove from the IFG. + LRG lrg = color_stack[sptr++]; + // If sptr was swork, then pulled an at-risk lrg + if( sptr > swork ) + swork = sptr; + + // Walk all neighbors and remove + if( lrg._adj != null ) { + for( LRG nlrg : lrg._adj ) { + // Remove and compress out neighbor + if( nlrg.removeCompress(lrg) ) { + // Neighbor is just exactly going trivial as 'lrg' is removed from IFG + // Find "j" position in the color_stack + int jx = swork; while( color_stack[jx] != nlrg ) jx++; + // Add trivial neighbor to trivial list. Pull lrg j out of + // unknown set, since its now in the trivial set + swap(color_stack, swork++, jx); + } + } + } + } + + // Reverse simplify (unstack the color stack), and set colors (registers) for live ranges + while( sptr > 0 ) { + LRG lrg = color_stack[--sptr]; + if( lrg==null ) continue; + RegMaskRW rmask = lrg._mask.copy(); + // Walk neighbors and remove adjacent colors + if( lrg._adj!=null ) { + for( LRG nlrg : lrg._adj ) { + nlrg.reinsert(lrg); + int reg = nlrg._reg; + if( reg!= -1 ) // Failed neighbors do not count + rmask.clr(reg); // Remove neighbor from my choices + } + } + // At-risk live-range did not color? + if( rmask.isEmpty() ) { + alloc.fail(lrg); + lrg._reg = -1; + } else { + // Pick first available register + short reg = rmask.firstReg(); + // Pick a "good" color from the choices. Typically, biased-coloring + // removes some over-spilling. + if( rmask.size() > 1 ) reg = biasColor(alloc,lrg,reg,rmask); + lrg._reg = reg; // Assign the register + } + } + + return alloc.success(); + } + + // Pick LRG from color stack + private static void pickColor(LRG[] color_stack, int sptr, int swork, RegAlloc alloc) { + // Out of trivial colorable, pick an at-risk to pull + if( sptr==swork ) + swap(color_stack,sptr,pickRisky(color_stack,sptr,alloc)); + // When coloring, we'd like to give more choices; so when coloring we'd + // like to see the single-def first (since no choices anyway), then + // non-split related (so more live ranges get colored), then + // split-related last, so they have more colors to bias towards. + + // Working in reverse, pick first split-related with many regs, then + // those with some regs, then single-def. + int bidx=sptr; + LRG best=color_stack[bidx]; + for( int idx = sptr+1; idx < swork; idx++ ) + if( betterLRG(best,color_stack[idx]) ) + best = color_stack[bidx=idx]; + if( bidx != sptr ) + swap(color_stack,sptr,bidx); // Pick best at sptr + } + + // Returns true if 'lrg' is better than 'best' + private static boolean betterLRG( LRG best, LRG lrg ) { + // If single-def varies, keep the not-single-def + if( best.size1() != lrg.size1() ) + return best.size1(); + // If hasSplit varies, keep the hasSplit + if( best.hasSplit() != lrg.hasSplit() ) + return lrg.hasSplit(); + // If both are single register (so both sizes are 1), + // return a multi-use over a multi-def + if( best.size1() ) + return !best._multiUse && lrg._multiUse; + // Keep large register count + return best.size() < lrg.size(); + } + + // Pick a live range that hasn't already spilled, or has a single-def- + // single-use that are not adjacent. + private static int pickRisky( LRG[] color_stack, int sptr, RegAlloc alloc ) { + int best=sptr; + int bestScore = pickRiskyScore(color_stack[best],alloc); + for( int i=sptr+1; i<color_stack.length; i++ ) { + if( bestScore == 1000000 ) return best; // Already max score + int iScore = pickRiskyScore(color_stack[i],alloc); + if( iScore > bestScore ) + { best = i; bestScore = iScore; } + } + return best; + } + + // Pick a live range to pull, that might not color. + // + // Picking a live range with a very large span, with defs and uses outside + // loops means spilling a relative cheap live range and getting that + // register over a large area. + // + // Picking a live range that is very close to coloring might allow it to + // color despite being risky. + private static int pickRiskyScore( LRG lrg, RegAlloc alloc ) { + // Callee-save live ranges cover the whole function and are cheap to + // split. Rank scheduled area recovered against the loop-scaled cost + // of the def- and use-side moves. + if( lrg._machDef instanceof CalleeSaveNode ) + return areaScore(functionArea(lrg._fun,alloc), + splitCost((Node)lrg._machDef,(Node)lrg._machUse)); + if( lrg._splitDef != null && lrg._splitDef. in(1) instanceof CalleeSaveNode && + lrg._splitUse != null && lrg._splitUse.out(0) instanceof ReturnNode ) + return areaScore(functionArea(lrg._fun,alloc), + splitCost(lrg._splitDef,lrg._splitUse)); + + // Pick single-def clonables that are not right next to their single-use. + // Rank by the area recovered by cloning closer to the use. + if( !lrg._multiDef && lrg._machDef!=null && lrg._machUse!=null && lrg._machDef.isClone() ) { + Node def = ((Node)lrg._machDef); + Node use = ((Node)lrg._machUse); + CFGNode cfg = def.cfg0(); + int area = cfg != use.cfg0() + ? functionArea(lrg._fun,alloc) + : cfg._outputs.find(use)-cfg._outputs.find(def)-1; + // A single-use function address can move next to its use without + // adding an instruction. Prefer that to saving a callee-save + // register just to hold the address. Stay in the same block and + // require a gap so the clone makes progress. + if( def instanceof FunPtrNode && !lrg._multiUse && + cfg == use.cfg0() && area > 0 ) + return 1000000; + if( area > 0 ) + return areaScore(area,loopCost(use)); + } + + // Default for ranges without a specific area/cost estimate. + return 1000 + (lrg._multiUse ? -100 : 0); + } + + private static int functionArea(FunNode fun, RegAlloc alloc) { + int area=0; + for( CFGNode bb : alloc._code._cfg ) + if( bb.fun()==fun ) + area += bb.nOuts(); + return area; + } + + private static int areaScore(int area, int cost) { + return 1000 + (area<<10)/cost; + } + + private static int splitCost(Node def, Node use) { + return loopCost(def) + (use==null ? 0 : loopCost(use)); + } + + private static int loopCost(Node n) { + return 1 << (n.cfg0().loopDepth()*3); + } + + private static short biasColor( RegAlloc alloc, LRG lrg, short reg, RegMask mask ) { + if( mask.size1() ) return reg; + // Check chain of splits up the def-chain. Take first allocated + // register, and if it's available in the mask, take it. + Node def = lrg._splitDef, use = lrg._splitUse; + int tidx=0, cnt=0; + + while( def != null || use != null ) { + if( cnt++ > 10 ) break; + + if( def != null ) { + short bias = biasColor( alloc, def, mask ); + if( bias >= 0 ) return bias; // Good bias + if( bias == -2 ) def = null; // Kill this side, no more searching + else if( (tidx=biasable(def)) == 0 ) def = null; + } + + if( use != null ) { + short bias = biasColor( alloc, use, mask ); + if( bias >= 0 ) return bias; // Good bias + if( bias == -2 ) use = null; // Kill this side, no more searching + else if( biasable(use)==0 ) use = null; + } + + if( def != null ) { + short bias = biasColorNeighbors( alloc, def, mask ); + if( bias >= 0 ) return bias; + // Advance def side + def = def.in(tidx); + if( alloc.lrg(def)==null ) def=null; + } + + if( use != null ) { + short bias = biasColorNeighbors( alloc, use, mask ); + if( bias >= 0 ) return bias; + use = use.out(0); + if( biasable(use)==0 ) use=null; + } + + } + return mask.firstReg(); + } + + private static int biasable(Node split) { + if( split instanceof SplitNode ) return 1; // Yes biasable, advance is slot 1 + if( split instanceof PhiNode phi ) return phi.region() instanceof LoopNode ? 2 : 1; // Prefer the backedge in loops + if( !(split instanceof MachNode mach) ) return 0; // Not biasable + return mach.twoAddress(); // Only biasable if 2-addr + } + + // 3-way return: + // - good bias reg, take it & exit + // - this path is cutoff; do not search here anymore + // - advance this side + private static short biasColor( RegAlloc alloc, Node split, RegMask mask ) { + short bias = alloc.lrg(split)._reg; + if( bias != -1 ) { + if( mask.test(bias) ) return bias; // Good bias + else return -2; // Kill this side + } else return -1; // Advance this side + } + + // Check if we can match "split" color, or else trim "mask" to colors + // "split" might get. + private static short biasColorNeighbors( RegAlloc alloc, Node split, RegMask mask ) { + LRG slrg = alloc.lrg(split); + if( slrg._adj == null ) return -1; // No trimming + + // Can I limit my own choices to valid neighbor choices? + for( LRG alrg : slrg._adj ) { + short reg = alrg._reg; + if( reg == -1 && alrg._mask.size1() ) + reg = alrg._mask.firstReg(); + if( reg != -1 ) { + mask.clr(reg); + if( mask.size1() ) + return mask.firstReg(); + } + } + return -1; // No obvious color choice, but mask got trimmed + } + + private static void swap( LRG[] ary, int x, int y ) { + LRG tmp = ary[x]; ary[x] = ary[y]; ary[y] = tmp; + } +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/InstSelect.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/InstSelect.java new file mode 100644 index 00000000..5fc002a1 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/InstSelect.java @@ -0,0 +1,126 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.*; +import java.util.BitSet; +import java.util.IdentityHashMap; + +// Replace ideal nodes with machine nodes while preserving CFG/loop metadata. +// Selection also owns target ABI masks and rebuilding the copied graph's uses. +class InstSelect { + private final CodeGen _code; + private InstSelect(CodeGen code) { _code=code; } + + static CodeGen select(CodeGen code, String cpu, String callingConv, String base) { + // Ideal evaluation schedules the ideal graph without a CPU port. + if( cpu==null ) return code; + return code.phase(CodeGen.Phase.Select, code._phase==CodeGen.Phase.Unlink, + () -> new InstSelect(code).run(cpu,callingConv,base)); + } + + private void run(String cpu, String callingConv, String base) { + _code._callingConv = callingConv; + + // Look for CPU in fixed named place: + // com.seaofnodes.simple.node.cpus."cpu"."cpu.class" + String clzFile = base+"."+cpu+"."+cpu; + try { _code._mach = Class.forName(clzFile).asSubclass(Machine.class).getDeclaredConstructor(CodeGen.class).newInstance(_code); } + catch( Exception e ) { throw new RuntimeException(e); } + + _code._cCallingConv = _code._mach.cCallingConv(callingConv); + + // Build global copies of common register masks. + long callerSave = _code._mach.callerSave(); + long cCallerSave= _code._mach.callerSave(_code._cCallingConv); + long neverSave = _code._mach. neverSave(); + int maxReg = Math.min(64,_code._mach.regs().length); + assert maxReg>=64 || (-1L << maxReg & callerSave)==0; // No stack slots in callerSave + assert maxReg>=64 || (-1L << maxReg & cCallerSave)==0; // No stack slots in cCallerSave + _code._callerSave = new RegMask(callerSave); + _code._cCallerSave= new RegMask(cCallerSave); + + // Build a Return RegMask array. All returns have the following inputs: + // 0 - ctrl + // 1 - memory + // 2 - varies with returning GPR,FPR + // 3 - RPC + // 4+- Caller save registers + _code._retMasks = new RegMask[(maxReg-_code._callerSave.size()-Long.bitCount( neverSave ))+4]; + for( int reg=0, i=4; reg<maxReg; reg++ ) + if( !_code._callerSave.test(reg) && ((1L<<reg)&neverSave)==0 ) + _code._retMasks[i++] = new RegMask(reg); + _code._rpcMask = new RegMask(_code._mach.rpc()); + _code._retMasks[3] = _code._rpcMask; + + // Convert to machine ops + _code._uid = 1; // All new machine nodes reset numbering + var map = new IdentityHashMap<Node,Node>(); + _instSelect( _code._stop, map ); + _code._stop = (StopNode)map.get(_code._stop ); + StartNode start = (StartNode)map.get(_code._start); + _code._start = start==null ? new StartNode(_code._start) : start; + _instOuts(_code._stop,_code.visit()); + _code._visit.clear(); + + // Replace the CompUnit stop (and list of functions) + // with hardware-specific ones + for( CompUnit cu : _code._compunits.values() ) { + cu._start= (StartCUNode)map.get(cu._start); + cu._stop = (StopCUNode )map.get(cu._stop ); + } + } + + private Node _instSelect( Node n, IdentityHashMap<Node,Node> map ) { + if( n==null ) return null; + Node x = map.get(n); + if( x !=null ) return x; // Been there, done that + + // If n is a MachNode already, then its part of a multi-node expansion. + // It does not need instruction selection (already selected!) + // but it does need its inputs walked. + if( n instanceof MachNode ) { + for( int i=0; i < n.nIns(); i++ ) + n._inputs.set(i, _instSelect(n.in(i),map) ); + return n; + } + + // Produce a machine node from n; map it to flag as done so stops cycles. + if( n instanceof MemOpNode mem && mem.off() instanceof ExternOffsetNode off ) { + // External fields replace base+offset with symbol+0. Keep the ideal + // declaring type on the address for machine load/store width selection. + TypeMemPtr ptr = (TypeMemPtr)mem.ptr()._type; + n._inputs.set(2,new ExternNode(TypeMemPtr.make((byte)2,ptr._obj,true),off._extern)); + n._inputs.set(3,ConstantNode.raw(TypeInteger.ZERO)); + } + map.put(n, x=_code._mach.instSelect(n) ); + // Carry loop-tree and pre-order info across the ideal->mach transition + if( n instanceof CFGNode ncfg && x instanceof CFGNode xcfg ) { + xcfg._ltree = ncfg._ltree; + xcfg._pre = ncfg._pre; + } + // Walk machine op and replace inputs with mapped inputs + for( int i=0; i < x.nIns(); i++ ) + x._inputs.set(i, _instSelect(x.in(i),map) ); + // Post selection action + if( x instanceof MachNode mach ) { + if( n instanceof ReturnNode ret ) + ((ReturnNode)mach)._fun = (FunNode)map.get(ret._fun); + mach.postSelect(_code); // Post selection action + } + + return x; + } + + private void _instOuts( Node n, BitSet visit ) { + if( visit.get(n._nid) ) return; + visit.set(n._nid); + for( Node in : n._inputs ) + if( in!=null ) { + in._outputs.push(n); + // CallNode special: outputs are partially ordered; CallEnd in slot 0 + if( in instanceof CallNode call && n instanceof CallEndNode cend && call.out(0) != cend ) + call._outputs.swap(0,call.nOuts()-1); + _instOuts(in,visit); + } + } +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/Iter.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Iter.java new file mode 100644 index 00000000..4085ebff --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Iter.java @@ -0,0 +1,101 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.*; +import com.seaofnodes.simple.util.Ary; + +// Establish a pessimistic fixed point after parsing/loading. Later Iter runs +// reuse the worklist and types produced by the preceding phase. +public abstract class Iter { + static void run(CodeGen code) { + assert code._globalEscapes==null; + code._globalEscapes = new GlobalEscapeNode(); + code._start.setDef(2,code._globalEscapes); + // The phase shift closes all parser-in-progress states and freezes + // unknown external inputs. Imported nodes carry post-Opto escape + // precision, but Iter is a fresh pessimistic solve: memory escape + // summaries must restart at FULL so newly discovered private escapes + // can only be removed, never added. Seed the complete graph once. + boolean hasImports = code._externPaths != null; + code._start.walk(n -> { + // Type import can reconcile a constant's declaration. + n._type = n instanceof ConstantNode ? n.compute() + : hasImports ? iterImportReset(n._type) : iterEscapeReset(n._type); + code.add(n); + return null; + }); + code._iter.iterate(code); + // Expensive assert. + assert !CodeGen.expensiveAssert() || Opto.fixedPointCheck(code); + } + + private static Type iterEscapeReset(Type t) { + if( t instanceof TypeMem mem ) + return mem._one ? mem : mem.makeFrom(XInt.FULL,XInt.FULL); + if( t instanceof TypeTuple tt ) { + TypeTuple rez = tt; + for( int i=0; i<tt._types.length; i++ ) { + Type tx = iterEscapeReset(tt._types[i]); + if( tx != tt._types[i] ) rez = rez.makeFrom(i,tx); + } + return rez; + } + return t; + } + + private static Type iterImportReset(Type t) { + if( t.isHigh() ) + return t.dual(); + if( t instanceof TypeMem mem ) + return TypeMem.make(mem._alias,iterImportReset(mem._t),mem._one,mem._clz,mem._final,XInt.FULL,XInt.FULL); + if( t instanceof TypeMemPtr ) + return t.makeStorage(); + if( t instanceof TypeStruct ) + return t.makeStorage(); + if( t instanceof TypeTuple tt ) { + TypeTuple rez = tt; + for( int i=0; i<tt._types.length; i++ ) { + Type tx = iterImportReset(tt._types[i]); + if( tx != tt._types[i] ) rez = rez.makeFrom(i,tx); + } + return rez; + } + return t; + } + + // Saved types belong to a different caller context. Reset only the new + // graph, then repair any affected bounds before peepholes consume them. + static void imports(CodeGen code, Ary<Node> nodes) { + for( Node n : nodes ) + if( !n.isDead() ) + n._type = n instanceof ConstantNode ? n.compute() : iterImportReset(n._type); + // Stop normalizes unknown memory into the unit's visible escapes. + // Do not publish a transient FULL Stop merely because we reset its body. + for( Node n : nodes ) + if( n instanceof StopCUNode && !n.isDead() ) n._type = n.compute(); + code.addAll(nodes); + repair(code); + } + + // Repair the monotonicity assert after loading new nodes (from a file via + // Serialize). Only widening types here makes it safe to resume the usual + // lifting peeps. The worklist already contains changed edges and their + // neighbors. + public static void repair(CodeGen code) { + // Gather changes for a normal IterPeeps once the graph has its + // monotonicity property back. + Ary<Node> peeps = new Ary<>(Node.class); + Node n; + while( (n = code._iter._work.pop())!=null ) { + if( n.isDead() ) continue; + peeps.add(n); + // Meet over existing and new; only drops + Type t = n._type.meet(n.compute()); + if( t==n._type ) continue; + n._type = t; + code.addAll(n._outputs); + n.moveDepsToWorklist(); + } + code.addAll(peeps); + } +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/IterPeeps.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/IterPeeps.java new file mode 100644 index 00000000..543a69df --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/IterPeeps.java @@ -0,0 +1,349 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.type.Type; +import com.seaofnodes.simple.type.TypeInteger; +import com.seaofnodes.simple.util.Ary; +import com.seaofnodes.simple.node.*; +import java.util.Arrays; +import java.util.BitSet; +import java.util.IdentityHashMap; +import java.util.Random; + +/** + * The IterPeeps runs after parsing. It iterates the peepholes to a fixed point + * so that no more peepholes apply. This should be linear because peepholes rarely + * (never?) increase code size. The graph should monotonically reduce in some + * dimension, which is usually size. It might also reduce in e.g. number of + * MulNodes or Load/Store nodes, swapping out more "expensive" Nodes for cheaper + * ones. + * <br> + * The theoretical overall worklist is mindless just grabbing the next thing and + * doing it. If the graph changes, put the neighbors on the worklist. + * <br> + * Lather, Rinse, Repeat until the worklist runs dry. + * <p> + * The main issues we have to deal with: + * + * <ul> + * <li>Nodes have uses; replacing some set of Nodes with another requires more graph + * reworking. Not rocket science, but it can be fiddly. Its helpful to have a + * small set of graph munging utilities, and the strong invariant that the graph + * is stable and correct between peepholes. In our case `Node.subsume` does + * most of the munging, building on our prior stable Node utilities.</li> + * + * <li>Changing a Node also changes the graph "neighborhood". The neighbors need to + * be checked to see if THEY can also peephole, and so on. After any peephole + * or graph update we put a Nodes uses and defs on the worklist.</li> + * + * <li>Our strong invariant is that for all Nodes, either they are on the worklist + * OR no peephole applies. This invariant is easy to check, although expensive. + * Basically the normal "iterate peepholes to a fixed point" is linear, and this + * check is linear at each peephole step... so quadratic overall. It's a useful + * assert, but one we can disable once the overall algorithm is stable - and + * then turn it back on again when some new set of peepholes is misbehaving. + * The code for this is turned on in `IterPeeps.iterate` as `assert + * progressOnList(stop);`</li> + * </ul> + */ +public class IterPeeps { + + public final WorkList<Node> _work; + final WorkList<CallEndNode> _workInline; + private int _cnt; + + public IterPeeps( long seed ) { + _work = new WorkList<>(seed); + _workInline = new WorkList<>(seed); + } + + @SuppressWarnings("unchecked") + public <N extends Node> N add( N n ) { return (N)_work.push(n); } + + public void addAll( Ary<Node> ary ) { _work.addAll(ary); } + + /** + * Iterate peepholes and inlining to a fixed point + */ + public void iterate( CodeGen code ) { + boolean didInline = false; + Ary<Node> defer = new Ary<>(Node.class); + while( true ) { + // Clean up everything that does not grow the code + iteratePeeps(code); + + // Pick an inline candidate, no real heuristic, first come, first served + boolean inlined = false; + CallEndNode cend; + while( (cend = _workInline.pop()) != null) { + if( cend.isDead() ) continue; + if( cend.loadForInline() ) { inlined = true; break; } + byte inline = cend.maybeInline(); + if( inline == -1 ) defer.add(cend); + if( inline > 0 ) { + inlined = true; + break; + } + } + // Inlined, run peeps until clean again + if( inlined ) { didInline = true; continue; } + // No more candidates, check the defer list + if( !didInline ) break; // No more progress, so all the "maybe inline after cleanup" do not progress + // Some inlining happened, retry all the "try again after cleanup" calls + didInline = false; + code.addAll(defer); + defer.clear(); + } + } + + // Run all the code-reduction and type-lifting peeps as possible + private void iteratePeeps( CodeGen code ) { + assert !CodeGen.expensiveAssert(0) || (progressOnList(code, _work) && schedulableUses(code)); + + Node n; + while( (n = _work.pop()) != null ) { + if( n.isDead() ) continue; + _cnt++; // Useful for debugging, searching which peephole broke things + var obs = code._obs; // Collect peephole neighbors for graph display + if( obs != null ) obs.before(n); + Node x = n.peepholeOpt(); + if( n instanceof CallEndNode cend ) + _workInline.push(cend); + if( x != null ) { + assert !x.isDead(); // Peepholes return alive answers + // peepholeOpt can return brand-new nodes, needing an initial type set + if( x._type==null ) x.setType(x.compute()); + // Changes require neighbors onto the worklist + if( x != n || !(x instanceof ConstantNode) ) { + // All outputs of n (changing node) not x (prior existing node). + for( Node z : n._outputs ) _work.push(z); + // Everybody gets a free "go again" in case they didn't get + // made in their final form. + _work.push(x); + // A self-returning peephole can have rewritten its input + // edges. The new defs gained a user and may have + // backwards, user-sensitive peepholes of their own. + for( Node z : x._inputs ) _work.push(z); + // If the result is not self, revisit all inputs (because + // there's a new user), and replace in the graph. + if( x != n ) { + //for( Node z : n. _inputs ) _work.push(z); + for( Node z : x._outputs ) _work.push(z); + n.subsume(x); + } + } + // If there are distant neighbors, move to worklist + n.moveDepsToWorklist(); + // Very expensive assert. + assert !CodeGen.expensiveAssert(_cnt) || (progressOnList(code, _work) && schedulableUses(code)); + } + boolean unused = n.isUnused(); + if( unused ) { + assert !(n instanceof StopNode); // StopNodes can die if all code in the compunit dies + n.kill(); // Just plain dead + } + if( obs != null ) obs.after(n, x!=null ? x : unused ? n : null, true); + } + assert !CodeGen.expensiveAssert(0) || schedulableUses(code); + + } + + // Visit ALL nodes and confirm the invariant: + // Either you are on the _work worklist OR running `iter()` makes no progress. + + // This invariant ensures that no progress is missed, i.e., when the + // worklist is empty we have indeed done all that can be done. To help + // with debugging, the {@code assert} is broken out in a place where it is easy to + // stop if a change is found. + + // Also, the normal usage of `iter()` may attempt peepholes with distance + // neighbors and these should fail, but will then try to add dependencies + // {@link #Node.addDep} which is a side effect in an assert. The {@link + // #midAssert} is used to stop this side effect. + // Pessimistic solver assert + public static boolean progressOnList(CodeGen code, WorkList<Node> list ) { + code._midAssert = true; + Node changed = code._stop.walk( n -> { + // Ignore most in-progress things + if( n.iskeep() ) return null; + // Types must be forwards, even if on the worklist. + Type nval = n.compute(); + assert nval.isa(n._type) : "Non-monotonic peep: "+n+"#"+n._nid+" old="+n._type+" new="+nval+" inputs="+inputTypes(n)+" peep="+n; + if( list.on(n) ) + return null; // On worklist is ok! + if( n instanceof CallEndNode cend ) { + if( code._iter._workInline.on(cend) ) + return null; // On inline worklist is ok! + assert cend.maybeInline() <= 0 : "Inline fired and not on worklist, CallEndNode#"+cend._nid; + } + // Off list: must be live, in GVN table, and no peeps. + assert n.nOuts() > 0 : "Unused live node: "+n; + assert code._gvn.get(n)==n : "Not in GVN table: "+n; + Node m = n.peepholeOpt(); + assert m==null : "Peep fired and not on worklist, "+n.getClass().getSimpleName()+"#"+n._nid+" -> "+m; + return m; + }); + code._midAssert = false; + return changed==null; + } + + private static String inputTypes(Node n) { + StringBuilder sb = new StringBuilder("["); + for( Node in : n._inputs ) + sb.append(in==null ? "null" : in.getClass().getSimpleName()+"#"+in._nid+"="+in+":"+in._type+":keep="+in.iskeep()).append(','); + return sb.append(']').toString(); + } + + // GCM assumes every movable node can be placed no earlier than its inputs + // and no later than the LCA of its uses. Catch graphs where a use escapes + // above an input-defined branch before the later scheduling pass walks off + // the top of the idom tree. + private static final IdentityHashMap<Node,CFGNode> earlyCache = new IdentityHashMap<>(); + private static final BitSet VISIT = new BitSet(); + public static boolean schedulableUses(CodeGen code) { + earlyCache.clear(); + Node bad = code._stop.walk( n -> { + if( n.iskeep() || n.isDead() || n.isConst() || + n instanceof CFGNode || n instanceof PhiNode || n instanceof ProjNode ) + return null; + assert VISIT.isEmpty(); + CFGNode early = CFGNode.earlyCFG(n,earlyCache,VISIT); + VISIT.clear(); + if( early == null ) return null; + CFGNode lca = null; + for( Node use : n._outputs ) { + CFGNode ublk = useBlock(n,use); + if( ublk != null ) + lca = ublk.domLCA(lca,null); + } + if( lca == null ) return null; + if( !early.sameFun(lca) ) return null; + assert early.dominates(lca,null) : badSchedule(n,early,lca); + return null; + }); + return bad == null; + } + + private static CFGNode useBlock(Node n, Node use) { + if( use == null ) + return null; + if( use instanceof PhiNode phi ) { + CFGNode found = null; + for( int i = 1; i<phi.nIns(); i++ ) + if( phi.in(i)==n ) { + if( i >= phi.region().nIns() ) + return null; + found = phi.region().cfg(i).domLCA(found,null); + } + return found; + } + return CFGNode.safeCFG(use); + } + + private static String badSchedule(Node n, CFGNode early, CFGNode lca) { + StringBuilder sb = new StringBuilder("Unschedulable data node ") + .append(n.getClass().getSimpleName()).append('#').append(n._nid) + .append(" early=").append(cfg(early)) + .append(" use-lca=").append(cfg(lca)) + .append(" node=").append(n).append("\ninputs:"); + for( int i = 0; i<n.nIns(); i++ ) { + Node in = n.in(i); + sb.append("\n in").append(i).append(": ").append(node(in)) + .append(" cfg=").append(cfg(CFGNode.safeCFG(in))); + } + sb.append("\noutputs:"); + for( Node use : n._outputs ) + if( use != null ) + sb.append("\n use ").append(node(use)) + .append(" block=").append(cfg(useBlock(n,use))); + return sb.toString(); + } + + private static String node(Node n) { + return n==null ? "null" : n.getClass().getSimpleName()+"#"+n._nid+" "+n; + } + + private static String cfg(CFGNode cfg) { + return cfg==null ? "null" : cfg.getClass().getSimpleName()+"#"+cfg._nid+" "+cfg; + } + + /** + * Classic WorkList, with a fast add/remove, dup removal, random pull. + * The Node's nid is used to check membership in the worklist. + */ + @SuppressWarnings("unchecked") + public static class WorkList<E extends Node> { + + private Node[] _es; + private int _len; + private final BitSet _on; // Bit set if Node._nid is on WorkList + private final Random _R; // For randomizing pull from the WorkList + private final long _seed; + + /* Useful stat - how many nodes are processed in the post parse iterative opt */ + private long _totalWork = 0; + + public WorkList() { this(123); } + WorkList(long seed) { + _es = new Node[1]; + _len=0; + _on = new BitSet(); + _seed = seed; + _R = new Random(); + _R.setSeed(_seed); + } + + /** + * Pushes a Node on the WorkList, ensuring no duplicates + * If Node is null it will not be added. + */ + public E push( E x ) { + if( x==null ) return null; + int idx = x._nid; + if( !_on.get(idx) ) { + _on.set(idx); + if( _len==_es.length ) + _es = Arrays.copyOf(_es,_len<<1); + _es[_len++] = x; + _totalWork++; + } + return x; + } + + public void addAll( Ary<E> ary ) { + for( E n : ary ) + push(n); + } + public void addAll( E[] es ) { + for( E n : es ) + push(n); + } + + + /** + * True if Node is on the WorkList + */ + public boolean on( E x ) { return _on.get(x._nid); } + + /** + * Removes a random Node from the WorkList; null if WorkList is empty + */ + public E pop() { + if( _len == 0 ) return null; + int idx = _R.nextInt(_len); + E x = (E)_es[idx]; + _es[idx] = _es[--_len]; // Compress array + _es[_len] = null; + _on.clear(x._nid); + return x; + } + + public boolean isEmpty() { return _len==0; } + public void clear() { + Arrays.fill(_es,0,_len,null); + _len = 0; + _on.clear(); + _R.setSeed(_seed); + _totalWork = 0; + } + } +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/LRG.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/LRG.java new file mode 100644 index 00000000..fdde8838 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/LRG.java @@ -0,0 +1,240 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.util.Ary; +import com.seaofnodes.simple.util.SB; +import com.seaofnodes.simple.util.Utils; +import java.util.IdentityHashMap; + +/** A Live Range + * <p> + * A live range is a set of nodes and edges which must get the same register. + * Live ranges form an interconnected web with almost no limits on their + * shape. Live ranges also gather the set of register constraints from all + * their parts. + * <p> + * Most of the fields in this class are for spill heuristics, but there are a + * few key ones that define what a LRG is. LRGs have a unique dense integer + * `_lrg` number which names the LRG. New `_lrg` numbers come from the + * `RegAlloc._lrg_num` counter. LRGs can be unioned together - + * (<a href="https://en.wikipedia.org/wiki/Disjoint-set_data_structure">this is the Union-Find algorithm</a>) + * - and when this happens the lower numbered `_lrg` wins. Unioning only + * happens during `BuildLRG` and happens because either a `Phi` is forcing all + * its inputs and outputs into the same register, or because of a 2-address + * instruction. LRGs have matching `union` and `find` calls, and a set + * `_leader` field. + */ +public class LRG { + + // Dense live range numbers + final short _lrg; + + // U-F leader; null if leader + LRG _leader; + + // Choosen register + short _reg; + + // Count of single-register defs and uses + short _1regDefCnt, _1regUseCnt; + // Count of all defs, uses. Mostly interested 1 vs many + boolean _multiDef, _multiUse; + + // A sample MachNode def in the live range + public MachNode _machDef, _machUse; + short _uidx; // _machUse input + + // Some splits used in biased coloring + MachConcreteNode _splitDef, _splitUse; + + // All the self-conflicting defs for this live range + IdentityHashMap<Node,String> _selfConflicts; + + // AND of all masks involved; null if none have been applied yet + RegMask _mask; + + // Adjacent Live Range neighbors. Only valid during coloring + Ary<LRG> _adj; + + // Function fully containing the live range; used to find the stack frame + // layout, for spill op offsets. + final public FunNode _fun; + + public int nadj() { return _adj==null ? 0 : _adj._len; } + + public RegMask useMask() { return _machUse==null ? null : _machUse.regmap(_uidx); } + + void addNeighbor(LRG lrg) { + if( _adj==null ) _adj = new Ary<>(LRG.class); + _adj.push(lrg); + } + + // Remove and compress neighbor list; store ex-neighbor past + // end for unwinding Simplify. True if exactly going low-degree + boolean removeCompress( LRG lrg ) { + _adj.swap(_adj.find(lrg),_adj._len-1); + return _adj._len-- == _mask.size(); + } + + void reinsert( LRG lrg ) { + assert _adj._es[_adj._len]==lrg; + _adj._len++; + } + + // More registers than neighbors + boolean lowDegree() { return nadj() < _mask.size(); } + + LRG( short lrg, FunNode fun ) { _lrg = lrg; _reg = -1; _fun = fun; } + + boolean leader() { return _leader == null; } + + // Fast-path Find from the Union-Find algorithm + LRG find() { + if( _leader==null ) return this; // I am the leader + if( _leader._leader==null ) // I point to the leader + return _leader; + return _rollup(); + } + // Slow-path rollup of U-F + private LRG _rollup() { + LRG ldr = _leader._leader; + // Roll-up + while( ldr._leader!=null ) ldr = ldr._leader; + LRG l2 = this; + while( l2 != ldr ) { + LRG l3 = l2._leader; + l2._leader = ldr; + l2 = l3; + } + return ldr; + } + + // Union `this` and `lrg`, keeping the lower numbered _lrg. + // Includes a number of fast-path cutouts. + LRG union( LRG lrg ) { + assert leader(); + if( lrg==null ) return this; + lrg = lrg.find(); + if( lrg==this ) return this; + return _lrg < lrg._lrg ? _union(lrg) : lrg._union(this); + } + // Union `this` and `lrg`, folding together all stats. + private LRG _union( LRG lrg ) { + assert _fun == lrg._fun; + // Set U-F leader + lrg._leader = this; + // Fold together stats + if( _machDef==null ) { + _machDef = lrg._machDef; + } else if( lrg._machDef!=null ) { + if( _machDef != lrg._machDef ) _multiDef=true; + if( _1regDefCnt==0 ) + _machDef = lrg._machDef; + else if( _machDef==lrg._machDef ) + _1regDefCnt--; + } + _1regDefCnt += lrg._1regDefCnt; + _multiDef |= lrg._multiDef; + + if( _machUse==null ) { + _machUse = lrg._machUse; + _uidx = lrg._uidx; + } else if( lrg._machUse!=null ) { + if( _machUse != lrg._machUse ) _multiUse=true; + if( _1regUseCnt==0 ) { + _machUse = lrg._machUse; + _uidx = lrg._uidx; + } + else if( _machUse==lrg._machUse ) + _1regUseCnt--; + } + _1regUseCnt += lrg._1regUseCnt; + _multiUse |= lrg._multiUse; + + // Fold deepest Split + _splitDef = deepSplit(_splitDef,lrg._splitDef); + _splitUse = deepSplit(_splitUse,lrg._splitUse); + + // Fold together masks + RegMask mask = _mask.and(lrg._mask); + if( mask==null ) + mask = _mask.copy().and(lrg._mask); + _mask = mask; + return this; + } + + private static MachConcreteNode deepSplit( MachConcreteNode s0, MachConcreteNode s1 ) { + return s0==null || (s1!=null && s0.cfg0().loopDepth() < s1.cfg0().loopDepth()) ? s1 : s0; + } + + // Record any Mach def for spilling heuristics + LRG machDef( MachNode def, boolean size1 ) { + if( _machDef!=null && _machDef!=def ) + _multiDef = true; + if( _machDef==null || size1 ) + _machDef = def; + if( size1 ) + _1regDefCnt++; + if( def instanceof SplitNode split ) + _splitDef = deepSplit(_splitDef,split); + return this; + } + + // Record any Mach use for spilling heuristics + LRG machUse( MachNode use, short uidx, boolean size1 ) { + if( _machUse!=null && _machUse!=use ) + _multiUse = true; + if( _machUse==null || size1 ) + { _machUse = use; _uidx = uidx; } + if( size1 ) + _1regUseCnt++; + if( use instanceof SplitNode split ) + _splitUse = deepSplit(_splitUse,split); + return this; + } + + boolean hasSplit() { return _splitDef != null || _splitUse != null; } + short size() { return _mask.size(); } + boolean size1() { return _mask.size1(); } + + + void selfConflict( Node def ) { + if( _selfConflicts == null ) _selfConflicts = new IdentityHashMap<>(); + _selfConflicts.put(def,""); + } + + + // Record intersection of all register masks. + // True if still has registers + boolean and( RegMask mask ) { + RegMask mask2 = mask.and(_mask); + if( mask2==null ) + mask2 = _mask.copy().and(mask); + _mask = mask2; + return !_mask.isEmpty(); + } + // Remove this singular register + // True if still has registers + boolean clr( int reg ) { + if( _mask.clr(reg) ) return true; + _mask = _mask.copy(); // Need a mutable copy + return _mask.clr(reg); + } + // Subtract mask (AND complement) + // True if still has registers + boolean sub( RegMask mask ) { + RegMask mask2 = _mask.sub(mask); + if( mask2==null ) + mask2 = _mask.copy().sub(mask); + _mask = mask2; + return !_mask.isEmpty(); + } + + @Override public String toString() { return toString(new SB()).toString(); } + + public SB toString( SB sb ) { + sb.p("V").p(_lrg); + if( _mask!=null ) _mask.toString(sb); + return sb; + } +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/LinkMem.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/LinkMem.java new file mode 100644 index 00000000..252b5bf0 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/LinkMem.java @@ -0,0 +1,33 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.Node; +import com.seaofnodes.simple.type.TypeStruct; +import com.seaofnodes.simple.util.SB; +import com.seaofnodes.simple.util.Utils; + +// In-memory linking +public class LinkMem { + final CodeGen _code; + LinkMem( CodeGen code ) { _code = code; } + + public CodeGen link(Encoding enc) { + // Patch external calls internally + enc.patchGlobalRelocations(); + + // Place constant pool after code, static data after constants + int code = 0; // Code start + int cpool = (enc._bits .size() + code + 15) & -16; // pad to 16 + int sdata = (enc._cpool.size() + cpool + 15) & -16; + + + // Patch local ops, e.g. loading float constants from the constant pool + for( Node op : enc._bigCons.keySet() ) { + Encoding.Relo relo = enc._bigCons.get(op); + int target = enc._data.address(enc._data.object(relo._t),cpool,sdata); + ((RIPRelSize)relo._op).patch(enc, relo._opStart, enc.opLen(relo._op), target - relo._opStart); + } + + enc._data.link(cpool,sdata); + return _code; + } +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/ListScheduler.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/ListScheduler.java new file mode 100644 index 00000000..c3b603e5 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/ListScheduler.java @@ -0,0 +1,265 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.codegen.IterPeeps.WorkList; +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.*; +import com.seaofnodes.simple.util.Ary; +import com.seaofnodes.simple.util.Utils; +import java.util.*; + +public abstract class ListScheduler { + + // eXtra per-node stuff for scheduling. + // Nodes are all block-local, and projections fold into their parent. + private static final IdentityHashMap<Node,XSched> XS = new IdentityHashMap<>(); + private static class XSched { + + Node _n; // Node this extra is for + int _bcnt; // Not-ready not-scheduled inputs + int _rcnt; // Ready not-scheduled inputs + boolean _ruse; // Node IS a "remote use", uses a other-block value + boolean _rdef; // Node IS a "remote def" of a value used in other block + boolean _single; // Defines a single register, likely to conflict + + static final Ary<XSched> FREE = new Ary<>(XSched.class); + static void alloc(CFGNode bb, Node n) { + XSched x = FREE.pop(); + XS.put(n, (x==null ? new XSched() : x).init(bb,n)); + } + void free() { _n = null; FREE.push(this); } + + static XSched get(Node n) { return n==null ? null : XS.get(n instanceof ProjNode && !(n.in(0) instanceof CFGNode) ? n.in(0) : n); } + + private XSched init(CFGNode bb, Node n) { + _n = n; + _bcnt = 0; + _ruse = _rdef = _single = false; + + // Count block-locals and remote inputs + if( !(n instanceof PhiNode) ) + for( Node def : n._inputs ) + if( def != null && def != bb ) { + if( def.cfg0()==bb ) _bcnt++; // Raise block-local input count + else if( !def.isMem() ) _ruse = true; // Remote register user + } + // Count remote outputs and allowed registers + if( n instanceof MultiNode ) + for( Node use : n._outputs ) + computeSingleRDef(bb,use); + else + computeSingleRDef(bb,n); + + if( n instanceof NewNode || n instanceof CallNode ) + bb.fun()._hasCalls = true; + return this; + } + + private void computeSingleRDef(CFGNode bb, Node n) { + // See if this is 2-input, and that input is single-def + if( n instanceof MachNode mach && mach.twoAddress() != 0 ) { + XSched xs = XS.get(n.in(mach.twoAddress())); + if( xs != null ) + _single = xs._single; + } + // Visit all outputs + RegMaskRW rmask = !_single && n instanceof MachNode mach && mach.outregmap() != null ? mach.outregmap().copy() : null; + for( Node use : n._outputs ) { + // Remote use, so this is a remote def + if( use!=null && use.cfg0()!=bb ) _rdef = true; + // And mask between n's def mask and all n's users masks' + if( use instanceof MachNode mach && rmask != null ) + for( int i=1; i<use.nIns(); i++ ) + if( use.in(i)==n ) + rmask.and(mach.regmap(i)); + } + // Defines in a single register + _single |= rmask!=null && rmask.size1(); + } + + // If _bcnt==0, declare ready; move user bcnts into rcnts. + boolean isReady() { + if( _bcnt > 0 || _rcnt > 0 ) return false; + if( _n instanceof MultiNode ) + for( Node use : _n._outputs ) + if( use instanceof ProjNode ) readyUp(use); + // anti-dep from a multireg directly + else _readyUp(use); + else + readyUp(_n); + return true; + } + private static void readyUp(Node n) { + for( Node use : n._outputs ) + _readyUp(use); + } + private static void _readyUp(Node use) { + XSched xs = get(use); + if( xs!=null && !(use instanceof PhiNode) ) + { assert xs._bcnt>0; xs._bcnt--; xs._rcnt++; } + } + + boolean decIsReady() { + assert _rcnt > 0; + _rcnt--; + return isReady(); + } + } + + + // List schedule every block + public static void sched( CodeGen code ) { + sched(code._cfg); + } + + public static void sched(Ary<CFGNode> cfgs) { + XS.clear(); + for( CFGNode bb : cfgs ) + if( bb.blockHead() ) + local(bb,false); + } + + // Classic list scheduler + private static void local( CFGNode bb, boolean trace ) { + assert XS.isEmpty(); + int len = bb.nOuts(); + + // Count block-locals, remote inputs and outputs + for( int i=0; i<len; i++ ) + XSched.alloc(bb,bb.out(i)); + + // Nodes are ready if they are not used by other nodes in this block. + // Move ready up front. + int ready = 0; + for( int i=0; i<len; i++ ) + if( XSched.get(bb.out(i)).isReady() ) // No block-local inputs? + bb._outputs.swap(ready++,i); // Move into ready set + assert len==0 || ready > 0; + + // Classic list scheduler. Behind sched is scheduled. Between sched + // and ready have zero counts and are ready to schedule. Ahead of + // ready is undefined. + // + // As nodes are scheduled, the nodes they use decrement their bcnts[]. Once + // that gets to 0, all the nodes using it are scheduled, so it's ready. + int sched = 0; + while( sched < len ) { + assert sched < ready; + int pick = best(bb._outputs,sched,ready,trace); // Pick best + Node best = bb._outputs.swap(pick,sched++); // Swap best to head of ready and move into the scheduled set + // Lower ready count of users, and since projections are ready as + // soon as their multi is ready, lower those now too. + for( Node use : best._outputs ) + if( use instanceof ProjNode ) + for( Node useuse : use._outputs ) + ready = ready(bb,useuse,ready); + else + ready = ready(bb,use,ready); + } + + // Reset for next block + for( XSched x : XS.values() ) + x.free(); + XS.clear(); + } + + private static int ready(CFGNode bb, Node use, int ready) { + if( use!=null && use.in(0)==bb && !(use instanceof PhiNode) && XSched.get(use).decIsReady() ) + bb._outputs.set(ready++, use); // Became ready, move into ready set + //bb._outputs.swap(ready++,use); // Became ready, move into ready set + return ready; + } + + + // Pick best between sched and ready. + private static int best( Ary<Node> blk, int sched, int ready, boolean trace ) { + int pick = sched; + Node p = blk.at(pick); + int score = score(p); + if( trace ) { System.out.printf("%4d N%d ",score,p._nid); System.out.println(p); } + for( int i=sched+1; i<ready; i++ ) { + Node n = blk.at(i); + int nscore = score(n); + if( trace ) { System.out.printf("%4d N%d ",nscore,n._nid); System.out.println(n); } + if( nscore > score ) + { score = nscore; pick = i; p = n; } + } + if( trace ) { System.out.printf("Pick: N%d %s\n\n",p._nid,p); } + return pick; + } + + // Highest score wins. Max at 1000, min at 10, except specials. + private static final int[] CNT = new int[3]; + static int score( Node n ) { + if( n instanceof ProjNode ) return 1001; // Pinned at block entry + if( n instanceof CProjNode ) return 1001; // Pinned at block entry + if( n instanceof PhiNode ) return 1000; + if( n instanceof CFGNode ) return 1; // Pinned at block exit + + int score = 500; + // Ideal nodes ignore register pressure scheduling + if( !(n instanceof MachNode) ) + return score; + + // Register pressure local scheduling: avoid overlapping specialized + // registers usages. + + // Subtract 10 (delay) if this op forces a live range to exist that + // cannot be resolved immediately; i.e., live count goes up. Scale to + // 20 if multi-def. + + // Subtract 100 if this op forces a single-def live range to exist + // which might conflict on the same register with other live ranges. + // Defines a single register based on def & uses, and the output is not + // ready. Scale to 200 for multi-def. + CNT[1]=CNT[2]=0; + XSched xn = XSched.get(n); + // If defining a remote value, just generically stall alot. Value is + // used in a later block, can we delay until the end of this block? + if( xn._rdef ) score = 200; // If defining a remote value, just generically stall alot + boolean flags = false; + if( n instanceof MultiNode ) { + for( Node use : n._outputs ) + flags |= singleUseNotReady( use, xn._single ); + } else + flags |= singleUseNotReady( n, xn._single ); + score += -10 * Math.min( CNT[1], 2 ); + score += -100 * Math.min( CNT[2], 2 ); + if( flags ) return 10; + + // Add 10 if this op ends a single-def live range, add 20 for 2 or more. + // Scale to 100,200 if the single-def is also single-register. + CNT[1]=CNT[2]=0; + for( int i=1; i<n.nIns(); i++ ) { + XSched xd = XSched.get( n.in(i) ); + if( xd != null && n.in(i).nOuts()==1 ) + CNT[xd._single || ((MachNode)n).regmap(i)!=null && ((MachNode)n).regmap(i).size1() ? 2 : 1]++; + } + score += 10 * Math.min( CNT[1], 2 ); + score += 100 * Math.min( CNT[2], 2 ); + + // Nothing special + assert 10 <= score && score <= 990; + return score; + } + + // Single use, and that use is not ready to fire once this one does. If + // true, stalling 'n' might reduce 'n's lifetime. Stall by 1 (2 if single) + // per not-ready. + + // If CFG user in this block, it must be last, so definitely stall + // single-def (which stalls flags on x86,arm until the jmp). + private static boolean singleUseNotReady( Node n, boolean single ) { + // If n can rematerialize, assume allocator will split into private uses + // as needed. No impact on local scheduling. + if( n.nOuts()>1 && n instanceof MachNode mach && mach.isClone() ) + return false; + for( Node use : n.outs() ) { + XSched xu = XSched.get(use); + if( xu != null && xu._n instanceof CFGNode ) return true; + if( xu != null && xu._bcnt > 0 ) + CNT[single ? 2 : 1]++; // Since bcnt>0 or CFG, stall until user is more ready + } + return false; + } + +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/Machine.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Machine.java new file mode 100644 index 00000000..3eee8fc3 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Machine.java @@ -0,0 +1,127 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.TypeFunPtr; +import com.seaofnodes.simple.type.TypeInteger; + +abstract public class Machine { + // Human readable machine name. Something like "x86-64" or "arm" or "risc5" + public abstract String name(); + + // Default opcode size for printing; 4 bytes for most 32-bit risc chips + public int defaultOpSize() { return 4; } + + // Human-readable name for a register number, e.g. "RAX" or "R0" + public abstract String[] regs(); + + // Create a split op; any register to any register, including stack slots. + // The FunNode holds the stack layout after RegAlloc, which in turn + // dictates the stack slot (the frame offset) for stack splits. + public abstract SplitNode split( LRG lrg, String kind, byte round ); + + // List of caller-save registers, as a 64bit mask + public abstract long callerSave(); + public long callerSave(String conv) { return callerSave(); } + + // List of never-save registers, e.g. RSP or a ZERO register if you have one + public abstract long neverSave(); + + // Call Argument Mask. Passed in the function signature and argument + // number (2-based; 0 is for control and 1 for memory). Also passed in a 0 + // for the function itself, or for *outgoing* calls, the maximum stack slot + // given to the incoming function arguments (stack slots reserved for + // incoming arguments). + public abstract RegMask callArgMask(TypeFunPtr tfp, int arg, int maxArgSlot); + public RegMask callArgMask(TypeFunPtr tfp, int arg, int maxArgSlot, String conv) { return callArgMask(tfp,arg,maxArgSlot); } + + // Return register mask, based on signature (GPR vs FPR) + public abstract RegMask retMask(TypeFunPtr tfp); + + // Return PC register + public abstract int rpc(); + + // Return a MachNode unconditional branch + public abstract CFGNode jump(); + + // Instruction select from ideal nodes + public abstract Node instSelect( Node n ); + + /** Stack Slot Numbering + +RA gives all registers a number, starting at 0. Stack slots continue this +numbering going up from the last register number. Common register numbers are +0-15 for GPRs, 16-31 for FPRs, 32 for flags, and stack slots starting at +register#33 going up. These numbers are machine-specific, YMMV, etc; e.g. RPC +is only in stack slot 0 on X86; other cpus start with the rpc in a register +which may spill into any generic spill slot. + +For ease of reading and printing, stack slots start again at slot#0 - but +during RA they are actually biased by CPU.MAX_REG. + +Stack Layout during Reg Alloc: + + Slot Value Description + ---- ------- ----------- +MAXREG+ R+1 argR maxArgSlot(calleR tfp), can be 0 + ... +MAXREG+ 1 arg0 + ---- -- Caller ------ +MAXREG+ 0 RPC +MAXREG+ R+E+N PAD optional alignment + +MAXREG+ + Spills Caller Frame Space; N spills +MAXREG+ R+E+1 Spills + +MAXREG+ R+E argE maxArgSlot(calleE tfp), can be 0 + ... +MAXREG+ R+2 arg0 + ---- -- Callee ------- + ... + +Slots defined by maxArgSlot do not have to contain the actual passed argument; +they can be "shadow slots". If they exist, they are property of the frame / +FunNode where the TFP comes from. + +For a given calleR, there are generally many calleE's and they all share the +low-range stack slots. + +Example:, Win64, main(int arg) { calloc(1,size); } + + Post-Warp: SP+56 shadow-main-arg +Pre-alloc Post-Alloc SP+48 RPC +--- ------- ---------- ------------- +40 spill#3 SP+40 SP+40 spill#3 +39 spill#2 SP+32 "" +38 spill#1 SP+24 "" +37 spill#0 SP+16 "" spill#0 +36 shadow-calloc#size SP+ 8 SP+ 8 shadow-calloc#size +35 shadow-calloc#1 SP+ 0 SP+ 0 shadow-calloc#1 +--- --- FRAME BASE ---------- ---- FRAME BASE ------- +34 shadow-main-arg SP+56 +33 RPC SP+48 + + + */ + + + // Maximum stack slot (or 0) for the args in this TFP. This will include + // shadow slots if defined in the ABI, even if all arguments are passed in + // registers. + public abstract short maxArgSlot(TypeFunPtr tfp); + public short maxArgSlot(TypeFunPtr tfp, String conv) { return maxArgSlot(tfp); } + + // Calling convention used for external C symbols. + public String cCallingConv(String callingConv) { return callingConv; } + + // Only the declared C result width describes the physical return register. + // Simple functions already return canonical full-width integer values. + protected static TypeInteger cReturn(ProjNode prj) { + if( prj._idx==2 && prj.in(0) instanceof CallEndNode cend && + cend.call().fptr()._type instanceof TypeFunPtr tfp && tfp.isConstant() && + CodeGen.CODE.externFunc(tfp.fidx())!=null && + tfp.ret() instanceof TypeInteger ret && ret.log_size()<3 ) + return ret; + return null; + } + +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/Module.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Module.java new file mode 100644 index 00000000..a5d85dbc --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Module.java @@ -0,0 +1,98 @@ +package com.seaofnodes.simple.codegen; + + +/*** + Modules!?!?!? + + Goal: under no circumstances can you load from an uninitialized variable + (although you can see default inits for fields that have defaults). + Goal: nested tree-structured name spaces, to control complexity. + Goal: field decl order within a file is fairly flexible. + + A Module a root-level SMP file, and the full file path must be provided on + the command line. All other Simple structs are relative to this file. + + Example: "simple /user/Dave/hello.smp" + + Defines a module "hello" in directory "/user/Dave", compiles and produces + a `hello` or `hello.exe` output file, and runs it. + + A SMP file has some <finit> file-init code which is static-global. + This code is executed on "first touch" of the file (defined below). + + Example: hello.smp contains: + sys.io.p("Hello World!"); + On first-touch, will print "Hello World!". + + A SMP file is a lexical scope, and code inside it can reference those names. + + Example: hello.smp contains: + val greet = "Hello World!"; // Declare file-scoped variable + sys.io.p(greet); // Reference file-scoped variable + + Static code is NOT allowed to have forward-references AND be executed + before the <finit> finishes initializing all variables. After var + initialization, a singleton instance of the implied file struct is made, + and then <finit> code can call anything. + + Example: + sys.io.p(greet); // Illegal forward reference + val greet = "Oops!"; + + Example: + val delay = { -> greet }; // Legal forward ref, but cannot execute (yet) + val greet = "Hello World!"; // Ref is resolved + sys.io.p(delay()); // Allowed to call delay() + val nprimes = { N -> + // compute number of primes < N + } + // Can otherwise execute any code in a <finit> + sys.io.p(nprimes(100)); + + Inside a directory, you can have sibling SMP files: + File hello.smp contains: + sys.io.p(I9A.GREETING); // File hello references sibling file I9A static globals + File I9A.smp contains: + val GREETING = translate("Hello World!"); + val translate = { str msg -> ... }; + + <finit> code cannot reference sibling files (nor child files, see below) + until all static variables are initialized. This prevents cyclic + initialization bugs, where file A uses vars from file B and vice-versa. + + File Chicken.smp contains: + val lay = Egg.hatch; // Illegal, defining var 'lay' after calling sibling Egg + File Egg.smp contains: + val hatch = Chicken.lay; // Illegal + + You can make cross references, if they get delayed until after all fields + are declared. + + File Chicken.smp contains: + int lay = 2; + lay = Egg.egg; + + File Egg.smp contains: + int egg = 3; + egg = Chicken.lay; + + The problem here, of course, is that you need to touch one of Chicken or + Egg first. + + + Inside a SMP file, you can define structs; structs are also a lexical + scope. Bare code in the struct is part of the struct's <init> and only + runs when an instance is created. Structs do not have static <clinit> (nor + <finit>). Structs <init> cannot have any forward references, and locally + declared functions cannot have both forward references and be executed (or + passed along to a possibly-executing function). + + In the same directory as a SMP file, Simple allows other directories with + names matching SMP files. Inside these dirs can be more SMP files that are + lexically nested, similar to embedded structs. + + +*/ + +public abstract class Module { +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/Opto.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Opto.java new file mode 100644 index 00000000..aa997204 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Opto.java @@ -0,0 +1,331 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.codegen.IterPeeps.WorkList; +import com.seaofnodes.simple.type.*; +import com.seaofnodes.simple.util.Ary; + +import java.util.BitSet; + +// Optimistic Algorithm +abstract public class Opto { + + // Optimistic analysis - extended SCCP + + // Start all Types at TOP, and propagate changes until stable. Unlike a + // normal Sparse Conditional Constant Propagation, this version is + // interprocedural, computing effectively CFA0 in linear time using the + // normal SCCP algorithm. + + // This algo also builds a Call Graph; as Calls become reachable (CONTROL + // not TOP nor XCONTROL), they "link" against the reaching function + // pointers - adding edges from the Call to the Fun, and from the Return to + // the CallEnd. This allows the normal algorithm to use the normal edge + // propagation rules as function argument types stabilize. In the end, we + // get a fairly precise set of Call Graph edges between CallNodes and + // FunNodes. + public static void opto(CodeGen code) { + // Reset Types before running the worklist algorithm + Ary<Type> oldTypes = new Ary<>(Type.class); + WorkList<Node> deps = new WorkList<>(); + WorkList<Node> changes = new WorkList<>(); + Ary<Node> modes = new Ary<>(Node.class); + resetToTop(code,oldTypes,modes,deps); + unlinkStart(code); + + // OPTIMISTIC INTERPROCEDURAL SCCP + sccp(code, oldTypes, changes, deps); + + // Settle unresolved modes using SCCP's int/float evidence. Some + // computes wait at TOP until a mode is known; others already return + // BOTTOM and stay there. Resume SCCP only on the affected nodes; + // types are not reset, and any further changes must still fall. + resolveNumericModes(code,modes); + + if( !code._iter._work.isEmpty() ) + sccp(code, oldTypes, changes, deps); + Node dep; + while( (dep=deps.pop())!=null ) code.add(dep); + + // After SCCP add everything changed to the worklist and see if + // peepholes can make more progress. + moveChangesToWorklist(code, oldTypes, changes); + } + + // Reset Types before running the worklist algorithm + private static void resetToTop(CodeGen code, Ary<Type> oldTypes, Ary<Node> modes, WorkList<Node> deps ) { + code._start.walk( x -> { + // Opto ends Parm.inProgress(), enabling constant folding even + // when a parameter's type was already constant before SCCP. + if( x instanceof ParmNode && x._type.isHighOrConst() ) deps.push(x); + // SCCP adds call edges and default arguments, not numeric + // operations. Gather unresolved modes on this required walk. + if( x instanceof ModeNode mode && mode.mode()==0 ) modes.push(x); + oldTypes.setX(x._nid,x._type); // Compare the final SCCP type with Iter's type + x._type = Type.TOP; // Reset all Nodes to TOP + code._iter.add(x); // Visit everybody at least once; most Nodes produce better than TOP + return null; + } ); + } + + + // Only entry initializers start with unknown callers. Discover every + // other function through calls or escaping values during SCCP. + private static void unlinkStart(CodeGen code) { + // CodeGen owns these until FinalizeIR; linkStart may need them again. + for( CompUnit cu : code._compunits.values() ) { + ProjNode mem = cu._start.proj(1); + if( mem!=null ) code.keepUntilFinalize(mem); + } + + // Iterate all functions. + // TODO: use CodeGen _link list + for( Node stop : code._stop._inputs ) { + for( int i=0; i<stop.nIns(); i++ ) { + ReturnNode ret = (ReturnNode)stop.in(i); + FunNode fun = ret.fun(); + if( !fun.isRoot() && + fun.in(1) instanceof StartNode ) { + fun.removeDeadPath(1); + // Unhook from Stop without treating the ReturnNode as + // DEAD. It stays in limbo until Opto links when a caller + // is discovered (or not, and the function is really dead). + stop._inputs.del(i--); + ret.delUse(stop); + } + // Unlink the existing conservative call linkages. + for( int j=1; j<fun.nIns(); j++ ) + if( fun.in(j) instanceof CallNode call ) + call.unlink(fun,j--); + } + } + } + + // As part of building a CallGraph, when opto finds a function ptr flowing + // into a Call, link the Call and Fun. + private static void linkCG(CodeGen code, TypeFunPtr tfp, CallNode call) { + if( tfp.nargs() != call.nargs() ) return; // Error calls hit this + int[] fidxs = tfp.fidxs(); + // An unresolved forward call can still carry the infinite, inferred + // function set into Opto. There is no finite call graph to link yet; + // type checking will report the unresolved reference afterwards. This + // can yet be valid if the call is eventually dead. + // `if( false ) frefCall(args);` + if( XInt.isHigh(fidxs) ) return; + for( int fidx = XInt.next(fidxs,0); fidx >=0; fidx = XInt.next(fidxs,fidx) ) { + // unlinkStart deliberately leaves uncalled functions in a + // temporarily dead limbo. A newly discovered call revives them; + // do not use the cleanup lookup which removes dead linker entries. + FunNode fun = code._link(fidx); + // null here means an external function; i.e. this Call + // calls to an outside library and all its arguments escape. + if( fun != null && !call.linked(fun) ) { + call.link( fun ); + code._iter.add(fun); + code._iter.addAll(fun._outputs); + } + } + } + + // Start added some FIDXs, the same functions now can be reached from start + // and called by anybody. Add the CG edges. + private static void linkStart( CodeGen code, TypeTuple tt ) { + TypeMem tmem = (TypeMem)tt._types[1]; + int[] fidxs = tmem._escFs; + if( XInt.isHigh(fidxs) ) { + // A scalar BOTTOM can flow into memory while compiling an error + // program. It exposes all public functions, not private fields. + for( FunNode fun : code._linker ) + if( fun != null && code.publicFIDX(fun.sig().fidx()) ) + linkStart(code,fun); + return; + } + for( int fidx = XInt.next(fidxs,0); fidx >=0; fidx = XInt.next(fidxs,fidx) ) { + FunNode fun = code._linker.atX(fidx); + if( fun==null ) assert code._externFunc.containsKey(fidx); + else linkStart(code,fun); + } + } + + // Function is now reachable by unknown callers through Start. The + // default inputs must therefore use conservative declared types. + static void linkStart( CodeGen code, FunNode fun ) { + assert !fun.isDead(); + StartCUNode start = fun._compunit._start; + if( !fun.unknownCallers() ) { + assert !start.isDead(); + code.keepInCompUnit(fun); + // Function is reachable by any *remote* caller who gets the pointer! + fun.insertDef(1,start); + code._iter.add(fun); + for( Node p : fun._outputs ) + if( p instanceof ParmNode parm ) { + Node defalt = parm._idx==1 ? start.proj(1) : ConstantNode.seed(parm._declaredType).peephole(); + parm.insertDef(1,defalt); + code.add(parm); + } + } + } + + + // The core SCCP algorithm + private static void sccp( CodeGen code, Ary<Type> oldTypes, WorkList<Node> changes, WorkList<Node> deps ) { + // Iter workList is empty at this point + // OPTIMISTIC PASS + Node n; + int count = 0; // Debug counter + while( (n = code._iter._work.pop()) != null ) { + if( n.isDead() ) continue; + count++; + Type oval = n._type, nval = n.compute(); + if( n instanceof CLInitNode init ) init.link(); + // Remember possible peephole work without walking the graph again. + // SCCP can still fall back to the old type, so filter at the end. + if( nval!=oldTypes.atX(n._nid) || n.isUnlocked() ) changes.push(n); + if( oval == nval ) continue; + assert oval.isa(nval) : "Non-monotonic SCCP: "+n.getClass().getSimpleName()+"#"+n._nid+" "+n+" old="+oval+" new="+nval; // Types start high and always fall + Type pesiVal = oldTypes.atX(n._nid); + assert pesiVal==null || nval.isa(pesiVal) : "SCCP below pessimistic type: "+n.getClass().getSimpleName()+"#"+n._nid+" "+n+" pessimistic="+pesiVal+" new="+nval; // Never fall worse than the pessimistic pass + n._type = nval; + + // Now we have a series of stanzas where we lazily create the Call + // Graph - adding graph edges between every call site and called + // function, including external (unknown) callers. These edges are + // not in from the start because they are *too many*; we would get + // O(n^2) edges. So all along we have been treating them as + // "virtual" CFG edges - a FunNode taking a Start input is treated + // as if every call on the planet might call it. Now we make these + // CG edges *concrete*, adding them back as we discover that a call + // calls a particular function. + + // If a TFP adds a new function input to a call, link to the new + // Fun. This adds new edges to the graph allowing argument values + // to flow from calls into functions. The added edges are + // effectively a refined Call Graph. + if( nval instanceof TypeFunPtr tfp ) { + for( Node use : n._outputs ) + if( !use._type.isHigh() && use instanceof CallNode call && call.fptr() == n ) + linkCG(code,tfp,call); + } + + // Link a Call which becomes alive. Like above, adds new Call + // Graph edges to the graph. + if( n instanceof CallNode call && nval == Type.CONTROL && call.fptr()._type instanceof TypeFunPtr ) + linkCG(code,call.tfp(),call); + + // If an otherwise-dead function pointer escapes, any future linked + // caller might find and call it. Force the function to be alive + // and called by Start. + if( n instanceof StartNode ) + linkStart(code,(TypeTuple)n._type); + + // Since n._type changed, visit all output neighbors + code._iter.addAll(n._outputs); + // SCCP consumes these without running peepholes. Save them for + // the following Iter pass before moving clears the dependency list. + for( int i=0; i<n.nDeps(); i++ ) deps.push(n.dep(i)); + n.moveDepsToWorklist(code._iter); + // Quadratic (expensive) small-step assert. + assert !CodeGen.expensiveAssert(count) || worklistCheck(code); + } + } + + // Quadratic (expensive) fixed-point assert + public static boolean fixedPointCheck(CodeGen code) { + return fixedPointCheck(code,code._stop); + } + public static boolean fixedPointCheck(CodeGen code, Node root) { + root.walk( x -> { + Type xval = x.compute(); + assert xval == x._type; + return null; + }); + return true; + } + + // Quadratic (expensive) small-step assert for SCCP, where changed nodes + // may already be on the worklist. + private static boolean worklistCheck(CodeGen code) { + code._midAssert = true; + Node root = code._start; + root.walk( x -> { + Type xval = x.compute(); + assert xval == x._type || (x._type.isa( xval ) && code._iter._work.on( x )) + : "Non-monotonic or missed SCCP work: "+x.getClass().getSimpleName()+"#"+x._nid+" "+x+" old="+x._type+" new="+xval; + return null; + }); + code._midAssert = false; + return true; + } + + // Types can stay unchanged while SCCP edits edges or numeric modes. Such + // edits unlock the node from GVN. Revisit both kinds of change and their + // neighbors; distant dependencies consumed by SCCP were saved separately. + private static void moveChangesToWorklist(CodeGen code, Ary<Type> oldTypes, WorkList<Node> changes) { + Node n; + while( (n=changes.pop())!=null ) { + if( n.isDead() ) continue; + assert n.compute() == n._type; // Hit the fixed point. + Type old = oldTypes.atX(n._nid); + assert old==null || n._type.isa(old); + if( n._type!=old || n.isUnlocked() ) { + code.addAll(n); + code.addAll(n._outputs); + n.moveDepsToWorklist(); + } + } + } + + // Resolve modes left undecided by SCCP, following recursive inputs for + // evidence. Float wins because integer operands can be converted to float. + private static void resolveNumericModes(CodeGen code, Ary<Node> modes) { + final BitSet visited = new BitSet(); + for( Node n : modes ) { + if( !n.isDead() && n instanceof ModeNode modeNode && modeNode.mode()==0 ) { + visited.clear(); + visited.set(n._nid); // Inspect inputs, not Bool's integer result + int mode = 0; + for( int i=1; i<n.nIns(); i++ ) + mode |= numericEvidence(n.in(i),visited); + // A closed recursive numeric component can have no remaining + // concrete evidence after SCCP, e.g. dead all-integer loops. + // With no float evidence, settle to integer. + modeNode.setMode((byte)((mode&2)==2 ? 2 : 1)); + code.add(n); + code.addAll(n.outs()); + } + } + } + + // Gather numeric evidence through Phi and interprocedural return cycles. + // The visited set makes self- and mutually-recursive calls finite. + private static int numericEvidence(Node n, BitSet visited) { + if( n == null ) return 0; + if( visited.get(n._nid) ) return 0; + visited.set(n._nid); + + Type t = n._type; + if( t instanceof TypeFloat ) return 2; + if( t instanceof TypeInteger ) return 1; + if( t!=Type.BOTTOM && !(t instanceof TypeTuple) ) + return 0; // Something unrelated, like a ptr + + int evidence = 0; + assert !(n instanceof ConvertNode); // No test case + switch( n ) { + case ModeNode modeNode when modeNode.mode() != 0 -> { return modeNode.mode(); } + case ModeNode modeNode -> {} + case PhiNode phi -> {} + case ReturnNode ret -> {} + case ProjNode proj when proj.in(0) instanceof CallEndNode cend -> { + for( int i=1; i<cend.nIns(); i++ ) + evidence |= numericEvidence(cend.in(i),visited); + } + default -> { return 0; } + } + for( int i=1; i<n.nIns(); i++ ) + evidence |= numericEvidence(n.in(i),visited); + + return evidence; + } +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/ParseAll.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/ParseAll.java new file mode 100644 index 00000000..1218c956 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/ParseAll.java @@ -0,0 +1,449 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.parser.Parser; +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.print.*; +import com.seaofnodes.simple.type.*; +import com.seaofnodes.simple.util.*; +import java.io.File; +import java.io.IOException; +import java.util.*; + +// Parse a given file, and all its dependencies +public abstract class ParseAll { + + static void parse(CodeGen code, String src) { + Parser.TYPES.clear(); + Parser.TYPES.putAll(Parser.INIT_TYPES); + if( src != null ) { + // No source file, just the source itself. + parseSource(code,code._srcName==null ? "Test" : code._srcName,src); + } else { + // Path from module root to source file + parsePath(code,code._srcName); + } + } + + // Worklist of symbols to search and files to parse + private static final Ary<CompUnit> WORK = new Ary<>(CompUnit.class); + + private static final Ary<CompUnit> NEEDS_LOAD = new Ary<>(CompUnit.class); + + // Return a compilation unit (cached or new). If new, it does not necessarily + // need to be compiled. + public static CompUnit makeCUnit( CodeGen code, CompUnit par, String name ) throws IOException { + return makeCUnit(code,par,name,null,false); + } + + private static CompUnit makeCUnit( CodeGen code, CompUnit par, String name, File obj ) throws IOException { + return makeCUnit(code,par,name,obj,obj!=null); + } + + private static CompUnit makeCUnit( CodeGen code, CompUnit par, String name, File obj, boolean external ) throws IOException { + // Slashed file name? Move leading parts into parent CUs. + int idx = name.indexOf("/"); + if( idx >= 0 ) { + String head = name.substring(0,idx); + CompUnit sub = makeCUnit(code,par,head,external ? externalObj(code,par,head) : null,external); + return makeCUnit(code,sub,name.substring(idx+1).intern(),obj,external); + } + + String fname = par==null ? name : (par._fname + "/" + name).intern(); + CompUnit cunit = code._compunits.get(fname); + if( cunit != null ) return cunit; + String cname = par==null ? name : (par._cname + "." + name).intern(); + code._compunits.put(fname, cunit = external ? new CompUnit(obj,par,fname,cname,name) : new CompUnit(par,fname,cname,name)); + return cunit; + } + + // Parse a top-level string, used for e.g. testing + static void parseSource( CodeGen code, String src ) { + parseSource(code,"Test",src); + } + + // Parse a top-level string with an explicit compilation-unit name. + static void parseSource( CodeGen code, String fname, String src ) { + CompUnit cunit = new CompUnit(fname,src); + code._compunits.put(cunit._fname,cunit); + parseAll(code,cunit); + } + + // module_root/.../src.smp is source file name. + static void parsePath(CodeGen code, String fname) { + try { + // Split and build the parent-path down to the source file. + CompUnit cu = makeCUnit( code, null, fname ); + // Compile this CompUnit + parseAll(code,cu); + } catch( IOException ioe ) { + throw new RuntimeException(ioe); + } + } + + private static void enqueue(CompUnit cu) { + if( cu._par!=null && (cu._par._src!=null || (cu._par._obj!=null && cu._par._obj.exists())) ) enqueue(cu._par); + if( cu._start==null && WORK.find(cu)==-1 ) WORK.add(cu); + } + + private static void parseAll(CodeGen code, CompUnit cunit) { + WORK.clear(); // Cleanup from prior tests + NEEDS_LOAD.clear(); + // Prime the pump with the first CompUnit + enqueue(cunit); + // Work through all unparsed compilation units and parse + while( !WORK.isEmpty() ) { + CompUnit cu = WORK.remove(0); + // No source code? Means we do not need to parse, but instead need to load the symbols as-if we parsed + if( cu._src==null || cu._src.isEmpty() ) + loadDeps(code,cu); // Load recursive dependents + else + parseOne( code, cu ); // Parse + } + + // Load all pre-compiled CompUnits + for( CompUnit cu : NEEDS_LOAD ) + loadSummaries(code,cu); + + // Many types will have been declared cyclically either internally or + // across multiple compilation units. + + // Close over all recursive types, and upgrade TYPES + Ary<TypeStruct> ary = new Ary<>(TypeStruct.class); + for( Type t : Parser.TYPES.values() ) + if( t instanceof TypeStruct ts ) + ary.add(ts); + TypeStruct[] tss = Type.closeOver(ary.asAry(),Parser.TYPES); + for( TypeStruct ts : tss ) + Parser.TYPES.put(ts._name,ts); + for( CompUnit cu : code._compunits.values() ) + if( cu._ext==null ) + cu._clz = (TypeStruct)Parser.TYPES.get(Parser.addClzPrefix(cu._cname)); + + // Walk over all Nodes, and upgrade the internal constants to the + // closed-over types. + code._stop.walk( (Node n) -> { + n.upgradeType(Parser.TYPES); + if( n instanceof FRefNode fref ) { + boolean alloc = fref._name.endsWith(".<new>"); + String clzName = alloc + ? fref._name.substring(0,fref._name.length()-".<new>".length()).intern() + : fref._name; + TypeStruct clz = (TypeStruct)Parser.TYPES.get(clzName); + if( clz == null ) + return null; + Node con; + if( alloc ) { + Field ctor = clz.field("<new>"); + if( ctor == null || !(ctor._t instanceof TypeFunPtr tfp) ) { + String name = clzName.startsWith(Parser.CLZ) + ? clzName.substring(Parser.CLZ.length()) : clzName; + int dot = name.lastIndexOf('.'); + throw Parser.error("Unknown struct type '"+(dot==-1 ? name : name.substring(dot+1))+"'",fref._loc); + } + FunNode fun = code.link(tfp); + if( fun == null ) return null; + con = new FunPtrNode(tfp,code._start,fun.ret()).peephole(); + } else { + // Class pointer constant: + TypeMemPtr clzptr = TypeMemPtr.make((byte)2,clz,true); + con = code.con(clzptr); + } + // Replace and optimize + fref.addDef(con); + code.add(fref); + } + // Adds sort by NIDs, which just got shuffled + if( n instanceof AddNode add ) + code.add(add); + // All imported bodies are loaded; link them before Iter checks types. + if( n instanceof CallNode call && !NEEDS_LOAD.isEmpty() ) + code.add(call).idealize(); + return null; + } ); + + // The complete public interface is now known. Keep this independent + // of later graph/linker liveness so SCCP computations remain local and + // monotonic. + code.freezePublicInterface(); + } + + // Parse one Simple source code file. Add all the FRefs produced to the worklist. + private static void parseOne( CodeGen code, CompUnit cunit ) { + cunit.setStart(code); + // Parse a source file, return missing external references + Ary<FRefNode> frefs = code.P.parse(cunit); + + // Find all missing external references, or complain. This can trigger more parses. + while( !frefs.isEmpty() ) { + FRefNode fref = frefs.remove(0); + // Symbol not found in source code (or we would not be here). + // Search the module for the fref + try { + CompUnit xcunit = findCUnitModule(code,cunit,fref._name); + if( xcunit == null ) + xcunit = findCUnitExternalSimple(code,fref._name); + if( xcunit != null ) { + // Replace the FRef with the discovered class name. + fref._name = Parser.addClzPrefix(xcunit._cname); + cunit.addDep(xcunit); + } else { + throw new RuntimeException("Undefined name '" + fref._name +"'"); + } + } catch( IOException ioe ) { + throw new RuntimeException(ioe); + } + } + addRequiredTypes(code,cunit); + } + + private static void addRequiredTypes(CodeGen code, CompUnit cunit) { + for( String typeName : code.P._requiredTypes ) { + // A forward use can have been resolved later in this source file. + if( Parser.TYPES.get(typeName) instanceof TypeStruct ts && !ts._fref ) continue; + try { + CompUnit xcunit = findRequiredType(code,cunit,typeName); + if( xcunit != null ) + cunit.addDep(xcunit); + else + assert false : "expected to find required type"; + } catch( IOException ioe ) { + throw new RuntimeException(ioe); + } + } + } + + private static CompUnit findRequiredType(CodeGen code, CompUnit cunit, String typeName) throws IOException { + for( CompUnit cu = cunit; cu != null; cu = cu._par ) { + String symbol = typeName.startsWith(cu._cname+".") + ? typeName.substring(cu._cname.length()+1) + : typeName; + CompUnit xcunit = findCUnitModule(code,cu,symbol); + if( xcunit != null ) + return xcunit; + } + return findCUnitExternalSimple(code,typeName); + } + + // Load the public symbols for this CompUnit, and add dependent CompUnits to the NEEDS_LOAD list. + private static void loadDeps(CodeGen code, CompUnit cunit) { + // Load IR after parsing + if( NEEDS_LOAD.find(cunit) != -1 ) + return; // Already on the to-be-loaded list? + for( CompUnit loaded : NEEDS_LOAD ) + if( loaded._obj.equals(cunit._obj) ) return; // Another class in the same object. + try { + NEEDS_LOAD.add(cunit); + // Load dependent classes + ElfReader elf = code.getElf(cunit._obj); + elf.loadPublicSymbols(); + // Register bundled definitions before resolving dependencies or IR. + for( String fname : elf._units ) + makeCUnit(code,null,fname,cunit._obj); + for( String fname : elf._deps ) { + // Check CompUnit for being up to date + CompUnit sub = code._compunits.get(fname); + if( sub==null ) sub=makeCUnit(code,null,fname,depObj(code,cunit,fname)); + // If not seen before or out of date, parse the new nested cunit + if( sub._clz == null && WORK.find(sub)==-1 && NEEDS_LOAD.find(sub)==-1 ) + enqueue(sub); + } + } catch( IOException ioe ) { + throw new RuntimeException(ioe); + } + } + + private static File depObj(CodeGen code, CompUnit cunit, String fname) { + File obj = CompUnit.dependencyObj(fname,cunit._smp!=null); + if( obj == null ) + throw new RuntimeException("Cannot find Simple dependency '"+fname+"'"); + return cunit._smp!=null && obj.equals(new File(code._buildDir+"/"+fname+".o")) ? null : obj; + } + + private static File externalObj(CodeGen code, CompUnit par, String name) { + String cname = par==null ? name : par._cname + "." + name; + ElfReader elf = code.findExternalSimple(Parser.addClzPrefix(cname)); + return elf == null ? null : elf._file; + } + + private static void loadSummaries(CodeGen code, CompUnit cu) { + ElfReader elf = code.getElf(cu._obj); + elf.loadPublicTypes(code); + cu._clz = elf._clz; + cu.setStart(code); + new ProjNode(cu._start,1,"$mem").init(); + for( FunctionSummary summary : elf._functions ) summary.install(code,cu); + } + + public static void loadCompUnit(CodeGen code, CompUnit cunit) { + // Expecting a previously compiled object file + assert cunit._obj != null; + // Extract published symbols and put them in the internal tables; + // cross-references will pick them up there. + ElfReader elf = code.getElf(cunit._obj); + if( elf._nodes!=null ) return; + StartCUNode oldStart = cunit._start; + StopCUNode oldStop = cunit._stop; + ProjNode oldMem = oldStart==null ? null : oldStart.proj(1); + if( oldMem!=null ) code.releaseReplacedNode(oldMem); + elf.loadSimple(code); + cunit._clz = (TypeStruct)Parser.TYPES.get(Parser.addClzPrefix(cunit._cname)); + StartNode lstart = (StartNode)elf._nodes.at(0); + StopNode lstop = ( StopNode)elf._nodes.last(); + + // Add all new loaded functions to the linker table + for( Node n : lstop._inputs ) { + StopCUNode stop = (StopCUNode)n; + StartCUNode start = stop.start(); + CompUnit cu = code._compunits.get(start._fname); + cu._start = start; + cu._stop = stop ; + + for( Node ret : stop._inputs ) { + FunNode fun = ((ReturnNode)ret).fun(); + FunNode old = code._link(fun.sig().fidx()); + code.link(fun); + if( old!=null && old.summary()!=null && !old.isDead() ) { + old.ret().keep(); + for( int i = 1; i<old.nIns(); i++ ) + if( old.in(i) instanceof CallNode call ) { + call.unlink(old,i--); + call.link(fun); + } else if( old.in(i) instanceof CProjNode proj && proj.in(0) instanceof CLInitNode init ) { + init.unlink(); + init.link(); + i--; + } + old.ret().unkeep().subsume(fun.ret()); + } + } + } + + // Hook the new code into the existing code graph + for( Node lstopcu : lstop._inputs ) + code._stop.addDef(lstopcu); + lstart.subsume(code._start); + if( oldStart!=null && oldStart!=cunit._start ) { + code.releaseReplacedNode(oldStart); + int idx = code._stop._inputs.find(oldStop); + if( idx!=-1 ) code._stop.delDef(idx); + oldStart.subsume(cunit._start); + } + // Saved types describe the library's original compilation. Repair + // only the newly loaded graph; ordinary dependencies carry changes + // into its callers without restarting whole-program analysis. + ProjNode startMem = cunit._start.proj(1); + if( startMem!=null ) code.keepUntilFinalize(startMem); + Iter.imports(code,elf._nodes); + + // Add the new top-level loaded class + Parser.TYPES.put(cunit._clz._name,cunit._clz); + Field inst = cunit._clz.field(cunit._name); + if( inst != null && inst._t instanceof TypeFunPtr tfp && tfp._ret instanceof TypeMem mem ) + Parser.TYPES.put(cunit._cname,mem._t); + } + + // Symbol not found in source code (or we would not be here). + // Search the module for the fref + public static CompUnit findCompUnitOrThrow(CodeGen code, CompUnit cu, String symbol) { + CompUnit cu2 = findCompUnit(code,cu,symbol); + if( cu2 != null ) return cu2; + throw new RuntimeException("Undefined name '" + symbol +"'"); + } + + public static CompUnit findCompUnit(CodeGen code, CompUnit cu, String symbol) { + // Symbol not found in source code (or we would not be here). + // Search the module for the fref + try { + CompUnit local = findCUnitModule(code,cu,symbol); + if( local != null ) + return local; + return findCUnitExternalSimple(code,symbol); + } catch( IOException ioe ) { + throw new RuntimeException(ioe); + } + } + + // Find the symbol in module, starting from cunit. This searches up the nested + // class chain and sideways 1 step, so that any prefix of the class chain can + // be skipped as a shortcut. Symbol can have dotted names, which then go + // down into classes. e.g. + // A/X/Y exists. + // A/B contains X.Y, which is a shortcut for A.X.Y + + // However if A/B contains "Y", the search would be for A.B.Y then A.Y both of + // which do not exist. + private static CompUnit findCUnitModule( CodeGen code, CompUnit cunit, String symbol ) throws IOException { + if( cunit == null ) // Ran off top of module scope? + return null; // Failed search + + // Check self + if( cunit._name.equals(symbol) ) + return cunit; + + // If symbol is itself nested (has embedded dots), move the leading part + // over to the CompUnit path. So e.g.: + // CompUnit{A.B}, Symbol="X.Y" + // becomes + // CompUnit{A.B.X}, Symbol="Y" + int dotIdx = symbol.indexOf("."); + if( dotIdx != -1 ) { + String pre = symbol.substring(0,dotIdx); + String rest = symbol.substring(dotIdx+1 ); + String fname = cunit._fname + "/" + pre; + File f = new File( code._modDir + "/" + fname + ".smp" ); + if( f.exists() ) { + CompUnit sub = makeCUnit(code,cunit,pre); + // Note that "sub" itself is not put on the to-be-parsed-list. We're + // referring to a child class and do not need to parse the middles. + CompUnit dcunit = findCUnitModule(code,sub,rest); + if( dcunit != null ) + return dcunit; // Nested dot search found cunit + } + } + + // Look in mod / fname / name / symbol.smp + String fname = cunit._fname + "/" + symbol; + File f = new File( code._modDir + "/" + fname + ".smp" ); + if( f.exists() ) { + CompUnit sub = makeCUnit(code,cunit,symbol); + // If not seen before or out of date, parse the new nested cunit + if( sub._clz == null && WORK.find(sub) == -1 ) + enqueue(sub); + return sub; + } + + // Look up the scope + return findCUnitModule( code, cunit._par, symbol ); + } + + // Search external Simple objects only. C linkage uses explicit "C" + // declarations, which produce ExternNodes and are resolved by the linker. + public static CompUnit findCUnitExternalSimple( CodeGen code, String symbol ) { + String clzName = symbol.startsWith(Parser.CLZ) ? symbol : Parser.addClzPrefix(symbol); + String cname = clzName.substring(Parser.CLZ.length()); + String fname = cname.replace('.','/'); + + CompUnit cunit = code._compunits.get(fname); + if( cunit != null ) + return cunit; + + ElfReader elf = code.findExternalSimple(clzName); + if( elf == null ) + return null; + + try { + cunit = makeCUnit(code,null,fname,elf._file); + } catch( IOException ioe ) { + throw new RuntimeException(ioe); + } + ElfReader typedElf = code.getElf(cunit._obj); + if( typedElf != null ) typedElf.loadPublicTypes(code); + TypeStruct clz = (TypeStruct)Parser.TYPES.get(clzName); + if( clz == null ) + Parser.TYPES.put(clzName,clz = TypeStruct.make(clzName,true)); + cunit._clz = clz; + if( WORK.find(cunit) == -1 && NEEDS_LOAD.find(cunit) == -1 ) + enqueue(cunit); + return cunit; + } +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/RIPRelSize.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/RIPRelSize.java new file mode 100644 index 00000000..48319006 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/RIPRelSize.java @@ -0,0 +1,14 @@ +package com.seaofnodes.simple.codegen; + +// This Node rip-relative encodings has sizes that vary by delta +public interface RIPRelSize { + // delta is the distance from the opcode *start* to the target start. Each + // hardware target adjusts as needed. X86 measures from the opcode *end* + // and small jumps are 2 bytes, so they'll need to subtract 2. + byte encSize(int delta); + + // Patch full encoding in. Used for both short and long forms. delta + // again is from the opcode *start*. + void patch( Encoding enc, int opStart, int opLen, int delta ); + +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/RegAlloc.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/RegAlloc.java new file mode 100644 index 00000000..ac792988 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/RegAlloc.java @@ -0,0 +1,780 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.util.Ary; +import com.seaofnodes.simple.util.Utils; +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.Type; +import java.util.Arrays; +import java.util.BitSet; +import java.util.IdentityHashMap; + +/** + * "Briggs/Chaitin/Click". + * Graph coloring. + * <p> + * Every def and every use have a bit-set of allowed registers. + * Fully general to bizarre chips. + * Multiple outputs fully supported, e.g. add-with-carry or combined div/rem or "xor rax,rax" setting both rax and flags. + * <p> + * Intel 2-address accumulator-style ops fully supported. + * Op-to-stack-spill supported. + * All addressing modes supported. + * Register pairs could be supported with some grief. + * <p> + * Splitting instead of spilling (simpler implementation). + * Stack slots are "just another register" (tiny stack frames, simpler implementation). + * Stack/unstack during coloring with a marvelous trick (simpler implementation). + * <p> + * Both bitset and adjacency list formats for the interference graph; one of + * the few times it's faster to change data structures mid-flight rather than + * just wrap one of the two. + * <p> + * Liveness computation and interference graph built in the same one pass (one + * fewer passes per round of coloring). + * <p> + * Single-register def or use live ranges deny neighbors their required + * register and thus do not interfere, vs interfering and denying the color + * and coloring time. Basically all function calls do this, but many oddball + * registers also, e.g. older div/mod/mul ops. (5x smaller IFG, the only + * O(n^2) part of this operation). + */ + +public class RegAlloc { + // Main Coloring Algorithm: + // Repeat until colored: + // Build Live Ranges (LRGs) + // - Intersect allowed registers + // If hard conflicts (LRGs with no allowed registers) + // - Pre-split conflicted LRGs, and repeat + // Build Interference Graph (IFG) + // - Self conflicts split now + // Color (remove trivial first then any until empty; reverse assign colors + // - If color fails: + // - - Split uncolorable LRGs + + // Top-level program graph structure + final CodeGen _code; + + public int _spills, _spillScaled; + + private boolean _done; + + // ----------------------- + // Live ranges with self-conflicts or no allowed registers + private final IdentityHashMap<LRG,String> _failed = new IdentityHashMap<>(); + // Loop Phis get one chance to color after only cold self-conflict splits. + // If the same Phi self-conflicts again, split its hot def/backedge also. + private final IdentityHashMap<PhiNode,String> _deferredLoopSelf = new IdentityHashMap<>(); + void fail( LRG lrg ) { + assert lrg.leader(); + _failed.put(lrg,""); + } + boolean success() { return _failed.isEmpty(); } + public boolean done() { return _done; } + + + // ----------------------- + // Map from Nodes to Live Ranges + private final IdentityHashMap<Node,LRG> _lrgs = new IdentityHashMap<>(); + short _lrg_num; + + // Define a new LRG, and assign n + LRG newLRG( Node n, FunNode fun ) { + LRG lrg = lrg(n); + if( lrg!=null ) return lrg; + lrg = new LRG(_lrg_num++, fun); + LRG old = _lrgs.put(n,lrg); assert old==null; + return lrg; + } + + // LRG for n + public LRG lrg( Node n ) { + LRG lrg = _lrgs.get(n); + if( lrg==null ) return null; + LRG lrg2 = lrg.find(); + if( lrg != lrg2 ) + _lrgs.put(n,lrg2); + return lrg2; + } + + // Find LRG for n.in(idx), and also map n to it + LRG lrg2( Node n, int idx ) { + LRG lrg = lrg(n.in(idx)); + return union(lrg,n); + } + + // Union any lrg for n with lrg and map to the union + LRG union( LRG lrg, Node n ) { + LRG lrgn = _lrgs.get(n); + LRG lrg3 = lrg.union(lrgn); + _lrgs.put(n,lrg3); + return lrg3; + } + + // Force all unified to roll up; collect live ranges + LRG[] _LRGS; + void unify() { + Ary<LRG> lrgs = new Ary<>(LRG.class); + for( Node n : _lrgs.keySet() ) { + LRG lrg = lrg(n); + lrgs.setX(lrg._lrg,lrg); + } + _LRGS = lrgs.asAry(); + // Remove unified lrgs from failed set also + for( LRG lrg : _failed.keySet().toArray(new LRG[0]) ) { + if( !lrg.leader() ) { + LRG lrg2 = lrg.find(); + _failed.remove(lrg); + _failed.put(lrg2,""); + } + } + } + + + // Diagnostic lookup: do not compress leaders or rewrite the node mapping. + private LRG _lrg(Node n) { + LRG lrg = _lrgs.get(n); + if( lrg != null ) + while( lrg._leader != null ) lrg = lrg._leader; + return lrg; + } + + public short regnum( Node n ) { + LRG lrg = _lrg(n); + return lrg==null ? -1 : lrg._reg; + } + + // Printable register number for node n + String reg( Node n ) { + LRG lrg = _lrg(n); + return lrg==null ? null : reg(lrg); + } + String reg( LRG lrg ) { + // No register yet, use LRG + if( lrg._reg == -1 ) return "V"+lrg._lrg; + // Chosen machine register unless stack-slot and past RA + String[] regs = _code._mach.regs(); + if( lrg._reg < regs.length || _code._phase.ordinal() <= CodeGen.Phase.RegAlloc.ordinal() || lrg._fun==null ) + return RegMask.reg(regs,lrg._reg); + // Stack-slot past RA uses the frame layout logic + return "[rsp+"+lrg._fun.computeStackOffset(_code,lrg._reg)+"]"; + } + + // ----------------------- + RegAlloc( CodeGen code ) { _code = code; } + + public void regAlloc() { + // Insert callee-save registers + String[] regs = _code._mach.regs(); + long neverSave= _code._mach.neverSave(); + for( CFGNode bb : _code._cfg ) + if( bb instanceof FunNode fun ) { + ReturnNode ret = fun.ret(); + int len = Math.min(regs.length,64); + for( int reg=0; reg<len; reg++ ) + if( !_code._callerSave.test(reg) && ((1L<<reg)&neverSave)==0 ) { + ret.addDef(new CalleeSaveNode(fun,reg,regs[reg])); + assert ret.regmap(ret.nIns()-1).firstReg()==reg; + } + } + // Cache reg masks for New and Call + for( CFGNode bb : _code._cfg ) { + if( bb instanceof CallEndMach cend ) cend.cacheRegs(_code); + for( Node n : bb._outputs ) + if( n instanceof NewNode nnn ) nnn.cacheRegs(_code); + } + // Expensive assert: useful when chasing cross-function allocator edges. + //assert verifyFunctionLocalEdges(); + + // Top driver: repeated rounds of coloring and splitting. + byte round=0; + while( !graphColor(round) ) { + split(round); + if( round >= 7 ) // Really expect to be done soon + throw new IllegalStateException("Register allocation made no progress after eight rounds"); + round++; + } + postColor(); // Remove no-op spills + _done = true; + } + + boolean verifyFunctionLocalEdges() { + for( CFGNode bb : _code._cfg ) + for( Node use : bb._outputs ) { + FunNode useFun = funOf(use); + for( int i=1; i<use.nIns(); i++ ) { + Node def = use.in(i); + if( def==null || def.isConst() || allowedCrossFunctionEdge(use,i,def) ) + continue; + FunNode defFun = funOf(def); + if( defFun != null && useFun != null && defFun != useFun ) + throw new AssertionError(crossFunctionMessage(def,defFun,use,useFun,i)); + } + } + return true; + } + + private static String crossFunctionMessage(Node def, FunNode defFun, Node use, FunNode useFun, int idx) { + StringBuilder sb = new StringBuilder("Cross-function value edge: "); + sb.append(def.uniqueName()).append("(").append(defFun.label()).append(") -> ") + .append(use.uniqueName()).append("(").append(useFun.label()).append(") input ").append(idx) + .append(" def=").append(def).append(" use=").append(use).append("\n all uses:"); + for( Node out : def.outs() ) { + FunNode outFun = funOf(out); + sb.append("\n ").append(out.uniqueName()).append("(") + .append(outFun==null ? "?" : outFun.label()).append(")"); + } + return sb.toString(); + } + + private static boolean allowedCrossFunctionEdge(Node use, int idx, Node def) { + // Function metadata/control edges are not value live ranges. + if( use instanceof FunNode || use instanceof StopNode || use instanceof StopCUNode ) + return true; + // Return-linked function pointers carry function identity/lifetime, not + // an allocator value flowing from the callee body into this function. + if( use instanceof FunPtrNode && def instanceof ReturnNode ) + return true; + // Parm defaults and call-linked argument edges are the entry protocol; + // BuildLRG also skips Parm input 1. + if( use instanceof ParmNode ) + return true; + // Linked CallEnd -> Return edges should be gone before RegAlloc, but + // diagnose allocator value edges rather than call graph metadata here. + if( use instanceof CallEndNode && def instanceof ReturnNode ) + return true; + return false; + } + + private static FunNode funOf(Node n) { + if( n instanceof ParmNode parm ) return parm.fun(); + if( n instanceof ReturnNode ret ) return ret.fun(); + if( n instanceof FunNode fun ) return fun; + if( n instanceof CFGNode cfg ) return cfg.fun(); + CFGNode cfg = n.cfg0(); + return cfg==null ? null : cfg.fun(); + } + + private boolean graphColor(byte round) { + _failed.clear(); + _lrgs.clear(); + _lrg_num = 1; + _LRGS=null; + + return + // Build Live Ranges + BuildLRG.run(round,this) && // if no hard register conflicts + // Build Interference Graph + IFG.build(round,this) && // If no self conflicts or uncolorable + // Conservative coalesce copies + Coalesce.coalesce(round,this) && + // Color attempt + IFG.color(round,this); // If colorable + } + + // ----------------------- + // Split conflicted live ranges. + void split(byte round) { + + // In C2, all splits are handling in one pass over the program. Here, + // in the name of clarity, we'll handle each failing live range + // independently... which generally requires a full pass over the + // program for each failing live range. i.e., might be a lot of + // passes. + + // Sort, to avoid non-deterministic HashMap ordering + LRG[] splits = _failed.keySet().toArray(new LRG[0]); + Arrays.sort(splits, (x,y) -> x._lrg - y._lrg ); + for( LRG lrg : splits ) + split(round,lrg); + } + + // Split this live range, top level heuristic + boolean split( byte round, LRG lrg ) { + assert lrg.leader(); // Already rolled up + + if( lrg._selfConflicts != null ) + return splitSelfConflict(round,lrg); + + // Register mask when empty; split around defs and uses with limited + // register masks. + if( lrg._mask.isEmpty() ) { + if( lrg._1regDefCnt <= 1 && + lrg._1regUseCnt <= 1 && + (lrg._1regDefCnt + lrg._1regUseCnt) > 0 && + // A fixed-register clone with flexible uses must split at the uses. + !(lrg._1regDefCnt==1 && lrg._machDef.isClone() && lrg._1regUseCnt==0) ) + return splitEmptyMaskSimple(round,lrg); + // Repeated single-reg uses from a single def. Special for archs + // with more fixed regs. + if( !lrg._multiDef && lrg._1regDefCnt <= 1 && lrg._1regUseCnt > 2 ) + if( splitEmptyMaskByUse(round,lrg) ) + return true; + } + if( lrg._mask.isEmpty() && lrg._multiDef && lrg._1regUseCnt > 1 && lrg._1regDefCnt <= 1 ) { + if( splitEmptyMaskSingleUses(round,lrg) ) + return true; + } + + // Generic split-by-loop depth. + return splitByLoop(round,lrg); + } + + // Split live range with an empty mask. Specifically forces splits at + // single-register defs or uses and not elsewhere. + boolean splitEmptyMaskSimple( byte round, LRG lrg ) { + // Live range has a single-def single-register, and/or a single-use + // single-register. Split after the def and before the use. Does not + // require a full pass. + + // Compute def nOuts *before* replacing those uses with a split + int defNouts = ((Node)lrg._machDef).nOuts(); + // Split just after def + if( lrg._1regDefCnt==1 && !lrg._machDef.isClone() ) + // Force must-split, even if a prior split same block because register + // conflicts. Example: + // alloc + // V1/rax - forced by alloc + // alloc + // V2/rax - kills prior RAX + // st4 [V1],len - No good, must split around + insertAfterAndReplace( makeSplit("def/empty1",round,lrg), (Node)lrg._machDef, false/*true*/); + // Split just before use + if( lrg._machUse!=null && (lrg._1regUseCnt==1 || (lrg._1regDefCnt==1 && defNouts==1)) ) + insertBefore((Node)lrg._machUse,lrg._uidx,"use/empty1",round,lrg); + return true; + } + + // Popular single-def value: share a split among uses with compatible masks. + boolean splitEmptyMaskByUse( byte round, LRG lrg ) { + Node def = (Node)lrg._machDef; + // Snapshot distinct users: rewriting an input changes def's output list, + // and a call can use the same value in more than one argument. + Ary<Node> uses = new Ary<>(Node.class); + for( Node use : def._outputs ) + if( uses.find(use)==-1 ) uses.push(use); + + Ary<RegMask> rclass = new Ary<>(RegMask.class); + int ncalls=0; + for( Node use : uses ) + if( use instanceof MachNode mach ) { + // Sharing copies across several calls usually loses to their kills. + if( use instanceof CallNode && ++ncalls > 1 ) return false; + for( int i=1; i<use.nIns(); i++ ) + if( use.in(i)==def && mach.regmap(i)!=null ) + putIntoRegClass(rclass,mach.regmap(i)); + } + if( rclass._len <= 1 ) return false; + + for( RegMask rmask : rclass ) { + Node split = makeSplit(def,"popular",round,lrg,rmask); + split.insertAfter(def); + if( split.nIns()>1 ) split.setDef(1,def); + for( Node use : uses ) + if( use instanceof MachNode mach ) + for( int i=1; i<use.nIns(); i++ ) { + RegMask mask = mach.regmap(i); + if( use.in(i)==def && mask!=null && mask.overlap(rmask) ) { + assert rmask.and(mask)==rmask; + use.setDefOrdered(i,split); + } + } + } + return true; + } + + // Empty-mask live range with several direct single-register uses, often + // through a Phi with multiple defs. Split those uses directly instead of + // doing a loop-depth split, which might keep isolating the wrong edge. + boolean splitEmptyMaskSingleUses( byte round, LRG lrg ) { + findAllLRG(lrg); + if( _ns._len <= 5 ) + return false; + boolean progress = false; + for( Node n : _ns ) { + if( !(n instanceof MachNode mach) ) continue; + for( int i=1; i<n.nIns(); i++ ) { + RegMask use = mach.regmap(i); + if( use != null && use.size1() && lrg(n.in(i))==lrg ) { + insertBefore(n,i,"use/emptyN",round,lrg); + progress = true; + } + } + } + return progress; + } + + // Narrowing preserves all earlier users of a class; new classes are disjoint. + // Each use contains its first overlapping class, so one pass suffices. + private static void putIntoRegClass( Ary<RegMask> rclass, RegMask rmask ) { + for( int i=0; i<rclass._len; i++ ) { + RegMask omask = rclass.at(i); + if( !omask.overlap(rmask) ) continue; + if( omask.and(rmask)!=omask ) + rclass.set(i,new RegMask(omask.copy().and(rmask))); + return; + } + rclass.push(rmask); + } + + + // Self conflicts require Phis (or two-address). + // Insert a split after every def. + boolean splitSelfConflict( byte round, LRG lrg ) { + // Sort conflict set, so we're deterministic + Node[] conflicts = lrg._selfConflicts.keySet().toArray(new Node[0]); + Arrays.sort(conflicts, (x,y) -> x._nid - y._nid ); + + // First encounter with a loop Phi: try only cold splits this round. + // Remember the Phi across rounds so a persistent self-conflict forces + // the hot def/backedge splits on the next encounter. + IdentityHashMap<PhiNode,String> deferred = new IdentityHashMap<>(); + for( Node def : conflicts ) { + if( def instanceof PhiNode phi && phi.region() instanceof LoopNode ) + deferLoopSelf(phi,deferred); + for( Node use : def._outputs ) + if( use instanceof PhiNode phi && phi.region() instanceof LoopNode && + phi._inputs.find(def)==2 ) + deferLoopSelf(phi,deferred); + } + + // For all conflicts + for( Node def : conflicts ) { + assert lrg(def)==lrg; // Might be conflict use-side + // Split before each use that extends the live range; i.e. is a + // Phi or two-address + for( int i=0; i<def._outputs._len; i++ ) { + Node use = def.out(i); + if( (use instanceof PhiNode phi && + !(deferred.containsKey(phi) && phi.region() instanceof LoopNode && phi._inputs.find(def)==2) && + !(phi.region() instanceof LoopNode loop && phi.in(2)==def && def.cfg0().idepth() > loop.idepth() ) ) || + (use instanceof MachNode mach && mach.twoAddress()!=0 && use.in(mach.twoAddress())==def) ) + insertBefore( use, use._inputs.find(def), "use/self/use",round,lrg ); + } + // Split after the Phi and before its entry. Also isolate inputs + // shared with another Phi: otherwise the shared input keeps the + // conflicting live ranges joined, even after splitting the entry. + if( def instanceof PhiNode phi && !(def instanceof ParmNode) ) { + if( !deferred.containsKey(phi) ) { + SplitNode split = makeSplit("def/self",round,lrg); + insertAfterAndReplace(split,def,false); + if( split.nOuts()==0 ) + split.killOrdered(); + } + for( int i=1; i<(deferred.containsKey(phi) ? 2 : phi.nIns()); i++ ) + if( i==1 || sharedPhiInput(phi,i) ) + insertBefore(phi,i,"use/self/phi",round,lrg); + } + // Split before two-address ops which extend the live range + if( def instanceof MachNode mach && mach.twoAddress()!= 0 ) + insertBefore(def,mach.twoAddress(),"use/self/two",round,lrg); + } + return true; + } + + private void deferLoopSelf(PhiNode phi, IdentityHashMap<PhiNode,String> deferred) { + if( !_deferredLoopSelf.containsKey(phi) ) { + _deferredLoopSelf.put(phi,""); + deferred.put(phi,""); + } + } + + + private static boolean sharedPhiInput(PhiNode phi, int i) { + for( Node use : phi.in(i)._outputs ) + if( use instanceof PhiNode && use!=phi ) return true; + return false; + } + + // Generic: split around the outermost loop with non-split def/uses. This + // covers both self-conflicts (once we split deep enough) and register + // pressure issues. + boolean splitByLoop( byte round, LRG lrg ) { + findAllLRG(lrg); + + // Find min loop depth for all non-split defs and uses. + long ld = (-1L<<32) | 9999; + for( Node n : _ns ) { + if( lrg(n)==lrg ) // This is a LRG def + ld = ldepth(ld,n,n.cfg0()); + // PhiNodes check all CFG inputs + if( n instanceof PhiNode phi ) { + for( int i=1; i<n.nIns(); i++ ) + if( phi.in(i)._type!=Type.TOP ) + ld = ldepth(ld, phi.in(i), phi.region().cfg(i)); + } else { + // Others check uses + for( int i=1; i<n.nIns(); i++ ) + if( lrgSame(n.in(i),lrg) ) // This is a LRG use + ld = ldepth(ld,n,n.cfg0()); + } + } + int min = (int)ld; + int max = (int)(ld>>32); + + boolean emptyMulti = lrg._mask.isEmpty() && lrg._multiDef && lrg._1regDefCnt <= 1 && + (lrg._1regUseCnt > 1 || _ns._len > 5); + // Multi-def empty-mask conflicts need the hot/deep side isolated from + // fixed register uses. Ordinary pressure splits keep the old shallow + // boundary behavior. + for( Node n : _ns ) { + if( n instanceof SplitNode split && !split.isNonStackSplit() && min!=max ) continue; // Ordinary splits already reach stack storage. + if( n.isDead() ) continue; // Some Cloneable went dead by other spill changes + // If this is a 2-address commutable op (e.g. AddX86, MulX86) and the rhs has only a single user, + // commute the inputs... which chops the LHS live ranges' upper bound to just the RHS. + if( n instanceof MachNode mach && lrg(n)==lrg && mach.twoAddress()==1 && mach.commutes() && n.in(2).nOuts()==1 ) + n.swap12(); + + if( lrg(n)==lrg && // This is a LRG def + // At loop boundary, or splitting in inner loop + (min==max || (emptyMulti ? n.cfg0().loopDepth() >= max : n.cfg0().loopDepth() <= min)) ) { + // Cloneable constants will be cloned at uses, not after def + if( !(n instanceof MachNode mach && mach.isClone()) && + // Single user is already a split adjacent + !(n.nOuts()==1 && n.out(0) instanceof SplitNode split && sameBlockNoClobber(split) ) ) + // Split after def in min loop nest + insertAfterAndReplace( makeSplit("def/loop",round,lrg), n,true); + } + + // PhiNodes check all CFG inputs + if( n instanceof PhiNode phi && !(n instanceof ParmNode)) { + for( int i=1; i<n.nIns(); i++ ) + // No split in front of a split + if( !(n.in(i) instanceof SplitNode) && + // splitting in inner loop or at loop border + (min==max || (emptyMulti ? phi.region().cfg(i).loopDepth() >= max : phi.region().cfg(i).loopDepth() <= min)) && + // and not around the backedge of a loop (bad place to force a split, hard to remove) + !(phi.region() instanceof LoopNode && i==2 && (phi.in(i) instanceof PhiNode pp && pp.region()==phi.region())) ) + // Split before phi-use in prior block + insertBefore(phi,i, "use/loop/phi",round,lrg); + + } else { + // Others check uses + for( int i=1; i<n.nIns(); i++ ) { + // This is a LRG use + // splitting in inner loop or at loop border + if( lrgSame( n.in( i ), lrg ) && + (min == max || (n.in(i) instanceof MachNode mach && mach.isClone()) || + (emptyMulti ? n.cfg0().loopDepth() >= max : n.cfg0().loopDepth() <= min)) ) + // Split before in this block + insertBefore( n, i, "use/loop/use", round,lrg, false ); + } + } + } + return true; + } + + private boolean lrgSame(Node x, LRG lrg) { + LRG xlrg = lrg(x); + while( xlrg==null && x instanceof SplitNode ) + xlrg=lrg(x = x.in(1)); + return xlrg==lrg; + } + + private static long ldepth( long ld, Node n, CFGNode cfg ) { + // Ordinary splits already reach stack storage; register-only transfers do not. + if( n instanceof SplitNode split && !split.isNonStackSplit() ) return ld; + // Collect min/max loop depth + int min = (int)ld; + int max = (int)(ld>>32); + int d = cfg.loopDepth(); + // if n will lower the min loop and is in the tail end of the loop + // header, splitting "around" the loop will not help. Treat n as being + // in the loop. + if( d < min ) { + if( cfg.uctrl() instanceof LoopNode loop && loop.entry()==cfg ) { + for( int i=cfg.nOuts()-2; i>=0; i-- ) { + Node out = cfg.out(i); + if( n==out ) + { d = loop.loopDepth(); break; } // Treat n as being "in the loop" + if( !((out instanceof MachNode mach && mach.isClone()) || out instanceof SplitNode ) ) + break; // Treat b as "normal", out of loop + } + } + } + + // lower min, raise max, and re-fold + min = Math.min(min,d); + max = Math.max(max,d); + return ((long)max<<32) | min; + } + + // Find all members of a live range, both defs and uses + private final Ary<Node> _ns = new Ary<>(Node.class); + void findAllLRG( LRG lrg ) { + _ns.clear(); + int wd = 0; + if( lrg._machDef != null ) _ns.push((Node)lrg._machDef); + if( lrg._machUse != null ) _ns.push((Node)lrg._machUse); + while( wd < _ns._len ) { + Node n = _ns.at(wd++); + if( lrg(n)!=lrg ) continue; + for( Node def : n._inputs ) + if( lrg(def)==lrg && _ns.find(def)== -1 ) + _ns.push(def); + for( Node use : n._outputs ) + if( _ns.find(use)== -1 ) + _ns.push(use); + } + for( Node n : _ns ) assert !n.isDead(); + } + + void insertBefore(Node n, int i, String kind, byte round, LRG lrg, boolean skip) { + Node def = n.in(i); + // Phi splitting may visit the synthetic never-exit arm. + if( def._type==Type.TOP ) return; + // Effective block for use + CFGNode cfg = n instanceof PhiNode phi ? phi.region().cfg(i) : n.cfg0(); + // Use-side RegMask, if available + RegMask umask = n instanceof MachNode mach ? mach.regmap(i) : null; + // Def is a split ? + if( skip && def instanceof SplitNode ) { + boolean singleReg = umask!=null && umask.size1(); + // Same block, multiple registers, split is only used by n, + // assume this is good enough and do not split again. + if( cfg==def.cfg0() && def.nOuts()==1 && !singleReg ) + return; + } + makeSplit(def,kind,round,lrg,umask).insertBefore(n, i); + // Skip split-of-split same block + if( skip && def instanceof SplitNode && cfg==def.cfg0() ) + n.in(i).setDefOrdered(1,def.in(1)); + } + void insertBefore(Node n, int i, String kind, byte round, LRG lrg) { + insertBefore(n,i,kind,round,lrg,true); + } + + // Replace uses of `def` with `split`, and insert `split` immediately after + // `def` in the basic block. + public void insertAfterAndReplace( Node split, Node def, boolean must ) { + split.insertAfter(def); + if( split.nIns()>1 ) split.setDef(1,def); + for( int j=def.nOuts()-1; j>=0; j-- ) { + Node use = def.out(j); + if( use==split ) continue; // Skip self + // Can we avoid a split of a split? 'this' split is used by + // another split in the same block. + if( !must && use instanceof SplitNode split2 && sameBlockNoClobber(split2) ) + continue; + int idx = use._inputs.find(def); + use.setDefOrdered(idx,split); + if( j < def.nOuts() ) j++; + } + } + + private Node makeSplit( Node def, String kind, byte round, LRG lrg, RegMask umask ) { + // Clone simple constants if possible + Node split = def instanceof MachNode mach && mach.isClone() && (umask==null || mach.outregmap().overlap(umask)) + ? mach.copy() + : _code._mach.split(lrg,kind,round); + _lrgs.put(split,lrg); + return split; + } + private SplitNode makeSplit( String kind, byte round, LRG lrg ) { + SplitNode split = _code._mach.split(lrg,kind,round); + _lrgs.put(split,lrg); + return split; + } + + + // ----------------------- + // POST PASS: Remove empty spills that biased-coloring made + private void postColor() { + int maxReg = -1; + for( CFGNode bb : _code._cfg ) { // For all ops + if( bb instanceof FunNode ) + maxReg = -1; // Reset for new function + // Compute frame size, based on arguments and largest reg seen + if( bb instanceof ReturnNode ret ) + ret.fun().computeFrameAdjust(_code,maxReg); + // Raise frame size by max stack args passed, even if ignored + if( bb instanceof CallEndMach cend ) + maxReg = Math.max(maxReg,cend._xslot); + + for( int j=0; j<bb.nOuts(); j++ ) { + Node n = bb.out(j); + if( lrg(n)!=null ) + maxReg = Math.max(maxReg,lrg(n)._reg+1); + // Raise frame size by max stack args passed to New + if( n instanceof NewNode nnn ) + maxReg = Math.max(maxReg,nnn._xslot); + + if( !(n instanceof SplitNode ) ) continue; + int defreg = lrg(n )._reg; + int usereg = lrg(n.in(1))._reg; + // Attempt to bypass split + if( defreg != usereg && splitBypass(bb,j,n,defreg) ) + usereg = lrg(n.in(1))._reg; + + // Same register, or the next instruction can use the source directly. + if( defreg == usereg || splitForward(bb,j,n,usereg) ) { + n.removeSplit(); + j--; + continue; + } + // Split is kept; count against split score + _spills++; + _spillScaled += (1<<bb.loopDepth()*3); + assert _spillScaled >= 0; + } + } + } + + // A sole, adjacent use can read the source register directly. Nothing can + // clobber it in between; only the operand mask and two-address tie remain. + private boolean splitForward( CFGNode bb, int j, Node split, int reg ) { + if( split.nOuts()!=1 || j+1==bb.nOuts() ) return false; + Node use = split.out(0); + if( use!=bb.out(j+1) || use instanceof PhiNode || use instanceof CFGNode || + !(use instanceof MachNode mach) ) return false; + int i = use._inputs.find(split); + RegMask mask = mach.regmap(i); + return mask!=null && mask.test(reg) && mach.twoAddress()!=i; + } + + private boolean splitBypass( CFGNode bb, int j, Node lo, int defreg ) { + // Attempt to bypass split + Node hi = lo.in(1); + while( true ) { + if( !(hi instanceof SplitNode && lo.cfg0() == hi.cfg0()) ) + return false; + if( lrg(hi.in(1))._reg==defreg ) + break; + hi = hi.in(1); + } + // Walk backward from the later copy, including instructions without results. + for( int idx = j-1; bb.out(idx) != hi; idx--) { + Node n = bb.out(idx); + if( clobbers(n,defreg) ) + return false; // Clobbered + } + lo.setDefOrdered(1,hi.in(1)); + return true; + } + + + private boolean clobbers(Node n, int reg) { + LRG lrg = lrg(n); + return lrg!=null && (lrg._reg==reg || lrg._reg==-1 && lrg._mask.size1() && lrg._mask.firstReg()==reg) || + n instanceof MachNode mach && mach.killmap()!=null && mach.killmap().test(reg); + } + + public boolean sameBlockNoClobber( SplitNode split ) { + Node def = split.in(1); + CFGNode cfg = def.cfg0(); + if( cfg != split.cfg0() ) return false; // Not same block + // Get multinode head + Node def0 = def instanceof ProjNode ? def.in(0) : def; + int defreg = lrg(def)._reg; + if( defreg == -1 ) defreg = lrg(def)._mask.firstReg(); + if( defreg == -1 ) return false; // no allowed registers -> clobbered + for( int idx = cfg._outputs.find(split) -1; idx >= 0; idx-- ) { + Node n = cfg.out(idx); + if( n==def0 ) return true; // No clobbers + if( lrg(n) == lrg(def) ) return false; // Self conflict + if( clobbers(n,defreg) ) + return false; // Clobbered + } + throw Utils.TODO("Should not reach here"); + } +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/RegMask.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/RegMask.java new file mode 100644 index 00000000..9cb9ec16 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/RegMask.java @@ -0,0 +1,121 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.util.SB; +import com.seaofnodes.simple.util.Utils; + +/** RegMask + * A "register mask" - 1 bit set for each allowed register. In addition, + * "stack slot" registers may be allowed, effectively making the set infinite. + * <p> + * For smaller and simpler machines it suffices to make such masks an i64 or + * i128 (64- or 128-bit integers), and this presentation is by far the better + * way to go... if all register allocations can fit in this bit limitation. + * The allocator will need bits for stack-based parameters and for splits + * which cannot get a register. For a 32-register machine like the X86, add 1 + * for flags - gives 33 registers. Using a Java `long` has 64 bits, leaving + * 31 for spills and stack passing. This is adequate for nearly all + * allocations; only the largest allocations will run this out. However, if + * we move to a chip with 64 registers we'll immediately run out, and need at + * least a 128 bit mask. Since you cannot *return* a 128 bit value directly + * in Java, Simple will pick up a `RegMask` class object. +*/ +public class RegMask { + + long _bits0, _bits1; + + private static final RegMask EMPTY = new RegMask(0L); + public static final RegMask FULL = new RegMask(-1L); + + public RegMask(int bit) { + if( bit < 64 ) _bits0 = 1L<<bit; + else _bits1 = 1L<<(bit-64); + } + public RegMask(long bits ) { _bits0 = bits; } + public RegMask(long bits0, long bits1 ) { _bits0 = bits0; _bits1 = bits1; } + public RegMask(RegMask r) { _bits0 = r._bits0; _bits1 = r._bits1; } + + // AND, with copy-on-write if changing + RegMask and( RegMask mask ) { + if( mask==null ) return this; + long bits0 = _bits0 & mask._bits0; + long bits1 = _bits1 & mask._bits1; + if( bits0==_bits0 && bits1==_bits1 ) return this; + if( bits0==mask._bits0 && bits1==mask._bits1 ) return mask; + if( bits0==0 && bits1==0 ) return EMPTY; + // Update-in-place a mutable mask, or make a defensive copy + return null; + } + RegMask sub( RegMask mask ) { + if( mask==null ) return this; + long bits0 = _bits0 & ~mask._bits0; + long bits1 = _bits1 & ~mask._bits1; + if( bits0==_bits0 && bits1==_bits1 ) return this; + if( bits0==mask._bits0 && bits1==mask._bits1 ) return mask; + if( bits0==0 && bits1==0 ) return EMPTY; + // Update-in-place a mutable mask, or make a defensive copy + return null; + } + + // Fails if bit is set, because this is immutable + public boolean clr( int reg ) { + if( reg < 64 ) return ((_bits0 >> reg)&1)==0; + if( reg < 128 ) return ((_bits1 >> reg)&1)==0; + return true; + } + + + public short firstReg() { + if( isEmpty() ) return -1; + return (short)(_bits0 != 0 + ? Long.numberOfTrailingZeros(_bits0) + : Long.numberOfTrailingZeros(_bits1)+64); + } + public short nextReg(short reg) { + if( reg < 63 ) { + long bits0 = _bits0 & (-1L << (reg+1)); + if( bits0!=0 ) return (short)Long.numberOfTrailingZeros(bits0); + } + long bits1 = reg < 64 ? _bits1 : reg < 127 ? _bits1 & (-1L << (reg-63)) : 0; + return bits1==0 ? -1 : (short)(64+Long.numberOfTrailingZeros(bits1)); + } + + boolean isEmpty() { return _bits0==0 && _bits1==0; } + + public boolean test( int reg ) { + return ((reg<64 ? (_bits0 >> reg) : (_bits1 >> (reg-64))) & 1) != 0; + } + + // checks if the 2 masks have at least 1 bit in common + public boolean overlap( RegMask mask ) { return (_bits0 & mask._bits0)!=0 || (_bits1 & mask._bits1)!=0; } + + // Defensive writable copy + public RegMaskRW copy() { return new RegMaskRW( _bits0, _bits1 ); } + + // Has exactly 1 bit set + public boolean size1() { + return !isEmpty() && (((_bits0 & -_bits0)==_bits0 && _bits1==0) || + ((_bits1 & -_bits1)==_bits1 && _bits0==0)); + } + + // Cardinality + public short size() { return (short)(Long.bitCount(_bits0)+Long.bitCount(_bits1)); } + + @Override public String toString() { return toString(new SB()).toString(); } + public SB toString(SB sb) { + Machine mach = CodeGen.CODE._mach; + if( _bits0==0 && _bits1==0 ) return sb.p("[]"); + sb.p("["); + String[] regs = mach.regs(); + for( int i=0; i<64; i++ ) + if( ((_bits0 >> i)&1) != 0 ) + sb.p(reg(regs,i)).p(","); + for( int i=0; i<64; i++ ) + if( ((_bits1 >> i)&1) != 0 ) + sb.p(reg(regs,i+64)).p(","); + return sb.unchar().p("]"); + } + + public static String reg(String[] regs, int i) { + return i<regs.length ? regs[i] : "[stk#"+i+"]"; + } +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/RegMaskRW.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/RegMaskRW.java new file mode 100644 index 00000000..1da9a3b1 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/RegMaskRW.java @@ -0,0 +1,36 @@ +package com.seaofnodes.simple.codegen; + +// Mutable regmask(writable) +public class RegMaskRW extends RegMask { + public RegMaskRW(long x, long y) { super(x,y); } + // clears bit at position r. Returns true if the mask is still not empty. + public boolean clr(int r) { + if( r < 64 ) _bits0 &= ~(1L<<(r )); + else _bits1 &= ~(1L<<(r-64)); + return _bits0!=0 || _bits1!=0; + } + // sets bit at position r. + public void set(int r) { + if( r < 64 ) _bits0 |= (1L<<(r )); + else _bits1 |= (1L<<(r-64)); + } + @Override RegMaskRW and( RegMask mask ) { + if( mask==null ) return this; + _bits0 &= mask._bits0; + _bits1 &= mask._bits1; + return this; + } + @Override RegMaskRW sub( RegMask mask ) { + if( mask==null ) return this; + _bits0 &= ~mask._bits0; + _bits1 &= ~mask._bits1; + return this; + } + public RegMaskRW or( RegMask mask ) { + if( mask!=null ) { + _bits0 |= mask._bits0; + _bits1 |= mask._bits1; + } + return this; + } +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/Serialize.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Serialize.java new file mode 100644 index 00000000..76829151 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Serialize.java @@ -0,0 +1,716 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.parser.Parser; +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.*; +import com.seaofnodes.simple.util.*; +import java.util.*; + +public class Serialize { + + // Shared by all objects in this serialization pass. The graph and its + // declarations stay unchanged while each compilation unit is written. + private final ClassInitDependencies _init; + private final HashSet<TypeStruct> _pubTypes = new HashSet<>(); + private final HashMap<CompUnit,Ary<TypeStruct>> _classes = new HashMap<>(); + + private Serialize(CodeGen code) { + _init = code._classInitDependencies; + assert _init!=null; // TypeCheck checked this same, still-unmodified graph. + // Preserve the canonical declarations, distinct from sharper graph + // types with the same names. Gather/sort them once for all objects. + TreeMap<String,TypeStruct> named = new TreeMap<>(); + for( Type type : Parser.TYPES.values() ) + if( type instanceof TypeStruct ts ) named.put(ts._name,ts); + for( TypeStruct ts : named.values() ) { + _pubTypes.add(ts); + CompUnit cu = classOwner(code,ts); + if( cu!=null && cu._src!=null ) + _classes.computeIfAbsent(cu,k -> new Ary<>(TypeStruct.class)).add(ts); + } + } + + // Encoding/emulation need only machine code. File export also needs ideal + // IR for a later compiler to import; an explicit Serialize request stops here. + static void forTarget(CodeGen code, CodeGen.Phase target, boolean inMemory) { + if( target==CodeGen.Phase.Serialize || (target.ordinal()>=CodeGen.Phase.Export.ordinal() && !inMemory) ) + code.serialize(); + } + + static void serialize( CodeGen code ) { + // Anonymous functions also need names when another object refers to them. + for( FunNode fun : code._linker ) + if( fun!=null && !fun.isDead() && fun._name==null ) + fun._name=code._fidxs.symbol(fun.sig().fidx()); + ArrayList<CompUnit> units = new ArrayList<>(code._compunits.values()); + units.sort(Comparator.comparing(cu -> cu._fname)); + Serialize writer = new Serialize(code); + for( CompUnit cu : units ) { + if( cu._src==null ) continue; + BAOS serial = writer.serialize(code,cu); + code._serials.put(cu,serial); + if( cu._fname.equals(code._srcName==null ? "Test" : code._srcName) ) code._serial=serial; + } + } + + private BAOS serialize(CodeGen code, CompUnit top) { + Ary<Node> nodes = nodeOrder(code,top); + HashSet<String> xdeps = new HashSet<>(); + // Retain build dependencies even when their calls were inlined away. + if( top._deps!=null ) for( CompUnit dep : top._deps ) + if( dep!=top ) xdeps.add(dep._fname); + if( top._par!=null ) xdeps.add(top._par._fname); + String[] deps = xdeps.toArray(new String[0]); + Arrays.sort(deps); + String[] units = {top._fname}; + TypeStruct clz = top._clz; + + // Compress into bytes + Ary<TypeStruct> classes = _classes.get(top); + BAOS baos = write(nodes, clz, classes, deps, units, code._aliases, code._fidxs, code._rpcs, code._externFunc); + + // --- Expensive bijection assert + if( false ) { + // Inflate into POJOs; renumbers everything + ElfReader elf = new ElfReader(new BAOS(baos.toByteArray())); + readAll(code,elf, code._aliases, code._fidxs, code._rpcs); + BAOS baos2 = write(elf._nodes,elf._clz,classes,elf._deps,elf._units, code._aliases, code._fidxs, code._rpcs, code._externFunc); + + // Bi-jection + for( int i=0; i<baos.size(); i++ ) + assert baos.buf()[i]==baos2.buf()[i] : "bijection mismatch at "+i+": "+(baos.buf()[i]&0xFF)+" != "+(baos2.buf()[i]&0xFF); + assert baos.size()==baos2.size() : "bijection size mismatch: "+baos.size()+" != "+baos2.size(); + } + // --- Expensive bijection assert + + // Record serialized IR for later ELF writing + return baos; + } + + // -------------------------------------------------- + private BAOS write(Ary<Node> nodes, TypeStruct clz, Ary<TypeStruct> classes, String[] depobjs, String[] units, GlobalBits aliases, GlobalBits fidxs, GlobalBits rpcs, HashMap<Integer,String> externFunc) { + // Initialize the mapping from bits/tags to types + Type.TAGOFFS(); + + IdentityHashMap<Node,Integer> anodes = new IdentityHashMap<>(); + for( Node n : nodes ) anodes.put(n,anodes.size()+1); // 0 is null. + // The saved wrappers summarize this object's units only. Keep the + // live whole-program graph intact for subsequent lowering. + TypeTuple stopType = TypeTuple.STOP_HIGH; + for( Node n : nodes.last()._inputs ) + if( anodes.containsKey(n) ) stopType = (TypeTuple)stopType.meet(n._type); + Type startType = TypeTuple.make(Type.CONTROL,stopType.mem(),((StartNode)nodes.at(0)).arg()); + + BAOS baos = new BAOS(); + // A - Print the C0DE header. The format is not versioned yet. + baos.write('C').write('0').write('D').write('E'); + + // Count unique Types + var types = new HashMap<Type,Integer>(); + IdentityHashMap<FunNode,TypeTuple> summaries = new IdentityHashMap<>(); + // Count published types *first*, so can align with published strings + types.put(Type.TOP,0); + types.put( clz, types.size() ); + // Have to visit the innards specially, since they did not get the + // normal recursive visit when first touching a type, because the + // published types are packed early to align with their string names so + // lookup in ElfReader can be fast. + for( int i=0; i<clz.nkids(); i++ ) + clz.at(i).gather(types); + // Gather types from all sources + for( Node n : nodes ) { + n._type.gather(types); + if( n instanceof CallEndNode cend ) cend._rpc.gather(types); + if( n instanceof FunNode fun ) { + fun.sig().gather(types); + TypeTuple state = (TypeTuple)fun.ret()._type; + TypeMem mem = (TypeMem)state.mem(); + // Retention can happen on a path that never returns. The + // unit's escape summary includes those native-call effects. + TypeMem escaped = (TypeMem)((TypeTuple)fun._compunit._stop._type).mem(); + mem=mem.makeFrom(XInt.meet(mem._escFs,escaped._escFs),XInt.meet(mem._escAs,escaped._escAs)); + state=state.makeFrom(1,mem); + summaries.put(fun,state); + state.gather(types); + } + if( n instanceof ParmNode parm ) parm._declaredType.gather(types); + if( n instanceof TypeNode con ) con._con .gather(types); + } + stopType.gather(types); + startType.gather(types); + // A defined class remains importable even when its initializer body dies. + // Nested structs also need their class declaration: <new> and <ctor> + // can be absent from graph types even while their function bodies live. + // Keep the nearest file owner, so an imported child is not re-exported + // merely because its parent was compiled here. + if( classes!=null ) + for( TypeStruct ts : classes ) + ts.gather(types); + + // Only record canonical declarations already reachable from this + // object's graph. Pulling unrelated parser types into this closure + // can include open or partially built types from other units. + Ary<TypeStruct> writtenPubs = new Ary<>(TypeStruct.class); + for( Type type : types.keySet() ) + if( type instanceof TypeStruct ts && _pubTypes.contains(ts) ) + writtenPubs.add(ts); + Arrays.sort(writtenPubs._es,0,writtenPubs._len,Comparator.comparing(ts -> ts._name)); + // Radix sort by Type ID# + Type[] atypes = new Type[types.size()]; + for( Type t : types.keySet() ) + atypes[types.get(t)] = t; + + // Count all unique Strings + var strs = new HashMap<String,Integer>(); + strs.put("",0); // Null string is always 0 + // Find and count *published* Strings first + gather(strs,clz._name); + // Find and count dependent object file string names + if( depobjs != null ) + for( String fname : depobjs ) + gather(strs,fname); + for( String fname : units ) gather(strs,fname); + // Count strings from all types + for( Type t : atypes ) { + if( t instanceof Field fld ) gather(strs,fld._fname); + if( t instanceof TypeStruct ts ) gather(strs,ts . _name); + } + // Count strings used by local->global bit mappings. + aliases.gather(strs); + fidxs .gather(strs); + rpcs .gather(strs); + for( String extern : externFunc.values() ) + gather(strs,extern); + // Count strings from nodes + for( Node n : nodes ) + n.gather(strs); + // Radix sort by string ID# + String[] astrs = new String[strs.size()]; + for( String s : strs.keySet() ) + astrs[strs.get(s)] = s; + + // Check that published strings and types align + assert astrs[1] == ((TypeStruct)atypes[1])._name; + if( depobjs != null ) + for( int i=0; i<depobjs.length; i++ ) + assert astrs[1+i+1] == depobjs[i]; + + + // B - Write unique strings + baos.packed4(depobjs==null ? 0 : depobjs.length); + baos.packed4(strs.size()); + // Write in ID# order; the first depobjs.length of these are dependent strings + for( String s : astrs ) + baos.packedS(s); + + // Defined units are loader metadata, not initialization dependencies. + baos.packed4(units.length); + for( String fname : units ) baos.packed4(strs.get(fname)); + + // B.2 - Write the local->global mappings + aliases.packed(baos,strs); + fidxs .packed(baos,strs); + rpcs .packed(baos,strs); + ArrayList<Integer> externFidxs = new ArrayList<>(externFunc.keySet()); + Collections.sort(externFidxs); + baos.packed2(externFidxs.size()); + for( int fidx : externFidxs ) { + baos.packed2(fidx); + baos.packed2(strs.get(externFunc.get(fidx))); + } + + // C - Write unique Types + baos.packed4(atypes.length); + baos.packed2(writtenPubs._len); + for( TypeStruct ts : writtenPubs ) + baos.packed4(types.get(ts)); + // Write Types in ID# order, no children + for( Type t : atypes ) + t.packed(baos,strs); + // Write Types in ID# order, only child IDs + for( Type t : atypes ) { + int nkids = t.nkids(); + for( int i=0; i<nkids; i++ ) + baos.packed4(types.get(t.at(i))); + } + + + // Function summaries precede the graph, so declarations and call + // behavior are available without allocating the implementation nodes. + int nfuncs = 0; + for( Node n : nodes ) if( n instanceof FunNode ) nfuncs++; + baos.packed4(nfuncs); + for( Node n : nodes ) if( n instanceof FunNode fun ) { + baos.packed2(types.get(fun.sig())); + baos.packed2(types.get(summaries.get(fun))); + baos.packed2(fun._name==null ? 0 : strs.get(fun._name)); + baos.packed4(fun._approxUIDs); + int[] edges = _init._edgess[fun.sig().fidx()]; + baos.packed4(edges==null ? 0 : edges.length); + if( edges!=null ) for( int edge : edges ) baos.packed4(edge); + } + + // D - Write the nodes. Since this is expected to be the bulk of the + // data, we might want to explore various options. + + // How many nodes? + baos.packed4(nodes._len); + + // First cut: 1 byte opcode, optional per-node info, includes variable nIns + for( Node n : nodes ) { + baos.packed1(n.serialTag().ordinal()); + baos.packed2(types.get(n==nodes.at(0) ? startType : n==nodes.last() ? stopType : n._type)); + n.packed(baos,strs,types,anodes); + } + // Write out the input indices packed. + for( Node n : nodes ) { + for( int j=0; j<n.nIns(); j++ ) { + if( !writeInput(n,j,anodes) ) continue; + Node def = n.in(j); + Integer idx = def==null ? Integer.valueOf(0) : anodes.get(def); + // Imported function constants retain their symbol, not a body. + if( idx==null && n instanceof FunPtrNode && j==1 ) idx=0; + // Shared escape analysis belongs to this compilation only. + if( def instanceof GlobalEscapeNode ) idx=0; + assert idx != null : "Missing serialized input for "+n+" input "+j+" def "+def; + baos.packed2(idx); + } + if( n instanceof FunNode fun ) + baos.packed2(anodes.get(fun.ret())); + } + + return baos; + } + + // Omit cross-object call-graph links without unlinking the live IR. + // A parameter follows exactly the same retained callers as its function. + private static boolean writeInput(Node n, int i, IdentityHashMap<Node,Integer> nodes) { + if( n instanceof ParmNode parm ) n=parm.fun(); + if( (n instanceof FunNode || n instanceof CallEndNode) && i>0 ) + return nodes.containsKey(n.in(i)); + if( n instanceof StopNode ) return nodes.containsKey(n.in(i)); + return true; + } + + public static int nIns(Node n, IdentityHashMap<Node,Integer> nodes) { + int len=0; + for( int i=0; i<n.nIns(); i++ ) if( writeInput(n,i,nodes) ) len++; + return len; + } + public static <T> void gather( HashMap<T,Integer> strs, T s) { + if( s!=null && !strs.containsKey(s) ) + strs.put(s,strs.size()); + } + public static void gather( AryInt is, int s) { + if( is.atX(s)==0 ) { + int cnt=0; + for( int i=1; i<is._len; i++ ) + if( is._es[i] != 0 ) + cnt++; + is.setX(s,cnt+1); + } + } + + // A nested declaration belongs to its nearest enclosing file. + private static CompUnit classOwner(CodeGen code, TypeStruct ts) { + if( !Parser.startsClzPrefix(ts._name) ) return null; + String name = ts._name.substring(Parser.CLZ.length()).replace('.','/'); + while( true ) { + CompUnit cu = code._compunits.get(name); + if( cu!=null ) return cu; + int slash = name.lastIndexOf('/'); + if( slash<0 ) return null; + name = name.substring(0,slash); + } + } + + static void read_public_strs(ElfReader elf) { + BAOS bais = elf._bais; + Type.TAGOFFS(); + + // A - Read a header + if( bais.read()!='C' || bais.read()!='0' || bais.read()!='D' || bais.read()!='E' ) + throw new IllegalArgumentException("Unsupported serialized IR format; rebuild the object file"); + + // B - Packed read of #strings, then strings + int ndependents = bais.packed4(); // Number of dependent object files + int nstrs = bais.packed4(); // Total number of strings + String[] strs = new String[nstrs]; + for( int i=0; i<nstrs; i++ ) + strs[i] = bais.packedS(); + elf._strs = strs; + + // Collect dependent file names + elf._deps = new String[ndependents]; + for( int i=0; i<ndependents; i++ ) + elf._deps[i] = strs[1+i+1]; + elf._units = new String[bais.packed4()]; + for( int i=0; i<elf._units.length; i++ ) + elf._units[i] = strs[bais.packed4()]; + } + + // -------------------------------------------------- + record TypeRead(Type[] types, String[] strs, GlobalBits fileAliases, GlobalBits fileFidxs, GlobalBits fileRpcs) {} + + static TypeRead readTypes( CodeGen code, ElfReader elf, GlobalBits aliases, GlobalBits fidxs, GlobalBits rpcs ) { + if( elf._types!=null ) return elf._types; + BAOS bais = elf._bais; + // Initialize the mapping from bits/tags to types + Type.TAGOFFS(); + + // If not read the strings already, read them now + String[] strs = elf._strs; + if( strs == null ) { + // A - Read a header + if( bais.read() != 'C' || bais.read() != '0' || bais.read() != 'D' || bais.read() != 'E' ) + throw new IllegalArgumentException( "Unsupported serialized IR format; rebuild the object file" ); + + // B - Packed read of #strings, then strings + int ndependents = bais.packed4(); // Number of dependent object files + int nstrs = bais.packed4(); // Total number of strings + elf._strs = strs = new String[nstrs]; + for( int i = 0; i < nstrs; i++ ) + strs[i] = bais.packedS(); + + // Collect dependent file names + elf._deps = new String[ndependents]; + for( int i = 0; i < ndependents; i++ ) + elf._deps[i] = strs[1 + i + 1]; + elf._units = new String[bais.packed4()]; + for( int i=0; i<elf._units.length; i++ ) + elf._units[i] = strs[bais.packed4()]; + } + + // B.2 - Read the local->global mappings. Further down we will map + // Elf file-local -> global, then global -> CodeGen.CODE local + GlobalBits fileAliases = GlobalBits.packed(bais,strs); + GlobalBits fileFidxs = GlobalBits.packed(bais,strs); + GlobalBits fileRpcs = GlobalBits.packed(bais,strs); + int nexterns = bais.packed2(); + for( int i=0; i<nexterns; i++ ) { + int fileFidx = bais.packed2(); + String extern = strs[bais.packed2()]; + int fidx = fileFidx < GlobalBits.RESERVED ? fileFidx : fidxs.map(fileFidxs,fileFidx); + String old = code.externFunc(fidx); + if( old == null ) + code.externFunc(fidx,extern); + else + assert old.equals(extern); + } + + + // C - Packed read of #types, then types + int ntypes = bais.packed4(); + int npubTypes = bais.packed2(); + int[] pubTypeIds = new int[npubTypes]; + for( int i=0; i<npubTypes; i++ ) + pubTypeIds[i] = bais.packed4(); + Type[] types = Type.packed(bais,strs,ntypes, Parser.TYPES, + fileAliases,aliases, + fileFidxs ,fidxs, + fileRpcs ,rpcs); + + // aliases.set(0,0); + // Check and collect first published symbols map to TypeStructs + elf._clz = (TypeStruct)types[1/*skip zero*/]; + assert elf._clz._name==strs[1/*skip null ptr*/]; + for( Type type : types ) + if( type instanceof TypeStruct ts ) + Parser.TYPES.put(ts._name,ts); + for( Type type : types ) { + if( !(type instanceof TypeStruct ts) || !Parser.startsClzPrefix(ts._name) ) + continue; + String cname = ts._name.substring(Parser.CLZ.length()); + int dot = cname.lastIndexOf('.'); + String name = dot == -1 ? cname : cname.substring(dot+1); + Field inst = ts.field(name.intern()); + if( inst != null && inst._t instanceof TypeFunPtr tfp && tfp._ret instanceof TypeMem mem && mem._t instanceof TypeStruct its ) + Parser.TYPES.put(cname,its); + } + elf._pubTypes = new TypeStruct[npubTypes]; + for( int i=0; i<npubTypes; i++ ) { + TypeStruct ts = elf._pubTypes[i] = (TypeStruct)types[pubTypeIds[i]]; + Parser.TYPES.put(ts._name,ts); + } + elf._functions = new FunctionSummary[bais.packed4()]; + for( int i = 0; i<elf._functions.length; i++ ) { + TypeFunPtr sig = (TypeFunPtr)types[bais.packed2()]; + TypeTuple state = (TypeTuple)types[bais.packed2()]; + String name = strs[bais.packed2()]; + int size = bais.packed4(); + int[] edges = new int[bais.packed4()]; + for( int j = 0; j<edges.length; j++ ) edges[j]=fidxs.map(fileFidxs,bais.packed4()); + elf._functions[i]=new FunctionSummary(sig,state,name.isEmpty() ? fidxs.symbol(sig.fidx()) : name,size,edges); + } + return elf._types = new TypeRead(types,strs,fileAliases,fileFidxs,fileRpcs); + } + + static void readAll( CodeGen code, ElfReader elf, GlobalBits aliases, GlobalBits fidxs, GlobalBits rpcs ) { + BAOS bais = elf._bais; + TypeRead tread = readTypes(code, elf, aliases, fidxs, rpcs); + Type[] types = tread.types(); + String[] strs = tread.strs(); + GlobalBits fileAliases = tread.fileAliases(); + + // D - Read the nodes + + // Number of nodes + int num = bais.packed4(); + // Packed array of nodes + Ary<Node> nodes = new Ary<>(Node.class); + CompUnit cu = null; + for( int i = 0; i<num; i++ ) { + Node.Tag tag = Node.Tag.VALS[bais.packed1()]; + Type t = types[bais.packed2()]; + Node n = tag.make(bais,strs,types,fileAliases,aliases); + n._type = t; + nodes.push(n); + // A little extra for FunNodes + if( n instanceof StartCUNode startcu ) { + cu = code._compunits.get(startcu._fname==null ? (code._srcName==null ? "Test" : code._srcName) : startcu._fname); + assert cu != null; + } + if( n instanceof StopCUNode stopcu ) { + cu = null; + } + if( n instanceof FunNode fun ) { + assert cu != null; + fun._compunit = cu; + if( fun._name==null || fun._name.isEmpty() ) fun._name=fidxs.symbol(fun.sig().fidx()); + } + if( n instanceof ExternNode ext && ext._con instanceof TypeFunPtr tfp && tfp.isConstant() ) { + String old = code.externFunc(tfp.fidx()); + if( old == null ) + code.externFunc(tfp.fidx(), ext._extern); + else + assert old.equals(ext._extern); + } + } + + // Node edges + for( Node n : nodes ) { + int len = n.nIns(); + for( int i = 0; i<len; i++ ) { + int idx = bais.packed2(); + Node def = idx==0 ? null : nodes.at(idx-1); + // MemMerge precise aliases are encoded by their input slot. + // Remap populated file-local slots into this compilation's + // local alias numbering; null holes carry no identity. + int local = i; + if( n instanceof MemMergeNode && i >= GlobalBits.RESERVED ) { + if( def == null ) continue; + local = aliases.map(fileAliases,i); + } + n.setDefX(local,def); + // Call's output[0] is always the CallEnd + if( n instanceof CallEndNode cend && i==0 ) { + var outs = cend.call()._outputs; + outs.swap(0,outs._len-1); + } + } + // A little extra for FunNodes + if( n instanceof FunNode fun ) { + // Return is hooked like an edge + ReturnNode ret = (ReturnNode)nodes.at(bais.packed2()-1); + fun.setRet(ret); + ret._fun = fun; + } + } + // Add edge from Start to Stop + StartNode start = (StartNode)nodes.at(0); + StopNode stop = (StopNode)nodes.last(); + start.setDef(1,stop); + + elf._nodes = nodes; + } + + // -------------------------------------------------- + public static Ary<Node> nodeOrder( CodeGen code, CompUnit cu ) { + BitSet visit = code.visit(); + Ary<Node> nodes = new Ary<>(Node.class); + visit.set(code._stop._nid); + // FinalizeIR attached every surviving function to its unit's Stop. + for( Node ret : cu._stop._inputs ) + _funWalk(((ReturnNode)ret).fun(),nodes,visit); + + // Gather constants and shared expressions through this unit's inputs. + // Walking their users instead would search other units and miss new + // kinds of shared expressions unless we special-cased each one. + Ary<Node> cons = new Ary<>(Node.class); + cons.add(code._start); + visit.set(code._start._nid); + visit.set(cu._start._nid); + for( Node n : cu._start._outputs ) + if( n instanceof ProjNode ) visit.set(n._nid); + for( Node n : nodes ) + for( int i = 0; i<n.nIns(); i++ ) { + // Parameters omit arguments from callers in other objects, + // just as writeInput omits those call-graph edges. + if( n instanceof ParmNode parm && i>0 && !visit.get(parm.fun().in(i)._nid) ) continue; + constantInputs(n.in(i),cons,visit); + } + // StartCU and memory projection + cons.add(cu._start); + for( Node n : cu._start._outputs ) + if( n instanceof ProjNode ) cons.add(n); + // All the innards of functions + cons.addAll(nodes); + // StopCU, Stop + cons.add(cu._stop); + cons.add(code._stop); + visit.clear(); + return cons; + } + + private static void constantInputs(Node n, Ary<Node> cons, BitSet visit) { + if( n==null || visit.get(n._nid) ) return; + // Call graph edges lead to other functions, not shared values. + if( n instanceof CFGNode && !n.isConst() ) return; + visit.set(n._nid); + cons.add(n); + // A FunPtr's Return edge must not pull in the callee's body. + if( !n.isConst() ) + for( Node def : n._inputs ) constantInputs(def,cons,visit); + } + + // -------------------------------------------------- + public static Ary<Node> nodeOrder( CodeGen code ) { + Ary<Node> nodes = new Ary<>(Node.class); + BitSet visit = code.visit(); + nodes.add(code._start); + + // All the global constants + for( Node n : code._start._outputs ) { + if( n.isConst() ) { + visit.set( n._nid ); + nodes.add( n ); + // Special for Cast:BOT + for( Node cast : n._outputs ) + if( cast != null && cast.nIns() == 2 && cast.in( 0 ) == null ) { + visit.set( cast._nid ); + nodes.add( cast ); + // Array pointer conversion adds the body offset after + // the zero-code PtrToInt. Both nodes are global-value + // descendants of the constant and otherwise have no + // function-local root from which to walk them. + if( cast instanceof PtrToIntNode ) + for( Node add : cast.outs() ) + if( add instanceof AddNode && !visit.get(add._nid) ) { + visit.set(add._nid); + nodes.add(add); + } + } + } + } + + // All the CompUnits + for( CompUnit cu : code._compunits.values() ) { + // Whole CompUnits have no users, went dead + if( cu._src!=null && cu._start != null && !cu._start.isDead() ) { + nodes.add(cu._start); + // A memory-free, nonreturning unit can lose its memory projection. + ProjNode mem=cu._start.proj(1); + if( mem!=null ) nodes.add(mem); + // All the functions, including internal ones, in comp-unit order + for( FunNode fun : code._linker ) + if( fun!=null && !fun.isDead() && fun._compunit == cu ) + _funWalk(fun,nodes,visit); + nodes.add(cu._stop); + } + } + + nodes.add(code._stop); + visit.clear(); + return nodes; + } + + // Walk and print whole functions at a time, in CG order + private static void _funWalk( FunNode fun, Ary<Node> rpo, BitSet visit ) { + int start = rpo._len; + _funRPO(fun,rpo,visit); + // A nonreturning function still owns a Return, even before LoopTree + // inserts synthetic exits. No loop-depth ordering is needed on disk. + _funRPO(fun.ret(),rpo,visit); + int len = rpo._len - start; + for( int i=0; i< len>>1; i++ ) + rpo.swap(start+i,start+len-1-i); + } + + // Walk and gather RPO nodes. + private static void _funRPO(Node n, Ary<Node> rpo, BitSet visit ) { + // Return-linked constants can have users in other compilation units. + if( n.isConst() ) return; + if( visit.get(n._nid) ) return; // Been there, done that + visit.set(n._nid); // Stop recursion + // Walk outputs ordered + if( n instanceof CFGNode ncfg ) { + if( n instanceof IfNode iff ) { + CProjNode c0 = iff.cproj(0); + CProjNode c1 = iff.cproj(1); + _funRPO(c0,rpo,visit); + _funRPO(c1,rpo,visit); + } else if( n instanceof StopNode ) { + // nothing + } else { + // CFG to CFG before CFG to data + for( Node use : n._outputs ) + if( use instanceof CFGNode usecfg && !ncfg.skip(usecfg) ) // Do not walk from Call to Fun + _funRPO(use,rpo,visit); + } + // Walk CFG to data eventually + for( Node use : n._outputs ) + if( !(use instanceof CFGNode) ) + _funRPO(use,rpo,visit); + } else { + // Do not walk from a non-CFG to a CFG; CFGs walk to CFGs to + // preserve CFG order. Do not walk *into* Phis; wait for the + // Region to walk the Phis (otherwise you visit some of the + // Phis ahead of others forcing the Phis to be spread around). + for( Node use : n._outputs ) { + if( use!=null && !(use instanceof CFGNode) && !(use instanceof PhiNode) ) + _funRPO(use,rpo,visit); + } + } + + // If parent is a Multi, do not add (yet). + // If self is a Multi, add self and children. + if( multiChild(n) || n instanceof CallEndNode ) { + // nothing; delay until Multi-head or CallNode + } else if( n instanceof MultiNode ) { + // Now lump all multi/projections together + printMulti(n,rpo); + } else if( n instanceof CallNode call ) { + printMulti(call.cend(),rpo); + rpo.add(call); + } else if( n instanceof RegionNode ) { + // Now lump all Phis together + int old = rpo._len; + for( Node use : n._outputs ) + if( use instanceof PhiNode ) + rpo.add(use); + // Sort by label? Want Phis before control outputs + Arrays.sort( rpo._es, old, rpo._len, (x,y) -> y.label().compareTo(x.label()) ); + rpo.add(n); + } else if( n instanceof StopNode ) { + // nothing + } else { + rpo.add(n); // Post-order add + } + } + + private static boolean multiChild(Node n) { + return n instanceof Proj || n instanceof PhiNode; + } + + private static void printMulti(Node n, Ary<Node> rpo) { + // Now lump all multi/projections together + for( Node use : n._outputs ) + rpo.add(use); + // Sort by projection order + if( !(n instanceof StartNode) ) + Arrays.sort( rpo._es,rpo._len-n.nOuts(),rpo._len, + (x,y) -> x instanceof ProjNode px && y instanceof ProjNode py ? (py.idx() - px.idx()) : (y._nid - x._nid) ); + rpo.add(n); + } + +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/SourceError.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/SourceError.java new file mode 100644 index 00000000..f2f46146 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/SourceError.java @@ -0,0 +1,49 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.util.SB; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +/** Format one parser byte offset. Source is read and text built only on error. */ +public abstract class SourceError { + public static String format(CompUnit cu, int pos, String message) { + String file=cu.sourceFile(); + String name=file==null ? cu._fname+".smp" : file; + byte[] source; + try { + source=cu._src!=null ? cu._src.getBytes(StandardCharsets.UTF_8) + : file==null ? null : Files.readAllBytes(Path.of(file)); + } catch( IOException ex ) { source=null; } + if( source==null || pos<0 || pos>source.length ) + return name+": error: "+message+" (source unavailable at byte offset "+pos+")"; + + int line=1, start=0; + for( int i=0; i<pos; i++ ) + if( source[i]=='\n' ) { line++; start=i+1; } + int end=pos; + while( end<source.length && source[end]!='\n' ) end++; + if( end>start && source[end-1]=='\r' ) end--; + int point=Math.min(pos,end); + String before=new String(source,start,point-start,StandardCharsets.UTF_8); + String text=new String(source,start,end-start,StandardCharsets.UTF_8); + int column=before.codePointCount(0,before.length())+1; + SB sb=new SB().p(name).p(':').p(line).p(':').p(column).p(": error: ").p(message).p("\n "); + int width=0, caret=0; + for( int i=0; i<text.length(); ) { + if( i==before.length() ) caret=width; + int ch=text.codePointAt(i); + if( ch=='\t' ) { + int spaces=8-(width%8); + for( int j=0; j<spaces; j++ ) sb.p(' '); + width+=spaces; + } else { sb.p(new String(Character.toChars(ch))); width++; } + i+=Character.charCount(ch); + } + if( before.length()==text.length() ) caret=width; + sb.p("\n "); + for( int i=0; i<caret; i++ ) sb.p('-'); + return sb.p('^').toString(); + } +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/StaticData.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/StaticData.java new file mode 100644 index 00000000..f813fab5 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/StaticData.java @@ -0,0 +1,177 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.parser.Parser; +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.*; +import com.seaofnodes.simple.util.BAOS; +import java.util.*; + +// Pool objects have identity independently of their initial field values. +// References in their contents use the same symbols in ELF and emulator images. +public final class StaticData { + public static final class ObjectData { + public final Type type; + public final String symbol; + public final boolean readOnly, external; + public final int alignment, size; + public int offset; + ObjectData(Type t, String name, boolean ro, boolean ext) { + type=t; symbol=name; readOnly=ro; external=ext; + alignment=t.alignment(); + int align=1<<alignment; + size=t instanceof TypeStruct ts ? (Math.max(1,ts.size())+align-1)&-align : align; + } + } + public record Relocation(ObjectData object, int offset, int logSize, Type value) {} + + final Encoding enc; + final LinkedHashMap<Object,ObjectData> objects = new LinkedHashMap<>(); + final ArrayList<Relocation> relocations = new ArrayList<>(); + final String prefix; + + public ArrayList<ObjectData> entries() { + ArrayList<ObjectData> entries=new ArrayList<>(objects.values()); + entries.sort(Comparator.comparingInt((ObjectData obj) -> obj.alignment).reversed()); + return entries; + } + + StaticData(Encoding enc) { + this.enc=enc; + prefix=(enc._code._emitUnit==null ? enc._code.entryClinitName() : enc._code._emitUnit._cname)+"$DATA$"; + } + + static boolean isClass(Type t) { + return t instanceof TypeStruct ts && Parser.startsClzPrefix(ts._name); + } + + static CompUnit owner(CodeGen code,TypeStruct ts) { + return owner(code,ts._name.substring(Parser.CLZ.length())); + } + static CompUnit owner(CodeGen code,String name) { + if(name==null) return null; + CompUnit owner=null; + for( CompUnit cu : code._compunits.values() ) + if( cu._cname!=null && (name.equals(cu._cname) || name.startsWith(cu._cname+".")) && + (owner==null || cu._cname.length()>owner._cname.length()) ) owner=cu; + return owner; + } + + ObjectData object(Type t) { + Object key=isClass(t) ? ((TypeStruct)t)._name + : t instanceof TypeFunPtr fp ? functionSymbol(fp) : t; + ObjectData old=objects.get(key); + if( old!=null ) return old; + boolean clz=isClass(t); + if( clz && Parser.TYPES.get(((TypeStruct)t)._name) instanceof TypeStruct canonical ) t=canonical; + CompUnit owner=clz ? owner(enc._code,(TypeStruct)t) + : t instanceof TypeFunPtr fp ? owner(enc._code,enc._code._fidxs.owner(fp.fidx())) : null; + String symbol=clz ? ((TypeStruct)t)._name.substring(Parser.CLZ.length())+".$class" + : t instanceof TypeFunPtr fp ? functionSymbol(fp) : prefix+objects.size(); + ObjectData obj=new ObjectData(t,symbol,!clz && (!(t instanceof TypeStruct ts) || ts.isConstant()), + owner!=null && !enc._code.emits(owner)); + objects.put(key,obj); + // Register before following fields: class objects may refer to each other. + if( !obj.external && t instanceof TypeStruct ts ) + for( Field f : ts._fields ) + if( !f._extern && f._t.isConstant() ) target(f._t); + return obj; + } + + private static boolean pointerConstant(TypeMemPtr ptr) { + return ptr.notNull() && (isClass(ptr._obj) || (ptr._obj.isConstant() && (!ptr._obj.isAry() || ptr._obj.isConAry()))); + } + + private void target(Type t) { + if( t instanceof TypeMemPtr ptr && pointerConstant(ptr) ) object(ptr._obj); + if( t instanceof TypeFunPtr fp && fp.notNull() && + enc._code.link(fp)==null && !enc._code._externFunc.containsKey(fp.fidx()) ) + // A dead function still needs a unique non-null identity for comparisons. + object(fp); + } + + void write() { + ArrayList<Encoding.Relo> refs=new ArrayList<>(enc._bigCons.values()); + refs.sort(Comparator.comparingInt(r -> r._op._nid)); + for( Encoding.Relo ref : refs ) object(ref._t); + // Definitions belong to their compilation unit, even if every local use folded. + ArrayList<CompUnit> units=new ArrayList<>(enc._code._compunits.values()); + units.sort(Comparator.comparing(cu -> cu._cname==null ? "" : cu._cname)); + for( CompUnit cu : units ) if( enc._code.emits(cu) && cu._clz!=null ) object(cu._clz); + for( ObjectData obj : entries() ) { + if( obj.external ) continue; + BAOS bits=obj.readOnly ? enc._cpool : enc._sdata; + Encoding.padN(1<<obj.alignment,bits); + obj.offset=bits.size(); + if( obj.type instanceof TypeStruct ts ) writeStruct(obj,ts,bits); + else Encoding.addN(obj.alignment,obj.type,bits); + while( bits.size()<obj.offset+obj.size ) bits.write(0); + } + for( Encoding.Relo ref : refs ) { + ObjectData obj=object(ref._t); + ref._target=obj.offset; + ref._opStart=enc.opStart(ref._op); + } + } + + private void writeStruct(ObjectData obj, TypeStruct ts, BAOS bits) { + int[] layout=ts.layout(); + for( int idx : layout ) { + if( idx==ts._fields.length ) continue; + Field f=ts._fields[idx]; + if( f._extern ) continue; + int off=ts.offset(idx); + while( bits.size()<obj.offset+off ) bits.write(0); + if( f._fname.equals("[]") ) ((TypeConAry)f._t).write(bits); + else { + Type value=f._t.isConstant() ? f._t : f._t.makeZero(); + int log=f._t.log_size(); + Encoding.addN(log,value,bits); + if( (value instanceof TypeFunPtr fp && fp.notNull()) || + (value instanceof TypeMemPtr ptr && pointerConstant(ptr)) ) + relocations.add(new Relocation(obj,off,log,value)); + } + } + while( bits.size()<obj.offset+ts.size() ) bits.write(0); + } + + String functionSymbol(TypeFunPtr fp) { + String ext=enc._code._externFunc.get(fp.fidx()); + return ext!=null ? ext : enc._code._fidxs.symbol(fp.fidx()); + } + + String targetSymbol(Type value) { + if( value instanceof TypeMemPtr ptr ) return object(ptr._obj).symbol; + TypeFunPtr fp=(TypeFunPtr)value; + return enc._code.link(fp)==null && !enc._code._externFunc.containsKey(fp.fidx()) + ? object(fp).symbol : functionSymbol(fp); + } + + void link(int cpool, int sdata) { + for( Relocation rel : relocations ) { + long address; + if( rel.value instanceof TypeMemPtr ptr ) address=address(object(ptr._obj),cpool,sdata); + else { + TypeFunPtr fp=(TypeFunPtr)rel.value; + String ext=enc._code._externFunc.get(fp.fidx()); + FunNode fun=enc._code.link(fp); + if( ext!=null ) address=enc.externalAddress(ext); + else if( fun==null ) address=address(object(fp),cpool,sdata); + else { + if( !fun.isFromSource() ) throw new IllegalArgumentException("Unlinked function "+functionSymbol(fp)); + address=enc.opStart(fun); + } + } + // Emulator native-call sentinels are signed 32-bit addresses. + if( rel.logSize==2 && (address<Integer.MIN_VALUE || address>0xffff_ffffL) ) + throw new IllegalArgumentException("Function address outside 32 bits: "+address); + BAOS bits=rel.object.readOnly ? enc._cpool : enc._sdata; + int off=rel.object.offset+rel.offset; + for( int i=0; i<(1<<rel.logSize); i++ ) bits.buf()[off+i]=(byte)(address>>>(8*i)); + } + } + + int address(ObjectData obj,int cpool,int sdata) { + if( obj.external ) throw new IllegalArgumentException("Unlinked class object "+obj.symbol); + return obj.offset+(obj.readOnly ? cpool : sdata); + } +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/TypeCheck.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/TypeCheck.java new file mode 100644 index 00000000..c9e53311 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/TypeCheck.java @@ -0,0 +1,59 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.parser.Parser; +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.Type; +import com.seaofnodes.simple.util.Ary; + +// Reject invalid optimized programs before lowering erases their source-level +// obligations. Queue surviving CLInits during the same walk for Lower's Iter. +abstract class TypeCheck { + static void run(CodeGen code) { + // Check for class-init cycles first; such a cycle can prevent other + // things from resolving types. Use the graph and summaries left by + // optimization; do not load more bodies to disprove a possible cycle. + ClassInitDependencies deps = new ClassInitDependencies(code); + String cycle = deps.check(); + if( cycle!=null ) throw Parser.error(cycle,null); + code._classInitDependencies = deps; // Serialize reuses this checked graph. + + Ary<Node> errs = new Ary<>(Node.class); + code._stop.walk( n -> { + if( n.err() != null ) + errs.add(n); + if( n instanceof CLInitNode ) code.add(n); + return null; + }); + + if( !errs.isEmpty() ) { + // Prefer an originating error over a secondary error caused by a + // BOTTOM input. If all errors are downstream-poisoned, retain + // the old first-error behavior. + Node min = null; + for( Node n : errs ) + if( noBottomInputs(n) ) + { min = n; break; } + if( min==null ) + min = errs.at(0); + for( Node n : errs ) + if( n!=min && + noBottomInputs(n) && + ((n instanceof MemOpNode nmop && + min instanceof MemOpNode mmop && + nmop._loc.before(mmop._loc)) || + (n instanceof CFGNode ncfg && + min instanceof CFGNode mincfg && + ncfg.idepth() < mincfg.idepth())) ) + min = n; + if( code._obs != null ) code._obs.error(min, min.err().getMessage()); + throw min.err(); + } + } + + private static boolean noBottomInputs(Node n) { + for( Node def : n._inputs ) + if( def!=null && !(def instanceof FldOffNode) && def._type==Type.BOTTOM ) + return false; + return true; + } +} diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/Unlink.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Unlink.java new file mode 100644 index 00000000..bb9a6196 --- /dev/null +++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Unlink.java @@ -0,0 +1,174 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.util.Ary; + +// Discard call-graph edges after LoopTree. Machine selection keeps only this +// object's bodies; ideal evaluation keeps imports so their calls can execute. +abstract class Unlink { + // Disconnect compiler-only interprocedural edges before scheduling. Keep + // signatures in the linker; runtime calls resolve FIDXs, not graph edges. + static Ary<FunNode> forEval(CodeGen code, StartNode start, StopNode stop) { + Ary<Node> nodes = new Ary<>(Node.class); + start.walk(n -> { nodes.add(n); return null; }); + Ary<FunNode> funs = new Ary<>(FunNode.class); + for( Node n : nodes ) + if( n instanceof FunNode fun ) { + fun.ret().keep(); + if( fun.summary()==null ) funs.add(fun); + } + for( Node n : nodes ) { + if( n instanceof CallNode call ) call.unlink_all(); + if( n instanceof CLInitNode init ) init.unlink(); + if( n instanceof FunPtrNode ptr && ptr.ret()!=null ) ptr.setDef(1,null); + } + for( FunNode fun : funs ) { + // Keep bodies rooted for scheduling after their callers detach. + if( fun.nIns()==1 ) { + fun.addDef(fun._compunit._start); + for( Node use : fun._outputs ) + if( use instanceof ParmNode parm ) { + ConstantNode con = ConstantNode.raw(parm._type); + con.setDef(0,start); + parm.addDef(con); + } + } + restoreRPC(fun); + } + code.releaseUnlinkedNodes(); + for( Node n : nodes ) + if( n instanceof StartCUNode unitStart ) detachEmptyStart(unitStart); + // A summary is a dispatch target, not a schedulable implementation. + // Retain its own wrapper graph without including it in this schedule. + for( int i = 0; i<stop.nIns(); i++ ) { + StopCUNode unit = (StopCUNode)stop.in(i); + boolean summary = false; + for( Node r : unit._inputs ) + summary |= ((ReturnNode)r).fun().summary()!=null; + if( summary || unit.nIns()==0 ) { + unit.start().setDef(0,null); + stop.delDef(i--); + } + } + for( Node n : nodes ) + if( n instanceof FunNode fun ) fun.ret().unkeep(); + return funs; + } + + private static void unlinkImports(CodeGen code) { + // Imported function constants keep their symbolic identity, not an + // embedded copy of the other object's Return and body. + for( Node n : code._start._outputs.asAry() ) + if( n instanceof FunPtrNode ptr && ptr.ret()!=null && !ptr.fun().isFromSource() ) { + ptr.liftType(ptr._type); + ptr.setDef(1,null); + } + for( FunNode fun : code._linker ) { + if( fun==null || fun.isDead() || fun.isFromSource() ) + continue; + for( int i=1; i<fun.nIns(); i++ ) + if( fun.in(i) instanceof CallNode call ) + call.unlink(fun,i--); + } + for( int i=0; i<code._stop.nIns(); i++ ) { + StopCUNode stop=(StopCUNode)code._stop.in(i); + CompUnit cu=code._compunits.get(stop.start()._fname); + if( cu._src==null ) code._stop.delDef(i--); + } + // The whole-object wrappers now summarize only the emitted units. + code._stop ._type=code._stop .compute(); + code._start._type=code._start.compute(); + } + + static void run(CodeGen code, boolean removeImports) { + if( removeImports ) unlinkImports(code); + + // The remaining passes assume all calls are unlinked; i.e. we are throwing + // away the Call Graph here. Functions only reachable from internal calls + // need to be re-hooked to stop/start less they go dead. + for( FunNode fun : code._linker ) { + // Already linked to start, not going dead + if( fun==null || fun.isDead() ) + continue; + + // Imported functions are not emitted by this object. Ideal + // evaluation retains them so their calls can actually execute. + if( removeImports && !fun.isFromSource() ) { + while( fun.nIns() > 1 ) { + if( fun.in(1) instanceof CallNode call ) + call.unlink(fun,1); + else + fun.removeDeadPath(1); + } + StartCUNode start = fun._compunit._start; + StopCUNode stop = fun._compunit._stop; + // Unhook the Stop->Return edge also + ReturnNode ret = fun.ret(); + int idx = stop._inputs.find(ret); + if( idx != -1 ) { + stop._inputs.del(idx); + ret.delUse(stop); + } + // + if( stop.nIns()==0 ) { + idx = code._stop._inputs.find(stop); + if( idx != -1 ) + code._stop.delDef(idx); + if( start.nIns() > 0 && start.in(0) != null ) + start.setDef(0,null); + if( start.nIns() > 1 && start.in(1) != null ) + start.setDef(1,null); + } + continue; + } + + // Insert the post-Opto code-generation hook. Unlike an Opto-time + // Start input, this no longer carries unknown-caller semantics. + ReturnNode ret = fun.ret(); + StartNode start = fun._compunit._start; + assert start!=null; + if( fun.nIns() <= 1 || fun.in(1) != start ) { + fun.insertDef(1,start); + for( Node use : fun._outputs ) + if( use instanceof ParmNode parm ) + parm.insertDef(1,ConstantNode.raw(parm._type)); + if( fun._compunit._stop._inputs.find(ret) == -1 ) + fun._compunit._stop.addDef(ret); + } + // Unlink from Call + for( int i=2; i<fun.nIns(); i++ ) + ((CallNode)fun.in(i)).unlink(fun,i--); + restoreRPC(fun); + } + code.releaseUnlinkedNodes(); + for( CompUnit cu : code._compunits.values() ) + if( cu._start!=null && !cu._start.isDead() ) detachEmptyStart(cu._start); + + code._stop.walk(n -> { + if( n instanceof CallEndNode cend ) + for( int i=1; i<cend.nIns(); i++ ) + if( cend.in(i) instanceof ReturnNode ) + cend.delDef(i--); + if( n instanceof CallNode call ) + call.unlink_all(); + return null; + }); + } + + private static void detachEmptyStart(StartCUNode start) { + // An unused unit can lose its outer Stop edge during optimization. + // Detach its empty wrapper from the scheduling root as well. + if( start.in(1) instanceof StopCUNode stop && stop.nIns()==0 ) start.setDef(0,null); + } + + private static void restoreRPC(FunNode fun) { + if( fun.rpc()!=null ) return; + // A single-call target can have a folded RPC even when a multi-target + // call cannot inline it. Restore the parameter for codegen and for + // Eval2's continuation, now that calls no longer supply graph edges. + ReturnNode ret = fun.ret(); + ParmNode rpc = new ParmNode("$rpc",0,ret.rpc()._type,fun); + rpc.addDef(ret.rpc()); + ret.setDef(3,rpc.init()); + } +} |