summaryrefslogtreecommitdiff
path: root/chapter29/src/main/java/com/seaofnodes/simple/codegen
diff options
context:
space:
mode:
authorCliff Click <cliffc@acm.org>2026-11-01 10:33:55 -0800
committerCliff Click <cliffc@acm.org>2026-11-01 10:33:55 -0800
commitf4bd723d929e53475f7990374dca7b7dfcbbdc42 (patch)
tree7bb080119e7b88ad4ac9b30af019fe6a5e4ba9dd /chapter29/src/main/java/com/seaofnodes/simple/codegen
parent0ac0c9e10287972ac3af5a4ee43d26f0d5b318fe (diff)
downloadSimple-main.tar.gz
Simple-main.zip
Add Chapter 29 for tech debt cleanup with the parser splitmain
Keep Chapter 28b intact and carry the grammar-oriented parser package, Var move, IterPeeps move, and parser regression tests into Chapter 29. Defer privacy and exclude its experimental tests and fixture changes. Register the chapter in the indexes, IntelliJ modules and run configurations, Maven reactor, roadmap, and linear-history publisher. Windows and Linux each pass 573 tests, release builds, 199 allocation replays, and 29 native checks.
Diffstat (limited to 'chapter29/src/main/java/com/seaofnodes/simple/codegen')
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/BuildLRG.java101
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/ClassInitDependencies.java238
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/Coalesce.java105
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/CodeGen.java889
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/CompUnit.java170
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/ElfReader.java245
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/ElfWriter.java468
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/Encoding.java570
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/FinalizeIR.java44
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/FunctionSummary.java28
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/GlobalBits.java177
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/GlobalCodeMotion.java356
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/IFG.java586
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/InstSelect.java126
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/Iter.java101
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/IterPeeps.java349
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/LRG.java240
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/LinkMem.java33
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/ListScheduler.java265
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/Machine.java127
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/Module.java98
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/Opto.java331
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/ParseAll.java449
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/RIPRelSize.java14
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/RegAlloc.java780
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/RegMask.java121
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/RegMaskRW.java36
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/Serialize.java716
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/SourceError.java49
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/StaticData.java177
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/TypeCheck.java59
-rw-r--r--chapter29/src/main/java/com/seaofnodes/simple/codegen/Unlink.java174
32 files changed, 8222 insertions, 0 deletions
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/BuildLRG.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/BuildLRG.java
new file mode 100644
index 00000000..0b314978
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/BuildLRG.java
@@ -0,0 +1,101 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.type.Type;
+import com.seaofnodes.simple.type.TypeMem;
+
+abstract public class BuildLRG {
+ // Compute live ranges in a single forwards pass. Every def is a new live
+ // range except live ranges are joined at Phi. Also, gather the
+ // intersection of register masks in each live range. Example: "free
+ // always zero" register forces register zero, and calling conventions
+ // typically force registers.
+
+ // Sets FAILED to the set of hard-conflicts (means no need for an IFG,
+ // since it will not color, just split the conflicted ranges now). Returns
+ // true if no hard-conflicts, although we still might not color.
+ public static boolean run(int round, RegAlloc alloc) {
+ FunNode fun = null;
+ for( Node bb : alloc._code._cfg ) {
+ if( bb instanceof FunNode fun2 )
+ fun = fun2;
+ for( Node n : bb.outs() ) {
+ // A synthetic never-exit value has no runtime definition.
+ if( n._type==Type.TOP ) continue;
+ if( n instanceof PhiNode phi && !(phi._type instanceof TypeMem) ) {
+ if( phi instanceof MachNode mach && mach.outregmap()==null )
+ continue;
+ // All real Phi inputs end up with the same LRG.
+ // Pass 1: find any pre-existing LRG, to avoid make-then-Union a LRG
+ LRG lrg = alloc.lrg(phi);
+ if( lrg == null )
+ for( int i=1; i<phi.nIns(); i++ )
+ if( (lrg = alloc.lrg(phi.in(i))) != null )
+ break;
+ // If none, make one.
+ if( lrg==null ) lrg = alloc.newLRG(n,fun);
+ if( phi instanceof MachNode ) defLRG(alloc,n,fun);
+ // Pass 2: everybody uses the same LRG
+ lrg=alloc.union(lrg,phi);
+ for( int i=phi instanceof ParmNode ? 2 : 1; i<n.nIns(); i++ )
+ // TOP keeps the type merge neutral, but needs no register.
+ if( n.in(i)._type!=Type.TOP )
+ lrg=alloc.union(lrg,n.in(i));
+ if( lrg._mask!=null && lrg._mask.isEmpty() )
+ alloc.fail(lrg);
+
+ } else if( n instanceof MachNode mach ) {
+ // Attempt to commute ops to keep live ranges compatible.
+ if( mach.commutes() && n.nOuts()==1 ) {
+ int uidx = n.out(0)._inputs.find(n);
+ RegMask mask1 = n.in (1) instanceof MachNode machx ? machx.outregmap() : alloc.lrg(n.in(1))._mask;
+ RegMask mask2 = n.in (2) instanceof MachNode machx ? machx.outregmap() : alloc.lrg(n.in(2))._mask;
+ RegMask masko = n.out(0) instanceof MachNode machx ? machx.regmap(uidx): mach.outregmap();
+ if( masko!=null && !mask1.overlap(masko) && mask2.overlap(masko) )
+ n.swap12();
+ }
+
+ // Define live range
+ defLRG(alloc,n,fun);
+
+ // Now, look in the opposite direction. How are incoming
+ // LRGs affected by this node: For all uses, make live lrgs
+ for( int i=1; i<n.nIns(); i++ )
+ if( n.in(i)!=null ) {
+ LRG lrg2 = alloc.lrg(n.in(i));
+ if( lrg2 != null ) { // Anti-dep or other, no LRG
+ RegMask use_mask = mach.regmap(i); // use_mask is also null for anti-dep
+ if( use_mask!=null && !lrg2.machUse(mach,(short)i,use_mask.size1()).and(use_mask) )
+ alloc.fail(lrg2); // Empty register mask, must split
+ }
+ }
+
+ }
+
+ // MultiNodes have projections which set registers
+ if( n instanceof MultiNode && !(n instanceof CFGNode) )
+ for( Node proj : n.outs() )
+ if( proj instanceof MachNode )
+ defLRG(alloc,proj,fun);
+ }
+ }
+
+ // Collect live ranges
+ alloc.unify();
+
+ return alloc.success();
+ }
+
+ private static void defLRG( RegAlloc alloc, Node n, FunNode fun ) {
+ MachNode mach = (MachNode)n;
+ RegMask def_mask = mach.outregmap();
+ if( def_mask == null ) return;
+ LRG lrg = mach.twoAddress() == 0
+ ? alloc.newLRG(n,fun) // Define a new LRG for N
+ : alloc.lrg2(n,mach.twoAddress()); // Use the matching 2-adr input
+ // Record mask and mach
+ if( !lrg.machDef(mach,def_mask.size1()).and(def_mask) )
+ alloc.fail(lrg); // Empty register mask, must split
+ }
+
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/ClassInitDependencies.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/ClassInitDependencies.java
new file mode 100644
index 00000000..3547b86c
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/ClassInitDependencies.java
@@ -0,0 +1,238 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.parser.Lexer;
+import com.seaofnodes.simple.parser.Parser;
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.type.*;
+import com.seaofnodes.simple.util.*;
+import java.util.*;
+
+// Search for Class Initialization Cycles.
+
+// Done after Opto, so dead code does not make a cycle.
+
+// Build the inits-graph, a call-graph plus CLInits plus class parents.
+
+// (1) Scan all functions f, building _edges[f][]: the set of <clinit> fcns
+// from CompUnits from CLInitNodes, and the set of fcn call targets in f and
+// the class parent. These are all reported as fidxs (ints).
+//
+// (2) Walk the _edges graph using Tarjans SCC. Reject cycles containing more
+// than one <clinit>.
+//
+// (3) Report on <clinit> cycles, if any
+//
+public class ClassInitDependencies {
+ private final CodeGen _code;
+ // Function FIDX -> callees, touched initializers, and parent initializer.
+ public final int[][] _edgess;
+ private final int[] _pres;
+ private final int[] _lows;
+ private final int[] _stk ;
+ private final int[] _spos; // Stack position; FIDX -> stack slot + 1, or 0 when absent.
+ private int _sp;
+ private int _pre;
+
+
+ // Build once; tests can inspect the graph independently of cycle checking.
+ public ClassInitDependencies( CodeGen code ) {
+ _code = code;
+ assert code._phase.ordinal() >= CodeGen.Phase.IterPostOpto.ordinal();
+ _pres = new int[code._linker._len];
+ _lows = new int[code._linker._len];
+ _stk = new int[code._linker._len];
+ _spos = new int[code._linker._len];
+
+ //
+ BitSet visit = code.visit(); // Visit set per-fcn
+ Ary<CFGNode> work = new Ary<>(CFGNode.class); // Worklist per-fcn
+ IntHashMap iset = new IntHashMap(0); // dedup hash set per-fcn
+ _edgess = new int[code._linker._len][];
+
+ // For all functions, build the _edges array
+ for( FunNode fun : code._linker )
+ if( fun!=null && !fun.isDead() )
+ // Visit function and collect direct class touches and callees
+ _edgess[fun.sig().fidx()] = walk(code,fun,work,iset,visit,null);
+
+ // FIDX zero represents escapees - an unknown callee, including a
+ // callback supplied by another compilation. Its code can call any
+ // exposed function. Keep this edge in saved summaries so later
+ // applications contribute their callbacks too.
+
+ // Filter escapees by dead and not a class (class initializers are
+ // reached through bodies, not directly callable function pointers).
+ int[] escaped = ((TypeTuple)code._stop._type).mem()._escFs;
+ AryInt callees = new AryInt();
+ // An imported Stop can still carry FULL (only local bodies have edges)
+ // so limit by existing FIDXs.
+ int limit = code._linker._len;
+ for( int fidx = XInt.next(escaped,0,limit); fidx >= 0; fidx = XInt.next(escaped,fidx,limit) ) {
+ FunNode fun = code.link(fidx); // Discard dead bodies and external C functions.
+ if( fun!=null && !fun.isClz() ) callees.push(fidx);
+ }
+ _edgess[0] = callees.isEmpty() ? null : callees.asAry();
+ visit.clear();
+ }
+
+ // Return the outgoing init edge set for this function
+ private static int[] walk( CodeGen code, FunNode fun, Ary<CFGNode> work, IntHashMap iset, BitSet visit,
+ Ary<Lexer> sites) {
+ if( fun.summary()!=null ) return fun.summary()._classDependencyFIDXs;
+ // Walk function body, looking for outgoing init edges
+ assert iset .isEmpty(); // int hash set for de-dup
+ assert visit.isEmpty(); // needs a visit set, because loops
+ assert work .isEmpty(); // needs a worklist, because DAG, parallel explore paths
+
+ // Children require their parents to init first;
+ // add child->parent edge
+ if( fun.isClz() && fun._compunit._par != null )
+ iset.put(code.link(fun._compunit._par).sig().fidx(),1);
+
+ work.push(fun);
+ while( !work.isEmpty() ) {
+ CFGNode cfg = work.pop();
+ if( visit.get(cfg._nid) ) continue; // Been there, done that
+ visit.set(cfg._nid);
+ assert !cfg._type.isHigh();
+ // Do not walk off end of function
+ if( cfg instanceof ReturnNode ) continue;
+
+ // CLInit adds a class init requirement to this function
+ // (plus all parent classes)
+ if( cfg instanceof CLInitNode init ) {
+ FunNode clz = code.link(init.cu()); // <clinit> referred to
+ iset.put(clz.sig().fidx(),1); // Add to edge set
+ if( sites!=null && sites.atX(clz.sig().fidx())==null ) sites.setX(clz.sig().fidx(),init._loc);
+ }
+
+ // Calls union all their fptr fidxs into the callee set
+ if( cfg instanceof CallNode call && call.fptr()._type instanceof TypeFunPtr fp ) {
+ if( XInt.isHigh(fp.fidxs()) ) {
+ iset.put(0,1);
+ if( sites!=null && sites.atX(0)==null ) sites.setX(0,call._loc);
+ }
+ // Union call targets into the callee set
+ else for( int dstx = XInt.next(fp.fidxs(),0); dstx>=0; dstx=XInt.next(fp.fidxs(),dstx) ) {
+ iset.put(dstx,1);
+ if( sites!=null && sites.atX(dstx)==null ) sites.setX(dstx,call._loc);
+ }
+ }
+
+ // Forward recursion to walk the whole function
+ for( Node use : cfg._outputs ) {
+ // Do not walk into another function
+ if( !(use instanceof CFGNode cfg1) || use instanceof FunNode || use instanceof StopNode ) continue;
+ // Expect control flow edge is sensible
+ assert (use instanceof RegionNode r && r._inputs.find(cfg)>0) || use.in(0)==cfg;
+ if( !visit.get(cfg1._nid) ) // Small speed opt
+ work.add(cfg1);
+ }
+ }
+
+ visit.clear();
+ int[] edges = iset.isEmpty() ? null : iset.keySet();
+ iset.clear();
+ return edges;
+ }
+
+ // Return an error string, or null if no cycle contains multiple clinits.
+ public String check() {
+ Arrays.fill(_pres,0);
+ // A prior check may have stopped with a cycle still on the stack.
+ while( _sp>0 ) _spos[_stk[--_sp]] = 0;
+ _pre = 1;
+
+ int idx = -1;
+ for( int fidx = 0; fidx < _edgess.length; fidx++ )
+ if( _pres[fidx] == 0 && // Already visited
+ (idx = scc(fidx)) != -1 ) // And found a cycle
+ return report(idx); // Report cycle
+ return null;
+ }
+
+ // Every internal edge of an SCC belongs to a cycle. Print all of them:
+ // stack order is not a path, and enumerating all cycle paths can explode.
+ // This bookkeeping and formatting are only needed on an error.
+ private String report(int idx) {
+ BitSet members = new BitSet();
+ for( int i=idx; i<_sp; i++ ) members.set(_stk[i]);
+ SB sb = new SB().p("Cyclic class initialization:");
+ // Recover one surviving source site per edge only after finding an
+ // error. A representative touch is enough; runtime first-touch order
+ // need not be statically known. Lexer snapshots retain the source CU
+ // even when the node has been inlined into a different function.
+ Ary<CFGNode> work = new Ary<>(CFGNode.class);
+ IntHashMap iset = new IntHashMap(0);
+ BitSet visit = new BitSet();
+ Ary<Lexer> sites = new Ary<>(Lexer.class);
+ for( int i=idx; i<_sp; i++ ) {
+ int src = _stk[i];
+ FunNode fun = _code.link(src);
+ Arrays.fill(sites._es,null);
+ sites.clear();
+ if( fun!=null ) walk(_code,fun,work,iset,visit,sites);
+ if( _edgess[src]!=null )
+ for( int dst : _edgess[src] )
+ if( members.get(dst) ) {
+ sb.p("\n ").p(name(src)).p(" -> ").p(name(dst));
+ Lexer loc = sites.atX(dst);
+ if( loc!=null && loc._cu!=null )
+ sb.p('\n').p(SourceError.format(loc._cu,loc.pos(),"cyclic class initialization"));
+ else if( fun!=null && fun.isClz() && dst!=0 && fun._compunit._par==_code.link(dst)._compunit )
+ sb.p("\n parent initialization requirement");
+ }
+ }
+ return sb.toString();
+ }
+
+ private String name(int fidx) {
+ if( fidx==0 ) return "unknown call";
+ FunNode fun = _code.link(fidx);
+ String owner=fun._compunit._cname;
+ if( fun.isClz() ) return owner;
+ String label=fun.label();
+ return (label.startsWith(owner+".") ? label : owner+"."+label)+" [fidx "+fidx+"]";
+ }
+
+ // Tarjan SCC
+ int scc( int fidx ) {
+ // Assign pre-order number
+ _lows[fidx] = _pres[fidx] = _pre++;
+ _spos[fidx] = _sp+1;
+ _stk[_sp++] = fidx;
+ // walk outbound edges
+ if( _edgess[fidx]!=null ) {
+ for( int edge : _edgess[fidx] ) {
+ if( edge >= _pres.length ) continue; // Out of bounds happens when calling C fcns
+ if( _pres[edge] == 0 ) { // Not visited
+ int idx = scc(edge);
+ if( idx != -1 ) return idx; // Found a cycle
+ _lows[fidx] = Math.min(_lows[fidx],_lows[edge]);
+ } else if( _edgess[edge]!=null && _spos[edge]!=0 ) {
+ _lows[fidx] = Math.min(_lows[fidx],_pres[edge]);
+ }
+ }
+ }
+ // Found a cycle
+ if( _lows[fidx] == _pres[fidx] ) {
+ int idx = _spos[fidx]-1;
+ if( hasClzCycle(idx) )
+ return idx; // Found a class cycle
+ // Pop stack, including fidx, should be a really short loop
+ while( _sp>idx ) _spos[_stk[--_sp]] = 0;
+ }
+ return -1; // No cycle
+ }
+
+ private boolean hasClzCycle(int idx) {
+ int clinits = 0;
+ for( int i=idx; i<_sp; i++ ) {
+ FunNode fun = _code.link(_stk[i]);
+ if( fun!=null && fun.isClz() && ++clinits > 1 )
+ return true;
+ }
+ return false;
+ }
+
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/Coalesce.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Coalesce.java
new file mode 100644
index 00000000..689ceceb
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Coalesce.java
@@ -0,0 +1,105 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.util.Ary;
+import com.seaofnodes.simple.util.Utils;
+import java.util.BitSet;
+import java.util.IdentityHashMap;
+
+// Coalesce copies
+abstract public class Coalesce {
+
+ public static boolean coalesce( int round, RegAlloc alloc ) {
+
+ // Walk all the splits, looking for coalesce chances
+ boolean progress = false;
+ for( CFGNode bb : alloc._code._cfg ) {
+ outer:
+ for( int j=0; j<bb.nOuts(); j++ ) {
+ Node n = bb.out(j);
+ if( !(n instanceof SplitNode) ) continue;
+ LRG v1 = alloc.lrg(n);
+ LRG v2 = alloc.lrg(n.in(1));
+ if( v1 != v2 ) {
+ LRG ov1 = v1, ov2 = v2;
+ // Get the smaller neighbor count in v1
+ if( v1.nadj() > v2.nadj() )
+ { v1 = v2; v2 = alloc.lrg(n); }
+ int v2len = v2._adj==null ? 0 : v2._adj._len;
+ // See that they do not conflict (coalescing would make a self-conflict)
+ if( v1._adj!=null && v2._adj!=null && v1._mask.overlap(v2._mask) ) {
+ // Walk the smaller neighbors, and add to the larger.
+ // Check for direct conflicts along the way.
+ for( LRG v3 : v1._adj ) {
+ if( v3==v2 ) {
+ // Direct conflict, skip this LRG pair
+ v2._adj.setLen(v2len);
+ continue outer;
+ }
+ // Add missing LRGs from V1 to V2
+ if( v2._adj.find(v3)== -1 )
+ v2._adj.push(v3);
+ }
+ }
+ // Most constrained mask
+ RegMask mask = v1._mask==v2._mask ? v1._mask : v1._mask.copy().and(v2._mask);
+ // Check for capacity
+ if( v2.nadj() >= mask.size() ) {
+ // Fails capacity, will not be trivial colorable
+ if( v2._adj!=null ) v2._adj.setLen(v2len);
+ continue;
+ }
+
+ // Union lrgs.
+ progress = true;
+
+ // Since `n` is dying, remove from the LRGS BEFORE doing
+ // the union; this will let the union code keep the
+ // matching bits from the other side, preserving some
+ // info for biased coloring
+ if( ov1. _machDef==n ) ov1. _machDef = null;
+ if( ov1._splitDef==n ) ov1._splitDef = null;
+ if( ov2. _machUse==n ) ov2. _machUse = null;
+ if( ov2._splitUse==n ) ov2._splitUse = null;
+ // Keep larger adjacency list.
+ Ary<LRG> larger = v2._adj;
+ v2._adj = null;
+
+ // Union lrgs. Swaps again, based on small lrg
+ LRG vWin = v1.union(v2);
+ vWin._adj = larger;
+ LRG vLose = v1.leader() ? v2 : v1;
+
+ // LRGs below v2len were always in v2, but maybe not in v1.
+ // LRGs above v2len are in v1 but were not in v2.
+
+ // Walk v2's neighbors, replacing vLose with vWin.
+ if( larger != null )
+ for( LRG vn : larger ) {
+ int idx = vn._adj.find(vLose);
+ if( idx != -1 )
+ vn._adj.del(idx);
+ if( vn._adj.find(vWin) == -1 )
+ vn._adj.add(vWin);
+ }
+ } else {
+
+ // Since `n` is dying, remove from the LRGS
+ if( v1. _machDef==n ) v1. _machDef=null;
+ if( v1. _machUse==n ) v1. _machUse=null;
+ if( v1._splitDef==n ) v1._splitDef=null;
+ if( v1._splitUse==n ) v1._splitUse=null;
+ }
+
+ // Remove junk split
+ n.removeSplit();
+ j--;
+ }
+ }
+
+ if( progress )
+ alloc.unify();
+
+ return true;
+ }
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/CodeGen.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/CodeGen.java
new file mode 100644
index 00000000..bdb510b4
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/CodeGen.java
@@ -0,0 +1,889 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.graph.GraphObserver;
+import com.seaofnodes.simple.parser.Parser;
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.print.*;
+import com.seaofnodes.simple.type.*;
+import com.seaofnodes.simple.util.*;
+import java.io.File;
+import java.util.*;
+
+
+// Compile Driver for a Compilation Unit
+@SuppressWarnings("unchecked")
+public class CodeGen {
+
+ // Last created CodeGen as a global; used all over to avoid passing about a "context".
+ public static CodeGen CODE;
+
+ // Location of built-in CPU ports
+ public static final String PORTS = "com.seaofnodes.simple.node.cpus";
+
+ // Compile phases
+ public enum Phase {
+ Parse, // Parse ASCII text into Sea-of-Nodes IR
+ Iter, // Pessimistic peepholes after parse
+ Opto, // SCCP, numeric modes, seed the peephole worklist
+ IterPostOpto, // Peepholes and inlining using SCCP results
+ FinalizeIR, // Field layout, offset cleanup, function/CU roots
+ TypeCheck, // Last check for bad programs
+ Serialize, // Save ideal IR, including opaque class initialization
+ Lower, // Expand checked class initialization to ordinary IR
+ LoopTree, // Build a loop tree; break infinite loops
+ Unlink, // Unlink call sites before machine code generation
+ Select, // Convert to target hardware nodes
+ Schedule, // Global schedule (code motion) nodes
+ LocalSched, // Local schedule
+ RegAlloc, // Register allocation
+ Encoding, // Encoding
+ Export, // Export
+ LastPhase // After last phase
+ }
+ public Phase _phase;
+ // Null for normal compilation; owned by this compilation, not a global viewer.
+ public GraphObserver<Node> _obs;
+
+ /** True when tests deliberately randomize Iter worklist order. */
+ public static boolean iterSeedOverridden() { return System.getProperty("simple.iter.seed") != null; }
+
+ // Expensive assert sampling:
+ // -Dsimple.assert.expensive=0 disables these checks
+ // -Dsimple.assert.expensive=1 checks every step
+ // -Dsimple.assert.expensive=N checks once every 1<<N steps
+ private static final int EXPENSIVE_ASSERT_LOG = Integer.getInteger("simple.assert.expensive",8);
+ public static boolean expensiveAssert() { return EXPENSIVE_ASSERT_LOG > 0; }
+ public static boolean expensiveAssert( int trip ) {
+ if( EXPENSIVE_ASSERT_LOG <= 0 ) return false;
+ if( EXPENSIVE_ASSERT_LOG == 1 ) return true;
+ int shift = Math.min(EXPENSIVE_ASSERT_LOG,30);
+ return (trip & ((1<<shift)-1)) == 0;
+ }
+
+ // ---------------------------
+ // Module Source Root
+ public final String _modDir;
+ // Module Build Root
+ public final String _buildDir;
+ // Search path from CWD for `.o` files containing external symbols and types.
+ // This can contain archives, zips and other .o file containers.
+ public final Ary<String> _externPaths;
+
+ // Dotted path from module root to file containing the source, sans ".smp".
+ // e.g. module root: "sys", nested source name "sys.io".
+ public final String _srcName;
+
+ // Only available for tests, otherwise source comes from the CompUnit
+ private final String _src;
+
+ // Current Working Directory; default module base
+ private final String _cwd;
+
+ // Compilation Units used in this compile; typically one per source/object
+ // file, although you can nest structs and get more than one. These are all
+ // required to compile, and are expected to have their source timestamps
+ // ahead of their obj file. Some of these may entirely be dead or inlined
+ // in this CompUnit and thus not needed in any linking stage. All the
+ // alive ones (inlined away or not) participate in class init cycle checks.
+ public HashMap<String,CompUnit> _compunits;
+
+ // Test shortcut for only one compilation unit
+ public CompUnit compunit() {
+ assert _compunits.size()==1;
+ for( CompUnit cu : _compunits.values() )
+ return cu;
+ assert false : "no compilation unit";
+ return null;
+ }
+
+ // Frozen after ParseAll: public initializers and function-valued fields.
+ public boolean publicFIDX(int fidx) {
+ return fidx >= GlobalBits.RESERVED && XInt.bit(publicFIDXs(),fidx);
+ }
+
+ private void gatherPublicFields(TypeStruct clz, boolean classFields) {
+ boolean hasCtor = clz.field("<ctor>") != null;
+ for( Field fld : clz._fields ) {
+ // With a user constructor, class:N.N is the private <init>.
+ if( fld._fname.charAt(0)=='_' || fld._t==Type.TOP ||
+ (hasCtor && clz.hiddenInit(fld)) ) continue;
+ if( classFields ) _publicAliases = XInt.make(_publicAliases,fld._alias);
+ if( fld._t instanceof TypeFunPtr tfp )
+ _publicFIDXs = XInt.meet(_publicFIDXs,tfp._fidxs);
+ }
+ }
+
+ public static boolean hasUserConstructor(TypeStruct self) {
+ Type t = Parser.TYPES.get(Parser.addClzPrefix(self._name));
+ return t instanceof TypeStruct clz && clz.field("<ctor>") != null;
+ }
+
+ // A forward reference has a placeholder class, not a definition yet.
+ private static boolean resolvedClass(TypeStruct clz) {
+ Type t = Parser.TYPES.get(clz._name.substring(Parser.CLZ.length()));
+ return !(t instanceof TypeStruct ts) || !ts._fref;
+ }
+
+ private int[] _publicFIDXs, _publicAliases;
+
+ /**
+ * Collect every possible public escape - after ParseAll is done - in case
+ * some code calls a generic function. Example:
+ * var apply = = { {int->int} f, int x -> f(x); }; // Generic apply() call
+ * The CallEnd from `f(x)` has to use generic TypeMem.BOT, which
+ * immediately escapes *everything* including known private functions.
+ * As weak as this is, its still sharper than "everything".
+ */
+ void freezePublicInterface() {
+ assert _publicFIDXs==null && _publicAliases==null;
+ _publicFIDXs = _publicAliases = XInt.EMPTY;
+ for( Type t : Parser.TYPES.values() )
+ if( t instanceof TypeStruct clz &&
+ Parser.startsClzPrefix(clz._name) &&
+ Parser.isPublicClassName(clz._name) &&
+ resolvedClass(clz) ) {
+ // Gather function pointers from public fields of both class
+ // and instance objects.
+ gatherPublicFields(clz,true);
+ Type self = Parser.TYPES.get(clz._name.substring(Parser.CLZ.length()));
+ if( self instanceof TypeStruct ts ) gatherPublicFields(ts,false);
+ }
+ // Limit broad function types to known FIDXs, excluding reserved bits.
+ // Public class initializers are roots without needing a field reference.
+ int[] fidxs = XInt.EMPTY;
+ for( int fidx = GlobalBits.RESERVED; fidx<_linker._len; fidx++ ) {
+ FunNode fun = _linker.atX(fidx);
+ if( XInt.bit(_publicFIDXs,fidx) || (fun!=null && fun.isClz() && fun.isPublic()) )
+ fidxs = XInt.make(fidxs,fidx);
+ }
+ _publicFIDXs = fidxs;
+ }
+
+ public int[] publicFIDXs() { assert _publicFIDXs !=null; return _publicFIDXs; }
+ public int[] publicAliases(){ assert _publicAliases!=null; return _publicAliases; }
+ public boolean publicInterfaceFrozen() { return _publicFIDXs != null; }
+
+ // ---------------------------
+
+ // Very common nodes, cached here
+ public final ConstantNode ZERO;
+ public final XCtrlNode XCTRL;
+
+ public Node constant( Type t ) {
+ assert _phase==null || _phase.ordinal() <= Phase.Lower.ordinal();
+ return ConstantNode.make(t).peephole();
+ }
+ public ConstantNode con( Type t ) { return (ConstantNode)constant(t); }
+ public ConstantNode con( long con ) { return con==0 ? ZERO : con(TypeInteger.constant(con)); }
+
+
+ // ---------------------------
+ // Test setup; no module nor file with specific argument
+ public CodeGen( String src ) { this(src, 126L, true ); }
+
+ // Test setup; no module nor file; can alter seed & argument; can re-run same CodeGen
+ public CodeGen( String src, Type arg ) {
+ this(null,null,null,null, src, 126L, true, arg);
+ }
+ // Test setup; no module nor file; can alter seed & argument; can re-run same CodeGen
+ public CodeGen( String src, long workListSeed, boolean resetTypes ) {
+ this(null,null,null,null, src, workListSeed, resetTypes, TypeInteger.BOT);
+ }
+
+ // Generic CodeGen, including full module setup
+ public CodeGen( String modDir, String buildDir, Ary<String> externPaths,
+ String srcName, String src, long workListSeed, boolean resetTypes, Type arg ) {
+ // Public singleton to avoid passing about this state to a huge count
+ // of places. Probably becomes a TLS at some point.
+ CODE = this;
+ if( resetTypes ) Type.reset(); // Reset to recover from failed test
+
+ _cwd = System.getProperty("user.dir")+"/";
+ _modDir = modDir == null ? _cwd : modDir;
+ _buildDir = buildDir == null ? _cwd : buildDir;
+ _externPaths = externPaths;
+ _srcName = srcName;
+ _callingConv = null; // Calling convention
+ // Source code from test strings, not files
+ _src = src;
+ // All the compilation units
+ _compunits = new HashMap<>();
+ _phase = null;
+ // Start GVN table
+ _gvn = new HashMap<>();
+ // Allow whole-suite worklist-order sweeps without rewriting the many
+ // explicit seeds used by tests and command-line compilation.
+ String iterSeed = System.getProperty("simple.iter.seed");
+ _iter = new IterPeeps(iterSeed == null ? workListSeed : Long.parseLong(iterSeed));
+ // End points of graph
+ _stop = new StopNode().init();
+ _start = new StartNode(null,_stop,arg).init();
+ ZERO = con(TypeInteger.ZERO).keep();
+ XCTRL = new XCtrlNode().peephole().keep();
+ P = new Parser(this);
+ }
+
+
+ // Run requested phases. File export must request Serialize before Lower;
+ // requesting Export up front does that automatically. Encoding alone does
+ // not retain reloadable ideal IR. Choose combined in-memory output before
+ // Encoding; otherwise saved IR requests separate object encodings.
+
+ // No code emission, just IR generation
+ public CodeGen driver( Phase phase ) { return driver(phase,null,null,false,false,0); }
+ // No object file writing, but code generation for a specific cpu/os pair (allows emulation)
+ public CodeGen driver( Phase phase, String cpu, String callingConv ) {
+ if( phase.ordinal() < Phase.Export.ordinal() )
+ // No code outputted
+ return driver(phase,cpu,callingConv,false,false,0);
+ if( _srcName == null )
+ throw new RuntimeException("No source filename provided, so do not know how to name the obj file");
+ boolean emitEntrySymbol = !_srcName.contains(".");
+ return driver( phase, cpu, callingConv, false, emitEntrySymbol, 0 );
+ }
+ // Write an object file for a specific cpu/os pair
+
+ public CodeGen driver( String cpu, String callingConv, boolean inMemory, boolean emitEntrySymbol ) { return driver(Phase.Export,cpu,callingConv,inMemory,emitEntrySymbol,0); }
+
+ // The Generic driver
+ public CodeGen driver( Phase target, String cpu, String callingConv, boolean inMemory, boolean emitEntrySymbol, int dump ) {
+ // Parse a Simple file, and for mentioned external symbols either load
+ // from .o or recursively parse
+ step(target, Phase.Parse, this::parse, dump);
+ // Run peepholes to a fixed point
+ step(target, Phase.Iter, this::iter, dump);
+ // Run SCCP (and resolve int/flt expressions)
+ step(target, Phase.Opto, this::opto, dump);
+ // Run peepholes to a fixed point again, on the better types.
+ step(target, Phase.IterPostOpto, this::iterPostOpto, dump);
+ // Field layouts; field symbols become constant offsets
+ step(target, Phase.FinalizeIR, this::finalizeIR, dump);
+ // Final error check - bad programs fail here
+ step(target, Phase.TypeCheck, this::typeCheck, dump);
+ // Save reloadable ideal IR for a later seperate compilation
+ step(target, Phase.Serialize, () -> Serialize.forTarget(this,target,inMemory), dump);
+ // Eval2 executes ideal operations, including CLInit, directly. Only
+ // machine compilation expands them before scheduling.
+ step(target, Phase.Lower, () -> { if( cpu!=null ) lower(); }, dump);
+ // Compute the loop-tree, used for approximating exec frequencies for
+ // scheduling and register allocation
+ step(target, Phase.LoopTree, this::loopTree, dump);
+ // Detach call-graph edges; ideal execution retains unloaded summaries
+ // as dispatch targets and schedules their bodies on first use.
+ step(target, Phase.Unlink, () -> unlink(cpu!=null), dump);
+ // Instruction selection
+ step(target, Phase.Select, () -> instSelect(cpu,callingConv), dump);
+ // Global Code Motion
+ step(target, Phase.Schedule, this::GCM, dump);
+ // Local (basic block only) scheduling
+ step(target, Phase.LocalSched, this::localSched, dump);
+ // Register allocation
+ step(target, Phase.RegAlloc, this::regAlloc, dump);
+ // Encode machine instructions into bits
+ step(target, Phase.Encoding, () -> encode(inMemory), dump);
+ // Export bits (and serialized IR) into ELF files
+ step(target, Phase.Export, () -> exportELF(inMemory,emitEntrySymbol), dump);
+ return this;
+ }
+
+ // The driver names the order; optional phases decide locally whether to run.
+ private void step(Phase target, Phase next, Runnable work, int dump) {
+ if( next.ordinal() > target.ordinal() || (_phase!=null && _phase.ordinal()>=next.ordinal()) ) return;
+ Phase prior = _phase;
+ work.run();
+ if( _phase!=prior ) dump(dump);
+ }
+
+ // All phase entry points share ordering checks, timing, and viewer notification.
+ // Record elapsed time even on failure, but report completion only on success.
+ CodeGen phase(Phase next, boolean ordered, Runnable work) {
+ assert ordered : "Cannot run "+next+" after "+_phase;
+ // Serialize reads the checked graph; other phases can change it.
+ if( next!=Phase.Serialize ) _classInitDependencies = null;
+ _phase = next;
+ long t0 = System.currentTimeMillis();
+ work.run(); // Do The Phase
+ _times[next.ordinal()] = System.currentTimeMillis() - t0;
+ if( _obs!=null ) _obs.phase(next.name()); // Record for viewer
+ return this;
+ }
+
+ public String entryClinitName() {
+ return Parser.addClzPrefix(_srcName==null ? "Test" : _srcName.replace('/','.'))+".<clinit>";
+ }
+
+ // Verbose printing during compilation
+ private void dump(int dump) {
+ int p2 = _phase.ordinal();
+ if( (dump & (1<<p2)) == 0 ) return;
+ if( (dump & (1<<30)) != 0 )
+ System.err.println("After "+_phase+":");
+ System.err.println(_phase.ordinal() >= Phase.LocalSched.ordinal()
+ ? asm(new SB()).toString()
+ : IRPrinter.prettyPrint(this));
+ }
+
+
+ // Record times by phase
+ public final long[] _times = new long[Phase.LastPhase.ordinal()];
+
+ // ---------------------------
+ /**
+ * A counter, for unique node id generation. Starting with value 1, to
+ * avoid bugs confusing node ID 0 with uninitialized values.
+ */
+ public int _uid = 1;
+ public int UID() { return _uid; }
+ public int getUID() {
+ return _uid++;
+ }
+
+
+ // These next fields all 2-way map some *global* program feature to a
+ // local dense index, suitable for packing in BitSets. The mapping takes a
+ // global value - always a {source file/class} name, and an order number
+ // counting up per feature in the same file. The dense local index can be
+ // different in different compilations.
+
+ // The global info allows loading the same remote class info from different
+ // paths and aligning them. Example: Compiling A loads pre-compiled B and
+ // C, both of which load pre-compiled D. The 2 different D loads unify via
+ // this global info.
+
+ // These mappings are all trivial identities when compiling one file; they
+ // only become complex when loading separately compiled code.
+
+ // Compute local function index (FIDX) from global function info. This is
+ // called *in order* during parsing, and that order is part of the global
+ // unique mapping
+ public final GlobalBits _aliases = new GlobalBits();
+ public int alias(String clz) { return _aliases.next(clz); }
+
+ // C storage has one global alias per symbol, including across object files.
+ public int externAlias(String name) { return _aliases.next("C:"+name,GlobalBits.RESERVED); }
+
+ // External data addresses supplied by an in-memory linker (image offsets).
+ public final HashMap<String,Integer> _externDataAddresses = new HashMap<>();
+
+ // Compute local function index (FIDX) from global function info. This is
+ // called *in order* during parsing, and that order is part of the global
+ // unique mapping
+ public final GlobalBits _fidxs = new GlobalBits();
+ // Return local index for specific global file & order. Used to find fidxs for e.g. FREF class <init> fcns
+ public int fidx( String clz, int order ) { return _fidxs.next(clz, order); }
+ // New fidx in the current file
+ public int fidx( String clz ) { return fidx(clz, -1); }
+
+ // Compute local RPC index from global RPC info, one per call
+ public final GlobalBits _rpcs = new GlobalBits();
+ public int rpc( String clz ) { return _rpcs.next(clz); }
+
+
+ // Inlining can insert CFG edges and invalidate cached depth ordering.
+ private char _iDepthVersion;
+ public char iDepthVersion() { return _iDepthVersion; }
+ public void invalidateIDepthCaches() {
+ assert _iDepthVersion < Character.MAX_VALUE : "Dominator cache version exceeds 65535";
+ ++_iDepthVersion;
+ }
+
+ // Popular visit bitset, declared here, so it gets reused all over
+ public final BitSet _visit = new BitSet();
+ public BitSet visit() { assert _visit.isEmpty(); return _visit; }
+
+ // Start and Stop; end points of the generated IR
+ public StartNode _start;
+ public StopNode _stop;
+ // Iter creates this shared analysis result. Start's unused backedge slot
+ // roots it until Unlink; serialized Stops retain its escape facts instead.
+ public GlobalEscapeNode _globalEscapes;
+
+ // Global Value Numbering. Hash over opcode and inputs; hits in this table
+ // are structurally equal.
+ public final HashMap<Node,Node> _gvn;
+
+ // Reverse from a constant function pointer to the IR function being called.
+ // Error to call with a non-constant TFP
+ public FunNode link( TypeFunPtr tfp ) { return link(tfp.fidx()); }
+ // Read-only linker lookup, suitable for printers and debugger display.
+ // In particular, do not lazily remove a partially dead function.
+ public FunNode _link(int fidx) { return _linker.atX(fidx); }
+ public FunNode _link(TypeFunPtr tfp) { return _link(tfp.fidx()); }
+ // Return the FunNode from a fidx
+ public FunNode link( int fidx ) {
+ FunNode fun =_linker.atX(fidx);
+ if( fun!=null && fun.isDead() ) {
+ _linker.setX(fidx,null); fun = null; }
+ return fun;
+ }
+ // Return the <clinit> for a CompUnit
+ public FunNode link( CompUnit cu ) {
+ for( FunNode fun : _linker )
+ if( fun!=null && fun._compunit == cu && fun.isClz() )
+ return fun;
+ return null;
+ }
+
+ // Insert linker mapping from constant function signature to the function
+ // being called.
+ public void link(FunNode fun) {
+ int fidx = fun.sig().fidx();
+ _linker.setX(fidx,fun);
+ add(_globalEscapes); // A new or replacement summary may expose escapes.
+ }
+
+ // "Linker" mapping from constant TypeFunPtrs to heads of function. These
+ // TFPs all have exact single fidxs and their return is wiped to BOTTOM (so
+ // the return is not part of the match).
+ public final Ary<FunNode> _linker = new Ary<>(FunNode.class);
+
+ // Extern function declarations; input is the FIDX assigned to the name.
+ final HashMap<Integer,String> _externFunc = new HashMap<>();
+ public void externFunc(int fidx, String ex) {
+ assert !_externFunc.containsKey(fidx);
+ _externFunc.put(fidx,ex);
+ }
+ public String externFunc(int fidx) { return _externFunc.get(fidx); }
+
+ public String funcName(int fidx ) {
+ FunNode fun = _link(fidx);
+ if( fun!=null ) return fun._name;
+ return externFunc(fidx);
+ }
+
+ // Select which source CompUnit's object this encoding pass emits.
+ CompUnit _emitUnit;
+ boolean emits(CompUnit cu) { return cu!=null && cu._src!=null && (_emitUnit==null || _emitUnit==cu); }
+ boolean emits(FunNode fun) { return emits(fun._compunit); }
+ final LinkedHashMap<CompUnit,Encoding> _encodings = new LinkedHashMap<>();
+
+
+ // ---------------------------
+ // Parser object
+ public final Ary<CallNode> _calls = new Ary<>(CallNode.class);
+
+ public final Parser P;
+
+ // Parse ASCII text into Sea-of-Nodes IR
+ public CodeGen parse() { return phase(Phase.Parse, _phase==null, () -> ParseAll.parse(this,_src)); }
+
+
+ // ---------------------------
+ // Iterator peepholes.
+ public final IterPeeps _iter;
+ // Stop tracking deps while assertin
+ public boolean _midAssert;
+ // Statistics on peepholes
+ public int _iter_cnt, _iter_nop_cnt;
+ // Stat gathering
+ public void iterCnt() { if( !_midAssert ) _iter_cnt++; }
+ public void iterNop() { if( !_midAssert ) _iter_nop_cnt++; }
+
+ // Pessimistic peepholes after parsing. This lifts loaded and parsed
+ // types before the optimistic Opto pass resets types and lets them fall.
+ public CodeGen iter() { return phase(Phase.Iter, _phase==Phase.Parse, () -> Iter.run(this)); }
+
+ // SCCP and numeric-mode resolution; leave cleanup work queued for Iter.
+ public CodeGen opto() { return phase(Phase.Opto, _phase==Phase.Iter, () -> Opto.opto(this)); }
+
+ public CodeGen iterPostOpto() { return phase(Phase.IterPostOpto, _phase==Phase.Opto, () -> _iter.iterate(this)); }
+
+ // Materialize an inline candidate at an empty worklist. Loading queues
+ // the new body and affected callers before CallEnd considers inlining.
+ public void loadForInline(CompUnit cu) {
+ assert _phase==Phase.IterPostOpto && _iter._work.isEmpty();
+ ParseAll.loadCompUnit(this,cu);
+ }
+
+ // TypeCheck saves its successful analysis for Serialize. Loading bodies
+ // or entering another graph-changing phase discards it.
+ ClassInitDependencies _classInitDependencies;
+
+ public CodeGen finalizeIR() { return phase(Phase.FinalizeIR, _phase==Phase.IterPostOpto, () -> FinalizeIR.run(this)); }
+
+ // Cross-phase `keep()`s belong here! We have a few nodes which we can
+ // lose all uses of, but later find them and want to re-add output edges.
+ // If we don't `keep()` them, they'll dead-code-elimnate.
+
+ // These are mostly around adding and removal functions from other
+ // compilation units. E.g. a StartCU, when all his functions are exactly
+ // called with no escapes and no default input. Those functions will still
+ // go out in their own ELF, and will later hook on to the StartCU during
+ // code emission. Between e.g. Opto and code emission the StartCU has no
+ // direct users - but we're going to want the StartCU at elf-writer time!
+
+ // The Parser and `idealize()` `keep()`s remain locally balanced.
+
+ // We make a few collections of these cross-phase `keep()`s.
+
+ // A few more cross-phase example lifetimes:
+
+ // Until FinalizeIR: Opto and import loading retain StartCU's memory
+ // projection while unknown-caller edges disappear and may be rebuilt.
+ // CompUnit._start.proj(1) finds it; Opto.linkStart uses it for the memory
+ // Parm, found by scanning fun._outputs. FinalizeIR first holds the StartCU
+ // itself, then releases these projections and queues ordinary dead cleanup.
+
+ // Until Unlink: FinalizeIR retains StartCU even if it has no users left.
+ // CompUnit._start is the non-graph reference used by Serialize and Unlink.
+ // Unlink.run/forEval install scheduling edges before releasing these holds.
+ // A Java reference alone does not protect a node from graph deletion.
+ //
+ // Replacement is the exception to those release points: ParseAll must
+ // release our old wrapper/projection holds before subsuming the wrapper.
+ // It registers the replacement projection for the next FinalizeIR call.
+ private final Ary<Node> _keptUntilFinalize = new Ary<>(Node.class);
+ private final Ary<Node> _keptUntilUnlink = new Ary<>(Node.class);
+
+ void keepUntilFinalize(Node n) { keepOnce(_keptUntilFinalize,n); }
+ void keepUntilUnlink (Node n) { keepOnce(_keptUntilUnlink ,n); }
+ private static void keepOnce(Ary<Node> kept, Node n) {
+ if( kept.find(n)==-1 ) kept.add(n.keep());
+ }
+
+ // FinalizeIR has retained the wrappers; unused projections may now die.
+ void releaseFinalizedNodes() {
+ for( Node n : _keptUntilFinalize ) add(n.unkeep());
+ _keptUntilFinalize.clear();
+ }
+
+ // Unlink has installed the scheduling edges. No optimization is needed.
+ void releaseUnlinkedNodes() {
+ for( Node n : _keptUntilUnlink ) n.unkeep();
+ _keptUntilUnlink.clear();
+ }
+
+ // Only release our holds. A caller's temporary keep is a separate owner.
+ void releaseReplacedNode(Node n) {
+ releaseReplacedNode(_keptUntilFinalize,n);
+ releaseReplacedNode(_keptUntilUnlink,n);
+ }
+ private static void releaseReplacedNode(Ary<Node> kept, Node n) {
+ int idx = kept.find(n);
+ if( idx!=-1 ) kept.del(idx).unkeep();
+ }
+
+ // Root a surviving function without claiming that it has unknown callers.
+ public void keepInCompUnit(FunNode fun) {
+ StopCUNode stop = fun._compunit._stop;
+ if( stop._inputs.find(fun.ret()) == -1 )
+ add(stop).addDef(fun.ret());
+ }
+ public <N extends Node> N add( N n ) { return _iter.add(n); }
+ public void addAll( Ary<Node> ary ) { _iter.addAll(ary); }
+ public void addAll( Node n ) { _iter.add(n); _iter.addAll(n._inputs); }
+
+ // ---------------------------
+ // Last check for bad programs
+ public CodeGen typeCheck() { return phase(Phase.TypeCheck, _phase==Phase.FinalizeIR, () -> TypeCheck.run(this)); }
+
+ // ---------------------------
+ // TypeCheck queued surviving CLInits during its existing error walk.
+ public CodeGen lower() {
+ return phase(Phase.Lower, (_phase==Phase.TypeCheck || _phase==Phase.Serialize), () -> _iter.iterate(this));
+ }
+
+ // ---------------------------
+ // Build the loop tree; break never-exit loops
+ public CodeGen loopTree() {
+ return phase(Phase.LoopTree, (_phase==Phase.Lower || _phase==Phase.TypeCheck || _phase==Phase.Serialize),
+ () -> _start.buildLoopTree(_linker,_stop));
+ }
+
+ // ---------------------------
+ public BAOS _serial;
+ final LinkedHashMap<CompUnit,BAOS> _serials = new LinkedHashMap<>();
+ public void serialize() {
+ phase(Phase.Serialize, _phase==Phase.TypeCheck, () -> Serialize.serialize(this));
+ }
+
+ // ---------------------------
+ // Code generation CPU target
+ public Machine _mach;
+ // Chosen calling convention (usually either Win64 or SystemV)
+ public String _callingConv;
+ public String _cCallingConv;
+ // Callee save registers
+ public RegMask _callerSave;
+ public RegMask _cCallerSave;
+
+ // All returns have the following inputs:
+ // 0 - ctrl
+ // 1 - memory
+ // 2 - varies with returning GPR,FPR
+ // 3 - RPC
+ // 4+- Caller save registers
+ public RegMask[] _retMasks;
+ // Return Program Counter
+ public RegMask _rpcMask;
+
+ // Convert to target hardware nodes
+ public CodeGen instSelect( String cpu, String callingConv ) { return instSelect(cpu,callingConv,PORTS); }
+ public CodeGen instSelect( String cpu, String callingConv, String base ) {
+ return InstSelect.select(this,cpu,callingConv,base);
+ }
+
+ // ---------------------------
+ public void unlink() {
+ unlink(true);
+ }
+ private void unlink(boolean removeImports) {
+ phase(Phase.Unlink, _phase==Phase.LoopTree, () -> {
+ _start.setDef(2,null); // Releases the compiler-only escape analysis.
+ if( removeImports ) Unlink.run(this,true);
+ else Unlink.forEval(this,_start,_stop);
+ });
+ }
+
+ // ---------------------------
+ // Control Flow Graph in Reverse Post Order.
+ public Ary<CFGNode> _cfg = new Ary<>(CFGNode.class);
+
+ // Global schedule (code motion) nodes
+ public CodeGen GCM() {
+ return phase(Phase.Schedule, (_phase==Phase.Unlink || _phase==Phase.Select), () -> GlobalCodeMotion.buildCFG(this));
+ }
+
+ // ---------------------------
+ // Local (basic block) scheduler phase, a classic list scheduler
+ public CodeGen localSched() { return phase(Phase.LocalSched, _phase==Phase.Schedule, () -> ListScheduler.sched(this)); }
+
+
+ // ---------------------------
+ // Register Allocation
+ public RegAlloc _regAlloc;
+ public CodeGen regAlloc() {
+ return phase(Phase.RegAlloc, _phase==Phase.LocalSched, () -> {
+ _regAlloc = new RegAlloc(this);
+ _regAlloc.regAlloc();
+ });
+ }
+
+ // Human-readable register name
+ public String reg(Node n) {
+ if( _phase.ordinal() >= Phase.RegAlloc.ordinal() ) {
+ String s = _regAlloc.reg(n);
+ if( s!=null ) return s;
+ }
+ return "N"+ n._nid;
+ }
+
+
+ // ---------------------------
+ // Encoding
+ public Encoding _encoding;
+ public CodeGen encode() { return encode(false); }
+ private CodeGen encode(boolean inMemory) { return phase(Phase.Encoding, _phase==Phase.RegAlloc, () -> {
+ // Saved ideal IR requests one object per unit, including when the
+ // driver stopped at Serialize and resumed later. In-memory linking
+ // explicitly requests one combined image instead.
+ if( inMemory || _serials.isEmpty() ) { _encoding = new Encoding(this).encode(); return; }
+ for( CompUnit cu : _serials.keySet() ) {
+ _emitUnit = cu;
+ Encoding enc = new Encoding(this).encode();
+ _encodings.put(cu,enc);
+ if( cu._fname.equals(_srcName==null ? "Test" : _srcName) ) _encoding = enc;
+ }
+ _emitUnit = null;
+ }); }
+
+ // ---------------------------
+ // Exporting to external formats
+ ElfWriter _elf;
+ public CodeGen exportELF( boolean inMemory, boolean emitEntrySymbol ) {
+ return phase(Phase.Export, _phase==Phase.Encoding, () -> {
+ if( inMemory ) {
+ assert _encodings.isEmpty() : "In-memory output must be selected before Encoding";
+ new LinkMem(this).link(_encoding);
+ }
+ else {
+ if( _serials.isEmpty() )
+ throw new IllegalStateException("ELF export needs ideal IR: request Export from the driver, or Serialize before Lower");
+ assert !_encodings.isEmpty() : "File output must be selected before Encoding";
+ Encoding entry = _encoding;
+ BAOS serial = _serial;
+ try {
+ for( var e : _encodings.entrySet() ) {
+ _emitUnit = e.getKey();
+ _encoding = e.getValue();
+ _serial = _serials.get(_emitUnit);
+ _elf = new ElfWriter(this).export(emitEntrySymbol && _emitUnit._fname.equals(_srcName));
+ }
+ // Mutually dependent units built together must not become
+ // stale merely because their files were written in sequence.
+ long time = System.currentTimeMillis();
+ for( CompUnit cu : _encodings.keySet() ) {
+ File obj = new File(_buildDir+"/"+cu._fname+".o");
+ if( !obj.setLastModified(time) ) throw new IllegalStateException("Cannot timestamp "+obj);
+ }
+ } finally { _emitUnit=null; _encoding=entry; _serial=serial; }
+ }
+ });
+ }
+
+
+ // ---------------------------
+
+ // Search a external path list, each path is recursively searched for .o
+ // files, which are partially loaded and searched for public symbols.
+
+ // This is a state machine which lazily searches down the set of search
+ // paths until the requested module is class is found.
+
+ // Map from external Strings to either partially read Simple ElfFile or ExternNode
+ public final HashMap<String,Object> _externSymbols = new HashMap<>();
+
+ // State machine elements; the outermost element is _externPaths
+ private int _extPathIdx; // Search index into the extern path list
+ private final Ary<File> _files = new Ary<>(File.class); // Files in the current extern path being searched
+ private final HashSet<File> _seenFiles = new HashSet<>();
+ private int _extFileIdx; // Index into _files
+
+ public ElfReader findExternalSimple( String name ) {
+ return findExternalSymbol(name) instanceof ElfReader elf ? elf : null;
+ }
+
+ public ExternNode findExternal( String name ) {
+ return findExternalSymbol(name) instanceof ExternNode extern ? extern : null;
+ }
+
+ // Search the search-path for the name; return an ElfReader if found in a
+ // Simple-made ELF; return an ExternNode if from another ELF, or null if
+ // not found.
+ private Object findExternalSymbol( String name ) {
+ // State Machine!
+
+ while( true ) {
+ // Check for an immediate hit
+ switch( _externSymbols.get(name) ) {
+
+ case ExternNode extern:
+ // Name maps to an ExternNode
+ return extern;
+
+ case ElfReader elf:
+ return elf;
+
+ case null:
+ // Name is unknown. Advance the file-system search, pulling
+ // out new ELF files from the search directory list and loading
+ // their published symbols.
+
+ // The last directory was mined out, so get the next search
+ // directory and find all ELF-like files.
+ if( _extFileIdx >= _files._len ) {
+ if( _externPaths==null || _extPathIdx >= _externPaths._len )
+ return null; // A true miss in the whole search path
+ loadExtPath(); // Load next batch file of files
+ break;
+ }
+ // Load and parse an ELF header
+ ElfReader elf = getElf(_files.at(_extFileIdx++));
+ if( elf == null ) break; // Not an ELF
+ // Load public symbols from the ELF
+ elf.loadPublicSymbols();
+ if( elf._strs==null ) break; // No .simple section with strings
+ // Put public symbol in global table; first names shadow all others
+ // Map from symbol string to ELFReader; most lookups
+ // will miss and no need to unpack ElfReader types
+ _externSymbols.putIfAbsent(elf._strs[1],elf);
+
+ // Publish definitions only; referenced classes belong to other objects.
+ for( String fname : elf._units )
+ _externSymbols.putIfAbsent(Parser.addClzPrefix(fname.replace('/','.')),elf);
+ // A -L object can name dependencies alongside itself. They
+ // need metadata discovery even when no directory was supplied.
+ for( String dep : elf._deps ) {
+ File file = elf.dependencyFile(dep);
+ if( file.isFile() ) addExtFile(file);
+ }
+ break;
+
+ default: throw Utils.TODO("should not reach here");
+ }
+ }
+ }
+
+ // Load all the .o files in the next external path.
+ private void loadExtPath() {
+ _files.clear();
+ _extFileIdx = 0;
+ String extpath = _externPaths.at(_extPathIdx++);
+ File f = new File(extpath);
+ if( !f.isAbsolute() )
+ f = new File(_cwd+extpath);
+ fillExtFiles(f);
+ }
+
+ // Recursive search (TODO: gzip, archives) and gather all .o files.
+ private void fillExtFiles(File dir) {
+ if( dir.isDirectory() )
+ for( File f : dir.listFiles() )
+ fillExtFiles(f);
+ else if( dir.getName().endsWith(".o") )
+ addExtFile(dir);
+ }
+
+ private void addExtFile(File file) {
+ file=file.toPath().toAbsolutePath().normalize().toFile();
+ if( _seenFiles.add(file) ) _files.add(file);
+ }
+
+ // Map from object file names to ElfReaders
+ public final HashMap<String,ElfReader> _elfs = new HashMap<>();
+ public ElfReader getElf(File f) {
+ ElfReader elf = _elfs.get(f.getPath());
+ if( elf != null )
+ return elf;
+ _elfs.put(f.getPath(), elf = ElfReader.load(f,null));
+ return elf;
+ }
+
+ // ---------------------------
+ public boolean _asmLittle=true;
+ public String asm() { return asm(new SB()).toString(); }
+ SB asm(SB sb) {
+ ASMPrinter.print(sb,this);
+ return sb;
+ }
+
+
+ // Testing shortcuts
+ public Node ctrl() { return compunit()._stop.ret().ctrl(); }
+ public Node expr() { return compunit()._stop.ret().expr(); }
+ public String print() { return compunit()._stop.print(); }
+
+ // Debugging helper
+ @Override public String toString() {
+ if( _stop == null )
+ return "No StopNode";
+ return IRPrinter.prettyPrint(this);
+ }
+
+ // Debugging helper
+ public Node f(int idx) {
+ for( CompUnit cu : _compunits.values() ) {
+ Node n = (cu._stop == null ? _stop : cu._stop).find(idx);
+ if( n != null ) return n;
+ }
+ return null;
+ }
+
+
+ String printCFG() {
+ if( _cfg==null ) return "no CFG";
+ SB sb = new SB();
+ for( CFGNode cfg : _cfg ) {
+ sb.fix(8,""+(int)cfg._idepth);
+ IRPrinter.printLine( cfg, sb );
+ }
+ return sb.toString();
+ }
+
+ public static void print_as_hex(Encoding enc) {
+ for (byte b : enc._bits.toByteArray()) {
+ System.out.print(String.format("%02X", b));
+ }
+ System.out.println();
+ }
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/CompUnit.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/CompUnit.java
new file mode 100644
index 00000000..2b091b3c
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/CompUnit.java
@@ -0,0 +1,170 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.parser.Parser;
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.type.Type;
+import com.seaofnodes.simple.type.TypeStruct;
+import com.seaofnodes.simple.util.*;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.*;
+import static com.seaofnodes.simple.codegen.CodeGen.CODE;
+
+// A compilation-unit, which is an external reference and source and object
+// files. These are mirrored over a file system tree (so form a tree), both
+// the source tree and build/object tree.
+public class CompUnit {
+ public final CompUnit _par; // Parent, e.g. A/B
+ final String _fname; // Full slashed file name starting from module root, e.g. A/B/C
+ final public String _cname; // Full dotted class name starting from module root, e.g. A.B.C
+ final String _name; // Base name, e.g. C
+ final ExternNode _ext; // Found in external libs; e.g. libc
+ final File _smp; // Simple source file, e.g. module_root/A/B/C.smp
+ public final File _obj; // ELF output file, e.g. build/A/B/C.o
+ final public String _src; // Source code, from file or test case
+ // Original source path from an imported diagnostic location. Does not
+ // affect discovery/rebuild decisions, which use _smp and _src.
+ public String _sourceFile;
+
+ public String sourceFile() {
+ return _smp!=null ? _smp.getAbsolutePath() : _sourceFile;
+ }
+
+ public TypeStruct _clz; // The one TypeStruct being published
+
+ // _deps records units required for types, code, or class references,
+ // without requiring their initializers to run. These discovery/rebuild
+ // dependencies may form a cycle.
+ Ary<CompUnit> _deps;
+
+ // Per-Compilation-Unit Start/StopNodes, keeping alive all exported Nodes.
+ // Null means no code loaded (yet).
+ public StartCUNode _start;
+ public StopCUNode _stop ;
+
+
+ // Discovered source file. Source is loaded if required to parse, and null
+ // otherwise.
+ CompUnit( CompUnit par, String fname, String cname, String name ) throws IOException {
+ _par = par;
+ _fname= fname;
+ _cname= cname;
+ _name = name;
+ _ext = null;
+ _smp = new File( CODE. _modDir + "/" + fname + ".smp");
+ _obj = new File( CODE._buildDir + "/" + fname + ".o" );
+
+ boolean mustCompile = _smp.exists() && // Has source to compile
+ ( fname.equals(CODE._srcName) || // Explicit input is a compilation request
+ !_obj.exists() || // No object file
+ _smp.lastModified() > _obj.lastModified() || // Out-of-date with source?
+ !current(_obj,new HashSet<>()) ); // Out-of-date anywhere in its inputs?
+ _src = mustCompile ? new String(Files.readAllBytes(_smp.toPath())) : null;
+ }
+
+ // Test case. No source file. Source code passed in as a String. Fake
+ // class name.
+ CompUnit( String fname, String src ) {
+ _par = null;
+ _name = fname;
+ _fname= fname;
+ _cname= fname;
+ _ext = null;
+ _smp = null;
+ _obj = null;
+ _src = src;
+ }
+
+ // External CompUnit, pulled in from an external library like libc
+ CompUnit( ExternNode ext, String symbol ) {
+ _par = null;
+ _name = symbol;
+ _fname= _name;
+ _cname= null;
+ _ext = ext;
+ _smp = null;
+ _obj = null;
+ _src = null;
+ }
+
+ // External Simple object file, or namespace parent, found on an external search path.
+ CompUnit( File obj, CompUnit par, String fname, String cname, String name ) {
+ _par = par;
+ _name = name;
+ _fname= fname;
+ _cname= cname;
+ _ext = null;
+ _smp = null;
+ _obj = obj;
+ _src = null;
+ }
+
+ // Check source-backed inputs before importing any saved types or IR.
+ // A dependency's source can be newer while its object is unchanged.
+ // Use a fresh visited set for each root: encountering a cycle is safe, but
+ // caching a provisional "current" answer from inside it would not be.
+ private static boolean current(File obj, HashSet<File> visit) {
+ if( obj==null || !obj.isFile() ) return false;
+ if( !visit.add(obj.getAbsoluteFile()) ) return true;
+ ElfReader elf = CODE.getElf(obj);
+ if( elf==null ) return false;
+ elf.loadPublicSymbols();
+ if( elf._units==null ) return false;
+ long time = obj.lastModified();
+ for( String unit : elf._units ) {
+ // Never import another copy of a unit explicitly being compiled.
+ CompUnit cu = CODE._compunits.get(unit);
+ if( unit.equals(CODE._srcName) || (cu!=null && cu._src!=null) ) return false;
+ File src = new File(CODE._modDir+"/"+unit+".smp");
+ if( !src.isFile() || src.lastModified()>time ) return false;
+ }
+ for( String dep : elf._deps ) {
+ if( Arrays.asList(elf._units).contains(dep) ) continue; // Owned by this object.
+ File file = dependencyObj(dep,true);
+ if( file==null || !file.isFile() || file.lastModified()>time ) return false;
+ // Object-only libraries are supplied as a coherent build. Their
+ // input versions cannot be inferred from installation timestamps.
+ if( file.equals(new File(CODE._buildDir+"/"+dep+".o")) && !current(file,visit) ) return false;
+ }
+ return true;
+ }
+
+ // Resolve dependencies the same way for freshness checks and actual loads.
+ static File dependencyObj(String fname, boolean local) {
+ File obj = new File(CODE._buildDir+"/"+fname+".o");
+ if( local && (obj.isFile() || new File(CODE._modDir+"/"+fname+".smp").isFile()) ) return obj;
+ ElfReader elf = CODE.findExternalSimple(Parser.addClzPrefix(fname.replace('/','.')));
+ return elf==null ? null : elf._file;
+ }
+
+ // Add a discovered dependency, for later writing out in an ELF file, ".simple" section
+ void addDep(CompUnit dep) {
+ if( _deps == null ) _deps = new Ary<>(CompUnit.class);
+ _deps.add(dep);
+ }
+
+ public void addFun(CodeGen code, FunNode fun) {
+ assert fun.in(1)==_start;
+ _stop.addDef(fun.ret());
+ code.add(_stop);
+ _stop.addDep(fun);
+ }
+
+ void setStart(CodeGen code) {
+ _stop = new StopCUNode().init();
+ _start = new StartCUNode(code._start,_stop, Type.BOTTOM, _fname).init();
+ code._stop.addDef(_stop);
+ }
+
+ @Override public String toString() {
+ String x = "{"+_cname;
+ if( _smp!=null && _smp.exists()) x += ",hasSMP";
+ if( _obj!=null && _obj.exists()) x += ",hasObj";
+ if( _src!=null ) x += ",hasSrc";
+ if( _ext != null ) x += ",isExt";
+ x += "}";
+ return x;
+ }
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/ElfReader.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/ElfReader.java
new file mode 100644
index 00000000..1dced3a8
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/ElfReader.java
@@ -0,0 +1,245 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.node.Node;
+import com.seaofnodes.simple.type.TypeMemPtr;
+import com.seaofnodes.simple.type.TypeStruct;
+import com.seaofnodes.simple.util.Ary;
+import com.seaofnodes.simple.util.BAOS;
+import com.seaofnodes.simple.util.Utils;
+import java.io.*;
+import java.nio.file.Files;
+import java.util.HashMap;
+
+public class ElfReader {
+
+ public final File _file;
+ public final byte[] _buf;
+ int _x; // Cursor into buffer
+ // ELF Header
+ public final ElfHeader _header;
+ // All sections
+ public final Section[] _sections;
+ // Simple section, if available
+ public final Section _simple;
+ // Byte array in/out stream - used as a reader
+ public final BAOS _bais;
+ // Compilation unit; written into FunNodes
+ public final CompUnit _compunit;
+
+ // Strings from the Simple section, either small public set or whole set
+ public String[] _strs;
+ // Dependent file names
+ public String[] _deps;
+ // Compilation units defined by this object, independent of init analysis.
+ public String[] _units;
+ // Only published symbol this file
+ public TypeStruct _clz;
+ // Canonical named types, distinct from optimized same-named graph states.
+ public TypeStruct[] _pubTypes;
+ // Nodes read in this Elf
+ public Ary<Node> _nodes;
+ Serialize.TypeRead _types;
+ FunctionSummary[] _functions;
+
+ // Load an Elf file (mostly lazily)
+ public static ElfReader load( File f, CompUnit cu ) {
+ try { return new ElfReader(f,cu); }
+ catch( IOException ioe ) { return null; }
+ }
+
+ // Load the Simple section from the ELF file
+ private ElfReader( File f, CompUnit cu ) throws IOException {
+ _file = f;
+ _compunit = cu;
+ _buf = Files.readAllBytes(f.toPath());
+ _header = new ElfHeader();
+ // No attempt is made to parse the program header table.
+ _sections = new Section[_header._shnum];
+ for( int i=0; i<_header._shnum; i++ )
+ _sections[i] = new Section(i);
+
+ // Load section names
+ Section secstr = _sections[_header._shstrndx];
+ for( Section sec : _sections )
+ if( sec._type != 0 )
+ sec._name = secstr.str(sec._noff);
+
+ Section simple = null;
+ for( Section sec : _sections )
+ if( ".simple".equals(sec._name) )
+ simple = sec;
+ _simple = simple;
+ _bais = _simple==null ? null : new BAOS(_buf, (int)_simple._offset);
+ }
+
+ // For Serialize self-test
+ ElfReader( BAOS bais ) {
+ _file = null;
+ _compunit = null;
+ _buf = bais.buf();
+ _header = null;
+ _sections = null;
+ _simple = null;
+ _bais = bais;
+ }
+
+ // Load the public symbols and dependent objs, but not the whole IR
+ void loadPublicSymbols() {
+ // Check already loaded and cached
+ if( _strs == null && _simple != null )
+ // Read only the published public strings, for speed purposes when
+ // scanning a large count of ELF files but getting hits in only a
+ // few of them. i.e. optimizing (prematurely?) for the traditional
+ // case of linking a small file against a large library
+ Serialize.read_public_strs(this);
+ }
+
+ File dependencyFile(String unit) {
+ File root = _file;
+ for( String part : _units[0].split("/") ) root=root.getParentFile();
+ return new File(root,unit+".o");
+ }
+
+ // Loads the entire IR
+ public TypeStruct loadSimple(CodeGen code) {
+ code._classInitDependencies = null; // Loaded bodies can change init edges.
+ Serialize.readAll(code, this, code._aliases, code._fidxs, code._rpcs);
+ return _clz;
+ }
+
+ // Loads strings, mappings, and types, but not Nodes. Used after the
+ // public-string scan finds a matching Simple ELF, so parsing can resolve
+ // imported field/array aliases before the full graph is linked in later.
+ public TypeStruct loadPublicTypes(CodeGen code) {
+ Serialize.readTypes(code, this, code._aliases, code._fidxs, code._rpcs);
+ return _clz;
+ }
+
+ int u1() { return _buf[_x++]&0xFF; }
+ int magic() { // Big endian
+ return (u1() << 24) | (u1() << 16) | (u1() << 8) | u1();
+ }
+ int u2() { return u1() | (u1()<<8); } // Little endian
+ int u4() { // Little endian
+ return u1() | (u1() << 8) | (u1() << 16) | (u1() << 24);
+ }
+ long u8() { return u4() | ((long)u4()<<32); }
+ void skip(int skip) { _x += skip; }
+
+
+ public class ElfHeader {
+ final int _magic;
+ final boolean _format32; // Format32 or format64
+ final byte _osabi;
+ final boolean _little; // True if little endian, false if big endian
+ final char _type;
+ final char _machine;
+ final long _entry; // Process entry point
+ final long _phoff; // Program header table offset, from file start
+ final long _shoff; // Section header table offset, from file start
+ final int _flags; // Flags
+ final int _phnum; // Number of program header entries
+ final int _shentsize;
+ final int _shnum; // Number of section header entries
+ final int _shstrndx; // Section header index contains section names
+
+
+ ElfHeader() throws IOException {
+ _x = 0;
+ if( (_magic = magic()) != 0x7F454C46 ) bad("magic number",_magic);
+ byte EI_CLASS = (byte)u1();
+ if( EI_CLASS != 1 && EI_CLASS != 2 ) bad("EI_CLASS",EI_CLASS);
+ _format32 = (EI_CLASS==1);
+ byte EI_DATA = (byte)u1();
+ if( EI_DATA != 1 && EI_DATA != 2 ) bad("EI_DATA",EI_DATA);
+ _little = EI_DATA == 1;
+ if( u1() != 1 ) bad("version",1);
+ if( (_osabi=(byte)u1()) != 0 ) bad("EI_VERSION",_osabi);
+ byte abiversion = (byte)u1();
+ if( abiversion != 0 ) bad("EI_ABIVERSION",abiversion);
+ skip(7); // Pad
+
+ // Endian matters now
+ if( !_little ) throw new IOException("TODO: Big endian");
+ if( (_type = (char)u2()) != 1 ) bad("ET_REL",_type);
+ if( (_machine = (char)u2()) != 0x3E ) bad("AMD x86-64",_machine);
+ int ver = u4();
+ if( ver != 1 ) bad("Version",ver);
+ // Format32 matters now
+ if( _format32 ) bad("Format",1);
+ _entry = u8(); // Process entry point
+ _phoff = u8(); // Program header table offset
+ _shoff = u8(); // Section header table offset
+ _flags = u4(); // Flags
+ int ehsize = u2();
+ if( ehsize != 0x40 ) bad("EHSIZE",ehsize);
+ int phentsize = u2();
+ if( phentsize != 0x38 && phentsize != 0 ) bad("PHENTSIZE",phentsize);
+ _phnum = u2();
+ _shentsize = u2();
+ if( _shentsize != 0x40 ) bad("SHENTSIZE",_shentsize);
+ _shnum = u2();
+ // String section
+ _shstrndx = u2();
+ assert _x == ehsize;
+ }
+ }
+
+ public class Section {
+ final int _noff; // Offset in string section for name
+ final int _type; // Section type
+ final long _flags;
+ final long _addr; // Offset in virtual mem, if loaded
+ final long _offset; // Offset in file
+ final long _size; // Size in bytes
+ final int _link; // Associated section index, varies
+ final int _info; // Varies
+ final byte _align; // Log2 alignment
+ final int _esize; // Element size, if non-zero
+ String _name;
+
+ Section(int sidx) throws IOException {
+ _x = (int)(_header._shoff + (sidx * _header._shentsize));
+ _noff = u4();
+ _type = u4();
+ _flags = u8();
+ _addr = u8();
+ _offset= u8();
+ _size = u8();
+ _link = u4();
+ _info = u4();
+ long align = u8();
+ if( align==0 ) align=1;
+ _align = (byte)Long.numberOfTrailingZeros(align);
+ if( (1L << _align ) != align )
+ throw new IOException("Bad section alignment: "+align);
+ _esize = (int)u8();
+ }
+
+ String str(int off) {
+ assert _type == 3/*SHT_STRTAB*/; // String table section header
+ int start = (int)(_offset+off);
+ int x = start;
+ while( _buf[x++] !=0 ) ;
+ return new String(_buf,start,x-start-1);
+ }
+ }
+
+ private static void bad(String msg, int val) throws IOException {
+ throw new IOException("Bad "+msg+": "+val);
+ }
+
+
+
+ //public TypeMemPtr lookup( String name ) {
+ // if( _simple == null ) return null;
+ // BAOS baos = new BAOS(_buf, (int)_simple._offset);
+ // Ary<String> strs = Serialize.read_strs(baos);
+ //
+ //
+ //
+ // Ary<Node> nodes = Serialize.read_types(baos);
+ //
+ // throw Utils.TODO("Elf load symbol "+name);
+ //}
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/ElfWriter.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/ElfWriter.java
new file mode 100644
index 00000000..fcf5c287
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/ElfWriter.java
@@ -0,0 +1,468 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.type.TypeFunPtr;
+import com.seaofnodes.simple.type.TypeStruct;
+import com.seaofnodes.simple.util.Ary;
+import com.seaofnodes.simple.util.BAOS;
+import java.io.*;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.HashMap;
+
+public class ElfWriter {
+
+ final CodeGen _code;
+ Ary<Section> _sections;
+ StringSection _strtab;
+
+ ElfWriter( CodeGen code ) {
+ _code = code;
+ }
+
+ // ------------------------------------------------------------------------
+ abstract public class Section {
+ static final int SHT_NULL = 0;
+ static final int SHT_PROGBITS= 1;
+ static final int SHT_SYMTAB = 2;
+ static final int SHT_STRTAB = 3;
+ static final int SHT_RELA = 4;
+ static final int SHT_SIMPLE = 10; // Simple section
+
+ final String _name;
+ final int _type, _entsize, _index, _name_pos, _flags;
+ int _link, _info;
+ int _file_offset;
+
+ Section(String name, int type, int entsize, int flags) {
+ _name = name;
+ _type = type;
+ _entsize = entsize;
+ _flags = flags;
+ _index = _sections._len;
+ _sections.push(this);
+ _name_pos = name.isEmpty()
+ ? 0 // Null section
+ : _strtab == null
+ ? 1 // StringSection
+ // All the other sections
+ : _strtab.writeCString(name);
+ }
+
+ void writeHeader(ByteBuffer out) {
+ out.putInt(_name_pos); // name
+ out.putInt(_type); // type
+ out.putLong(_flags); // flags
+ out.putLong(0); // addr
+ out.putLong(_file_offset); // offset
+ out.putLong(size()); // size
+ out.putInt(_link); // link
+ out.putInt(_info); // info
+ out.putLong(16); // addralign
+ out.putLong(_entsize); // Fixed-size element size
+ }
+
+ abstract void write(ByteBuffer out);
+ abstract int size();
+ }
+
+ // ------------------------------------------------------------------------
+ // Generic data sections; carries about a BAOS for bits
+ public class DataSection extends Section {
+ static final int SHF_WRITE = 1;
+ static final int SHF_ALLOC = 2;
+ static final int SHF_EXECINSTR = 4;
+ static final int SHF_INFO_LINK = 64;
+
+ final BAOS _contents;
+
+ DataSection( String name, int type, int flags, int entsize ) { this(name, type, flags, entsize, new BAOS() ); }
+ DataSection( String name, int type, BAOS contents, int flags ) { this(name, type, flags, 0, contents ); }
+ private DataSection( String name, int type, int flags, int entsize, BAOS contents ) {
+ super(name, type, entsize, flags);
+ _contents = contents;
+ }
+
+ @Override void write(ByteBuffer out) { out.put(_contents.toByteArray()); }
+ @Override int size() { return _contents.size(); }
+ public void write2( int op ) {
+ _contents.write(op );
+ _contents.write(op>> 8);
+ }
+ public void write4( int op ) {
+ _contents.write(op );
+ _contents.write(op>> 8);
+ _contents.write(op>>16);
+ _contents.write(op>>24);
+ }
+ public void write8( long i64 ) {
+ write4( (int) i64 );
+ write4( (int)(i64>>32));
+ }
+
+ }
+
+ // ------------------------------------------------------------------------
+ // Section to collect and de-dup strings
+ public class StringSection extends DataSection {
+ final HashMap<String,Integer> _dedup = new HashMap<>();
+
+ StringSection() {
+ super(".strtab",SHT_STRTAB, 0, 0 );
+ int emptyoff = writeCString(""); // Always empty string at offset 0
+ assert emptyoff == 0;
+ int namepos = writeCString(".strtab");
+ assert namepos == _name_pos;
+ }
+
+ public int writeCString(String name) {
+ if( name==null ) return 0; // Anonymous functions have no name
+ Integer ii = _dedup.get(name);
+ if( ii != null ) return ii;
+ // place name in the string table
+ int pos = _contents.size();
+ _dedup.put(name,pos);
+ byte[] name_bytes = name.getBytes();
+ _contents.write(name_bytes);
+ // the strings are null terminated
+ _contents.write(0);
+ return pos;
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // The relocations; 3 words per relocation
+ public class ReloSection extends DataSection {
+ ReloSection( int info ) {
+ this(".rela.text",info);
+ }
+ ReloSection(String name,int info) {
+ super(name, SHT_RELA, SHF_INFO_LINK, 24/*entsize*/);
+ _link = 2;
+ _info = info;
+ }
+
+ void writeReloOffPlus(int offset, int symidx, byte relo) {
+ writeRelo(offset,symidx,relo,-4);
+ }
+ void writeRelo(int offset,int symidx,byte relo,long addend) {
+ assert symidx > 0;
+ write8(offset);
+ write8(((long)symidx << 32L) | relo);
+ write8(addend);
+ }
+ }
+
+
+ // ------------------------------------------------------------------------
+ // Symbols section; emit the symbols into the data section directly
+ public class SymbolSection extends DataSection {
+ public static final int SYMBOL_SIZE = 4+1+1+2+8+8;
+ final BAOS _globals;
+ final int _nlocals;
+
+ SymbolSection(int nlocals) {
+ super(".symtab", SHT_SYMTAB, 0/*flags*/, SYMBOL_SIZE);
+ _globals = new BAOS();
+ _nlocals = nlocals;
+ }
+
+ // Global symbol index
+ int gidx() { return _nlocals+(_globals.size()/SYMBOL_SIZE); }
+
+ public int symbol( String name, int shndx, int bind, int type, long value, long size ) {
+ int gidx = gidx(); // Global index before I write to _globals
+ BAOS baos = bind==SYM_BIND_LOCAL ? _contents : _globals;
+ write4(baos,_strtab.writeCString(name));
+ int info = (bind << 4) | (type & 0xF);
+ baos.write(info);
+ baos.write(0); // other/visibility
+ write2(baos,shndx); // section index
+ write8(baos,value);
+ write8(baos,size);
+ return bind==SYM_BIND_LOCAL ? -99 : gidx;
+ }
+
+ public int symbol( String name, int shndx, int bind, int type ) {
+ return symbol(name,shndx,bind,type,0,0);
+ }
+
+ // creates function and stores where it starts.
+ // Filters to just this class.obj file.
+ void encodeFunctions(StopNode stop, Encoding enc, int text_idx, boolean emitEntrySymbol) {
+ for( Node stopcu : stop._inputs ) {
+ for( Node ret : stopcu._inputs ) {
+ FunNode fun = ((ReturnNode)ret).fun();
+ if( !_code.emits(fun) )
+ continue;
+ int end = enc.opStart(fun.ret()) + enc.opLen(fun.ret());
+ long value = enc.opStart(fun);
+ long size = end - value;
+ // Stable FIDX names are exported separately below.
+ if( fun._name != null && !fun._name.equals(_code._fidxs.symbol(fun.sig().fidx())) ) {
+ // The C runtime owns `main` and adapts argc/argv into
+ // a normal Simple array. Keep the Simple function on
+ // its native ABI under a private entry symbol.
+ String name = emitEntrySymbol && "main".equals(fun._name)
+ ? "simple_main"
+ : fun._name;
+ symbol(name, text_idx, SYM_BIND_GLOBAL, SYM_TYPE_FUNC, value, size);
+ }
+ }
+ }
+ }
+
+ // Insert a default-main for top-level compilations, if the top-level
+ // does not declare a main.
+ void symbolEntryClinit(int text_idx, Encoding enc) {
+ FunNode clinit = null;
+ String entryName = _code.entryClinitName();
+ for( FunNode fun : _code._linker )
+ if( fun != null && _code.emits(fun) ) {
+ if( "main".equals(fun._name) )
+ return; // encodeFunctions exported this as simple_main
+ if( entryName.equals(fun._name) )
+ clinit = fun; // Keep top-level entry, if we don't find an explicit main
+ }
+ // No top-level main, so the top-level clinit becomes a default main
+ if( clinit != null ) {
+ assert enc.opStart(clinit) == 0;
+ symbol("simple_main",text_idx, SYM_BIND_GLOBAL, SYM_TYPE_FUNC, enc.opStart(clinit), 0);
+ return;
+ }
+ throw new IllegalStateException("Missing entry <clinit> "+entryName);
+ }
+
+ static void write2( BAOS bits, int op ) {
+ bits.write(op );
+ bits.write(op>> 8);
+ }
+ static void write4( BAOS bits, int op ) {
+ bits.write(op );
+ bits.write(op>> 8);
+ bits.write(op>>16);
+ bits.write(op>>24);
+ }
+ static void write8( BAOS bits, long i64 ) {
+ write4(bits, (int) i64 );
+ write4(bits, (int)(i64>>32));
+ }
+
+ @Override
+ void writeHeader(ByteBuffer out) {
+ // points to string table section
+ _link = _strtab._index;
+ _info = _nlocals;
+ super.writeHeader(out);
+ }
+
+ @Override void write(ByteBuffer out) {
+ out.put(_contents.toByteArray());
+ out.put(_globals .toByteArray());
+ }
+ @Override int size() {
+ return _contents.size() + _globals.size();
+ }
+ }
+
+ public static final int SYM_BIND_LOCAL = 0;
+ public static final int SYM_BIND_GLOBAL = 1;
+ public static final int SYM_BIND_WEAK = 2;
+
+ public static final int SYM_TYPE_NOTYPE = 0;
+ public static final int SYM_TYPE_OBJECT = 1;
+ public static final int SYM_TYPE_FUNC = 2;
+ public static final int SYM_TYPE_SECTION = 3;
+
+ public static final int SECTION_HDR_SIZE = 64;
+
+ // ------------------------------------------------------------------------
+ public class SimpleSection extends DataSection {
+ SimpleSection(BAOS serial) { super(".simple", SHT_SIMPLE, serial, 0); }
+ }
+
+
+ // ------------------------------------------------------------------------
+
+ // Native code can live above 4G, outside our four-byte function pointers.
+ // Give address-taken natives a low entry which tail-jumps through the PLT.
+ // A weak, stable name makes references from separate objects agree on one
+ // entry. Do not patch references straight to the local copy: that would
+ // bypass the linker's choice and break pointer equality across objects.
+ private int nativeEntry(String name, HashMap<String,Integer> entries,
+ DataSection text, SymbolSection symbols, ReloSection relos) {
+ Integer old = entries.get(name);
+ if( old!=null ) return old;
+ int offset = text.size();
+ text._contents.write(0xE9); // jmp rel32; preserve arguments and return address
+ text.write4(0);
+ int target = symbols.symbol(name,0,SYM_BIND_GLOBAL,SYM_TYPE_FUNC);
+ relos.writeReloOffPlus(offset+1,target,(byte)4/*PLT32*/);
+ int entry = symbols.symbol("__simple_native$"+name,text._index,
+ SYM_BIND_WEAK,SYM_TYPE_FUNC,offset,5);
+ entries.put(name,entry);
+ return entry;
+ }
+
+ public ElfWriter export(boolean emitEntrySymbol) {
+ if( _code._serial==null )
+ throw new IllegalStateException("ELF export needs ideal IR: request Export from the driver, or Serialize before Lower");
+ // Sections are created in the order they are emitted.
+ _sections = new Ary<>(Section.class);
+
+ // Null section
+ Section nullSection = new DataSection("",Section.SHT_NULL,0,0);
+
+ // String section next; written out early and available to record/dedup
+ // strings for all remaining sections
+ _strtab = new StringSection();
+
+ // place all the symbols
+ int nlocals = 1/*null*/+9/*number of sections*/+1/*Simple section*/;
+ SymbolSection symbols = new SymbolSection(nlocals);
+
+ // we've already constructed these entire sections in the encoding phase
+ Encoding enc = _code._encoding;
+ // Append ELF-only thunks to a copy. Export must not change instruction
+ // offsets or the image used by the in-memory linker and emulators.
+ BAOS textBits = new BAOS(enc._bits.toByteArray(),enc._bits.size());
+ DataSection text = new DataSection(".text" , Section.SHT_PROGBITS, textBits, DataSection.SHF_ALLOC | DataSection.SHF_WRITE | DataSection.SHF_EXECINSTR );
+ // Constant pool
+ DataSection rodata = new DataSection(".rodata", Section.SHT_PROGBITS, enc._cpool, DataSection.SHF_ALLOC );
+ // Static/global pool
+ DataSection rwdata = new DataSection(".data" , Section.SHT_PROGBITS, enc._sdata, DataSection.SHF_ALLOC | DataSection.SHF_WRITE );
+
+
+ // populate function symbols
+ symbols.encodeFunctions(_code._stop, _code._encoding, text._index, emitEntrySymbol);
+ // Export the source compilation unit <clinit> as the C runtime "main".
+ // Encoding keeps this function at text offset zero.
+ if( emitEntrySymbol )
+ symbols.symbolEntryClinit(text._index,enc);
+
+ // create external .text relocations
+ ReloSection relocations = new ReloSection(text._index);
+ HashMap<String,Integer> nativeEntries = new HashMap<>();
+
+ for( Node n : enc._externals.keySet()) {
+ String extern = enc._externals.get(n);
+ boolean nativeAddress = n instanceof FunPtrNode &&
+ _code.externFunc(((TypeFunPtr)n._type).fidx())!=null;
+ int symidx = nativeAddress
+ ? nativeEntry(extern,nativeEntries,text,symbols,relocations)
+ : symbols.symbol(extern,0,SYM_BIND_GLOBAL,SYM_TYPE_FUNC);
+ int offset = enc.opStart(n) + enc.opLen(n) - 4;
+ // Address materialization and static fields name the same thunk;
+ // direct calls still go straight to the native symbol's PLT entry.
+ relocations.writeReloOffPlus(offset,symidx,(byte)(nativeAddress ? 2/*PC32*/ : 4/*PLT32*/));
+ }
+
+ // Native ELF output is AMD64. Data addresses use PC32, not PLT calls.
+ for( Node n : enc._externData.keySet() ) {
+ String extern = enc._externData.get(n);
+ int symidx = symbols.symbol(extern, 0, SYM_BIND_GLOBAL, SYM_TYPE_OBJECT);
+ int offset = enc.opStart(n) + enc.opLen(n) - 4;
+ relocations.writeReloOffPlus(offset, symidx, (byte)2/*PC32*/);
+ }
+
+ // Stable function identities are also addressable from data, including
+ // anonymous functions which have no source-level exported name.
+ HashMap<String,Integer> dataSymbols=new HashMap<>();
+ for( FunNode fun : _code._linker )
+ if( fun!=null && !fun.isDead() && _code.emits(fun) && enc.opStart(fun)>=0 ) {
+ String name=_code._fidxs.symbol(fun.sig().fidx());
+ dataSymbols.put(name,symbols.symbol(name,text._index,SYM_BIND_GLOBAL,SYM_TYPE_FUNC,enc.opStart(fun),0));
+ }
+ for( StaticData.ObjectData obj : enc._data.objects.values() ) {
+ int section=obj.external ? 0 : obj.readOnly ? rodata._index : rwdata._index;
+ dataSymbols.put(obj.symbol,symbols.symbol(obj.symbol,section,SYM_BIND_GLOBAL,SYM_TYPE_OBJECT,
+ obj.external ? 0 : obj.offset,obj.external ? 0 : obj.size));
+ }
+ for( Encoding.Relo relo : enc._bigCons.values() ) {
+ StaticData.ObjectData obj=enc._data.object(relo._t);
+ relocations.writeReloOffPlus(relo._opStart+relo._off,dataSymbols.get(obj.symbol),relo._elf);
+ }
+ ReloSection roRelos=new ReloSection(".rela.rodata",rodata._index);
+ ReloSection rwRelos=new ReloSection(".rela.data",rwdata._index);
+ for( StaticData.Relocation relo : enc._data.relocations ) {
+ String name=enc._data.targetSymbol(relo.value());
+ String nativeName=relo.value() instanceof TypeFunPtr fp ? _code.externFunc(fp.fidx()) : null;
+ Integer sym=nativeName==null ? dataSymbols.get(name)
+ : nativeEntry(nativeName,nativeEntries,text,symbols,relocations);
+ if( sym==null ) {
+ sym=symbols.symbol(name,0,SYM_BIND_GLOBAL,SYM_TYPE_NOTYPE);
+ dataSymbols.put(name,sym);
+ }
+ // AMD64 absolute 32/64-bit addresses; linker rejects overflow.
+ byte kind=(byte)(relo.logSize()==2 ? 10 : 1);
+ (relo.object().readOnly ? roRelos : rwRelos).writeRelo(relo.object().offset+relo.offset(),sym,kind,0);
+ }
+
+ // Create a "Simple" section for Simple types and ir
+ SimpleSection simple = new SimpleSection(_code._serial);
+
+ // Write the local section header symbols
+ assert nlocals == 1+_sections._len;
+ symbols.symbol("",0,0,0); // Null symbol at index 0
+ for( Section s : _sections )
+ symbols.symbol(s._name, s._index, SYM_BIND_LOCAL, SYM_TYPE_SECTION, 0, s.size());
+
+ int size = 64; // ELF header size
+ // size of all section data
+ for( Section s : _sections ) {
+ s._file_offset = size;
+ size += s.size();
+ }
+ // headers go at the very end
+ int shdr_offset = size;
+ size += SECTION_HDR_SIZE*_sections.len();
+
+ ByteBuffer out = ByteBuffer.allocate(size);
+ out.order(ByteOrder.LITTLE_ENDIAN);
+
+ // ELF header
+ out.put(new byte[]{ 0x7F, 'E', 'L', 'F', 2, 1, 1, 0 });
+ out.put(new byte[8]);
+ out.putShort((short)1 /* ET_REL */); // type
+ out.putShort((short)62 /* AMD64 */); // machine
+ out.putInt(1); // version
+ out.putLong(0); // entry
+ out.putLong(0); // phoff
+ out.putLong(shdr_offset); // shoff
+ out.putInt(0); // flags
+ out.putShort((short)64); // ehsize
+ out.putShort((short)0); // phentsize
+ out.putShort((short)0); // phnum
+ out.putShort((short)64); // shentsize
+ out.putShort((short) _sections.len()); // shnum
+ out.putShort((short) _strtab._index); // shstrndx
+
+ // raw section data
+ for( Section s : _sections )
+ s.write(out);
+
+ // Section headers
+ for( Section s : _sections )
+ s.writeHeader(out);
+
+ assert out.position() == size;
+
+ String objName = _code._buildDir + "/" + (_code._emitUnit==null ? _code._srcName : _code._emitUnit._fname) + ".o";
+ File file = new File(objName);
+ if( file.getParentFile()!=null )
+ file.getParentFile().mkdirs();
+ try {
+ BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file));
+ bos.write(out.array());
+ bos.close();
+ } catch( IOException ioe ) {
+ // Caller does not want to deal, but it is a failed compilation in any case
+ throw new RuntimeException(ioe);
+ }
+ // Reset for next ELF
+ _sections = null;
+ _strtab = null;
+ return this;
+ }
+
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/Encoding.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Encoding.java
new file mode 100644
index 00000000..8b868218
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Encoding.java
@@ -0,0 +1,570 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.parser.Parser;
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.print.IRPrinter;
+import com.seaofnodes.simple.type.*;
+import com.seaofnodes.simple.util.*;
+import java.util.*;
+
+
+/**
+ * Instruction encodings
+ * <p>
+ * This class holds the encoding bits, plus the relocation information needed
+ * to move the code to a non-zero offset, or write to an external file (ELF).
+ * <p>
+ * There are also a bunch of generic utilities for managing bits and bytes
+ * common in all encodings.
+ */
+public class Encoding implements com.seaofnodes.isa.CodeSink {
+
+ // Top-level program graph structure
+ public final CodeGen _code;
+
+ // CFG Nodes in RPO order, per-encoding.
+ public Ary<CFGNode> _cfg;
+
+ // Instruction bytes. The registers are encoded already. Relocatable data
+ // is in a fixed format depending on the kind of relocation.
+
+ // - RIP-relative offsets into the same chunk are just encoded with their
+ // chunk relative offsets; no relocation info is required.
+
+ // - Fixed address targets in the same encoding will be correct for a zero
+ // base; moving the entire encoding requires adding the new base address.
+
+ // - RIP-relative to external chunks have a zero offset; the matching
+ // relocation info will be used to patch the correct value.
+
+ public final BAOS _bits = new BAOS(); // Instructions / encodings
+ public final BAOS _cpool = new BAOS(); // Constant r/o pool
+ public final BAOS _sdata = new BAOS(); // Static r/w pool
+ public final HashMap<Node,Relo> _bigCons = new HashMap<>();
+ public final StaticData _data;
+ public final HashMap<String,Integer> _externalAddresses = new HashMap<>();
+
+ // Big Constant relocation info.
+ public static class Relo {
+ public final Node _op;
+ public final Type _t; // Constant type
+ public final int _align; // Log alignment, captured for diagnostic display
+ public final int _structSize; // Capture layout during encoding, not printing
+ public final byte _off; // Offset from start of opcode
+ public final byte _elf; // ELF relocation type, e.g. 2/PC32
+ public int _target; // Where constant is finally placed
+ public int _opStart; // Opcode start
+ Relo( Node op, Type t, byte off, byte elf ) {
+ _op=op; _t=t; _off=off; _elf=elf;
+ _align = t.alignment();
+ _structSize = t instanceof TypeStruct ts ? ts.size() : 0;
+ _readOnly = readOnly(t);
+ }
+ public boolean readOnly() { return _readOnly; }
+ private final boolean _readOnly;
+ private static boolean readOnly(Type t) {
+ // Class initializers still write their fields, even when every
+ // final value is constant. Their storage must remain writable.
+ return !(t instanceof TypeStruct ts) ||
+ (!Parser.startsClzPrefix(ts._name) && ts.isConstant());
+ }
+ }
+
+ Encoding( CodeGen code ) {
+ _code = code;
+ _data = new StaticData(this);
+ }
+
+ // Shortcut to the defining register
+ public short reg(Node n) {
+ return _code._regAlloc.regnum(n);
+ }
+
+ public int read1(int idx) {
+ byte[] buf = _bits.buf();
+ return (buf[idx]&0xFF);
+ }
+ public int read2(int idx) {
+ byte[] buf = _bits.buf();
+ return (buf[idx]&0xFF) | (buf[idx+1]&0xFF) <<8;
+ }
+ public int read4(int idx) {
+ byte[] buf = _bits.buf();
+ return (buf[idx]&0xFF) | (buf[idx+1]&0xFF) <<8 | (buf[idx+2]&0xFF)<<16 | (buf[idx+3]&0xFF)<<24;
+ }
+ public long read8(int idx) {
+ return (read4(idx) & 0xFFFFFFFFL) | read4(idx+4);
+ }
+
+ // This buffer is invalid/moving until after all encodings are written
+ public byte[] bits() { return _bits.buf(); }
+
+ // 4 byte little-endian write
+ public void patch4( int idx, int val ) {
+ byte[] buf = _bits.buf();
+ buf[idx ] = (byte)(val );
+ buf[idx+1] = (byte)(val>> 8);
+ buf[idx+2] = (byte)(val>>16);
+ buf[idx+3] = (byte)(val>>24);
+ }
+
+ static void padN(int n, BAOS bits) {
+ while( (bits.size()+n-1 & -n) > bits.size() )
+ bits.write(0);
+ }
+
+ // Convenience for writing log-N
+ static void addN( int log, long x, BAOS bits ) {
+ for( int i=0; i < 1<<log; i++ ) {
+ bits.write((int)x);
+ x >>= 8;
+ }
+ }
+
+ public Encoding add1( int op ) { addN(0,op,_bits); return this; }
+ public Encoding add2( int op ) { addN(1,op,_bits); return this; }
+ public Encoding add4( int op ) { addN(2,op,_bits); return this; }
+ public Encoding add8(long op ) { addN(3,op,_bits); return this; }
+
+ private final static int SENTINEL = -1;
+ private final IntHashMap _opStart = new IntHashMap(SENTINEL);
+ private final IntHashMap _opLen = new IntHashMap(SENTINEL);
+
+
+ // Read op start for n
+ public int opStart(Node n) { return _opStart.get(n._nid); }
+ // Read op length for n
+ public byte opLen (Node n) { return (byte)_opLen.get(n._nid); }
+
+ // Set op start for n
+ public void opStart(Node n, int off) {
+ assert off != SENTINEL;
+ _opStart.put(n._nid,off);
+ }
+
+ // Set op start for n
+ public void opStartAdd(Node n, int off) {
+ if( off==0 ) return;
+ int old = _opStart.get(n._nid);
+ assert old != SENTINEL && old+off != SENTINEL;
+ _opStart.put(n._nid,old+off);
+ }
+
+ // Set op length for n
+ public void opLen(Node n, byte len) {
+ assert len != SENTINEL;
+ _opLen.put(n._nid,len);
+ }
+
+ // Nodes need "relocation" patching; things done after code is placed.
+ // Record src and dst Nodes.
+ private final HashMap<Node,CFGNode> _internals = new HashMap<>();
+ // Source is a Call, destination in the Fun.
+ public Encoding relo( CallNode call ) {
+ FunNode fun = _code.link(call.tfp());
+ if( fun != null && !_code.emits(fun) )
+ _externals.put(call,fun._name);
+ else
+ _internals.put(call,fun);
+ return this;
+ }
+ // Code address as a constant
+ public Encoding relo( FunPtrNode fptr ) {
+ TypeFunPtr tfp = (TypeFunPtr)fptr._type;
+ // Native functions have no local FunNode, but are still callable.
+ // Do not treat their addresses as identities of deleted functions.
+ String nativeName = _code.externFunc(tfp.fidx());
+ if( nativeName!=null ) return external(fptr,nativeName);
+ FunNode fun = _code.link(tfp);
+ if( fun != null && !_code.emits(fun) )
+ _externals.put(fptr,fun._name);
+ else
+ _internals.put(fptr,fun);
+ return this;
+ }
+ // Local jump
+ public void jump( CFGNode jmp, CFGNode dst ) {
+ _internals.put(jmp,dst.uctrlSkipEmpty());
+ }
+
+ // External references only located by Strings a link-time
+ public final HashMap<Node,String> _externals = new HashMap<>();
+ public Encoding external( Node n, String extern ) {
+ _externals.put(n,extern);
+ return this;
+ }
+
+ public final HashMap<Node,String> _externData = new HashMap<>();
+ public void externalData(Node n, String name) { _externData.put(n,name); }
+
+ // Store t as a 32/64 bit constant in the code space; generate RIP-relative
+ // addressing to load it. Type is stored in either the .rodata or .data.
+ public void largeConstant( Node relo, Type t, int off, int elf ) {
+ // TODO: Any-old struct in the cpool
+ assert t.isConstant() || (t instanceof TypeStruct ts && Parser.startsClzPrefix(ts._name));
+ assert (byte)off == off;
+ assert (byte)elf == elf;
+ _bigCons.put(relo,new Relo(relo,t,(byte)off,(byte)elf));
+ }
+
+ // --------------------------------------------------
+ Encoding encode( ) {
+ // Basic block layout: negate branches to keep blocks in-order; insert
+ // unconditional jumps. Attempt to keep backwards branches taken,
+ // forwards not-taken (this is the default prediction on most
+ // hardware). Layout is still Reverse Post Order but with more
+ // restrictions.
+ basicBlockLayout( );
+
+ // Write encoding bits in order into a big byte array.
+ // Record opcode start and length.
+ writeEncodings();
+
+ // Short-form RIP-relative support: replace long encodings with short
+ // encodings and compact the code, changing all the offsets.
+ compactShortForm();
+
+ // Place pool objects and record relocations within their contents.
+ _data.write();
+
+ // Patch RIP-relative and local encodings now.
+ patchLocalRelocations();
+
+ return this;
+ }
+
+ // --------------------------------------------------
+ // Basic block layout: negate branches to keep blocks in-order; insert
+ // unconditional jumps. Attempt to keep backwards branches taken, forwards
+ // not-taken (this is the default prediction on most hardware). Layout is
+ // still Reverse Post Order but with more restrictions.
+ private void basicBlockLayout( ) {
+ IdentityHashMap<LoopNode,Ary<CFGNode>> rpos = new IdentityHashMap<>();
+ _cfg = new Ary<>(CFGNode.class);
+ rpos.put(_code._start.loop(),_cfg);
+ BitSet visit = _code.visit();
+
+ // Do them all except the <clinit>s
+ Ary<FunNode> clinits = new Ary<>(FunNode.class);
+ FunNode entry = null;
+ String entryName = _code.entryClinitName();
+ for( FunNode fun : _code._linker ) {
+ if( fun != null && !fun.isDead() && _code.emits(fun) ) {
+ if( fun.isClz() ) {
+ if( fun._name.equals(entryName) )
+ entry = fun;
+ else
+ clinits.add(fun);
+ } else {
+ int x = _cfg._len;
+ _rpo_cfg(fun,visit,rpos);
+ assert _cfg.at(x) instanceof ReturnNode;
+ }
+ }
+ }
+ // Now the <clinit>s last, so when reversed they become first. The
+ // program entry <clinit> is last of all, so it lands at offset zero.
+ for( FunNode clinit : clinits ) {
+ int x = _cfg._len;
+ _rpo_cfg(clinit,visit,rpos);
+ assert _cfg.at(x) instanceof ReturnNode;
+ }
+ if( entry != null ) {
+ int x = _cfg._len;
+ _rpo_cfg(entry,visit,rpos);
+ assert _cfg.at(x) instanceof ReturnNode;
+ }
+
+
+ // Reverse in-place
+ for( int i=0; i< _cfg.size()>>1; i++ )
+ _cfg.swap(i,_cfg.size()-1-i);
+ visit.clear();
+ }
+
+
+ private void _rpo_cfg(CFGNode bb, BitSet visit, IdentityHashMap<LoopNode,Ary<CFGNode>> rpos ) {
+ if( bb==null || visit.get(bb._nid) ) return;
+ visit.set(bb._nid);
+ CFGNode next = bb.uctrl();
+
+ // Loops run an inner "rpo_cfg", then append the entire loop body in
+ // place. This keeps loop bodies completely contained, although if
+ // Some Day Later we have real profile data we ought to arrange the
+ // branch orderings based on frequency.
+ if( bb instanceof LoopNode loop ) {
+ Ary<CFGNode> body = new Ary<>(CFGNode.class); // Private RPO for the loop
+ rpos.put(loop.loop(),body); // Find it via loop tree
+ _rpo_cfg(next,visit,rpos); // RPO the loop
+ body.add(loop); // Include loop last (first in RPO)
+ Ary<CFGNode> outer = rpos.get(loop.cfg(1).loop());
+ outer.addAll(body); // Append to original CFG
+ return;
+ }
+
+ // IfNodes visit 2 sides, and may choose to reorder them in the RPO
+ if( bb instanceof IfNode iff ) {
+ // Pick out the T/F projections
+ CProjNode t = iff.cproj(0);
+ CProjNode f = iff.cproj(1);
+ // Invert the branch or not
+ if( shouldInvert(t,f,iff.loopDepth()) ) {
+ iff.negate();
+ t.invert();
+ f.invert();
+ CProjNode tmp=f; f=t; t=tmp; // Swap t/f
+ }
+
+ // Whichever side is visited first becomes last in the RPO. With
+ // no loops, visit the False side last (so True side first) so that
+ // when the False RPO visit returns, the IF is immediately next.
+ // When the RPO is reversed, the fall-through path will always be
+ // following the IF.
+ _rpo_cfg(t,visit,rpos); // True side first
+ next = f; // False side last
+ }
+
+ // If the *next* BB has already been visited, and we are not already a
+ // jump, we may need an unconditional forwards jump here
+ Ary<CFGNode> rpo = rpos.get(bb.loop());
+ if( next!=null && visit.get(next._nid) && !(bb instanceof IfNode) ) {
+ boolean needJump = next instanceof LoopNode
+ // Empty blocks from an IF will invert the IF and backwards
+ // branch to the loop head. If not empty, or not an IF
+ // will need a jump.
+ ? (bb.nOuts()>1 || !(bb.cfg0() instanceof IfNode) )
+ // Forwards jump. If all the blocks, in RPO order, to our
+ // target are empty, we will fall in and not need a jump.
+ : !backwardsEmptyScan(rpo,next);
+ if( needJump ) {
+ CFGNode jmp = _code._mach.jump();
+ jmp._ltree = bb._ltree;
+ jmp.setDefX(0,bb);
+ next.setDef(next._inputs.find(bb),jmp);
+ rpo.add(jmp);
+ }
+ }
+
+ _rpo_cfg(next,visit,rpos);
+ rpo.add(bb);
+ }
+
+ // Should this test be inverted?
+ private static boolean shouldInvert(CFGNode t, CFGNode f, int bld) {
+ int tld = t.loopDepth(), fld = f.loopDepth();
+ // These next two are symmetric and can happen in any order; if `tld <
+ // bld` is true, then `fld < bld` must be false, or else both directions
+ // exit the loop... and the IF test would not be in the loop.
+
+ // true to exit a loop usually (and false falls into Yet Another Loop
+ // Block), invert if false is taking an empty backwards branch.
+ if( tld < bld ) return forwardsEmptyScan(f,bld);
+ // false to exit a loop, normally invert to true (except empty back)
+ if( fld < bld ) return !forwardsEmptyScan(t,bld);
+
+ // Not exiting the loop; staying at this depth (or going deeper loop).
+
+ // Jump/true to an empty backwards block
+ if( forwardsEmptyScan(t,bld) ) return false;
+ if( forwardsEmptyScan(f,bld) ) return true ;
+
+ // Fall/false into a full block, Jump/true to an empty block.
+ if( f.nOuts()>1 && t.nOuts()==1 ) return false;
+ if( t.nOuts()>1 && f.nOuts()==1 ) return true ;
+ // Jump to a merge point, assuming other things are jumping there as well
+ if( f.out(0) instanceof RegionNode ) return true ;
+ if( t.out(0) instanceof RegionNode ) return false;
+ // Everything else equal, use pre-order
+ return t._pre > f._pre;
+ }
+
+ // Do we continue forwards from 'c' to the Loop back edge, without doing
+ // anything else? Then this branch can directly jump to the loop head.
+ private static boolean forwardsEmptyScan( CFGNode c, int bld ) {
+ if( c.nOuts()!=1 || c.loopDepth()!=bld ) return false;
+ return c.uctrl() instanceof RegionNode cfg &&
+ ((cfg instanceof LoopNode && cfg._ltree==c._ltree) || forwardsEmptyScan(cfg,bld));
+ }
+
+ // Is the CFG from "next" to the end empty? This means jumping to "next"
+ // will naturally fall into the end.
+ private static boolean backwardsEmptyScan(Ary<CFGNode> rpo, CFGNode next) {
+ for( int i=rpo._len-1; rpo.at(i)!=next; i-- )
+ if( rpo.at(i).nOuts()!=1 )
+ return false;
+ return true;
+ }
+
+ // --------------------------------------------------
+ // Write encoding bits in order into a big byte array.
+ // Record opcode start and length.
+
+ // Current function is used by the spill-op encodings to query the stack
+ // frame layout and get the spill offsets.
+ private void writeEncodings() {
+ for( CFGNode bb : _cfg ) {
+ if( !(bb instanceof MachNode mach0) )
+ opStart(bb, _bits.size());
+ else if( bb instanceof FunNode ) {
+ opStart(bb, _bits.size());
+ mach0.encoding( this );
+ opLen(bb, (byte) (_bits.size() - opStart(bb)));
+ }
+ for( Node n : bb._outputs ) {
+ if( n instanceof MachNode mach && !(n instanceof FunNode) &&
+ // FunPtrs are encoding in the block of their control, not
+ // the block of the ReturnNode input - which is also a CFG.
+ !(n instanceof FunPtrNode && bb instanceof ReturnNode) ) {
+ opStart(n, _bits.size());
+ mach.encoding( this );
+ opLen(n, (byte) (_bits.size() - opStart(n)));
+ }
+ }
+ }
+ }
+
+ // --------------------------------------------------
+ // Short-form RIP-relative support: replace short encodings with long
+ // encodings and expand the code, changing all the offsets.
+ private void compactShortForm() {
+ int len = _cfg._len;
+ int[] oldStarts = new int[len];
+ for( int i=0; i<len; i++ )
+ oldStarts[i] = opStart(_cfg.at(i));
+
+ // TODO: Rewrite this algo to use the small "_jmps" list of just the
+ // jumps instead of walking all blocks.
+ // TODO2: Allow short encodings on non-jumps. This algo doesn't
+ // really care jumps vs other rip-relative things.
+
+ // Check the size of short jumps, adjust the block starts for a whole
+ // pass over all blocks. If some jumps go long, they might stretch the
+ // distance for other jumps, so another pass is needed.
+ int slide = -1;
+ while( slide != 0) { // While no fails
+ slide = 0;
+ for( int i=0; i<len; i++ ) {
+ CFGNode bb = _cfg.at(i);
+ opStartAdd(bb, slide);
+ // Slide down all other (non-CFG) ops in the block
+ for( Node n : bb._outputs )
+ if( n instanceof MachNode && !(n instanceof CFGNode) &&
+ // Match writeEncodings(): FunPtrs are encoded in the
+ // block of their control, not again at the Return.
+ !(n instanceof FunPtrNode && bb instanceof ReturnNode) )
+ opStartAdd(n, slide);
+ if( bb instanceof RIPRelSize riprel ) {
+ CFGNode target = (bb instanceof IfNode iff ? iff.cproj(0) : (CFGNode)bb.out(0)).uctrlSkipEmpty();
+ // Delta is from opStart to opStart. X86 at least counts
+ // the delta from the opEnd, but we don't have the end until
+ // we decide the size - so the encSize has to deal
+ int delta = opStart(target) - opStart(bb);
+ byte opLen = riprel.encSize(delta);
+ // Recorded size is smaller than the current size?
+ if( opLen(bb) < opLen ) {
+ // Start sliding the code down; record slide amount and new size
+ slide += opLen - opLen(bb);
+ opLen(bb, opLen);
+ }
+ }
+ }
+ }
+
+
+ // Go again, inserting padding on function headers. Since no
+ // short-jumps span function headers, the padding will not make any
+ // short jumps fail.
+ for( int i=0; i<len; i++ ) {
+ CFGNode bb = _cfg.at(i);
+ // Functions pad to align 16
+ if( bb instanceof FunNode ) {
+ int newStart = opStart(bb)+slide;
+ slide += (newStart+15 & -16)-newStart;
+ }
+ opStartAdd(bb, slide);
+ for( Node n : bb._outputs )
+ if( n instanceof MachNode && !(n instanceof CFGNode) &&
+ !(n instanceof FunPtrNode && bb instanceof ReturnNode) )
+ opStartAdd(n, slide);
+ }
+
+ // Copy/slide the bits to make space for all the longer branches
+ int grow = opStart(_cfg.at(len-1)) - oldStarts[len-1];
+ if( grow > 0 ) { // If no short-form ops, nothing to do here
+ int end = _bits.size();
+ byte[] bits = new byte[end+grow];
+ for( int i=len-1; i>=0; i-- ) {
+ int start = oldStarts[i];
+ if( start==0 && i>1 ) continue;
+ int oldStart = oldStarts[i];
+ int newStart = opStart(_cfg.at(i));
+ System.arraycopy(_bits.buf(),oldStart,bits,newStart,end-start);
+ end = start;
+ }
+ _bits.set(bits,bits.length);
+ }
+
+ }
+
+
+ // --------------------------------------------------
+ // Patch local encodings now
+ void patchLocalRelocations() {
+ // Walk the local code-address relocations
+ for( Node src : _internals.keySet() ) {
+ int start = opStart(src);
+ Node dst = _internals.get(src);
+ // If function is entirely dead, only the function pointer remains
+ // and, it can only be used to test against zero or equals to
+ // another function pointer... i.e., there Is No Code Here.
+ int target = dst == null ? start : opStart(dst);
+ ((RIPRelSize)src).patch(this, start, opLen(src), target - start);
+ }
+ }
+
+ // Emit a single scalar as bits
+ static void addN( int log, Type t, BAOS bits ) {
+ long x = switch( t ) {
+ case TypeInteger ti -> ti.value();
+ case TypeFloat tf -> log==3
+ ? Double.doubleToRawLongBits( tf.value())
+ : Float.floatToRawIntBits((float)tf.value());
+ case TypeFunPtr tfp -> 0; // These need to be relocated
+ case TypeMemPtr tmp -> 0; // These need to be relocated
+ default -> { if( t==Type.NIL ) yield 0; else throw Utils.TODO(); }
+ };
+ addN(log,x,bits);
+ }
+
+ // A series of libc/external calls that Simple can link against in a JIT.
+ // Since no runtime in the JVM process, using magic numbers for the CPU
+ // emulators to pick up on.
+ public static int SENTINEL_CALLOC = -4;
+ public static int SENTINEL_WRITE = -8;
+
+ int externalAddress(String name) {
+ Integer address=_externalAddresses.get(name);
+ if( address!=null ) return address;
+ return switch(name) {
+ case "calloc" -> SENTINEL_CALLOC;
+ case "write" -> SENTINEL_WRITE;
+ default -> throw new IllegalArgumentException("Unresolved external function '"+name+"'");
+ };
+ }
+
+ void patchGlobalRelocations() {
+ for( Node src : _externData.keySet() ) {
+ String name = _externData.get(src);
+ Integer target = _code._externDataAddresses.get(name);
+ if( target==null ) throw new IllegalArgumentException("Unresolved C data symbol '"+name+"'");
+ int start = opStart(src);
+ ((RIPRelSize)src).patch(this,start,opLen(src),target-start);
+ }
+ for( Node src : _externals.keySet() ) {
+ int start = opStart(src);
+ String dst = _externals.get(src);
+ int target = externalAddress(dst);
+ ((RIPRelSize)src).patch(this, start, opLen(src), target - start);
+ }
+ }
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/FinalizeIR.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/FinalizeIR.java
new file mode 100644
index 00000000..69e0104d
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/FinalizeIR.java
@@ -0,0 +1,44 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.node.*;
+import java.util.Arrays;
+
+// Comnpute field layouts only after SCCP and its peepholes converge.
+// Keep surviving functions rooted without introducing unknown callers.
+abstract class FinalizeIR {
+ static void run(CodeGen code) {
+ // Compute field layouts and replace symbolic field offsets by constants.
+ for( int i=0; i<code._start.nOuts(); i++ ) {
+ Node use = code._start.out(i);
+ if( use instanceof ConFldOffNode off ) {
+ Node con = off.asOffset();
+ assert con != null : off._ts._name+"."+off._fname;
+ off.subsume(con);
+ i--;
+ }
+ }
+ code._iter.iterate(code); // Propagate field offsets and clean up.
+
+ // Retain surviving functions for emission without adding unknown callers.
+ for( FunNode fun : code._linker )
+ if( fun != null ) {
+ // A folded static function field can leave a detached body
+ // with internal cycles. It has no callers; later phases must
+ // not revive it merely because its nodes are still allocated.
+ if( fun.isDead() || fun._type.isHigh() ) {
+ code._linker.set(fun.sig().fidx(),null);
+ continue;
+ }
+ code.keepInCompUnit(fun);
+ }
+ // Hand off the wrapper lifetime before releasing its memory projection.
+ for( CompUnit cu : code._compunits.values() )
+ code.keepUntilUnlink(cu._start);
+ code.releaseFinalizedNodes();
+ // Stable print order: initializers first, then other Returns by NID.
+ for( Node stop : code._stop._inputs )
+ Arrays.sort(stop._inputs._es,0,stop.nIns(),
+ (x,y) -> ((((ReturnNode)x).fun().isClz() ? -1 : x._nid) -
+ (((ReturnNode)y).fun().isClz() ? -1 : y._nid) ));
+ }
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/FunctionSummary.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/FunctionSummary.java
new file mode 100644
index 00000000..6cd4035d
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/FunctionSummary.java
@@ -0,0 +1,28 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.type.*;
+
+// The small, eagerly readable part of a function's serialized implementation.
+record FunctionSummary(TypeFunPtr sig, TypeTuple state, String name, int size, int[] classDependencyFIDXs) {
+ void install(CodeGen code, CompUnit cu) {
+ FunNode fun = new FunNode(null,sig,name,cu,null,cu._start);
+ fun._approxUIDs=size;
+ fun.init();
+ // First user stays in output slot zero until the summary is removed.
+ SummaryNode body = new SummaryNode(state,classDependencyFIDXs,fun);
+ ParmNode rpc = new ParmNode("$rpc",0,TypeRPC.BOT,fun,code.con(TypeRPC.BOT)).init();
+ ParmNode mem = new ParmNode("$mem",1,TypeMem.BOT,fun,cu._start.proj(1)).init();
+ body.addDef(mem);
+ for( int i = 0; i<sig.nargs(); i++ )
+ body.addDef(new ParmNode("arg"+i,i+2,sig.arg(i),fun,code.con(sig.arg(i))).init());
+ body.init();
+ Node ctrl = new CProjNode(body,0,"ctrl").init();
+ Node memory = new ProjNode(body,1,"mem").init();
+ Node value = new ProjNode(body,2,"value").init();
+ ReturnNode ret = new ReturnNode(ctrl,memory,value,rpc,fun).init();
+ fun.setRet(ret);
+ cu._stop.addDef(ret);
+ code.link(fun);
+ }
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/GlobalBits.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/GlobalBits.java
new file mode 100644
index 00000000..bc9c9d39
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/GlobalBits.java
@@ -0,0 +1,177 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.util.*;
+import java.util.HashMap;
+import java.util.Arrays;
+import java.util.Objects;
+
+
+// This utility 2-way maps some *global* program feature to a local dense
+// index, suitable for packing in BitSets. The mapping takes a global value -
+// always a {source file/class} name and an order number counting up per
+// feature in the same file. The dense local index can be different in
+// different compilations.
+
+// The global info allows loading the same remote class info from different
+// paths and aligning them. Example: Compiling A loads pre-compiled B and C,
+// both of which load pre-compiled D. The 2 different D loads unify via this
+// global info.
+
+// These mappings are all trivial identities when compiling one file; they
+// only become complex when loading seperately compiled code.
+
+public class GlobalBits {
+ public static final int RESERVED = 2;
+ private final HashMap<String,int[]> _clzOrder2Local = new HashMap<>();
+ private final Ary<String> _local2Clz = new Ary<>(String.class);
+ private final AryInt _local2Order = new AryInt();
+ // Local dense index
+ private int _local;
+
+ GlobalBits() {
+ _local = RESERVED;
+ // Reserved null-class bits are global identities and are never
+ // remapped.
+ int[] xs = new int[RESERVED];
+ xs[0] = RESERVED; // Per-class next order counter
+ for( int i=0; i<RESERVED; i++ ) {
+ if( i > 0 ) xs[i] = i;
+ _local2Clz .setX(i,null);
+ _local2Order.setX(i,i);
+ }
+ _clzOrder2Local.put(null,xs);
+ }
+
+ // Get a next local index for this clz
+ int next( String clz ) { return next(clz,-1); }
+
+ // Given a clz and order, return an existing local index or make a new
+ // mapping and return that.
+ int next(String clz, int order ) {
+ // Get the per-clz Order->Local mapping
+ int[] xs = _clzOrder2Local.get(clz);
+ if( xs == null )
+ xs = new int[]{RESERVED,1}; // Make, if it does not exist
+ // Caller does not have an order in mind, just take the next one stored
+ // in xs[0]
+ if( order == -1 )
+ order = xs[0]++;
+ // Order exists from prior compilation, but not recorded locally
+ else if( order >= xs[0] )
+ xs[0] = order+1;
+ // Grow as needed
+ while( xs.length <= order )
+ xs = Arrays.copyOf(xs,xs.length*2);
+ // Have a local mapping already?
+ if( xs[order] != 0 )
+ return xs[order];
+ int local = _local++;
+ bind(local,clz,order);
+ return local; // The dense local index
+ }
+
+ // Bind a specific dense local index to {clz,order}. Used by both fresh
+ // allocation and deserialization, where local indexes must not compact
+ // across holes because XInt bitsets are serialized against this space.
+ private void bind( int local, String clz, int order ) {
+ if( _local <= local )
+ _local = local+1;
+ _local2Clz .setX(local,clz );
+ _local2Order.setX(local,order);
+
+ // Inline-only fidx holes are represented as {null,0}. They need the
+ // reverse mapping above, but no forward map because xs[0] is the
+ // per-class next-order counter.
+ if( clz==null && order==0 )
+ return;
+
+ int[] xs = _clzOrder2Local.get(clz);
+ if( xs == null )
+ xs = new int[]{RESERVED,1};
+ if( order >= xs[0] )
+ xs[0] = order+1;
+ while( xs.length <= order )
+ xs = Arrays.copyOf(xs,xs.length*2);
+ _clzOrder2Local.put(clz,xs);
+ xs[order] = local;
+ }
+
+ // A file-local function that will inline, and never write to disk
+ public int nextInline( ) {
+ int local = _local++;
+ _local2Clz .setX(local,null);
+ _local2Order.setX(local,0);
+ return local;
+ }
+ // Get a next local index for the same clz as this old index
+ public int next( int old ) { return next(_local2Clz.at(old)); }
+
+ // Map a file-local dense index to this compilation unit's local dense index.
+ public int map( GlobalBits fileBits, int fileLocal ) {
+ if( fileLocal >= fileBits._local2Clz.size() )
+ throw new ArrayIndexOutOfBoundsException(""+fileLocal+" >= "+fileBits._local2Clz.size());
+ return findOrNext(fileBits._local2Clz.at(fileLocal), fileBits._local2Order.at(fileLocal));
+ }
+
+ // Map a global {clz,order} to a local one; keep any existing mapping or
+ // make a new one as needed.
+ private int findOrNext(String clz, int order ) {
+ // Reserved null-class identities are not represented in xs[order],
+ // because xs[0] is the next-order counter.
+ if( clz==null && order < _local2Clz.size() && _local2Order.at(order)==order && _local2Clz.at(order)==null )
+ return order;
+ int[] xs = _clzOrder2Local.get(clz);
+ if( xs != null && order < xs.length ) {
+ int local = xs[order];
+ if( local < _local2Clz.size() && _local2Order.at(local)==order && Objects.equals(_local2Clz.at(local),clz) )
+ return local;
+ }
+ return next(clz,order);
+ }
+
+ // Stable native symbol identity, independent of this compilation's dense IDs.
+ String owner(int local) { return _local2Clz.at(local); }
+ String symbol(int local) {
+ return _local2Clz.at(local)+"$F$"+_local2Order.at(local);
+ }
+
+ // Write out to BAOS enough bits to unwind the local back to global index
+ void packed( BAOS baos, HashMap<String,Integer> strs ) {
+ // Unwind a local index to the clz.
+ baos.packed2(_local2Clz.size());
+ for( int i=RESERVED; i<_local2Clz.size(); i++ ) {
+ String clz = _local2Clz.at(i);
+ baos.packed2(clz==null ? 0 : strs.get(clz));
+ baos.packed2( _local2Order.at(i) );
+ }
+ }
+
+ void gather( HashMap<String,Integer> strs ) {
+ for( int i=RESERVED; i<_local2Clz.size(); i++ )
+ Serialize.gather(strs,_local2Clz.at(i));
+ }
+
+ // Read in the BAIS bits needed to make a file-local -> global mapping
+ static GlobalBits packed( BAOS bais, String[] strs ) {
+ var gb = new GlobalBits();
+ int n = bais.packed2();
+ for( int i=RESERVED; i<n; i++ ) {
+ int sx = bais.packed2();
+ String clz = strs[sx];
+ if( clz.isEmpty() ) clz = null;
+ int order = bais.packed2();
+ gb.bind(i,clz,order);
+ }
+ return gb;
+ }
+
+ @Override public String toString() {
+ SB sb = new SB().p('[');
+ for( int i=0; i<_local2Clz.size(); i++ ) {
+ if( i > 0 ) sb.p(',');
+ sb.p(i).p(" <-> ").p(_local2Clz.at(i)).p('#').p(_local2Order.at(i));
+ }
+ return sb.p(']').toString();
+ }
+
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/GlobalCodeMotion.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/GlobalCodeMotion.java
new file mode 100644
index 00000000..86cb2e98
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/GlobalCodeMotion.java
@@ -0,0 +1,356 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.codegen.IterPeeps.WorkList;
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.type.*;
+import com.seaofnodes.simple.util.Ary;
+import com.seaofnodes.simple.util.AryInt;
+import com.seaofnodes.simple.util.Utils;
+import java.util.*;
+
+public abstract class GlobalCodeMotion {
+
+ // Ideal execution needs a schedule, but neither CLInit lowering nor
+ // interprocedural optimization. Each loaded object is prepared once.
+ public static Ary<CFGNode> forEval(CodeGen code, StartNode start, StopNode stop) {
+ Ary<FunNode> funs = Unlink.forEval(code,start,stop);
+ start.buildLoopTree(funs,stop);
+ return buildCFG(code,start,stop);
+ }
+
+ // Arrange that the existing isCFG() Nodes form a valid CFG. The
+ // Node.use(0) is always a block tail (either IfNode or head of the
+ // following block). There are no unreachable infinite loops.
+ public static void buildCFG( CodeGen code ) {
+ code._cfg = buildCFG(code,code._start,code._stop);
+ }
+
+ // Schedule one graph, including an object loaded while Eval2 is running.
+ // Other graphs (and their active evaluation frames) are left untouched.
+ public static Ary<CFGNode> buildCFG(CodeGen code, StartNode start, StopNode stop) {
+ Ary<CFGNode> rpo = new Ary<>(CFGNode.class);
+ _rpo_cfg(null, start, code.visit(), rpo);
+ // Reverse in-place
+ for( int i=0; i< rpo.size()>>1; i++ )
+ rpo.swap(i,rpo.size()-1-i);
+ schedEarly(code,start,rpo);
+
+ // Break up shared global constants by functions
+ breakUpGlobalConstants(start);
+
+ code._visit.clear();
+ schedLate(code,stop);
+ return rpo;
+ }
+
+ // Post-Order of CFG
+ private static void _rpo_cfg(CFGNode def, Node use, BitSet visit, Ary<CFGNode> rpo) {
+ if( !(use instanceof CFGNode cfg) || visit.get(cfg._nid) )
+ return; // Been there, done that
+ // A shared dead-control node can also reach an unloaded function's
+ // Return. Its wrapper is a dispatch target, not executable code.
+ if( cfg instanceof FunNode fun && fun.summary()!=null ||
+ cfg instanceof ReturnNode ret && ret.fun().summary()!=null ) return;
+ assert !( def instanceof ReturnNode && use instanceof CallEndNode ); // All calls unwired now
+ assert !( def instanceof CallNode && use instanceof FunNode ); // All calls unwired now
+ visit.set(cfg._nid);
+ for( Node useuse : cfg._outputs )
+ _rpo_cfg(cfg,useuse,visit,rpo);
+ rpo.add(cfg);
+ }
+
+ // After early scheduling, every global chain member has Start in slot 0.
+ // Give each function one private copy of the entire dependency graph.
+ private static void breakUpGlobalConstants( StartNode start ) {
+ var globals = new IdentityHashMap<Node,Boolean>();
+ var cons = new ArrayList<Node>();
+ for( Node con : start._outputs )
+ if( con!=null && !(con instanceof CFGNode) &&
+ (con.isConst() || con instanceof MachNode mach && mach.outregmap()!=null) ) {
+ globals.put(con,true);
+ cons.add(con.keep()); // Preserve original inputs while rewiring users.
+ }
+
+ var copies = new IdentityHashMap<FunNode,IdentityHashMap<Node,Node>>();
+ for( Node con : cons )
+ for( Node use : con._outputs.asAry() ) {
+ if( use==null || globals.containsKey(use) ) continue;
+ FunNode fun = useFun(use);
+ if( fun==null || fun.summary()!=null ) continue; // Metadata has no executable owner.
+ assert CodeGen.CODE._mach==null || fun.isFromSource(); // Ideal evaluation includes imported bodies.
+ var local = copies.computeIfAbsent(fun,f -> new IdentityHashMap<>());
+ Node copy = cloneGlobal(con,fun,globals,local);
+ for( int i=0; i<use.nIns(); i++ )
+ if( use.in(i)==con )
+ use.setDef(i,copy);
+ }
+ for( Node con : cons ) con.unkill();
+ }
+
+ private static Node cloneGlobal(Node con, FunNode fun, IdentityHashMap<Node,Boolean> globals,
+ IdentityHashMap<Node,Node> copies) {
+ Node copy = copies.get(con);
+ if( copy!=null ) return copy;
+ copies.put(con,copy=con.copyEmpty());
+ copy.addDef(fun);
+ for( int i=1; i<con.nIns(); i++ ) {
+ Node def = con.in(i);
+ copy.addDef(globals.containsKey(def) ? cloneGlobal(def,fun,globals,copies) : def);
+ }
+ return copy;
+ }
+
+ private static FunNode useFun(Node use) {
+ if( use instanceof ReturnNode ret ) return ret.fun();
+ if( use instanceof ParmNode parm ) return parm.fun();
+ CFGNode cfg = use.cfg0();
+ while( cfg!=null && !(cfg instanceof FunNode) )
+ cfg = cfg.idom();
+ return (FunNode)cfg;
+ }
+
+
+ // ------------------------------------------------------------------------
+ // Visit all nodes in CFG Reverse Post-Order, essentially defs before uses
+ // (except at loops). Since defs are visited first - and hoisted as early
+ // as possible, when we come to a use we place it just after its deepest
+ // input.
+ private static void schedEarly(CodeGen code, StartNode start, Ary<CFGNode> cfgs) {
+ // Reverse Post-Order on CFG
+ for( CFGNode cfg : cfgs ) {
+ cfg.loopDepth();
+ for( Node n : cfg._inputs )
+ _schedEarly(n,code,start );
+ // In dead infinite loops, entire code blocks may be unreachable
+ // from below. Reach down from the CFG to their Phis so their
+ // inputs are scheduled too.
+ if( cfg instanceof RegionNode )
+ for( Node phi : cfg._outputs )
+ if( phi instanceof PhiNode )
+ _schedEarly(phi,code,start );
+ }
+ }
+
+ private static void _schedEarly(Node n, CodeGen code, StartNode start) {
+ if( n==null || code._visit.get(n._nid) ) return; // Been there, done that
+ assert !(n instanceof CFGNode) : n.getClass().getSimpleName()+"#"+n._nid+" outside scheduled CFG";
+ code._visit.set(n._nid);
+ // Schedule inputs first, except Phis: following their backedges would
+ // enter a data cycle before its control has been scheduled.
+ for( Node def : n._inputs )
+ if( def!=null && !(def instanceof PhiNode) )
+ _schedEarly(def,code,start);
+
+ // An existing edge 0 already supplies control (or a Phi/Proj binding).
+ if( n.in(0)==null ) {
+ // Schedule at deepest input
+ CFGNode early = start; // Maximally early, lowest idepth
+ for( int i=1; i<n.nIns(); i++ )
+ if( n.in(i)!=null && n.in(i).cfg0().idepth() > early.idepth() )
+ early = n.in(i).cfg0(); // Latest/deepest input
+ n.setDef(0,early);
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // Scheduling is not recursive. Retain scratch capacity between objects,
+ // but release graph references and reset only the entries we used.
+ private static CFGNode[] LATE = new CFGNode[1];
+ // Load NIDs at their CFG block choices, checked against Store blocks.
+ private static int[] ANTI = new int[1];
+ private static final AryInt TOUCHED = new AryInt();
+ private static final Ary<Node> PLACED = new Ary<>(Node.class);
+ private static final Ary<Node> ANTIDEPS = new Ary<>(Node.class);
+ private static final WorkList<Node> WORK = new WorkList<>();
+
+ private static void schedLate( CodeGen code, StopNode stop ) {
+ assert TOUCHED.isEmpty() && PLACED.isEmpty() && WORK.isEmpty();
+ if( code.UID() > LATE.length ) {
+ int len = Math.max(code.UID(),LATE.length<<1);
+ LATE = new CFGNode[len];
+ ANTI = new int[len];
+ }
+ WORK.clear();
+ // Breadth-first scheduling
+ // Early scheduling has released the shared visit bits.
+ breadth(stop,PLACED,LATE,ANTI,ANTIDEPS,code.visit());
+
+ // Copy the best placement choice into the control slot
+ for( Node n : PLACED )
+ if( !(n instanceof ProjNode) )
+ n.setDef(0,LATE[n._nid]);
+ for( int i=0; i<TOUCHED.len(); i++ )
+ LATE[TOUCHED.at(i)] = null;
+ TOUCHED.clear();
+ clear(PLACED);
+ clear(ANTIDEPS);
+ }
+
+ // Ary.clear retains references; static scratch must release them as well.
+ private static void clear(Ary<Node> nodes) {
+ Arrays.fill(nodes._es,0,nodes._len,null);
+ nodes.clear();
+ }
+
+ private static Ary<Node> antiDeps(MemOpNode load, Ary<Node> antideps) {
+ clear(antideps);
+ return load.antiDeps(antideps);
+ }
+
+ private static void breadth(Node stop, Ary<Node> placed, CFGNode[] late, int[] anti, Ary<Node> antideps, BitSet visit) {
+ // Things on the worklist have some (but perhaps not all) uses done.
+ WorkList<Node> work = WORK;
+ work.push(stop);
+ Node n;
+ outer:
+ while( (n = work.pop()) != null ) {
+ assert late[n._nid]==null; // No double visit
+ // These I know the late schedule of, and need to set early for loops
+ if( n instanceof CFGNode cfg ) late[n._nid] = cfg.blockHead() ? cfg : cfg.cfg(0);
+ else if( n instanceof PhiNode phi ) late[n._nid] = phi.region();
+ else if( n instanceof ProjNode && n.in(0) instanceof CFGNode cfg ) late[n._nid] = cfg;
+ else {
+
+ // All uses done?
+ for( Node use : n._outputs )
+ if( use!=null && late[use._nid]==null )
+ continue outer; // Nope, await all uses done
+
+ // Loads need their memory inputs' uses also done
+ if( n instanceof MemOpNode ld && ld._isLoad )
+ for( Node memuse : antiDeps(ld,antideps) )
+ if( late[memuse._nid]==null )
+ continue outer;
+
+ // All uses done, schedule
+ _doSchedLate(n,placed,late,anti,antideps);
+ }
+ TOUCHED.push(n._nid);
+
+ // A use just finished; reconsider its inputs and waiting loads,
+ // even when the shared memory input was already scheduled.
+ for( Node def : n._inputs ) {
+ if( def==null ) continue;
+ if( late[def._nid]==null ) work.push(def);
+ wakeLoads(def,late,work,visit);
+ }
+ visit.clear();
+ if( n instanceof LoopNode loop )
+ for( Node phi : loop._outputs )
+ if( phi instanceof PhiNode && late[phi._nid]==null )
+ work.push(phi);
+ }
+ }
+
+ private static void _doSchedLate(Node n, Ary<Node> placed, CFGNode[] late, int[] anti, Ary<Node> antideps) {
+ // Walk uses, gathering the LCA (Least Common Ancestor) of uses
+ CFGNode early = n.in(0) instanceof CFGNode cfg ? cfg : n.in(0).cfg0();
+ assert early != null;
+ CFGNode lca = null;
+ for( Node use : n._outputs )
+ if( use != null )
+ lca = use_block(n,use, late).domLCA(lca,null);
+
+ // Loads may need anti-dependencies, raising their LCA
+ if( n instanceof MemOpNode load && load._isLoad )
+ lca = find_anti_dep(lca,load,early,late,anti, antideps);
+
+ // Walk up from the LCA to the early, looking for best place. This is
+ // the lowest execution frequency, approximated by least loop depth and
+ // deepest control flow.
+ CFGNode best = lca;
+ lca = lca.idom(); // Already found best for starting LCA
+ for( ; lca != early.idom(); lca = lca.idom() )
+ if( better(lca,best) )
+ best = lca;
+ assert !(best instanceof IfNode);
+ placed.push(n);
+ late[n._nid] = best;
+ }
+
+ // Block of use. Normally from late[] schedule, except for Phis, which go
+ // to the matching Region input.
+ private static CFGNode use_block(Node n, Node use, CFGNode[] late) {
+ if( use instanceof ParmNode parm )
+ return late[use._nid];
+ if( !(use instanceof PhiNode phi) )
+ return late[use._nid];
+ CFGNode found=null;
+ for( int i=1; i<phi.nIns(); i++ )
+ if( phi.in(i)==n )
+ found = phi.region().cfg(i).domLCA(found,null); // Can be more than one matching input.
+
+ assert found!=null;
+ return found;
+ }
+
+
+ // Least loop depth first, then largest idepth
+ private static boolean better( CFGNode lca, CFGNode best ) {
+ return lca.loopDepth() < best.loopDepth() ||
+ lca instanceof NeverNode ||
+ lca.idepth() > best.idepth() ||
+ best instanceof IfNode;
+ }
+
+ // A writer may consume a load's memory through an aggregate.
+ private static void wakeLoads(Node def, CFGNode[] late, WorkList<Node> work, BitSet visit) {
+ if( visit.get(def._nid) ) return;
+ visit.set(def._nid);
+ for( Node out : def._outputs )
+ if( out instanceof MemOpNode ld && ld._isLoad && late[ld._nid]==null ) work.push(ld);
+ if( def instanceof MemMergeNode )
+ for( int i=1; i<def.nIns(); i++ )
+ if( def.in(i)!=null ) wakeLoads(def.in(i),late,work,visit);
+ }
+
+ private static CFGNode find_anti_dep(CFGNode lca, MemOpNode load, CFGNode early, CFGNode[] late, int[] anti, Ary<Node> antideps) {
+ // We could skip final-field loads here.
+ // Walk LCA->early, flagging Load's block location choices
+ CFGNode marked = lca;
+ for( CFGNode cfg = marked; cfg!=early.idom(); cfg = cfg.idom() )
+ anti[cfg._nid] = load._nid;
+ // Walk load->mem uses, looking for Stores causing an anti-dep
+ for( Node mem : antiDeps(load,antideps) ) {
+ if( mem instanceof PhiNode phi ) {
+ // Inputs carrying different memory might overwrite this alias.
+ // Raise the LCA for those paths; the unchanged-memory paths
+ // do not clobber the load. No scheduling edge to the Phi.
+ for( int i=1; i<phi.nIns(); i++ )
+ if( phi.in(i)!=load.mem() )
+ lca = anti_dep(load,phi.region().cfg(i),load.mem().cfg0(),lca,null,anti);
+ } else {
+ // Hunt for clobbering stores
+ assert mem instanceof MemOpNode || mem instanceof EscapeNode ||
+ mem instanceof CallNode || mem instanceof CLInitNode;
+ assert late[mem._nid]!=null;
+ lca = anti_dep(load,late[mem._nid],mem.cfg0(),lca,mem,anti);
+ }
+ }
+ // This load's marks are no longer needed; leave scratch clean even
+ // when the next compilation reuses these node IDs.
+ for( CFGNode cfg = marked; cfg!=early.idom(); cfg = cfg.idom() )
+ anti[cfg._nid] = 0;
+ return lca;
+ }
+
+ //
+ private static CFGNode anti_dep( MemOpNode load, CFGNode stblk, CFGNode defblk, CFGNode lca, Node st, int[] anti ) {
+ // A conditional writer can precede the load without dominating its late block.
+ // Walk back to the first overlap with the load's placement range.
+ // A different Phi input need not be dominated by this memory's
+ // definition. Stop at its depth rather than looking for that ancestor.
+ // Retain the sentinel too: a function entry has a null idom.
+ for( ; stblk != defblk.idom() && stblk.idepth() >= defblk.idepth(); stblk = stblk.idom() ) {
+ // Store and Load overlap, need anti-dependence
+ if( anti[stblk._nid]==load._nid ) {
+ lca = stblk.domLCA(lca,null); // Raise Loads LCA
+ if( lca == stblk && st != null && !(st instanceof CallNode) && st._inputs.find(load) == -1 ) // And if something moved,
+ st.addDef(load); // Add anti-dep as well (calls already end their block)
+ return lca; // Cap this stores' anti-dep to here
+ }
+ }
+ return lca;
+ }
+
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/IFG.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/IFG.java
new file mode 100644
index 00000000..d15e66da
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/IFG.java
@@ -0,0 +1,586 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.type.Type;
+import com.seaofnodes.simple.util.Ary;
+import com.seaofnodes.simple.util.Utils;
+import java.util.BitSet;
+import java.util.IdentityHashMap;
+
+// Interference Graph
+abstract public class IFG {
+ // Interference Graph Support
+
+ // Map from a Basic Block to Live-Out: {a map from a Live Range to a Def}
+ private static final IdentityHashMap<CFGNode,IdentityHashMap<LRG,Node>> BBOUTS = new IdentityHashMap<>();
+ static void resetBBLiveOut() {
+ for( IdentityHashMap<LRG,Node> bbout : BBOUTS.values() )
+ bbout.clear();
+ }
+ static final IdentityHashMap<LRG,Node> TMP = new IdentityHashMap<>();
+
+ // Inteference Graph: Array of Bitsets
+ static final Ary<BitSet> IFG = new Ary<>(BitSet.class);
+ static void resetIFG() {
+ for( BitSet bs : IFG )
+ if( bs!=null ) bs.clear();
+ }
+
+ // Set matching bit
+ private static void addIFG( LRG lrg0, LRG lrg1 ) {
+ short x0 = lrg0._lrg, x1 = lrg1._lrg;
+ // Triangulate
+ if( x0 < x1 ) _addIFG(x0,x1);
+ else _addIFG(x1,x0);
+ }
+ // Add lrg1 to lrg0's conflict set
+ private static void _addIFG( short x0, short x1 ) {
+ BitSet ifg = IFG.atX(x0);
+ if( ifg==null )
+ IFG.setX(x0,ifg = new BitSet());
+ ifg.set(x1);
+ }
+
+
+ private static final Ary<CFGNode> WORK = new Ary<>(CFGNode.class);
+ static void pushWork(CFGNode bb) {
+ if( WORK.find(bb) == -1 )
+ WORK.push(bb);
+ }
+
+ // ------------------------------------------------------------------------
+ // Visit all blocks, using the live-out LRGs per-block and doing a backwards
+ // walk over each block. At the end of the walk, push the live-out sets to
+ // prior blocks. Due to loops, no single visitation order suffices. This
+ // problem can be fairly efficiently solved with a proper LIVE calculation
+ // and bitsets. In the interests of simplicity (and assumption of smaller
+ // programs) I am skipping LIVE and directly computing it during the IFG
+ // building.
+
+ // Start by setting the live-out of the exit block (to empty), and putting
+ // it on a worklist.
+
+ // - Pull from worklist a block who's live-out has changed
+ // - Walk backwards, adding interferences and computing live.
+ // - At block head, "push" the new live-out to prior blocks.
+ // - - If they pick up new live-outs, put them on the worklist.
+
+ public static boolean build( int round, RegAlloc alloc ) {
+ // Reset all to empty
+ resetBBLiveOut();
+ resetIFG();
+ WORK.clear();
+
+ // Last block has nothing live out
+ for( CFGNode bb : alloc._code._cfg )
+ if( bb.blockHead())
+ WORK.push( bb );
+
+ // Process blocks until no more changes
+ while( !WORK.isEmpty() )
+ do_block(round,alloc,WORK.pop());
+
+ if( alloc.success() )
+ convert2DAdjacency(alloc);
+ return alloc.success();
+ }
+
+ // Walk one block backwards, compute live-in from live-out, and build IFG
+ static void do_block( int round, RegAlloc alloc, CFGNode bb ) {
+ assert bb.blockHead();
+ IdentityHashMap<LRG,Node> live_out = BBOUTS.get(bb);
+ TMP.clear();
+ if( live_out != null )
+ TMP.putAll(live_out); // Copy bits to temp
+
+ // A backwards walk over instructions in the basic block
+ for( int inum = bb.nOuts()-1; inum >= 0; inum-- ) {
+ Node n = bb.out(inum);
+ if( n.in(0) != bb ) continue;
+ // In a backwards walk, proj users come before the node itself
+ if( n instanceof MultiNode )
+ for( Node proj : n.outs() )
+ if( proj instanceof ProjNode )
+ do_node(alloc,proj);
+ if( n instanceof MachNode )
+ do_node(alloc,n);
+ }
+
+ // The bb head kills register, e.g. a CallEnd and caller-save registers
+ if( bb instanceof MachNode mach )
+ kills(alloc,mach);
+
+ // Push live-sets backwards to priors in CFG.
+ if( bb instanceof RegionNode )
+ for( int i=1; i<bb.nIns(); i++ )
+ mergeLiveOut(alloc,bb,i);
+ else
+ mergeLiveOut(alloc,bb,0);
+ }
+
+ private static void do_node(RegAlloc alloc, Node n) {
+ // Defining means killing live LRG
+ LRG lrg = alloc.lrg(n);
+ if( lrg != null ) {
+ // Check for def-side self-conflict live ranges. These must split,
+ // and only happens during the first round a particular LRG splits.
+ selfConflict(alloc,n,lrg);
+ TMP.remove(lrg); // Kill def
+ }
+
+ // Phis use and define the same live range, i.e. these LRGs already
+ // marked conflicted, no need to mark again
+ if( n instanceof PhiNode )
+ return;
+
+ // Kill any killed registers; milli-code routines like New can kill
+ // will not being a CFG.
+ if( n instanceof MachNode m )
+ kills(alloc,m);
+
+ // Interfere n with all live
+ if( lrg!=null ) {
+ // Interfere n with all live
+ for( LRG tlrg : TMP.keySet() ) {
+ assert tlrg.leader();
+ // Skip self
+ if( lrg != tlrg &&
+ // And register sets overlap
+ lrg._mask.overlap(tlrg._mask) )
+ // Then tlrg and lrg interfere. If lrg needs the single
+ // last tlrg register at some point, either tlrg or lrg
+ // must fail. If *n* (a subset of lrg) needs the single
+ // last tlrg register then only tlrg must fail.
+ if( ((MachNode)n).outregmap().size1() ) {
+ if( !tlrg.clr(lrg._mask.firstReg()) ) // Clear bit, no interference
+ alloc.fail(tlrg); // Clearing drives mask to empty
+ } else addIFG(lrg,tlrg); // Add interference
+ }
+ }
+
+
+ // Check for self-conflict live ranges. These must split, and only
+ // happens during the first round when a particular LRG splits.
+ // Also record use-side spills for biased coloring.
+ // Then make all inputs live.
+ for( int i=1; i<n.nIns(); i++ ) {
+ Node def = n.in(i);
+ if( def==null ) continue;
+ LRG lrg1 = alloc.lrg(def);
+ if( lrg1==null ) continue; // Anti-dependence, etc, no register
+
+ // Check use-side for self-conflict; if so we MUST split
+ selfConflict(alloc,def,lrg1);
+
+ // All operands are live together, regardless of their input order.
+ TMP.put(lrg1,def);
+ }
+
+ // Check fixed-register uses after gathering every operand. An indirect
+ // call's target comes after its arguments, but must also avoid their
+ // fixed registers (and may need to move from its incoming register).
+ for( int i=1; i<n.nIns(); i++ ) {
+ Node def = n.in(i);
+ if( def==null || alloc.lrg(def)==null ) continue;
+ // Look for a must-use single register conflicting with some other must-def.
+ if( n instanceof MachNode m ) {
+ RegMask ni_mask = m.regmap(i);
+ if( ni_mask!=null && ni_mask.size1() ) { // Must-use single register
+ // Search all current live
+ for( LRG tlrg : TMP.keySet() ) {
+ assert tlrg.leader();
+ Node live = TMP.get(tlrg);
+ if( live != def && live instanceof MachNode lmach && lmach.outregmap().overlap(ni_mask) ) {
+ // Deny the register, since it absolutely must be used here
+ if( !tlrg.clr(ni_mask.firstReg()) )
+ // Then direct reg-reg conflict between use here (at n.in(i)) and def (of tlrg) there.
+ // Fail the older live range, it must move its register.
+ alloc.fail( tlrg );
+ }
+ }
+ }
+ }
+ }
+ }
+
+ // Single-register defines *must* have their register; other live ranges
+ // *must* avoid this register, so instead of interfering we remove the
+ // single register from the other rmask.
+ private static void kills( RegAlloc alloc, MachNode m ) {
+ RegMask killMask = m.killmap();
+ if( killMask==null ) return;
+ // Kill registers with all live
+ for( LRG tlrg : TMP.keySet() ) {
+ assert tlrg.leader();
+ // Always, tlrg cannot use kills
+ if( tlrg._mask.overlap(killMask) ) {
+ Node live = TMP.get(tlrg);
+ // Disallow clone-ables from living across killed registers.
+ // Fail them and let splitting re-clone closer to the target use.
+ if( live instanceof MachNode mach && mach.isClone() )
+ alloc.fail(tlrg);
+ else if( !tlrg.sub(killMask) )
+ alloc.fail(tlrg);
+ }
+ }
+ }
+
+ // Check for self-conflict live ranges. These must split, and only happens
+ // during the first round a particular LRG splits.
+ private static void selfConflict(RegAlloc alloc, Node n, LRG lrg) {
+ selfConflict(alloc,n,lrg,TMP.get(lrg));
+ }
+ private static void selfConflict(RegAlloc alloc, Node n, LRG lrg, Node prior) {
+ if( prior!=null && prior != n ) {
+ lrg.selfConflict(prior);
+ lrg.selfConflict(n);
+ alloc.fail(lrg); // 2 unrelated values live at once same live range; self-conflict
+ }
+ }
+
+ // Merge TMP into bb's live-out set; if changes put bb on WORK
+ private static void mergeLiveOut( RegAlloc alloc, CFGNode priorbb, int i ) {
+ CFGNode bb = priorbb.cfg(i);
+ if( bb == null ) return; // Start has no prior
+ while( !bb.blockHead() ) bb = bb.cfg0();
+
+ // Lazy get live-out set for bb
+ IdentityHashMap<LRG, Node> lrgs = BBOUTS.computeIfAbsent( bb, k -> new IdentityHashMap<>() );
+
+ for( LRG lrg : TMP.keySet() ) {
+ Node def = TMP.get(lrg);
+ // Effective def comes from phi input from prior block
+ if( def instanceof PhiNode phi && phi.cfg0()==priorbb ) {
+ assert i!=0;
+ def = phi.in( i );
+ }
+ // The synthetic exit carries no value on this Phi arm.
+ if( def._type==Type.TOP ) continue;
+ Node def_bb = lrgs.get(lrg);
+ if( def_bb==null ) {
+ lrgs.put(lrg,def);
+ pushWork(bb);
+ } else {
+ // Alive twice with different definitions; self-conflict
+ selfConflict(alloc,def,lrg,def_bb);
+ }
+ }
+ }
+
+
+ static void convert2DAdjacency( RegAlloc alloc ) {
+ // Convert the 2-D array of bits (a 1-D array of BitSets) into an
+ // adjacency matrix.
+ int maxlrg = alloc._LRGS.length;
+ for( int i=1; i<maxlrg; i++ ) {
+ BitSet ifg = IFG.atX(i);
+ if( ifg != null ) {
+ LRG lrg0 = alloc._LRGS[i];
+ for( int lrg = ifg.nextSetBit(0); lrg>=0; lrg=ifg.nextSetBit(lrg+1) ) {
+ LRG lrg1 = alloc._LRGS[lrg];
+ lrg0.addNeighbor(lrg1);
+ lrg1.addNeighbor(lrg0);
+ }
+ }
+ }
+ }
+
+
+ // ------------------------------------------------------------------------
+ // Color the inference graph.
+
+ // Coloring works by removing "trivial" live ranges from the IFG - those
+ // live ranges with fewer neighbors than available colors (registers).
+ // These are trivial because even if every neighbor takes a unique color,
+ // there's at least one more available to color this live range.
+
+ // If we hit a clique of non-trivial live ranges, we pick one to be "at
+ // risk" of not-coloring. Good choices include live ranges with a large
+ // area and few hot uses.
+
+ // Then we reverse and put-back live ranges - picking a color as we go.
+ // If there's no spare color we'll have to spill this at-risk live range.
+
+ public static boolean color(int round, RegAlloc alloc) {
+ int maxlrg = alloc._LRGS.length, nlrgs=0;
+ for( int i=1; i<maxlrg; i++ )
+ if( alloc._LRGS[i] != null )
+ nlrgs++;
+
+ // Simplify
+
+ // Walk all the LRGS looking for some trivial ones to start with.
+ // During this pass all LRGS are broken into 3 disjoint sets:
+ // - trivial, removed from IFG; color_stack[0 to sptr]
+ // - trivial, not (yet) removed from IFG; color_stack[sptr to swork]
+ // - unknown; color_stack[work to maxlrg]
+
+ // Gather all not-unified (not-null); separate trivial and non-trivial set.
+ int sptr = 0, swork = 0;
+ LRG[] color_stack = new LRG[nlrgs];
+ for( int i=0, j=0; i<maxlrg; i++ ) {
+ LRG lrg = alloc._LRGS[i];
+ if( lrg==null ) continue; // Unified lrgs are null here
+ color_stack[j++] = lrg;
+ if( lrg.lowDegree() ) swap(color_stack, swork++, j-1);
+ }
+
+ // Pull all lrgs from IFG, in trivial order if possible
+ while( sptr < color_stack.length ) {
+ // Swap best color up front
+ pickColor(color_stack,sptr,swork,alloc);
+
+ // Pick a trivial lrg, and (temporarily) remove from the IFG.
+ LRG lrg = color_stack[sptr++];
+ // If sptr was swork, then pulled an at-risk lrg
+ if( sptr > swork )
+ swork = sptr;
+
+ // Walk all neighbors and remove
+ if( lrg._adj != null ) {
+ for( LRG nlrg : lrg._adj ) {
+ // Remove and compress out neighbor
+ if( nlrg.removeCompress(lrg) ) {
+ // Neighbor is just exactly going trivial as 'lrg' is removed from IFG
+ // Find "j" position in the color_stack
+ int jx = swork; while( color_stack[jx] != nlrg ) jx++;
+ // Add trivial neighbor to trivial list. Pull lrg j out of
+ // unknown set, since its now in the trivial set
+ swap(color_stack, swork++, jx);
+ }
+ }
+ }
+ }
+
+ // Reverse simplify (unstack the color stack), and set colors (registers) for live ranges
+ while( sptr > 0 ) {
+ LRG lrg = color_stack[--sptr];
+ if( lrg==null ) continue;
+ RegMaskRW rmask = lrg._mask.copy();
+ // Walk neighbors and remove adjacent colors
+ if( lrg._adj!=null ) {
+ for( LRG nlrg : lrg._adj ) {
+ nlrg.reinsert(lrg);
+ int reg = nlrg._reg;
+ if( reg!= -1 ) // Failed neighbors do not count
+ rmask.clr(reg); // Remove neighbor from my choices
+ }
+ }
+ // At-risk live-range did not color?
+ if( rmask.isEmpty() ) {
+ alloc.fail(lrg);
+ lrg._reg = -1;
+ } else {
+ // Pick first available register
+ short reg = rmask.firstReg();
+ // Pick a "good" color from the choices. Typically, biased-coloring
+ // removes some over-spilling.
+ if( rmask.size() > 1 ) reg = biasColor(alloc,lrg,reg,rmask);
+ lrg._reg = reg; // Assign the register
+ }
+ }
+
+ return alloc.success();
+ }
+
+ // Pick LRG from color stack
+ private static void pickColor(LRG[] color_stack, int sptr, int swork, RegAlloc alloc) {
+ // Out of trivial colorable, pick an at-risk to pull
+ if( sptr==swork )
+ swap(color_stack,sptr,pickRisky(color_stack,sptr,alloc));
+ // When coloring, we'd like to give more choices; so when coloring we'd
+ // like to see the single-def first (since no choices anyway), then
+ // non-split related (so more live ranges get colored), then
+ // split-related last, so they have more colors to bias towards.
+
+ // Working in reverse, pick first split-related with many regs, then
+ // those with some regs, then single-def.
+ int bidx=sptr;
+ LRG best=color_stack[bidx];
+ for( int idx = sptr+1; idx < swork; idx++ )
+ if( betterLRG(best,color_stack[idx]) )
+ best = color_stack[bidx=idx];
+ if( bidx != sptr )
+ swap(color_stack,sptr,bidx); // Pick best at sptr
+ }
+
+ // Returns true if 'lrg' is better than 'best'
+ private static boolean betterLRG( LRG best, LRG lrg ) {
+ // If single-def varies, keep the not-single-def
+ if( best.size1() != lrg.size1() )
+ return best.size1();
+ // If hasSplit varies, keep the hasSplit
+ if( best.hasSplit() != lrg.hasSplit() )
+ return lrg.hasSplit();
+ // If both are single register (so both sizes are 1),
+ // return a multi-use over a multi-def
+ if( best.size1() )
+ return !best._multiUse && lrg._multiUse;
+ // Keep large register count
+ return best.size() < lrg.size();
+ }
+
+ // Pick a live range that hasn't already spilled, or has a single-def-
+ // single-use that are not adjacent.
+ private static int pickRisky( LRG[] color_stack, int sptr, RegAlloc alloc ) {
+ int best=sptr;
+ int bestScore = pickRiskyScore(color_stack[best],alloc);
+ for( int i=sptr+1; i<color_stack.length; i++ ) {
+ if( bestScore == 1000000 ) return best; // Already max score
+ int iScore = pickRiskyScore(color_stack[i],alloc);
+ if( iScore > bestScore )
+ { best = i; bestScore = iScore; }
+ }
+ return best;
+ }
+
+ // Pick a live range to pull, that might not color.
+ //
+ // Picking a live range with a very large span, with defs and uses outside
+ // loops means spilling a relative cheap live range and getting that
+ // register over a large area.
+ //
+ // Picking a live range that is very close to coloring might allow it to
+ // color despite being risky.
+ private static int pickRiskyScore( LRG lrg, RegAlloc alloc ) {
+ // Callee-save live ranges cover the whole function and are cheap to
+ // split. Rank scheduled area recovered against the loop-scaled cost
+ // of the def- and use-side moves.
+ if( lrg._machDef instanceof CalleeSaveNode )
+ return areaScore(functionArea(lrg._fun,alloc),
+ splitCost((Node)lrg._machDef,(Node)lrg._machUse));
+ if( lrg._splitDef != null && lrg._splitDef. in(1) instanceof CalleeSaveNode &&
+ lrg._splitUse != null && lrg._splitUse.out(0) instanceof ReturnNode )
+ return areaScore(functionArea(lrg._fun,alloc),
+ splitCost(lrg._splitDef,lrg._splitUse));
+
+ // Pick single-def clonables that are not right next to their single-use.
+ // Rank by the area recovered by cloning closer to the use.
+ if( !lrg._multiDef && lrg._machDef!=null && lrg._machUse!=null && lrg._machDef.isClone() ) {
+ Node def = ((Node)lrg._machDef);
+ Node use = ((Node)lrg._machUse);
+ CFGNode cfg = def.cfg0();
+ int area = cfg != use.cfg0()
+ ? functionArea(lrg._fun,alloc)
+ : cfg._outputs.find(use)-cfg._outputs.find(def)-1;
+ // A single-use function address can move next to its use without
+ // adding an instruction. Prefer that to saving a callee-save
+ // register just to hold the address. Stay in the same block and
+ // require a gap so the clone makes progress.
+ if( def instanceof FunPtrNode && !lrg._multiUse &&
+ cfg == use.cfg0() && area > 0 )
+ return 1000000;
+ if( area > 0 )
+ return areaScore(area,loopCost(use));
+ }
+
+ // Default for ranges without a specific area/cost estimate.
+ return 1000 + (lrg._multiUse ? -100 : 0);
+ }
+
+ private static int functionArea(FunNode fun, RegAlloc alloc) {
+ int area=0;
+ for( CFGNode bb : alloc._code._cfg )
+ if( bb.fun()==fun )
+ area += bb.nOuts();
+ return area;
+ }
+
+ private static int areaScore(int area, int cost) {
+ return 1000 + (area<<10)/cost;
+ }
+
+ private static int splitCost(Node def, Node use) {
+ return loopCost(def) + (use==null ? 0 : loopCost(use));
+ }
+
+ private static int loopCost(Node n) {
+ return 1 << (n.cfg0().loopDepth()*3);
+ }
+
+ private static short biasColor( RegAlloc alloc, LRG lrg, short reg, RegMask mask ) {
+ if( mask.size1() ) return reg;
+ // Check chain of splits up the def-chain. Take first allocated
+ // register, and if it's available in the mask, take it.
+ Node def = lrg._splitDef, use = lrg._splitUse;
+ int tidx=0, cnt=0;
+
+ while( def != null || use != null ) {
+ if( cnt++ > 10 ) break;
+
+ if( def != null ) {
+ short bias = biasColor( alloc, def, mask );
+ if( bias >= 0 ) return bias; // Good bias
+ if( bias == -2 ) def = null; // Kill this side, no more searching
+ else if( (tidx=biasable(def)) == 0 ) def = null;
+ }
+
+ if( use != null ) {
+ short bias = biasColor( alloc, use, mask );
+ if( bias >= 0 ) return bias; // Good bias
+ if( bias == -2 ) use = null; // Kill this side, no more searching
+ else if( biasable(use)==0 ) use = null;
+ }
+
+ if( def != null ) {
+ short bias = biasColorNeighbors( alloc, def, mask );
+ if( bias >= 0 ) return bias;
+ // Advance def side
+ def = def.in(tidx);
+ if( alloc.lrg(def)==null ) def=null;
+ }
+
+ if( use != null ) {
+ short bias = biasColorNeighbors( alloc, use, mask );
+ if( bias >= 0 ) return bias;
+ use = use.out(0);
+ if( biasable(use)==0 ) use=null;
+ }
+
+ }
+ return mask.firstReg();
+ }
+
+ private static int biasable(Node split) {
+ if( split instanceof SplitNode ) return 1; // Yes biasable, advance is slot 1
+ if( split instanceof PhiNode phi ) return phi.region() instanceof LoopNode ? 2 : 1; // Prefer the backedge in loops
+ if( !(split instanceof MachNode mach) ) return 0; // Not biasable
+ return mach.twoAddress(); // Only biasable if 2-addr
+ }
+
+ // 3-way return:
+ // - good bias reg, take it & exit
+ // - this path is cutoff; do not search here anymore
+ // - advance this side
+ private static short biasColor( RegAlloc alloc, Node split, RegMask mask ) {
+ short bias = alloc.lrg(split)._reg;
+ if( bias != -1 ) {
+ if( mask.test(bias) ) return bias; // Good bias
+ else return -2; // Kill this side
+ } else return -1; // Advance this side
+ }
+
+ // Check if we can match "split" color, or else trim "mask" to colors
+ // "split" might get.
+ private static short biasColorNeighbors( RegAlloc alloc, Node split, RegMask mask ) {
+ LRG slrg = alloc.lrg(split);
+ if( slrg._adj == null ) return -1; // No trimming
+
+ // Can I limit my own choices to valid neighbor choices?
+ for( LRG alrg : slrg._adj ) {
+ short reg = alrg._reg;
+ if( reg == -1 && alrg._mask.size1() )
+ reg = alrg._mask.firstReg();
+ if( reg != -1 ) {
+ mask.clr(reg);
+ if( mask.size1() )
+ return mask.firstReg();
+ }
+ }
+ return -1; // No obvious color choice, but mask got trimmed
+ }
+
+ private static void swap( LRG[] ary, int x, int y ) {
+ LRG tmp = ary[x]; ary[x] = ary[y]; ary[y] = tmp;
+ }
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/InstSelect.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/InstSelect.java
new file mode 100644
index 00000000..5fc002a1
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/InstSelect.java
@@ -0,0 +1,126 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.type.*;
+import java.util.BitSet;
+import java.util.IdentityHashMap;
+
+// Replace ideal nodes with machine nodes while preserving CFG/loop metadata.
+// Selection also owns target ABI masks and rebuilding the copied graph's uses.
+class InstSelect {
+ private final CodeGen _code;
+ private InstSelect(CodeGen code) { _code=code; }
+
+ static CodeGen select(CodeGen code, String cpu, String callingConv, String base) {
+ // Ideal evaluation schedules the ideal graph without a CPU port.
+ if( cpu==null ) return code;
+ return code.phase(CodeGen.Phase.Select, code._phase==CodeGen.Phase.Unlink,
+ () -> new InstSelect(code).run(cpu,callingConv,base));
+ }
+
+ private void run(String cpu, String callingConv, String base) {
+ _code._callingConv = callingConv;
+
+ // Look for CPU in fixed named place:
+ // com.seaofnodes.simple.node.cpus."cpu"."cpu.class"
+ String clzFile = base+"."+cpu+"."+cpu;
+ try { _code._mach = Class.forName(clzFile).asSubclass(Machine.class).getDeclaredConstructor(CodeGen.class).newInstance(_code); }
+ catch( Exception e ) { throw new RuntimeException(e); }
+
+ _code._cCallingConv = _code._mach.cCallingConv(callingConv);
+
+ // Build global copies of common register masks.
+ long callerSave = _code._mach.callerSave();
+ long cCallerSave= _code._mach.callerSave(_code._cCallingConv);
+ long neverSave = _code._mach. neverSave();
+ int maxReg = Math.min(64,_code._mach.regs().length);
+ assert maxReg>=64 || (-1L << maxReg & callerSave)==0; // No stack slots in callerSave
+ assert maxReg>=64 || (-1L << maxReg & cCallerSave)==0; // No stack slots in cCallerSave
+ _code._callerSave = new RegMask(callerSave);
+ _code._cCallerSave= new RegMask(cCallerSave);
+
+ // Build a Return RegMask array. All returns have the following inputs:
+ // 0 - ctrl
+ // 1 - memory
+ // 2 - varies with returning GPR,FPR
+ // 3 - RPC
+ // 4+- Caller save registers
+ _code._retMasks = new RegMask[(maxReg-_code._callerSave.size()-Long.bitCount( neverSave ))+4];
+ for( int reg=0, i=4; reg<maxReg; reg++ )
+ if( !_code._callerSave.test(reg) && ((1L<<reg)&neverSave)==0 )
+ _code._retMasks[i++] = new RegMask(reg);
+ _code._rpcMask = new RegMask(_code._mach.rpc());
+ _code._retMasks[3] = _code._rpcMask;
+
+ // Convert to machine ops
+ _code._uid = 1; // All new machine nodes reset numbering
+ var map = new IdentityHashMap<Node,Node>();
+ _instSelect( _code._stop, map );
+ _code._stop = (StopNode)map.get(_code._stop );
+ StartNode start = (StartNode)map.get(_code._start);
+ _code._start = start==null ? new StartNode(_code._start) : start;
+ _instOuts(_code._stop,_code.visit());
+ _code._visit.clear();
+
+ // Replace the CompUnit stop (and list of functions)
+ // with hardware-specific ones
+ for( CompUnit cu : _code._compunits.values() ) {
+ cu._start= (StartCUNode)map.get(cu._start);
+ cu._stop = (StopCUNode )map.get(cu._stop );
+ }
+ }
+
+ private Node _instSelect( Node n, IdentityHashMap<Node,Node> map ) {
+ if( n==null ) return null;
+ Node x = map.get(n);
+ if( x !=null ) return x; // Been there, done that
+
+ // If n is a MachNode already, then its part of a multi-node expansion.
+ // It does not need instruction selection (already selected!)
+ // but it does need its inputs walked.
+ if( n instanceof MachNode ) {
+ for( int i=0; i < n.nIns(); i++ )
+ n._inputs.set(i, _instSelect(n.in(i),map) );
+ return n;
+ }
+
+ // Produce a machine node from n; map it to flag as done so stops cycles.
+ if( n instanceof MemOpNode mem && mem.off() instanceof ExternOffsetNode off ) {
+ // External fields replace base+offset with symbol+0. Keep the ideal
+ // declaring type on the address for machine load/store width selection.
+ TypeMemPtr ptr = (TypeMemPtr)mem.ptr()._type;
+ n._inputs.set(2,new ExternNode(TypeMemPtr.make((byte)2,ptr._obj,true),off._extern));
+ n._inputs.set(3,ConstantNode.raw(TypeInteger.ZERO));
+ }
+ map.put(n, x=_code._mach.instSelect(n) );
+ // Carry loop-tree and pre-order info across the ideal->mach transition
+ if( n instanceof CFGNode ncfg && x instanceof CFGNode xcfg ) {
+ xcfg._ltree = ncfg._ltree;
+ xcfg._pre = ncfg._pre;
+ }
+ // Walk machine op and replace inputs with mapped inputs
+ for( int i=0; i < x.nIns(); i++ )
+ x._inputs.set(i, _instSelect(x.in(i),map) );
+ // Post selection action
+ if( x instanceof MachNode mach ) {
+ if( n instanceof ReturnNode ret )
+ ((ReturnNode)mach)._fun = (FunNode)map.get(ret._fun);
+ mach.postSelect(_code); // Post selection action
+ }
+
+ return x;
+ }
+
+ private void _instOuts( Node n, BitSet visit ) {
+ if( visit.get(n._nid) ) return;
+ visit.set(n._nid);
+ for( Node in : n._inputs )
+ if( in!=null ) {
+ in._outputs.push(n);
+ // CallNode special: outputs are partially ordered; CallEnd in slot 0
+ if( in instanceof CallNode call && n instanceof CallEndNode cend && call.out(0) != cend )
+ call._outputs.swap(0,call.nOuts()-1);
+ _instOuts(in,visit);
+ }
+ }
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/Iter.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Iter.java
new file mode 100644
index 00000000..4085ebff
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Iter.java
@@ -0,0 +1,101 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.type.*;
+import com.seaofnodes.simple.util.Ary;
+
+// Establish a pessimistic fixed point after parsing/loading. Later Iter runs
+// reuse the worklist and types produced by the preceding phase.
+public abstract class Iter {
+ static void run(CodeGen code) {
+ assert code._globalEscapes==null;
+ code._globalEscapes = new GlobalEscapeNode();
+ code._start.setDef(2,code._globalEscapes);
+ // The phase shift closes all parser-in-progress states and freezes
+ // unknown external inputs. Imported nodes carry post-Opto escape
+ // precision, but Iter is a fresh pessimistic solve: memory escape
+ // summaries must restart at FULL so newly discovered private escapes
+ // can only be removed, never added. Seed the complete graph once.
+ boolean hasImports = code._externPaths != null;
+ code._start.walk(n -> {
+ // Type import can reconcile a constant's declaration.
+ n._type = n instanceof ConstantNode ? n.compute()
+ : hasImports ? iterImportReset(n._type) : iterEscapeReset(n._type);
+ code.add(n);
+ return null;
+ });
+ code._iter.iterate(code);
+ // Expensive assert.
+ assert !CodeGen.expensiveAssert() || Opto.fixedPointCheck(code);
+ }
+
+ private static Type iterEscapeReset(Type t) {
+ if( t instanceof TypeMem mem )
+ return mem._one ? mem : mem.makeFrom(XInt.FULL,XInt.FULL);
+ if( t instanceof TypeTuple tt ) {
+ TypeTuple rez = tt;
+ for( int i=0; i<tt._types.length; i++ ) {
+ Type tx = iterEscapeReset(tt._types[i]);
+ if( tx != tt._types[i] ) rez = rez.makeFrom(i,tx);
+ }
+ return rez;
+ }
+ return t;
+ }
+
+ private static Type iterImportReset(Type t) {
+ if( t.isHigh() )
+ return t.dual();
+ if( t instanceof TypeMem mem )
+ return TypeMem.make(mem._alias,iterImportReset(mem._t),mem._one,mem._clz,mem._final,XInt.FULL,XInt.FULL);
+ if( t instanceof TypeMemPtr )
+ return t.makeStorage();
+ if( t instanceof TypeStruct )
+ return t.makeStorage();
+ if( t instanceof TypeTuple tt ) {
+ TypeTuple rez = tt;
+ for( int i=0; i<tt._types.length; i++ ) {
+ Type tx = iterImportReset(tt._types[i]);
+ if( tx != tt._types[i] ) rez = rez.makeFrom(i,tx);
+ }
+ return rez;
+ }
+ return t;
+ }
+
+ // Saved types belong to a different caller context. Reset only the new
+ // graph, then repair any affected bounds before peepholes consume them.
+ static void imports(CodeGen code, Ary<Node> nodes) {
+ for( Node n : nodes )
+ if( !n.isDead() )
+ n._type = n instanceof ConstantNode ? n.compute() : iterImportReset(n._type);
+ // Stop normalizes unknown memory into the unit's visible escapes.
+ // Do not publish a transient FULL Stop merely because we reset its body.
+ for( Node n : nodes )
+ if( n instanceof StopCUNode && !n.isDead() ) n._type = n.compute();
+ code.addAll(nodes);
+ repair(code);
+ }
+
+ // Repair the monotonicity assert after loading new nodes (from a file via
+ // Serialize). Only widening types here makes it safe to resume the usual
+ // lifting peeps. The worklist already contains changed edges and their
+ // neighbors.
+ public static void repair(CodeGen code) {
+ // Gather changes for a normal IterPeeps once the graph has its
+ // monotonicity property back.
+ Ary<Node> peeps = new Ary<>(Node.class);
+ Node n;
+ while( (n = code._iter._work.pop())!=null ) {
+ if( n.isDead() ) continue;
+ peeps.add(n);
+ // Meet over existing and new; only drops
+ Type t = n._type.meet(n.compute());
+ if( t==n._type ) continue;
+ n._type = t;
+ code.addAll(n._outputs);
+ n.moveDepsToWorklist();
+ }
+ code.addAll(peeps);
+ }
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/IterPeeps.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/IterPeeps.java
new file mode 100644
index 00000000..543a69df
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/IterPeeps.java
@@ -0,0 +1,349 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.type.Type;
+import com.seaofnodes.simple.type.TypeInteger;
+import com.seaofnodes.simple.util.Ary;
+import com.seaofnodes.simple.node.*;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.IdentityHashMap;
+import java.util.Random;
+
+/**
+ * The IterPeeps runs after parsing. It iterates the peepholes to a fixed point
+ * so that no more peepholes apply. This should be linear because peepholes rarely
+ * (never?) increase code size. The graph should monotonically reduce in some
+ * dimension, which is usually size. It might also reduce in e.g. number of
+ * MulNodes or Load/Store nodes, swapping out more "expensive" Nodes for cheaper
+ * ones.
+ * <br>
+ * The theoretical overall worklist is mindless just grabbing the next thing and
+ * doing it. If the graph changes, put the neighbors on the worklist.
+ * <br>
+ * Lather, Rinse, Repeat until the worklist runs dry.
+ * <p>
+ * The main issues we have to deal with:
+ *
+ * <ul>
+ * <li>Nodes have uses; replacing some set of Nodes with another requires more graph
+ * reworking. Not rocket science, but it can be fiddly. Its helpful to have a
+ * small set of graph munging utilities, and the strong invariant that the graph
+ * is stable and correct between peepholes. In our case `Node.subsume` does
+ * most of the munging, building on our prior stable Node utilities.</li>
+ *
+ * <li>Changing a Node also changes the graph "neighborhood". The neighbors need to
+ * be checked to see if THEY can also peephole, and so on. After any peephole
+ * or graph update we put a Nodes uses and defs on the worklist.</li>
+ *
+ * <li>Our strong invariant is that for all Nodes, either they are on the worklist
+ * OR no peephole applies. This invariant is easy to check, although expensive.
+ * Basically the normal "iterate peepholes to a fixed point" is linear, and this
+ * check is linear at each peephole step... so quadratic overall. It's a useful
+ * assert, but one we can disable once the overall algorithm is stable - and
+ * then turn it back on again when some new set of peepholes is misbehaving.
+ * The code for this is turned on in `IterPeeps.iterate` as `assert
+ * progressOnList(stop);`</li>
+ * </ul>
+ */
+public class IterPeeps {
+
+ public final WorkList<Node> _work;
+ final WorkList<CallEndNode> _workInline;
+ private int _cnt;
+
+ public IterPeeps( long seed ) {
+ _work = new WorkList<>(seed);
+ _workInline = new WorkList<>(seed);
+ }
+
+ @SuppressWarnings("unchecked")
+ public <N extends Node> N add( N n ) { return (N)_work.push(n); }
+
+ public void addAll( Ary<Node> ary ) { _work.addAll(ary); }
+
+ /**
+ * Iterate peepholes and inlining to a fixed point
+ */
+ public void iterate( CodeGen code ) {
+ boolean didInline = false;
+ Ary<Node> defer = new Ary<>(Node.class);
+ while( true ) {
+ // Clean up everything that does not grow the code
+ iteratePeeps(code);
+
+ // Pick an inline candidate, no real heuristic, first come, first served
+ boolean inlined = false;
+ CallEndNode cend;
+ while( (cend = _workInline.pop()) != null) {
+ if( cend.isDead() ) continue;
+ if( cend.loadForInline() ) { inlined = true; break; }
+ byte inline = cend.maybeInline();
+ if( inline == -1 ) defer.add(cend);
+ if( inline > 0 ) {
+ inlined = true;
+ break;
+ }
+ }
+ // Inlined, run peeps until clean again
+ if( inlined ) { didInline = true; continue; }
+ // No more candidates, check the defer list
+ if( !didInline ) break; // No more progress, so all the "maybe inline after cleanup" do not progress
+ // Some inlining happened, retry all the "try again after cleanup" calls
+ didInline = false;
+ code.addAll(defer);
+ defer.clear();
+ }
+ }
+
+ // Run all the code-reduction and type-lifting peeps as possible
+ private void iteratePeeps( CodeGen code ) {
+ assert !CodeGen.expensiveAssert(0) || (progressOnList(code, _work) && schedulableUses(code));
+
+ Node n;
+ while( (n = _work.pop()) != null ) {
+ if( n.isDead() ) continue;
+ _cnt++; // Useful for debugging, searching which peephole broke things
+ var obs = code._obs; // Collect peephole neighbors for graph display
+ if( obs != null ) obs.before(n);
+ Node x = n.peepholeOpt();
+ if( n instanceof CallEndNode cend )
+ _workInline.push(cend);
+ if( x != null ) {
+ assert !x.isDead(); // Peepholes return alive answers
+ // peepholeOpt can return brand-new nodes, needing an initial type set
+ if( x._type==null ) x.setType(x.compute());
+ // Changes require neighbors onto the worklist
+ if( x != n || !(x instanceof ConstantNode) ) {
+ // All outputs of n (changing node) not x (prior existing node).
+ for( Node z : n._outputs ) _work.push(z);
+ // Everybody gets a free "go again" in case they didn't get
+ // made in their final form.
+ _work.push(x);
+ // A self-returning peephole can have rewritten its input
+ // edges. The new defs gained a user and may have
+ // backwards, user-sensitive peepholes of their own.
+ for( Node z : x._inputs ) _work.push(z);
+ // If the result is not self, revisit all inputs (because
+ // there's a new user), and replace in the graph.
+ if( x != n ) {
+ //for( Node z : n. _inputs ) _work.push(z);
+ for( Node z : x._outputs ) _work.push(z);
+ n.subsume(x);
+ }
+ }
+ // If there are distant neighbors, move to worklist
+ n.moveDepsToWorklist();
+ // Very expensive assert.
+ assert !CodeGen.expensiveAssert(_cnt) || (progressOnList(code, _work) && schedulableUses(code));
+ }
+ boolean unused = n.isUnused();
+ if( unused ) {
+ assert !(n instanceof StopNode); // StopNodes can die if all code in the compunit dies
+ n.kill(); // Just plain dead
+ }
+ if( obs != null ) obs.after(n, x!=null ? x : unused ? n : null, true);
+ }
+ assert !CodeGen.expensiveAssert(0) || schedulableUses(code);
+
+ }
+
+ // Visit ALL nodes and confirm the invariant:
+ // Either you are on the _work worklist OR running `iter()` makes no progress.
+
+ // This invariant ensures that no progress is missed, i.e., when the
+ // worklist is empty we have indeed done all that can be done. To help
+ // with debugging, the {@code assert} is broken out in a place where it is easy to
+ // stop if a change is found.
+
+ // Also, the normal usage of `iter()` may attempt peepholes with distance
+ // neighbors and these should fail, but will then try to add dependencies
+ // {@link #Node.addDep} which is a side effect in an assert. The {@link
+ // #midAssert} is used to stop this side effect.
+ // Pessimistic solver assert
+ public static boolean progressOnList(CodeGen code, WorkList<Node> list ) {
+ code._midAssert = true;
+ Node changed = code._stop.walk( n -> {
+ // Ignore most in-progress things
+ if( n.iskeep() ) return null;
+ // Types must be forwards, even if on the worklist.
+ Type nval = n.compute();
+ assert nval.isa(n._type) : "Non-monotonic peep: "+n+"#"+n._nid+" old="+n._type+" new="+nval+" inputs="+inputTypes(n)+" peep="+n;
+ if( list.on(n) )
+ return null; // On worklist is ok!
+ if( n instanceof CallEndNode cend ) {
+ if( code._iter._workInline.on(cend) )
+ return null; // On inline worklist is ok!
+ assert cend.maybeInline() <= 0 : "Inline fired and not on worklist, CallEndNode#"+cend._nid;
+ }
+ // Off list: must be live, in GVN table, and no peeps.
+ assert n.nOuts() > 0 : "Unused live node: "+n;
+ assert code._gvn.get(n)==n : "Not in GVN table: "+n;
+ Node m = n.peepholeOpt();
+ assert m==null : "Peep fired and not on worklist, "+n.getClass().getSimpleName()+"#"+n._nid+" -> "+m;
+ return m;
+ });
+ code._midAssert = false;
+ return changed==null;
+ }
+
+ private static String inputTypes(Node n) {
+ StringBuilder sb = new StringBuilder("[");
+ for( Node in : n._inputs )
+ sb.append(in==null ? "null" : in.getClass().getSimpleName()+"#"+in._nid+"="+in+":"+in._type+":keep="+in.iskeep()).append(',');
+ return sb.append(']').toString();
+ }
+
+ // GCM assumes every movable node can be placed no earlier than its inputs
+ // and no later than the LCA of its uses. Catch graphs where a use escapes
+ // above an input-defined branch before the later scheduling pass walks off
+ // the top of the idom tree.
+ private static final IdentityHashMap<Node,CFGNode> earlyCache = new IdentityHashMap<>();
+ private static final BitSet VISIT = new BitSet();
+ public static boolean schedulableUses(CodeGen code) {
+ earlyCache.clear();
+ Node bad = code._stop.walk( n -> {
+ if( n.iskeep() || n.isDead() || n.isConst() ||
+ n instanceof CFGNode || n instanceof PhiNode || n instanceof ProjNode )
+ return null;
+ assert VISIT.isEmpty();
+ CFGNode early = CFGNode.earlyCFG(n,earlyCache,VISIT);
+ VISIT.clear();
+ if( early == null ) return null;
+ CFGNode lca = null;
+ for( Node use : n._outputs ) {
+ CFGNode ublk = useBlock(n,use);
+ if( ublk != null )
+ lca = ublk.domLCA(lca,null);
+ }
+ if( lca == null ) return null;
+ if( !early.sameFun(lca) ) return null;
+ assert early.dominates(lca,null) : badSchedule(n,early,lca);
+ return null;
+ });
+ return bad == null;
+ }
+
+ private static CFGNode useBlock(Node n, Node use) {
+ if( use == null )
+ return null;
+ if( use instanceof PhiNode phi ) {
+ CFGNode found = null;
+ for( int i = 1; i<phi.nIns(); i++ )
+ if( phi.in(i)==n ) {
+ if( i >= phi.region().nIns() )
+ return null;
+ found = phi.region().cfg(i).domLCA(found,null);
+ }
+ return found;
+ }
+ return CFGNode.safeCFG(use);
+ }
+
+ private static String badSchedule(Node n, CFGNode early, CFGNode lca) {
+ StringBuilder sb = new StringBuilder("Unschedulable data node ")
+ .append(n.getClass().getSimpleName()).append('#').append(n._nid)
+ .append(" early=").append(cfg(early))
+ .append(" use-lca=").append(cfg(lca))
+ .append(" node=").append(n).append("\ninputs:");
+ for( int i = 0; i<n.nIns(); i++ ) {
+ Node in = n.in(i);
+ sb.append("\n in").append(i).append(": ").append(node(in))
+ .append(" cfg=").append(cfg(CFGNode.safeCFG(in)));
+ }
+ sb.append("\noutputs:");
+ for( Node use : n._outputs )
+ if( use != null )
+ sb.append("\n use ").append(node(use))
+ .append(" block=").append(cfg(useBlock(n,use)));
+ return sb.toString();
+ }
+
+ private static String node(Node n) {
+ return n==null ? "null" : n.getClass().getSimpleName()+"#"+n._nid+" "+n;
+ }
+
+ private static String cfg(CFGNode cfg) {
+ return cfg==null ? "null" : cfg.getClass().getSimpleName()+"#"+cfg._nid+" "+cfg;
+ }
+
+ /**
+ * Classic WorkList, with a fast add/remove, dup removal, random pull.
+ * The Node's nid is used to check membership in the worklist.
+ */
+ @SuppressWarnings("unchecked")
+ public static class WorkList<E extends Node> {
+
+ private Node[] _es;
+ private int _len;
+ private final BitSet _on; // Bit set if Node._nid is on WorkList
+ private final Random _R; // For randomizing pull from the WorkList
+ private final long _seed;
+
+ /* Useful stat - how many nodes are processed in the post parse iterative opt */
+ private long _totalWork = 0;
+
+ public WorkList() { this(123); }
+ WorkList(long seed) {
+ _es = new Node[1];
+ _len=0;
+ _on = new BitSet();
+ _seed = seed;
+ _R = new Random();
+ _R.setSeed(_seed);
+ }
+
+ /**
+ * Pushes a Node on the WorkList, ensuring no duplicates
+ * If Node is null it will not be added.
+ */
+ public E push( E x ) {
+ if( x==null ) return null;
+ int idx = x._nid;
+ if( !_on.get(idx) ) {
+ _on.set(idx);
+ if( _len==_es.length )
+ _es = Arrays.copyOf(_es,_len<<1);
+ _es[_len++] = x;
+ _totalWork++;
+ }
+ return x;
+ }
+
+ public void addAll( Ary<E> ary ) {
+ for( E n : ary )
+ push(n);
+ }
+ public void addAll( E[] es ) {
+ for( E n : es )
+ push(n);
+ }
+
+
+ /**
+ * True if Node is on the WorkList
+ */
+ public boolean on( E x ) { return _on.get(x._nid); }
+
+ /**
+ * Removes a random Node from the WorkList; null if WorkList is empty
+ */
+ public E pop() {
+ if( _len == 0 ) return null;
+ int idx = _R.nextInt(_len);
+ E x = (E)_es[idx];
+ _es[idx] = _es[--_len]; // Compress array
+ _es[_len] = null;
+ _on.clear(x._nid);
+ return x;
+ }
+
+ public boolean isEmpty() { return _len==0; }
+ public void clear() {
+ Arrays.fill(_es,0,_len,null);
+ _len = 0;
+ _on.clear();
+ _R.setSeed(_seed);
+ _totalWork = 0;
+ }
+ }
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/LRG.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/LRG.java
new file mode 100644
index 00000000..fdde8838
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/LRG.java
@@ -0,0 +1,240 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.util.Ary;
+import com.seaofnodes.simple.util.SB;
+import com.seaofnodes.simple.util.Utils;
+import java.util.IdentityHashMap;
+
+/** A Live Range
+ * <p>
+ * A live range is a set of nodes and edges which must get the same register.
+ * Live ranges form an interconnected web with almost no limits on their
+ * shape. Live ranges also gather the set of register constraints from all
+ * their parts.
+ * <p>
+ * Most of the fields in this class are for spill heuristics, but there are a
+ * few key ones that define what a LRG is. LRGs have a unique dense integer
+ * `_lrg` number which names the LRG. New `_lrg` numbers come from the
+ * `RegAlloc._lrg_num` counter. LRGs can be unioned together -
+ * (<a href="https://en.wikipedia.org/wiki/Disjoint-set_data_structure">this is the Union-Find algorithm</a>)
+ * - and when this happens the lower numbered `_lrg` wins. Unioning only
+ * happens during `BuildLRG` and happens because either a `Phi` is forcing all
+ * its inputs and outputs into the same register, or because of a 2-address
+ * instruction. LRGs have matching `union` and `find` calls, and a set
+ * `_leader` field.
+ */
+public class LRG {
+
+ // Dense live range numbers
+ final short _lrg;
+
+ // U-F leader; null if leader
+ LRG _leader;
+
+ // Choosen register
+ short _reg;
+
+ // Count of single-register defs and uses
+ short _1regDefCnt, _1regUseCnt;
+ // Count of all defs, uses. Mostly interested 1 vs many
+ boolean _multiDef, _multiUse;
+
+ // A sample MachNode def in the live range
+ public MachNode _machDef, _machUse;
+ short _uidx; // _machUse input
+
+ // Some splits used in biased coloring
+ MachConcreteNode _splitDef, _splitUse;
+
+ // All the self-conflicting defs for this live range
+ IdentityHashMap<Node,String> _selfConflicts;
+
+ // AND of all masks involved; null if none have been applied yet
+ RegMask _mask;
+
+ // Adjacent Live Range neighbors. Only valid during coloring
+ Ary<LRG> _adj;
+
+ // Function fully containing the live range; used to find the stack frame
+ // layout, for spill op offsets.
+ final public FunNode _fun;
+
+ public int nadj() { return _adj==null ? 0 : _adj._len; }
+
+ public RegMask useMask() { return _machUse==null ? null : _machUse.regmap(_uidx); }
+
+ void addNeighbor(LRG lrg) {
+ if( _adj==null ) _adj = new Ary<>(LRG.class);
+ _adj.push(lrg);
+ }
+
+ // Remove and compress neighbor list; store ex-neighbor past
+ // end for unwinding Simplify. True if exactly going low-degree
+ boolean removeCompress( LRG lrg ) {
+ _adj.swap(_adj.find(lrg),_adj._len-1);
+ return _adj._len-- == _mask.size();
+ }
+
+ void reinsert( LRG lrg ) {
+ assert _adj._es[_adj._len]==lrg;
+ _adj._len++;
+ }
+
+ // More registers than neighbors
+ boolean lowDegree() { return nadj() < _mask.size(); }
+
+ LRG( short lrg, FunNode fun ) { _lrg = lrg; _reg = -1; _fun = fun; }
+
+ boolean leader() { return _leader == null; }
+
+ // Fast-path Find from the Union-Find algorithm
+ LRG find() {
+ if( _leader==null ) return this; // I am the leader
+ if( _leader._leader==null ) // I point to the leader
+ return _leader;
+ return _rollup();
+ }
+ // Slow-path rollup of U-F
+ private LRG _rollup() {
+ LRG ldr = _leader._leader;
+ // Roll-up
+ while( ldr._leader!=null ) ldr = ldr._leader;
+ LRG l2 = this;
+ while( l2 != ldr ) {
+ LRG l3 = l2._leader;
+ l2._leader = ldr;
+ l2 = l3;
+ }
+ return ldr;
+ }
+
+ // Union `this` and `lrg`, keeping the lower numbered _lrg.
+ // Includes a number of fast-path cutouts.
+ LRG union( LRG lrg ) {
+ assert leader();
+ if( lrg==null ) return this;
+ lrg = lrg.find();
+ if( lrg==this ) return this;
+ return _lrg < lrg._lrg ? _union(lrg) : lrg._union(this);
+ }
+ // Union `this` and `lrg`, folding together all stats.
+ private LRG _union( LRG lrg ) {
+ assert _fun == lrg._fun;
+ // Set U-F leader
+ lrg._leader = this;
+ // Fold together stats
+ if( _machDef==null ) {
+ _machDef = lrg._machDef;
+ } else if( lrg._machDef!=null ) {
+ if( _machDef != lrg._machDef ) _multiDef=true;
+ if( _1regDefCnt==0 )
+ _machDef = lrg._machDef;
+ else if( _machDef==lrg._machDef )
+ _1regDefCnt--;
+ }
+ _1regDefCnt += lrg._1regDefCnt;
+ _multiDef |= lrg._multiDef;
+
+ if( _machUse==null ) {
+ _machUse = lrg._machUse;
+ _uidx = lrg._uidx;
+ } else if( lrg._machUse!=null ) {
+ if( _machUse != lrg._machUse ) _multiUse=true;
+ if( _1regUseCnt==0 ) {
+ _machUse = lrg._machUse;
+ _uidx = lrg._uidx;
+ }
+ else if( _machUse==lrg._machUse )
+ _1regUseCnt--;
+ }
+ _1regUseCnt += lrg._1regUseCnt;
+ _multiUse |= lrg._multiUse;
+
+ // Fold deepest Split
+ _splitDef = deepSplit(_splitDef,lrg._splitDef);
+ _splitUse = deepSplit(_splitUse,lrg._splitUse);
+
+ // Fold together masks
+ RegMask mask = _mask.and(lrg._mask);
+ if( mask==null )
+ mask = _mask.copy().and(lrg._mask);
+ _mask = mask;
+ return this;
+ }
+
+ private static MachConcreteNode deepSplit( MachConcreteNode s0, MachConcreteNode s1 ) {
+ return s0==null || (s1!=null && s0.cfg0().loopDepth() < s1.cfg0().loopDepth()) ? s1 : s0;
+ }
+
+ // Record any Mach def for spilling heuristics
+ LRG machDef( MachNode def, boolean size1 ) {
+ if( _machDef!=null && _machDef!=def )
+ _multiDef = true;
+ if( _machDef==null || size1 )
+ _machDef = def;
+ if( size1 )
+ _1regDefCnt++;
+ if( def instanceof SplitNode split )
+ _splitDef = deepSplit(_splitDef,split);
+ return this;
+ }
+
+ // Record any Mach use for spilling heuristics
+ LRG machUse( MachNode use, short uidx, boolean size1 ) {
+ if( _machUse!=null && _machUse!=use )
+ _multiUse = true;
+ if( _machUse==null || size1 )
+ { _machUse = use; _uidx = uidx; }
+ if( size1 )
+ _1regUseCnt++;
+ if( use instanceof SplitNode split )
+ _splitUse = deepSplit(_splitUse,split);
+ return this;
+ }
+
+ boolean hasSplit() { return _splitDef != null || _splitUse != null; }
+ short size() { return _mask.size(); }
+ boolean size1() { return _mask.size1(); }
+
+
+ void selfConflict( Node def ) {
+ if( _selfConflicts == null ) _selfConflicts = new IdentityHashMap<>();
+ _selfConflicts.put(def,"");
+ }
+
+
+ // Record intersection of all register masks.
+ // True if still has registers
+ boolean and( RegMask mask ) {
+ RegMask mask2 = mask.and(_mask);
+ if( mask2==null )
+ mask2 = _mask.copy().and(mask);
+ _mask = mask2;
+ return !_mask.isEmpty();
+ }
+ // Remove this singular register
+ // True if still has registers
+ boolean clr( int reg ) {
+ if( _mask.clr(reg) ) return true;
+ _mask = _mask.copy(); // Need a mutable copy
+ return _mask.clr(reg);
+ }
+ // Subtract mask (AND complement)
+ // True if still has registers
+ boolean sub( RegMask mask ) {
+ RegMask mask2 = _mask.sub(mask);
+ if( mask2==null )
+ mask2 = _mask.copy().sub(mask);
+ _mask = mask2;
+ return !_mask.isEmpty();
+ }
+
+ @Override public String toString() { return toString(new SB()).toString(); }
+
+ public SB toString( SB sb ) {
+ sb.p("V").p(_lrg);
+ if( _mask!=null ) _mask.toString(sb);
+ return sb;
+ }
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/LinkMem.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/LinkMem.java
new file mode 100644
index 00000000..252b5bf0
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/LinkMem.java
@@ -0,0 +1,33 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.node.Node;
+import com.seaofnodes.simple.type.TypeStruct;
+import com.seaofnodes.simple.util.SB;
+import com.seaofnodes.simple.util.Utils;
+
+// In-memory linking
+public class LinkMem {
+ final CodeGen _code;
+ LinkMem( CodeGen code ) { _code = code; }
+
+ public CodeGen link(Encoding enc) {
+ // Patch external calls internally
+ enc.patchGlobalRelocations();
+
+ // Place constant pool after code, static data after constants
+ int code = 0; // Code start
+ int cpool = (enc._bits .size() + code + 15) & -16; // pad to 16
+ int sdata = (enc._cpool.size() + cpool + 15) & -16;
+
+
+ // Patch local ops, e.g. loading float constants from the constant pool
+ for( Node op : enc._bigCons.keySet() ) {
+ Encoding.Relo relo = enc._bigCons.get(op);
+ int target = enc._data.address(enc._data.object(relo._t),cpool,sdata);
+ ((RIPRelSize)relo._op).patch(enc, relo._opStart, enc.opLen(relo._op), target - relo._opStart);
+ }
+
+ enc._data.link(cpool,sdata);
+ return _code;
+ }
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/ListScheduler.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/ListScheduler.java
new file mode 100644
index 00000000..c3b603e5
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/ListScheduler.java
@@ -0,0 +1,265 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.codegen.IterPeeps.WorkList;
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.type.*;
+import com.seaofnodes.simple.util.Ary;
+import com.seaofnodes.simple.util.Utils;
+import java.util.*;
+
+public abstract class ListScheduler {
+
+ // eXtra per-node stuff for scheduling.
+ // Nodes are all block-local, and projections fold into their parent.
+ private static final IdentityHashMap<Node,XSched> XS = new IdentityHashMap<>();
+ private static class XSched {
+
+ Node _n; // Node this extra is for
+ int _bcnt; // Not-ready not-scheduled inputs
+ int _rcnt; // Ready not-scheduled inputs
+ boolean _ruse; // Node IS a "remote use", uses a other-block value
+ boolean _rdef; // Node IS a "remote def" of a value used in other block
+ boolean _single; // Defines a single register, likely to conflict
+
+ static final Ary<XSched> FREE = new Ary<>(XSched.class);
+ static void alloc(CFGNode bb, Node n) {
+ XSched x = FREE.pop();
+ XS.put(n, (x==null ? new XSched() : x).init(bb,n));
+ }
+ void free() { _n = null; FREE.push(this); }
+
+ static XSched get(Node n) { return n==null ? null : XS.get(n instanceof ProjNode && !(n.in(0) instanceof CFGNode) ? n.in(0) : n); }
+
+ private XSched init(CFGNode bb, Node n) {
+ _n = n;
+ _bcnt = 0;
+ _ruse = _rdef = _single = false;
+
+ // Count block-locals and remote inputs
+ if( !(n instanceof PhiNode) )
+ for( Node def : n._inputs )
+ if( def != null && def != bb ) {
+ if( def.cfg0()==bb ) _bcnt++; // Raise block-local input count
+ else if( !def.isMem() ) _ruse = true; // Remote register user
+ }
+ // Count remote outputs and allowed registers
+ if( n instanceof MultiNode )
+ for( Node use : n._outputs )
+ computeSingleRDef(bb,use);
+ else
+ computeSingleRDef(bb,n);
+
+ if( n instanceof NewNode || n instanceof CallNode )
+ bb.fun()._hasCalls = true;
+ return this;
+ }
+
+ private void computeSingleRDef(CFGNode bb, Node n) {
+ // See if this is 2-input, and that input is single-def
+ if( n instanceof MachNode mach && mach.twoAddress() != 0 ) {
+ XSched xs = XS.get(n.in(mach.twoAddress()));
+ if( xs != null )
+ _single = xs._single;
+ }
+ // Visit all outputs
+ RegMaskRW rmask = !_single && n instanceof MachNode mach && mach.outregmap() != null ? mach.outregmap().copy() : null;
+ for( Node use : n._outputs ) {
+ // Remote use, so this is a remote def
+ if( use!=null && use.cfg0()!=bb ) _rdef = true;
+ // And mask between n's def mask and all n's users masks'
+ if( use instanceof MachNode mach && rmask != null )
+ for( int i=1; i<use.nIns(); i++ )
+ if( use.in(i)==n )
+ rmask.and(mach.regmap(i));
+ }
+ // Defines in a single register
+ _single |= rmask!=null && rmask.size1();
+ }
+
+ // If _bcnt==0, declare ready; move user bcnts into rcnts.
+ boolean isReady() {
+ if( _bcnt > 0 || _rcnt > 0 ) return false;
+ if( _n instanceof MultiNode )
+ for( Node use : _n._outputs )
+ if( use instanceof ProjNode ) readyUp(use);
+ // anti-dep from a multireg directly
+ else _readyUp(use);
+ else
+ readyUp(_n);
+ return true;
+ }
+ private static void readyUp(Node n) {
+ for( Node use : n._outputs )
+ _readyUp(use);
+ }
+ private static void _readyUp(Node use) {
+ XSched xs = get(use);
+ if( xs!=null && !(use instanceof PhiNode) )
+ { assert xs._bcnt>0; xs._bcnt--; xs._rcnt++; }
+ }
+
+ boolean decIsReady() {
+ assert _rcnt > 0;
+ _rcnt--;
+ return isReady();
+ }
+ }
+
+
+ // List schedule every block
+ public static void sched( CodeGen code ) {
+ sched(code._cfg);
+ }
+
+ public static void sched(Ary<CFGNode> cfgs) {
+ XS.clear();
+ for( CFGNode bb : cfgs )
+ if( bb.blockHead() )
+ local(bb,false);
+ }
+
+ // Classic list scheduler
+ private static void local( CFGNode bb, boolean trace ) {
+ assert XS.isEmpty();
+ int len = bb.nOuts();
+
+ // Count block-locals, remote inputs and outputs
+ for( int i=0; i<len; i++ )
+ XSched.alloc(bb,bb.out(i));
+
+ // Nodes are ready if they are not used by other nodes in this block.
+ // Move ready up front.
+ int ready = 0;
+ for( int i=0; i<len; i++ )
+ if( XSched.get(bb.out(i)).isReady() ) // No block-local inputs?
+ bb._outputs.swap(ready++,i); // Move into ready set
+ assert len==0 || ready > 0;
+
+ // Classic list scheduler. Behind sched is scheduled. Between sched
+ // and ready have zero counts and are ready to schedule. Ahead of
+ // ready is undefined.
+ //
+ // As nodes are scheduled, the nodes they use decrement their bcnts[]. Once
+ // that gets to 0, all the nodes using it are scheduled, so it's ready.
+ int sched = 0;
+ while( sched < len ) {
+ assert sched < ready;
+ int pick = best(bb._outputs,sched,ready,trace); // Pick best
+ Node best = bb._outputs.swap(pick,sched++); // Swap best to head of ready and move into the scheduled set
+ // Lower ready count of users, and since projections are ready as
+ // soon as their multi is ready, lower those now too.
+ for( Node use : best._outputs )
+ if( use instanceof ProjNode )
+ for( Node useuse : use._outputs )
+ ready = ready(bb,useuse,ready);
+ else
+ ready = ready(bb,use,ready);
+ }
+
+ // Reset for next block
+ for( XSched x : XS.values() )
+ x.free();
+ XS.clear();
+ }
+
+ private static int ready(CFGNode bb, Node use, int ready) {
+ if( use!=null && use.in(0)==bb && !(use instanceof PhiNode) && XSched.get(use).decIsReady() )
+ bb._outputs.set(ready++, use); // Became ready, move into ready set
+ //bb._outputs.swap(ready++,use); // Became ready, move into ready set
+ return ready;
+ }
+
+
+ // Pick best between sched and ready.
+ private static int best( Ary<Node> blk, int sched, int ready, boolean trace ) {
+ int pick = sched;
+ Node p = blk.at(pick);
+ int score = score(p);
+ if( trace ) { System.out.printf("%4d N%d ",score,p._nid); System.out.println(p); }
+ for( int i=sched+1; i<ready; i++ ) {
+ Node n = blk.at(i);
+ int nscore = score(n);
+ if( trace ) { System.out.printf("%4d N%d ",nscore,n._nid); System.out.println(n); }
+ if( nscore > score )
+ { score = nscore; pick = i; p = n; }
+ }
+ if( trace ) { System.out.printf("Pick: N%d %s\n\n",p._nid,p); }
+ return pick;
+ }
+
+ // Highest score wins. Max at 1000, min at 10, except specials.
+ private static final int[] CNT = new int[3];
+ static int score( Node n ) {
+ if( n instanceof ProjNode ) return 1001; // Pinned at block entry
+ if( n instanceof CProjNode ) return 1001; // Pinned at block entry
+ if( n instanceof PhiNode ) return 1000;
+ if( n instanceof CFGNode ) return 1; // Pinned at block exit
+
+ int score = 500;
+ // Ideal nodes ignore register pressure scheduling
+ if( !(n instanceof MachNode) )
+ return score;
+
+ // Register pressure local scheduling: avoid overlapping specialized
+ // registers usages.
+
+ // Subtract 10 (delay) if this op forces a live range to exist that
+ // cannot be resolved immediately; i.e., live count goes up. Scale to
+ // 20 if multi-def.
+
+ // Subtract 100 if this op forces a single-def live range to exist
+ // which might conflict on the same register with other live ranges.
+ // Defines a single register based on def & uses, and the output is not
+ // ready. Scale to 200 for multi-def.
+ CNT[1]=CNT[2]=0;
+ XSched xn = XSched.get(n);
+ // If defining a remote value, just generically stall alot. Value is
+ // used in a later block, can we delay until the end of this block?
+ if( xn._rdef ) score = 200; // If defining a remote value, just generically stall alot
+ boolean flags = false;
+ if( n instanceof MultiNode ) {
+ for( Node use : n._outputs )
+ flags |= singleUseNotReady( use, xn._single );
+ } else
+ flags |= singleUseNotReady( n, xn._single );
+ score += -10 * Math.min( CNT[1], 2 );
+ score += -100 * Math.min( CNT[2], 2 );
+ if( flags ) return 10;
+
+ // Add 10 if this op ends a single-def live range, add 20 for 2 or more.
+ // Scale to 100,200 if the single-def is also single-register.
+ CNT[1]=CNT[2]=0;
+ for( int i=1; i<n.nIns(); i++ ) {
+ XSched xd = XSched.get( n.in(i) );
+ if( xd != null && n.in(i).nOuts()==1 )
+ CNT[xd._single || ((MachNode)n).regmap(i)!=null && ((MachNode)n).regmap(i).size1() ? 2 : 1]++;
+ }
+ score += 10 * Math.min( CNT[1], 2 );
+ score += 100 * Math.min( CNT[2], 2 );
+
+ // Nothing special
+ assert 10 <= score && score <= 990;
+ return score;
+ }
+
+ // Single use, and that use is not ready to fire once this one does. If
+ // true, stalling 'n' might reduce 'n's lifetime. Stall by 1 (2 if single)
+ // per not-ready.
+
+ // If CFG user in this block, it must be last, so definitely stall
+ // single-def (which stalls flags on x86,arm until the jmp).
+ private static boolean singleUseNotReady( Node n, boolean single ) {
+ // If n can rematerialize, assume allocator will split into private uses
+ // as needed. No impact on local scheduling.
+ if( n.nOuts()>1 && n instanceof MachNode mach && mach.isClone() )
+ return false;
+ for( Node use : n.outs() ) {
+ XSched xu = XSched.get(use);
+ if( xu != null && xu._n instanceof CFGNode ) return true;
+ if( xu != null && xu._bcnt > 0 )
+ CNT[single ? 2 : 1]++; // Since bcnt>0 or CFG, stall until user is more ready
+ }
+ return false;
+ }
+
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/Machine.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Machine.java
new file mode 100644
index 00000000..3eee8fc3
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Machine.java
@@ -0,0 +1,127 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.type.TypeFunPtr;
+import com.seaofnodes.simple.type.TypeInteger;
+
+abstract public class Machine {
+ // Human readable machine name. Something like "x86-64" or "arm" or "risc5"
+ public abstract String name();
+
+ // Default opcode size for printing; 4 bytes for most 32-bit risc chips
+ public int defaultOpSize() { return 4; }
+
+ // Human-readable name for a register number, e.g. "RAX" or "R0"
+ public abstract String[] regs();
+
+ // Create a split op; any register to any register, including stack slots.
+ // The FunNode holds the stack layout after RegAlloc, which in turn
+ // dictates the stack slot (the frame offset) for stack splits.
+ public abstract SplitNode split( LRG lrg, String kind, byte round );
+
+ // List of caller-save registers, as a 64bit mask
+ public abstract long callerSave();
+ public long callerSave(String conv) { return callerSave(); }
+
+ // List of never-save registers, e.g. RSP or a ZERO register if you have one
+ public abstract long neverSave();
+
+ // Call Argument Mask. Passed in the function signature and argument
+ // number (2-based; 0 is for control and 1 for memory). Also passed in a 0
+ // for the function itself, or for *outgoing* calls, the maximum stack slot
+ // given to the incoming function arguments (stack slots reserved for
+ // incoming arguments).
+ public abstract RegMask callArgMask(TypeFunPtr tfp, int arg, int maxArgSlot);
+ public RegMask callArgMask(TypeFunPtr tfp, int arg, int maxArgSlot, String conv) { return callArgMask(tfp,arg,maxArgSlot); }
+
+ // Return register mask, based on signature (GPR vs FPR)
+ public abstract RegMask retMask(TypeFunPtr tfp);
+
+ // Return PC register
+ public abstract int rpc();
+
+ // Return a MachNode unconditional branch
+ public abstract CFGNode jump();
+
+ // Instruction select from ideal nodes
+ public abstract Node instSelect( Node n );
+
+ /** Stack Slot Numbering
+
+RA gives all registers a number, starting at 0. Stack slots continue this
+numbering going up from the last register number. Common register numbers are
+0-15 for GPRs, 16-31 for FPRs, 32 for flags, and stack slots starting at
+register#33 going up. These numbers are machine-specific, YMMV, etc; e.g. RPC
+is only in stack slot 0 on X86; other cpus start with the rpc in a register
+which may spill into any generic spill slot.
+
+For ease of reading and printing, stack slots start again at slot#0 - but
+during RA they are actually biased by CPU.MAX_REG.
+
+Stack Layout during Reg Alloc:
+
+ Slot Value Description
+ ---- ------- -----------
+MAXREG+ R+1 argR maxArgSlot(calleR tfp), can be 0
+ ...
+MAXREG+ 1 arg0
+ ---- -- Caller ------
+MAXREG+ 0 RPC
+MAXREG+ R+E+N PAD optional alignment
+
+MAXREG+ + Spills Caller Frame Space; N spills
+MAXREG+ R+E+1 Spills
+
+MAXREG+ R+E argE maxArgSlot(calleE tfp), can be 0
+ ...
+MAXREG+ R+2 arg0
+ ---- -- Callee -------
+ ...
+
+Slots defined by maxArgSlot do not have to contain the actual passed argument;
+they can be "shadow slots". If they exist, they are property of the frame /
+FunNode where the TFP comes from.
+
+For a given calleR, there are generally many calleE's and they all share the
+low-range stack slots.
+
+Example:, Win64, main(int arg) { calloc(1,size); }
+
+ Post-Warp: SP+56 shadow-main-arg
+Pre-alloc Post-Alloc SP+48 RPC
+--- ------- ---------- -------------
+40 spill#3 SP+40 SP+40 spill#3
+39 spill#2 SP+32 ""
+38 spill#1 SP+24 ""
+37 spill#0 SP+16 "" spill#0
+36 shadow-calloc#size SP+ 8 SP+ 8 shadow-calloc#size
+35 shadow-calloc#1 SP+ 0 SP+ 0 shadow-calloc#1
+--- --- FRAME BASE ---------- ---- FRAME BASE -------
+34 shadow-main-arg SP+56
+33 RPC SP+48
+
+
+ */
+
+
+ // Maximum stack slot (or 0) for the args in this TFP. This will include
+ // shadow slots if defined in the ABI, even if all arguments are passed in
+ // registers.
+ public abstract short maxArgSlot(TypeFunPtr tfp);
+ public short maxArgSlot(TypeFunPtr tfp, String conv) { return maxArgSlot(tfp); }
+
+ // Calling convention used for external C symbols.
+ public String cCallingConv(String callingConv) { return callingConv; }
+
+ // Only the declared C result width describes the physical return register.
+ // Simple functions already return canonical full-width integer values.
+ protected static TypeInteger cReturn(ProjNode prj) {
+ if( prj._idx==2 && prj.in(0) instanceof CallEndNode cend &&
+ cend.call().fptr()._type instanceof TypeFunPtr tfp && tfp.isConstant() &&
+ CodeGen.CODE.externFunc(tfp.fidx())!=null &&
+ tfp.ret() instanceof TypeInteger ret && ret.log_size()<3 )
+ return ret;
+ return null;
+ }
+
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/Module.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Module.java
new file mode 100644
index 00000000..a5d85dbc
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Module.java
@@ -0,0 +1,98 @@
+package com.seaofnodes.simple.codegen;
+
+
+/***
+ Modules!?!?!?
+
+ Goal: under no circumstances can you load from an uninitialized variable
+ (although you can see default inits for fields that have defaults).
+ Goal: nested tree-structured name spaces, to control complexity.
+ Goal: field decl order within a file is fairly flexible.
+
+ A Module a root-level SMP file, and the full file path must be provided on
+ the command line. All other Simple structs are relative to this file.
+
+ Example: "simple /user/Dave/hello.smp"
+
+ Defines a module "hello" in directory "/user/Dave", compiles and produces
+ a `hello` or `hello.exe` output file, and runs it.
+
+ A SMP file has some <finit> file-init code which is static-global.
+ This code is executed on "first touch" of the file (defined below).
+
+ Example: hello.smp contains:
+ sys.io.p("Hello World!");
+ On first-touch, will print "Hello World!".
+
+ A SMP file is a lexical scope, and code inside it can reference those names.
+
+ Example: hello.smp contains:
+ val greet = "Hello World!"; // Declare file-scoped variable
+ sys.io.p(greet); // Reference file-scoped variable
+
+ Static code is NOT allowed to have forward-references AND be executed
+ before the <finit> finishes initializing all variables. After var
+ initialization, a singleton instance of the implied file struct is made,
+ and then <finit> code can call anything.
+
+ Example:
+ sys.io.p(greet); // Illegal forward reference
+ val greet = "Oops!";
+
+ Example:
+ val delay = { -> greet }; // Legal forward ref, but cannot execute (yet)
+ val greet = "Hello World!"; // Ref is resolved
+ sys.io.p(delay()); // Allowed to call delay()
+ val nprimes = { N ->
+ // compute number of primes < N
+ }
+ // Can otherwise execute any code in a <finit>
+ sys.io.p(nprimes(100));
+
+ Inside a directory, you can have sibling SMP files:
+ File hello.smp contains:
+ sys.io.p(I9A.GREETING); // File hello references sibling file I9A static globals
+ File I9A.smp contains:
+ val GREETING = translate("Hello World!");
+ val translate = { str msg -> ... };
+
+ <finit> code cannot reference sibling files (nor child files, see below)
+ until all static variables are initialized. This prevents cyclic
+ initialization bugs, where file A uses vars from file B and vice-versa.
+
+ File Chicken.smp contains:
+ val lay = Egg.hatch; // Illegal, defining var 'lay' after calling sibling Egg
+ File Egg.smp contains:
+ val hatch = Chicken.lay; // Illegal
+
+ You can make cross references, if they get delayed until after all fields
+ are declared.
+
+ File Chicken.smp contains:
+ int lay = 2;
+ lay = Egg.egg;
+
+ File Egg.smp contains:
+ int egg = 3;
+ egg = Chicken.lay;
+
+ The problem here, of course, is that you need to touch one of Chicken or
+ Egg first.
+
+
+ Inside a SMP file, you can define structs; structs are also a lexical
+ scope. Bare code in the struct is part of the struct's <init> and only
+ runs when an instance is created. Structs do not have static <clinit> (nor
+ <finit>). Structs <init> cannot have any forward references, and locally
+ declared functions cannot have both forward references and be executed (or
+ passed along to a possibly-executing function).
+
+ In the same directory as a SMP file, Simple allows other directories with
+ names matching SMP files. Inside these dirs can be more SMP files that are
+ lexically nested, similar to embedded structs.
+
+
+*/
+
+public abstract class Module {
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/Opto.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Opto.java
new file mode 100644
index 00000000..aa997204
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Opto.java
@@ -0,0 +1,331 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.codegen.IterPeeps.WorkList;
+import com.seaofnodes.simple.type.*;
+import com.seaofnodes.simple.util.Ary;
+
+import java.util.BitSet;
+
+// Optimistic Algorithm
+abstract public class Opto {
+
+ // Optimistic analysis - extended SCCP
+
+ // Start all Types at TOP, and propagate changes until stable. Unlike a
+ // normal Sparse Conditional Constant Propagation, this version is
+ // interprocedural, computing effectively CFA0 in linear time using the
+ // normal SCCP algorithm.
+
+ // This algo also builds a Call Graph; as Calls become reachable (CONTROL
+ // not TOP nor XCONTROL), they "link" against the reaching function
+ // pointers - adding edges from the Call to the Fun, and from the Return to
+ // the CallEnd. This allows the normal algorithm to use the normal edge
+ // propagation rules as function argument types stabilize. In the end, we
+ // get a fairly precise set of Call Graph edges between CallNodes and
+ // FunNodes.
+ public static void opto(CodeGen code) {
+ // Reset Types before running the worklist algorithm
+ Ary<Type> oldTypes = new Ary<>(Type.class);
+ WorkList<Node> deps = new WorkList<>();
+ WorkList<Node> changes = new WorkList<>();
+ Ary<Node> modes = new Ary<>(Node.class);
+ resetToTop(code,oldTypes,modes,deps);
+ unlinkStart(code);
+
+ // OPTIMISTIC INTERPROCEDURAL SCCP
+ sccp(code, oldTypes, changes, deps);
+
+ // Settle unresolved modes using SCCP's int/float evidence. Some
+ // computes wait at TOP until a mode is known; others already return
+ // BOTTOM and stay there. Resume SCCP only on the affected nodes;
+ // types are not reset, and any further changes must still fall.
+ resolveNumericModes(code,modes);
+
+ if( !code._iter._work.isEmpty() )
+ sccp(code, oldTypes, changes, deps);
+ Node dep;
+ while( (dep=deps.pop())!=null ) code.add(dep);
+
+ // After SCCP add everything changed to the worklist and see if
+ // peepholes can make more progress.
+ moveChangesToWorklist(code, oldTypes, changes);
+ }
+
+ // Reset Types before running the worklist algorithm
+ private static void resetToTop(CodeGen code, Ary<Type> oldTypes, Ary<Node> modes, WorkList<Node> deps ) {
+ code._start.walk( x -> {
+ // Opto ends Parm.inProgress(), enabling constant folding even
+ // when a parameter's type was already constant before SCCP.
+ if( x instanceof ParmNode && x._type.isHighOrConst() ) deps.push(x);
+ // SCCP adds call edges and default arguments, not numeric
+ // operations. Gather unresolved modes on this required walk.
+ if( x instanceof ModeNode mode && mode.mode()==0 ) modes.push(x);
+ oldTypes.setX(x._nid,x._type); // Compare the final SCCP type with Iter's type
+ x._type = Type.TOP; // Reset all Nodes to TOP
+ code._iter.add(x); // Visit everybody at least once; most Nodes produce better than TOP
+ return null;
+ } );
+ }
+
+
+ // Only entry initializers start with unknown callers. Discover every
+ // other function through calls or escaping values during SCCP.
+ private static void unlinkStart(CodeGen code) {
+ // CodeGen owns these until FinalizeIR; linkStart may need them again.
+ for( CompUnit cu : code._compunits.values() ) {
+ ProjNode mem = cu._start.proj(1);
+ if( mem!=null ) code.keepUntilFinalize(mem);
+ }
+
+ // Iterate all functions.
+ // TODO: use CodeGen _link list
+ for( Node stop : code._stop._inputs ) {
+ for( int i=0; i<stop.nIns(); i++ ) {
+ ReturnNode ret = (ReturnNode)stop.in(i);
+ FunNode fun = ret.fun();
+ if( !fun.isRoot() &&
+ fun.in(1) instanceof StartNode ) {
+ fun.removeDeadPath(1);
+ // Unhook from Stop without treating the ReturnNode as
+ // DEAD. It stays in limbo until Opto links when a caller
+ // is discovered (or not, and the function is really dead).
+ stop._inputs.del(i--);
+ ret.delUse(stop);
+ }
+ // Unlink the existing conservative call linkages.
+ for( int j=1; j<fun.nIns(); j++ )
+ if( fun.in(j) instanceof CallNode call )
+ call.unlink(fun,j--);
+ }
+ }
+ }
+
+ // As part of building a CallGraph, when opto finds a function ptr flowing
+ // into a Call, link the Call and Fun.
+ private static void linkCG(CodeGen code, TypeFunPtr tfp, CallNode call) {
+ if( tfp.nargs() != call.nargs() ) return; // Error calls hit this
+ int[] fidxs = tfp.fidxs();
+ // An unresolved forward call can still carry the infinite, inferred
+ // function set into Opto. There is no finite call graph to link yet;
+ // type checking will report the unresolved reference afterwards. This
+ // can yet be valid if the call is eventually dead.
+ // `if( false ) frefCall(args);`
+ if( XInt.isHigh(fidxs) ) return;
+ for( int fidx = XInt.next(fidxs,0); fidx >=0; fidx = XInt.next(fidxs,fidx) ) {
+ // unlinkStart deliberately leaves uncalled functions in a
+ // temporarily dead limbo. A newly discovered call revives them;
+ // do not use the cleanup lookup which removes dead linker entries.
+ FunNode fun = code._link(fidx);
+ // null here means an external function; i.e. this Call
+ // calls to an outside library and all its arguments escape.
+ if( fun != null && !call.linked(fun) ) {
+ call.link( fun );
+ code._iter.add(fun);
+ code._iter.addAll(fun._outputs);
+ }
+ }
+ }
+
+ // Start added some FIDXs, the same functions now can be reached from start
+ // and called by anybody. Add the CG edges.
+ private static void linkStart( CodeGen code, TypeTuple tt ) {
+ TypeMem tmem = (TypeMem)tt._types[1];
+ int[] fidxs = tmem._escFs;
+ if( XInt.isHigh(fidxs) ) {
+ // A scalar BOTTOM can flow into memory while compiling an error
+ // program. It exposes all public functions, not private fields.
+ for( FunNode fun : code._linker )
+ if( fun != null && code.publicFIDX(fun.sig().fidx()) )
+ linkStart(code,fun);
+ return;
+ }
+ for( int fidx = XInt.next(fidxs,0); fidx >=0; fidx = XInt.next(fidxs,fidx) ) {
+ FunNode fun = code._linker.atX(fidx);
+ if( fun==null ) assert code._externFunc.containsKey(fidx);
+ else linkStart(code,fun);
+ }
+ }
+
+ // Function is now reachable by unknown callers through Start. The
+ // default inputs must therefore use conservative declared types.
+ static void linkStart( CodeGen code, FunNode fun ) {
+ assert !fun.isDead();
+ StartCUNode start = fun._compunit._start;
+ if( !fun.unknownCallers() ) {
+ assert !start.isDead();
+ code.keepInCompUnit(fun);
+ // Function is reachable by any *remote* caller who gets the pointer!
+ fun.insertDef(1,start);
+ code._iter.add(fun);
+ for( Node p : fun._outputs )
+ if( p instanceof ParmNode parm ) {
+ Node defalt = parm._idx==1 ? start.proj(1) : ConstantNode.seed(parm._declaredType).peephole();
+ parm.insertDef(1,defalt);
+ code.add(parm);
+ }
+ }
+ }
+
+
+ // The core SCCP algorithm
+ private static void sccp( CodeGen code, Ary<Type> oldTypes, WorkList<Node> changes, WorkList<Node> deps ) {
+ // Iter workList is empty at this point
+ // OPTIMISTIC PASS
+ Node n;
+ int count = 0; // Debug counter
+ while( (n = code._iter._work.pop()) != null ) {
+ if( n.isDead() ) continue;
+ count++;
+ Type oval = n._type, nval = n.compute();
+ if( n instanceof CLInitNode init ) init.link();
+ // Remember possible peephole work without walking the graph again.
+ // SCCP can still fall back to the old type, so filter at the end.
+ if( nval!=oldTypes.atX(n._nid) || n.isUnlocked() ) changes.push(n);
+ if( oval == nval ) continue;
+ assert oval.isa(nval) : "Non-monotonic SCCP: "+n.getClass().getSimpleName()+"#"+n._nid+" "+n+" old="+oval+" new="+nval; // Types start high and always fall
+ Type pesiVal = oldTypes.atX(n._nid);
+ assert pesiVal==null || nval.isa(pesiVal) : "SCCP below pessimistic type: "+n.getClass().getSimpleName()+"#"+n._nid+" "+n+" pessimistic="+pesiVal+" new="+nval; // Never fall worse than the pessimistic pass
+ n._type = nval;
+
+ // Now we have a series of stanzas where we lazily create the Call
+ // Graph - adding graph edges between every call site and called
+ // function, including external (unknown) callers. These edges are
+ // not in from the start because they are *too many*; we would get
+ // O(n^2) edges. So all along we have been treating them as
+ // "virtual" CFG edges - a FunNode taking a Start input is treated
+ // as if every call on the planet might call it. Now we make these
+ // CG edges *concrete*, adding them back as we discover that a call
+ // calls a particular function.
+
+ // If a TFP adds a new function input to a call, link to the new
+ // Fun. This adds new edges to the graph allowing argument values
+ // to flow from calls into functions. The added edges are
+ // effectively a refined Call Graph.
+ if( nval instanceof TypeFunPtr tfp ) {
+ for( Node use : n._outputs )
+ if( !use._type.isHigh() && use instanceof CallNode call && call.fptr() == n )
+ linkCG(code,tfp,call);
+ }
+
+ // Link a Call which becomes alive. Like above, adds new Call
+ // Graph edges to the graph.
+ if( n instanceof CallNode call && nval == Type.CONTROL && call.fptr()._type instanceof TypeFunPtr )
+ linkCG(code,call.tfp(),call);
+
+ // If an otherwise-dead function pointer escapes, any future linked
+ // caller might find and call it. Force the function to be alive
+ // and called by Start.
+ if( n instanceof StartNode )
+ linkStart(code,(TypeTuple)n._type);
+
+ // Since n._type changed, visit all output neighbors
+ code._iter.addAll(n._outputs);
+ // SCCP consumes these without running peepholes. Save them for
+ // the following Iter pass before moving clears the dependency list.
+ for( int i=0; i<n.nDeps(); i++ ) deps.push(n.dep(i));
+ n.moveDepsToWorklist(code._iter);
+ // Quadratic (expensive) small-step assert.
+ assert !CodeGen.expensiveAssert(count) || worklistCheck(code);
+ }
+ }
+
+ // Quadratic (expensive) fixed-point assert
+ public static boolean fixedPointCheck(CodeGen code) {
+ return fixedPointCheck(code,code._stop);
+ }
+ public static boolean fixedPointCheck(CodeGen code, Node root) {
+ root.walk( x -> {
+ Type xval = x.compute();
+ assert xval == x._type;
+ return null;
+ });
+ return true;
+ }
+
+ // Quadratic (expensive) small-step assert for SCCP, where changed nodes
+ // may already be on the worklist.
+ private static boolean worklistCheck(CodeGen code) {
+ code._midAssert = true;
+ Node root = code._start;
+ root.walk( x -> {
+ Type xval = x.compute();
+ assert xval == x._type || (x._type.isa( xval ) && code._iter._work.on( x ))
+ : "Non-monotonic or missed SCCP work: "+x.getClass().getSimpleName()+"#"+x._nid+" "+x+" old="+x._type+" new="+xval;
+ return null;
+ });
+ code._midAssert = false;
+ return true;
+ }
+
+ // Types can stay unchanged while SCCP edits edges or numeric modes. Such
+ // edits unlock the node from GVN. Revisit both kinds of change and their
+ // neighbors; distant dependencies consumed by SCCP were saved separately.
+ private static void moveChangesToWorklist(CodeGen code, Ary<Type> oldTypes, WorkList<Node> changes) {
+ Node n;
+ while( (n=changes.pop())!=null ) {
+ if( n.isDead() ) continue;
+ assert n.compute() == n._type; // Hit the fixed point.
+ Type old = oldTypes.atX(n._nid);
+ assert old==null || n._type.isa(old);
+ if( n._type!=old || n.isUnlocked() ) {
+ code.addAll(n);
+ code.addAll(n._outputs);
+ n.moveDepsToWorklist();
+ }
+ }
+ }
+
+ // Resolve modes left undecided by SCCP, following recursive inputs for
+ // evidence. Float wins because integer operands can be converted to float.
+ private static void resolveNumericModes(CodeGen code, Ary<Node> modes) {
+ final BitSet visited = new BitSet();
+ for( Node n : modes ) {
+ if( !n.isDead() && n instanceof ModeNode modeNode && modeNode.mode()==0 ) {
+ visited.clear();
+ visited.set(n._nid); // Inspect inputs, not Bool's integer result
+ int mode = 0;
+ for( int i=1; i<n.nIns(); i++ )
+ mode |= numericEvidence(n.in(i),visited);
+ // A closed recursive numeric component can have no remaining
+ // concrete evidence after SCCP, e.g. dead all-integer loops.
+ // With no float evidence, settle to integer.
+ modeNode.setMode((byte)((mode&2)==2 ? 2 : 1));
+ code.add(n);
+ code.addAll(n.outs());
+ }
+ }
+ }
+
+ // Gather numeric evidence through Phi and interprocedural return cycles.
+ // The visited set makes self- and mutually-recursive calls finite.
+ private static int numericEvidence(Node n, BitSet visited) {
+ if( n == null ) return 0;
+ if( visited.get(n._nid) ) return 0;
+ visited.set(n._nid);
+
+ Type t = n._type;
+ if( t instanceof TypeFloat ) return 2;
+ if( t instanceof TypeInteger ) return 1;
+ if( t!=Type.BOTTOM && !(t instanceof TypeTuple) )
+ return 0; // Something unrelated, like a ptr
+
+ int evidence = 0;
+ assert !(n instanceof ConvertNode); // No test case
+ switch( n ) {
+ case ModeNode modeNode when modeNode.mode() != 0 -> { return modeNode.mode(); }
+ case ModeNode modeNode -> {}
+ case PhiNode phi -> {}
+ case ReturnNode ret -> {}
+ case ProjNode proj when proj.in(0) instanceof CallEndNode cend -> {
+ for( int i=1; i<cend.nIns(); i++ )
+ evidence |= numericEvidence(cend.in(i),visited);
+ }
+ default -> { return 0; }
+ }
+ for( int i=1; i<n.nIns(); i++ )
+ evidence |= numericEvidence(n.in(i),visited);
+
+ return evidence;
+ }
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/ParseAll.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/ParseAll.java
new file mode 100644
index 00000000..1218c956
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/ParseAll.java
@@ -0,0 +1,449 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.parser.Parser;
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.print.*;
+import com.seaofnodes.simple.type.*;
+import com.seaofnodes.simple.util.*;
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+
+// Parse a given file, and all its dependencies
+public abstract class ParseAll {
+
+ static void parse(CodeGen code, String src) {
+ Parser.TYPES.clear();
+ Parser.TYPES.putAll(Parser.INIT_TYPES);
+ if( src != null ) {
+ // No source file, just the source itself.
+ parseSource(code,code._srcName==null ? "Test" : code._srcName,src);
+ } else {
+ // Path from module root to source file
+ parsePath(code,code._srcName);
+ }
+ }
+
+ // Worklist of symbols to search and files to parse
+ private static final Ary<CompUnit> WORK = new Ary<>(CompUnit.class);
+
+ private static final Ary<CompUnit> NEEDS_LOAD = new Ary<>(CompUnit.class);
+
+ // Return a compilation unit (cached or new). If new, it does not necessarily
+ // need to be compiled.
+ public static CompUnit makeCUnit( CodeGen code, CompUnit par, String name ) throws IOException {
+ return makeCUnit(code,par,name,null,false);
+ }
+
+ private static CompUnit makeCUnit( CodeGen code, CompUnit par, String name, File obj ) throws IOException {
+ return makeCUnit(code,par,name,obj,obj!=null);
+ }
+
+ private static CompUnit makeCUnit( CodeGen code, CompUnit par, String name, File obj, boolean external ) throws IOException {
+ // Slashed file name? Move leading parts into parent CUs.
+ int idx = name.indexOf("/");
+ if( idx >= 0 ) {
+ String head = name.substring(0,idx);
+ CompUnit sub = makeCUnit(code,par,head,external ? externalObj(code,par,head) : null,external);
+ return makeCUnit(code,sub,name.substring(idx+1).intern(),obj,external);
+ }
+
+ String fname = par==null ? name : (par._fname + "/" + name).intern();
+ CompUnit cunit = code._compunits.get(fname);
+ if( cunit != null ) return cunit;
+ String cname = par==null ? name : (par._cname + "." + name).intern();
+ code._compunits.put(fname, cunit = external ? new CompUnit(obj,par,fname,cname,name) : new CompUnit(par,fname,cname,name));
+ return cunit;
+ }
+
+ // Parse a top-level string, used for e.g. testing
+ static void parseSource( CodeGen code, String src ) {
+ parseSource(code,"Test",src);
+ }
+
+ // Parse a top-level string with an explicit compilation-unit name.
+ static void parseSource( CodeGen code, String fname, String src ) {
+ CompUnit cunit = new CompUnit(fname,src);
+ code._compunits.put(cunit._fname,cunit);
+ parseAll(code,cunit);
+ }
+
+ // module_root/.../src.smp is source file name.
+ static void parsePath(CodeGen code, String fname) {
+ try {
+ // Split and build the parent-path down to the source file.
+ CompUnit cu = makeCUnit( code, null, fname );
+ // Compile this CompUnit
+ parseAll(code,cu);
+ } catch( IOException ioe ) {
+ throw new RuntimeException(ioe);
+ }
+ }
+
+ private static void enqueue(CompUnit cu) {
+ if( cu._par!=null && (cu._par._src!=null || (cu._par._obj!=null && cu._par._obj.exists())) ) enqueue(cu._par);
+ if( cu._start==null && WORK.find(cu)==-1 ) WORK.add(cu);
+ }
+
+ private static void parseAll(CodeGen code, CompUnit cunit) {
+ WORK.clear(); // Cleanup from prior tests
+ NEEDS_LOAD.clear();
+ // Prime the pump with the first CompUnit
+ enqueue(cunit);
+ // Work through all unparsed compilation units and parse
+ while( !WORK.isEmpty() ) {
+ CompUnit cu = WORK.remove(0);
+ // No source code? Means we do not need to parse, but instead need to load the symbols as-if we parsed
+ if( cu._src==null || cu._src.isEmpty() )
+ loadDeps(code,cu); // Load recursive dependents
+ else
+ parseOne( code, cu ); // Parse
+ }
+
+ // Load all pre-compiled CompUnits
+ for( CompUnit cu : NEEDS_LOAD )
+ loadSummaries(code,cu);
+
+ // Many types will have been declared cyclically either internally or
+ // across multiple compilation units.
+
+ // Close over all recursive types, and upgrade TYPES
+ Ary<TypeStruct> ary = new Ary<>(TypeStruct.class);
+ for( Type t : Parser.TYPES.values() )
+ if( t instanceof TypeStruct ts )
+ ary.add(ts);
+ TypeStruct[] tss = Type.closeOver(ary.asAry(),Parser.TYPES);
+ for( TypeStruct ts : tss )
+ Parser.TYPES.put(ts._name,ts);
+ for( CompUnit cu : code._compunits.values() )
+ if( cu._ext==null )
+ cu._clz = (TypeStruct)Parser.TYPES.get(Parser.addClzPrefix(cu._cname));
+
+ // Walk over all Nodes, and upgrade the internal constants to the
+ // closed-over types.
+ code._stop.walk( (Node n) -> {
+ n.upgradeType(Parser.TYPES);
+ if( n instanceof FRefNode fref ) {
+ boolean alloc = fref._name.endsWith(".<new>");
+ String clzName = alloc
+ ? fref._name.substring(0,fref._name.length()-".<new>".length()).intern()
+ : fref._name;
+ TypeStruct clz = (TypeStruct)Parser.TYPES.get(clzName);
+ if( clz == null )
+ return null;
+ Node con;
+ if( alloc ) {
+ Field ctor = clz.field("<new>");
+ if( ctor == null || !(ctor._t instanceof TypeFunPtr tfp) ) {
+ String name = clzName.startsWith(Parser.CLZ)
+ ? clzName.substring(Parser.CLZ.length()) : clzName;
+ int dot = name.lastIndexOf('.');
+ throw Parser.error("Unknown struct type '"+(dot==-1 ? name : name.substring(dot+1))+"'",fref._loc);
+ }
+ FunNode fun = code.link(tfp);
+ if( fun == null ) return null;
+ con = new FunPtrNode(tfp,code._start,fun.ret()).peephole();
+ } else {
+ // Class pointer constant:
+ TypeMemPtr clzptr = TypeMemPtr.make((byte)2,clz,true);
+ con = code.con(clzptr);
+ }
+ // Replace and optimize
+ fref.addDef(con);
+ code.add(fref);
+ }
+ // Adds sort by NIDs, which just got shuffled
+ if( n instanceof AddNode add )
+ code.add(add);
+ // All imported bodies are loaded; link them before Iter checks types.
+ if( n instanceof CallNode call && !NEEDS_LOAD.isEmpty() )
+ code.add(call).idealize();
+ return null;
+ } );
+
+ // The complete public interface is now known. Keep this independent
+ // of later graph/linker liveness so SCCP computations remain local and
+ // monotonic.
+ code.freezePublicInterface();
+ }
+
+ // Parse one Simple source code file. Add all the FRefs produced to the worklist.
+ private static void parseOne( CodeGen code, CompUnit cunit ) {
+ cunit.setStart(code);
+ // Parse a source file, return missing external references
+ Ary<FRefNode> frefs = code.P.parse(cunit);
+
+ // Find all missing external references, or complain. This can trigger more parses.
+ while( !frefs.isEmpty() ) {
+ FRefNode fref = frefs.remove(0);
+ // Symbol not found in source code (or we would not be here).
+ // Search the module for the fref
+ try {
+ CompUnit xcunit = findCUnitModule(code,cunit,fref._name);
+ if( xcunit == null )
+ xcunit = findCUnitExternalSimple(code,fref._name);
+ if( xcunit != null ) {
+ // Replace the FRef with the discovered class name.
+ fref._name = Parser.addClzPrefix(xcunit._cname);
+ cunit.addDep(xcunit);
+ } else {
+ throw new RuntimeException("Undefined name '" + fref._name +"'");
+ }
+ } catch( IOException ioe ) {
+ throw new RuntimeException(ioe);
+ }
+ }
+ addRequiredTypes(code,cunit);
+ }
+
+ private static void addRequiredTypes(CodeGen code, CompUnit cunit) {
+ for( String typeName : code.P._requiredTypes ) {
+ // A forward use can have been resolved later in this source file.
+ if( Parser.TYPES.get(typeName) instanceof TypeStruct ts && !ts._fref ) continue;
+ try {
+ CompUnit xcunit = findRequiredType(code,cunit,typeName);
+ if( xcunit != null )
+ cunit.addDep(xcunit);
+ else
+ assert false : "expected to find required type";
+ } catch( IOException ioe ) {
+ throw new RuntimeException(ioe);
+ }
+ }
+ }
+
+ private static CompUnit findRequiredType(CodeGen code, CompUnit cunit, String typeName) throws IOException {
+ for( CompUnit cu = cunit; cu != null; cu = cu._par ) {
+ String symbol = typeName.startsWith(cu._cname+".")
+ ? typeName.substring(cu._cname.length()+1)
+ : typeName;
+ CompUnit xcunit = findCUnitModule(code,cu,symbol);
+ if( xcunit != null )
+ return xcunit;
+ }
+ return findCUnitExternalSimple(code,typeName);
+ }
+
+ // Load the public symbols for this CompUnit, and add dependent CompUnits to the NEEDS_LOAD list.
+ private static void loadDeps(CodeGen code, CompUnit cunit) {
+ // Load IR after parsing
+ if( NEEDS_LOAD.find(cunit) != -1 )
+ return; // Already on the to-be-loaded list?
+ for( CompUnit loaded : NEEDS_LOAD )
+ if( loaded._obj.equals(cunit._obj) ) return; // Another class in the same object.
+ try {
+ NEEDS_LOAD.add(cunit);
+ // Load dependent classes
+ ElfReader elf = code.getElf(cunit._obj);
+ elf.loadPublicSymbols();
+ // Register bundled definitions before resolving dependencies or IR.
+ for( String fname : elf._units )
+ makeCUnit(code,null,fname,cunit._obj);
+ for( String fname : elf._deps ) {
+ // Check CompUnit for being up to date
+ CompUnit sub = code._compunits.get(fname);
+ if( sub==null ) sub=makeCUnit(code,null,fname,depObj(code,cunit,fname));
+ // If not seen before or out of date, parse the new nested cunit
+ if( sub._clz == null && WORK.find(sub)==-1 && NEEDS_LOAD.find(sub)==-1 )
+ enqueue(sub);
+ }
+ } catch( IOException ioe ) {
+ throw new RuntimeException(ioe);
+ }
+ }
+
+ private static File depObj(CodeGen code, CompUnit cunit, String fname) {
+ File obj = CompUnit.dependencyObj(fname,cunit._smp!=null);
+ if( obj == null )
+ throw new RuntimeException("Cannot find Simple dependency '"+fname+"'");
+ return cunit._smp!=null && obj.equals(new File(code._buildDir+"/"+fname+".o")) ? null : obj;
+ }
+
+ private static File externalObj(CodeGen code, CompUnit par, String name) {
+ String cname = par==null ? name : par._cname + "." + name;
+ ElfReader elf = code.findExternalSimple(Parser.addClzPrefix(cname));
+ return elf == null ? null : elf._file;
+ }
+
+ private static void loadSummaries(CodeGen code, CompUnit cu) {
+ ElfReader elf = code.getElf(cu._obj);
+ elf.loadPublicTypes(code);
+ cu._clz = elf._clz;
+ cu.setStart(code);
+ new ProjNode(cu._start,1,"$mem").init();
+ for( FunctionSummary summary : elf._functions ) summary.install(code,cu);
+ }
+
+ public static void loadCompUnit(CodeGen code, CompUnit cunit) {
+ // Expecting a previously compiled object file
+ assert cunit._obj != null;
+ // Extract published symbols and put them in the internal tables;
+ // cross-references will pick them up there.
+ ElfReader elf = code.getElf(cunit._obj);
+ if( elf._nodes!=null ) return;
+ StartCUNode oldStart = cunit._start;
+ StopCUNode oldStop = cunit._stop;
+ ProjNode oldMem = oldStart==null ? null : oldStart.proj(1);
+ if( oldMem!=null ) code.releaseReplacedNode(oldMem);
+ elf.loadSimple(code);
+ cunit._clz = (TypeStruct)Parser.TYPES.get(Parser.addClzPrefix(cunit._cname));
+ StartNode lstart = (StartNode)elf._nodes.at(0);
+ StopNode lstop = ( StopNode)elf._nodes.last();
+
+ // Add all new loaded functions to the linker table
+ for( Node n : lstop._inputs ) {
+ StopCUNode stop = (StopCUNode)n;
+ StartCUNode start = stop.start();
+ CompUnit cu = code._compunits.get(start._fname);
+ cu._start = start;
+ cu._stop = stop ;
+
+ for( Node ret : stop._inputs ) {
+ FunNode fun = ((ReturnNode)ret).fun();
+ FunNode old = code._link(fun.sig().fidx());
+ code.link(fun);
+ if( old!=null && old.summary()!=null && !old.isDead() ) {
+ old.ret().keep();
+ for( int i = 1; i<old.nIns(); i++ )
+ if( old.in(i) instanceof CallNode call ) {
+ call.unlink(old,i--);
+ call.link(fun);
+ } else if( old.in(i) instanceof CProjNode proj && proj.in(0) instanceof CLInitNode init ) {
+ init.unlink();
+ init.link();
+ i--;
+ }
+ old.ret().unkeep().subsume(fun.ret());
+ }
+ }
+ }
+
+ // Hook the new code into the existing code graph
+ for( Node lstopcu : lstop._inputs )
+ code._stop.addDef(lstopcu);
+ lstart.subsume(code._start);
+ if( oldStart!=null && oldStart!=cunit._start ) {
+ code.releaseReplacedNode(oldStart);
+ int idx = code._stop._inputs.find(oldStop);
+ if( idx!=-1 ) code._stop.delDef(idx);
+ oldStart.subsume(cunit._start);
+ }
+ // Saved types describe the library's original compilation. Repair
+ // only the newly loaded graph; ordinary dependencies carry changes
+ // into its callers without restarting whole-program analysis.
+ ProjNode startMem = cunit._start.proj(1);
+ if( startMem!=null ) code.keepUntilFinalize(startMem);
+ Iter.imports(code,elf._nodes);
+
+ // Add the new top-level loaded class
+ Parser.TYPES.put(cunit._clz._name,cunit._clz);
+ Field inst = cunit._clz.field(cunit._name);
+ if( inst != null && inst._t instanceof TypeFunPtr tfp && tfp._ret instanceof TypeMem mem )
+ Parser.TYPES.put(cunit._cname,mem._t);
+ }
+
+ // Symbol not found in source code (or we would not be here).
+ // Search the module for the fref
+ public static CompUnit findCompUnitOrThrow(CodeGen code, CompUnit cu, String symbol) {
+ CompUnit cu2 = findCompUnit(code,cu,symbol);
+ if( cu2 != null ) return cu2;
+ throw new RuntimeException("Undefined name '" + symbol +"'");
+ }
+
+ public static CompUnit findCompUnit(CodeGen code, CompUnit cu, String symbol) {
+ // Symbol not found in source code (or we would not be here).
+ // Search the module for the fref
+ try {
+ CompUnit local = findCUnitModule(code,cu,symbol);
+ if( local != null )
+ return local;
+ return findCUnitExternalSimple(code,symbol);
+ } catch( IOException ioe ) {
+ throw new RuntimeException(ioe);
+ }
+ }
+
+ // Find the symbol in module, starting from cunit. This searches up the nested
+ // class chain and sideways 1 step, so that any prefix of the class chain can
+ // be skipped as a shortcut. Symbol can have dotted names, which then go
+ // down into classes. e.g.
+ // A/X/Y exists.
+ // A/B contains X.Y, which is a shortcut for A.X.Y
+
+ // However if A/B contains "Y", the search would be for A.B.Y then A.Y both of
+ // which do not exist.
+ private static CompUnit findCUnitModule( CodeGen code, CompUnit cunit, String symbol ) throws IOException {
+ if( cunit == null ) // Ran off top of module scope?
+ return null; // Failed search
+
+ // Check self
+ if( cunit._name.equals(symbol) )
+ return cunit;
+
+ // If symbol is itself nested (has embedded dots), move the leading part
+ // over to the CompUnit path. So e.g.:
+ // CompUnit{A.B}, Symbol="X.Y"
+ // becomes
+ // CompUnit{A.B.X}, Symbol="Y"
+ int dotIdx = symbol.indexOf(".");
+ if( dotIdx != -1 ) {
+ String pre = symbol.substring(0,dotIdx);
+ String rest = symbol.substring(dotIdx+1 );
+ String fname = cunit._fname + "/" + pre;
+ File f = new File( code._modDir + "/" + fname + ".smp" );
+ if( f.exists() ) {
+ CompUnit sub = makeCUnit(code,cunit,pre);
+ // Note that "sub" itself is not put on the to-be-parsed-list. We're
+ // referring to a child class and do not need to parse the middles.
+ CompUnit dcunit = findCUnitModule(code,sub,rest);
+ if( dcunit != null )
+ return dcunit; // Nested dot search found cunit
+ }
+ }
+
+ // Look in mod / fname / name / symbol.smp
+ String fname = cunit._fname + "/" + symbol;
+ File f = new File( code._modDir + "/" + fname + ".smp" );
+ if( f.exists() ) {
+ CompUnit sub = makeCUnit(code,cunit,symbol);
+ // If not seen before or out of date, parse the new nested cunit
+ if( sub._clz == null && WORK.find(sub) == -1 )
+ enqueue(sub);
+ return sub;
+ }
+
+ // Look up the scope
+ return findCUnitModule( code, cunit._par, symbol );
+ }
+
+ // Search external Simple objects only. C linkage uses explicit "C"
+ // declarations, which produce ExternNodes and are resolved by the linker.
+ public static CompUnit findCUnitExternalSimple( CodeGen code, String symbol ) {
+ String clzName = symbol.startsWith(Parser.CLZ) ? symbol : Parser.addClzPrefix(symbol);
+ String cname = clzName.substring(Parser.CLZ.length());
+ String fname = cname.replace('.','/');
+
+ CompUnit cunit = code._compunits.get(fname);
+ if( cunit != null )
+ return cunit;
+
+ ElfReader elf = code.findExternalSimple(clzName);
+ if( elf == null )
+ return null;
+
+ try {
+ cunit = makeCUnit(code,null,fname,elf._file);
+ } catch( IOException ioe ) {
+ throw new RuntimeException(ioe);
+ }
+ ElfReader typedElf = code.getElf(cunit._obj);
+ if( typedElf != null ) typedElf.loadPublicTypes(code);
+ TypeStruct clz = (TypeStruct)Parser.TYPES.get(clzName);
+ if( clz == null )
+ Parser.TYPES.put(clzName,clz = TypeStruct.make(clzName,true));
+ cunit._clz = clz;
+ if( WORK.find(cunit) == -1 && NEEDS_LOAD.find(cunit) == -1 )
+ enqueue(cunit);
+ return cunit;
+ }
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/RIPRelSize.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/RIPRelSize.java
new file mode 100644
index 00000000..48319006
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/RIPRelSize.java
@@ -0,0 +1,14 @@
+package com.seaofnodes.simple.codegen;
+
+// This Node rip-relative encodings has sizes that vary by delta
+public interface RIPRelSize {
+ // delta is the distance from the opcode *start* to the target start. Each
+ // hardware target adjusts as needed. X86 measures from the opcode *end*
+ // and small jumps are 2 bytes, so they'll need to subtract 2.
+ byte encSize(int delta);
+
+ // Patch full encoding in. Used for both short and long forms. delta
+ // again is from the opcode *start*.
+ void patch( Encoding enc, int opStart, int opLen, int delta );
+
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/RegAlloc.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/RegAlloc.java
new file mode 100644
index 00000000..ac792988
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/RegAlloc.java
@@ -0,0 +1,780 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.util.Ary;
+import com.seaofnodes.simple.util.Utils;
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.type.Type;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.IdentityHashMap;
+
+/**
+ * "Briggs/Chaitin/Click".
+ * Graph coloring.
+ * <p>
+ * Every def and every use have a bit-set of allowed registers.
+ * Fully general to bizarre chips.
+ * Multiple outputs fully supported, e.g. add-with-carry or combined div/rem or "xor rax,rax" setting both rax and flags.
+ * <p>
+ * Intel 2-address accumulator-style ops fully supported.
+ * Op-to-stack-spill supported.
+ * All addressing modes supported.
+ * Register pairs could be supported with some grief.
+ * <p>
+ * Splitting instead of spilling (simpler implementation).
+ * Stack slots are "just another register" (tiny stack frames, simpler implementation).
+ * Stack/unstack during coloring with a marvelous trick (simpler implementation).
+ * <p>
+ * Both bitset and adjacency list formats for the interference graph; one of
+ * the few times it's faster to change data structures mid-flight rather than
+ * just wrap one of the two.
+ * <p>
+ * Liveness computation and interference graph built in the same one pass (one
+ * fewer passes per round of coloring).
+ * <p>
+ * Single-register def or use live ranges deny neighbors their required
+ * register and thus do not interfere, vs interfering and denying the color
+ * and coloring time. Basically all function calls do this, but many oddball
+ * registers also, e.g. older div/mod/mul ops. (5x smaller IFG, the only
+ * O(n^2) part of this operation).
+ */
+
+public class RegAlloc {
+ // Main Coloring Algorithm:
+ // Repeat until colored:
+ // Build Live Ranges (LRGs)
+ // - Intersect allowed registers
+ // If hard conflicts (LRGs with no allowed registers)
+ // - Pre-split conflicted LRGs, and repeat
+ // Build Interference Graph (IFG)
+ // - Self conflicts split now
+ // Color (remove trivial first then any until empty; reverse assign colors
+ // - If color fails:
+ // - - Split uncolorable LRGs
+
+ // Top-level program graph structure
+ final CodeGen _code;
+
+ public int _spills, _spillScaled;
+
+ private boolean _done;
+
+ // -----------------------
+ // Live ranges with self-conflicts or no allowed registers
+ private final IdentityHashMap<LRG,String> _failed = new IdentityHashMap<>();
+ // Loop Phis get one chance to color after only cold self-conflict splits.
+ // If the same Phi self-conflicts again, split its hot def/backedge also.
+ private final IdentityHashMap<PhiNode,String> _deferredLoopSelf = new IdentityHashMap<>();
+ void fail( LRG lrg ) {
+ assert lrg.leader();
+ _failed.put(lrg,"");
+ }
+ boolean success() { return _failed.isEmpty(); }
+ public boolean done() { return _done; }
+
+
+ // -----------------------
+ // Map from Nodes to Live Ranges
+ private final IdentityHashMap<Node,LRG> _lrgs = new IdentityHashMap<>();
+ short _lrg_num;
+
+ // Define a new LRG, and assign n
+ LRG newLRG( Node n, FunNode fun ) {
+ LRG lrg = lrg(n);
+ if( lrg!=null ) return lrg;
+ lrg = new LRG(_lrg_num++, fun);
+ LRG old = _lrgs.put(n,lrg); assert old==null;
+ return lrg;
+ }
+
+ // LRG for n
+ public LRG lrg( Node n ) {
+ LRG lrg = _lrgs.get(n);
+ if( lrg==null ) return null;
+ LRG lrg2 = lrg.find();
+ if( lrg != lrg2 )
+ _lrgs.put(n,lrg2);
+ return lrg2;
+ }
+
+ // Find LRG for n.in(idx), and also map n to it
+ LRG lrg2( Node n, int idx ) {
+ LRG lrg = lrg(n.in(idx));
+ return union(lrg,n);
+ }
+
+ // Union any lrg for n with lrg and map to the union
+ LRG union( LRG lrg, Node n ) {
+ LRG lrgn = _lrgs.get(n);
+ LRG lrg3 = lrg.union(lrgn);
+ _lrgs.put(n,lrg3);
+ return lrg3;
+ }
+
+ // Force all unified to roll up; collect live ranges
+ LRG[] _LRGS;
+ void unify() {
+ Ary<LRG> lrgs = new Ary<>(LRG.class);
+ for( Node n : _lrgs.keySet() ) {
+ LRG lrg = lrg(n);
+ lrgs.setX(lrg._lrg,lrg);
+ }
+ _LRGS = lrgs.asAry();
+ // Remove unified lrgs from failed set also
+ for( LRG lrg : _failed.keySet().toArray(new LRG[0]) ) {
+ if( !lrg.leader() ) {
+ LRG lrg2 = lrg.find();
+ _failed.remove(lrg);
+ _failed.put(lrg2,"");
+ }
+ }
+ }
+
+
+ // Diagnostic lookup: do not compress leaders or rewrite the node mapping.
+ private LRG _lrg(Node n) {
+ LRG lrg = _lrgs.get(n);
+ if( lrg != null )
+ while( lrg._leader != null ) lrg = lrg._leader;
+ return lrg;
+ }
+
+ public short regnum( Node n ) {
+ LRG lrg = _lrg(n);
+ return lrg==null ? -1 : lrg._reg;
+ }
+
+ // Printable register number for node n
+ String reg( Node n ) {
+ LRG lrg = _lrg(n);
+ return lrg==null ? null : reg(lrg);
+ }
+ String reg( LRG lrg ) {
+ // No register yet, use LRG
+ if( lrg._reg == -1 ) return "V"+lrg._lrg;
+ // Chosen machine register unless stack-slot and past RA
+ String[] regs = _code._mach.regs();
+ if( lrg._reg < regs.length || _code._phase.ordinal() <= CodeGen.Phase.RegAlloc.ordinal() || lrg._fun==null )
+ return RegMask.reg(regs,lrg._reg);
+ // Stack-slot past RA uses the frame layout logic
+ return "[rsp+"+lrg._fun.computeStackOffset(_code,lrg._reg)+"]";
+ }
+
+ // -----------------------
+ RegAlloc( CodeGen code ) { _code = code; }
+
+ public void regAlloc() {
+ // Insert callee-save registers
+ String[] regs = _code._mach.regs();
+ long neverSave= _code._mach.neverSave();
+ for( CFGNode bb : _code._cfg )
+ if( bb instanceof FunNode fun ) {
+ ReturnNode ret = fun.ret();
+ int len = Math.min(regs.length,64);
+ for( int reg=0; reg<len; reg++ )
+ if( !_code._callerSave.test(reg) && ((1L<<reg)&neverSave)==0 ) {
+ ret.addDef(new CalleeSaveNode(fun,reg,regs[reg]));
+ assert ret.regmap(ret.nIns()-1).firstReg()==reg;
+ }
+ }
+ // Cache reg masks for New and Call
+ for( CFGNode bb : _code._cfg ) {
+ if( bb instanceof CallEndMach cend ) cend.cacheRegs(_code);
+ for( Node n : bb._outputs )
+ if( n instanceof NewNode nnn ) nnn.cacheRegs(_code);
+ }
+ // Expensive assert: useful when chasing cross-function allocator edges.
+ //assert verifyFunctionLocalEdges();
+
+ // Top driver: repeated rounds of coloring and splitting.
+ byte round=0;
+ while( !graphColor(round) ) {
+ split(round);
+ if( round >= 7 ) // Really expect to be done soon
+ throw new IllegalStateException("Register allocation made no progress after eight rounds");
+ round++;
+ }
+ postColor(); // Remove no-op spills
+ _done = true;
+ }
+
+ boolean verifyFunctionLocalEdges() {
+ for( CFGNode bb : _code._cfg )
+ for( Node use : bb._outputs ) {
+ FunNode useFun = funOf(use);
+ for( int i=1; i<use.nIns(); i++ ) {
+ Node def = use.in(i);
+ if( def==null || def.isConst() || allowedCrossFunctionEdge(use,i,def) )
+ continue;
+ FunNode defFun = funOf(def);
+ if( defFun != null && useFun != null && defFun != useFun )
+ throw new AssertionError(crossFunctionMessage(def,defFun,use,useFun,i));
+ }
+ }
+ return true;
+ }
+
+ private static String crossFunctionMessage(Node def, FunNode defFun, Node use, FunNode useFun, int idx) {
+ StringBuilder sb = new StringBuilder("Cross-function value edge: ");
+ sb.append(def.uniqueName()).append("(").append(defFun.label()).append(") -> ")
+ .append(use.uniqueName()).append("(").append(useFun.label()).append(") input ").append(idx)
+ .append(" def=").append(def).append(" use=").append(use).append("\n all uses:");
+ for( Node out : def.outs() ) {
+ FunNode outFun = funOf(out);
+ sb.append("\n ").append(out.uniqueName()).append("(")
+ .append(outFun==null ? "?" : outFun.label()).append(")");
+ }
+ return sb.toString();
+ }
+
+ private static boolean allowedCrossFunctionEdge(Node use, int idx, Node def) {
+ // Function metadata/control edges are not value live ranges.
+ if( use instanceof FunNode || use instanceof StopNode || use instanceof StopCUNode )
+ return true;
+ // Return-linked function pointers carry function identity/lifetime, not
+ // an allocator value flowing from the callee body into this function.
+ if( use instanceof FunPtrNode && def instanceof ReturnNode )
+ return true;
+ // Parm defaults and call-linked argument edges are the entry protocol;
+ // BuildLRG also skips Parm input 1.
+ if( use instanceof ParmNode )
+ return true;
+ // Linked CallEnd -> Return edges should be gone before RegAlloc, but
+ // diagnose allocator value edges rather than call graph metadata here.
+ if( use instanceof CallEndNode && def instanceof ReturnNode )
+ return true;
+ return false;
+ }
+
+ private static FunNode funOf(Node n) {
+ if( n instanceof ParmNode parm ) return parm.fun();
+ if( n instanceof ReturnNode ret ) return ret.fun();
+ if( n instanceof FunNode fun ) return fun;
+ if( n instanceof CFGNode cfg ) return cfg.fun();
+ CFGNode cfg = n.cfg0();
+ return cfg==null ? null : cfg.fun();
+ }
+
+ private boolean graphColor(byte round) {
+ _failed.clear();
+ _lrgs.clear();
+ _lrg_num = 1;
+ _LRGS=null;
+
+ return
+ // Build Live Ranges
+ BuildLRG.run(round,this) && // if no hard register conflicts
+ // Build Interference Graph
+ IFG.build(round,this) && // If no self conflicts or uncolorable
+ // Conservative coalesce copies
+ Coalesce.coalesce(round,this) &&
+ // Color attempt
+ IFG.color(round,this); // If colorable
+ }
+
+ // -----------------------
+ // Split conflicted live ranges.
+ void split(byte round) {
+
+ // In C2, all splits are handling in one pass over the program. Here,
+ // in the name of clarity, we'll handle each failing live range
+ // independently... which generally requires a full pass over the
+ // program for each failing live range. i.e., might be a lot of
+ // passes.
+
+ // Sort, to avoid non-deterministic HashMap ordering
+ LRG[] splits = _failed.keySet().toArray(new LRG[0]);
+ Arrays.sort(splits, (x,y) -> x._lrg - y._lrg );
+ for( LRG lrg : splits )
+ split(round,lrg);
+ }
+
+ // Split this live range, top level heuristic
+ boolean split( byte round, LRG lrg ) {
+ assert lrg.leader(); // Already rolled up
+
+ if( lrg._selfConflicts != null )
+ return splitSelfConflict(round,lrg);
+
+ // Register mask when empty; split around defs and uses with limited
+ // register masks.
+ if( lrg._mask.isEmpty() ) {
+ if( lrg._1regDefCnt <= 1 &&
+ lrg._1regUseCnt <= 1 &&
+ (lrg._1regDefCnt + lrg._1regUseCnt) > 0 &&
+ // A fixed-register clone with flexible uses must split at the uses.
+ !(lrg._1regDefCnt==1 && lrg._machDef.isClone() && lrg._1regUseCnt==0) )
+ return splitEmptyMaskSimple(round,lrg);
+ // Repeated single-reg uses from a single def. Special for archs
+ // with more fixed regs.
+ if( !lrg._multiDef && lrg._1regDefCnt <= 1 && lrg._1regUseCnt > 2 )
+ if( splitEmptyMaskByUse(round,lrg) )
+ return true;
+ }
+ if( lrg._mask.isEmpty() && lrg._multiDef && lrg._1regUseCnt > 1 && lrg._1regDefCnt <= 1 ) {
+ if( splitEmptyMaskSingleUses(round,lrg) )
+ return true;
+ }
+
+ // Generic split-by-loop depth.
+ return splitByLoop(round,lrg);
+ }
+
+ // Split live range with an empty mask. Specifically forces splits at
+ // single-register defs or uses and not elsewhere.
+ boolean splitEmptyMaskSimple( byte round, LRG lrg ) {
+ // Live range has a single-def single-register, and/or a single-use
+ // single-register. Split after the def and before the use. Does not
+ // require a full pass.
+
+ // Compute def nOuts *before* replacing those uses with a split
+ int defNouts = ((Node)lrg._machDef).nOuts();
+ // Split just after def
+ if( lrg._1regDefCnt==1 && !lrg._machDef.isClone() )
+ // Force must-split, even if a prior split same block because register
+ // conflicts. Example:
+ // alloc
+ // V1/rax - forced by alloc
+ // alloc
+ // V2/rax - kills prior RAX
+ // st4 [V1],len - No good, must split around
+ insertAfterAndReplace( makeSplit("def/empty1",round,lrg), (Node)lrg._machDef, false/*true*/);
+ // Split just before use
+ if( lrg._machUse!=null && (lrg._1regUseCnt==1 || (lrg._1regDefCnt==1 && defNouts==1)) )
+ insertBefore((Node)lrg._machUse,lrg._uidx,"use/empty1",round,lrg);
+ return true;
+ }
+
+ // Popular single-def value: share a split among uses with compatible masks.
+ boolean splitEmptyMaskByUse( byte round, LRG lrg ) {
+ Node def = (Node)lrg._machDef;
+ // Snapshot distinct users: rewriting an input changes def's output list,
+ // and a call can use the same value in more than one argument.
+ Ary<Node> uses = new Ary<>(Node.class);
+ for( Node use : def._outputs )
+ if( uses.find(use)==-1 ) uses.push(use);
+
+ Ary<RegMask> rclass = new Ary<>(RegMask.class);
+ int ncalls=0;
+ for( Node use : uses )
+ if( use instanceof MachNode mach ) {
+ // Sharing copies across several calls usually loses to their kills.
+ if( use instanceof CallNode && ++ncalls > 1 ) return false;
+ for( int i=1; i<use.nIns(); i++ )
+ if( use.in(i)==def && mach.regmap(i)!=null )
+ putIntoRegClass(rclass,mach.regmap(i));
+ }
+ if( rclass._len <= 1 ) return false;
+
+ for( RegMask rmask : rclass ) {
+ Node split = makeSplit(def,"popular",round,lrg,rmask);
+ split.insertAfter(def);
+ if( split.nIns()>1 ) split.setDef(1,def);
+ for( Node use : uses )
+ if( use instanceof MachNode mach )
+ for( int i=1; i<use.nIns(); i++ ) {
+ RegMask mask = mach.regmap(i);
+ if( use.in(i)==def && mask!=null && mask.overlap(rmask) ) {
+ assert rmask.and(mask)==rmask;
+ use.setDefOrdered(i,split);
+ }
+ }
+ }
+ return true;
+ }
+
+ // Empty-mask live range with several direct single-register uses, often
+ // through a Phi with multiple defs. Split those uses directly instead of
+ // doing a loop-depth split, which might keep isolating the wrong edge.
+ boolean splitEmptyMaskSingleUses( byte round, LRG lrg ) {
+ findAllLRG(lrg);
+ if( _ns._len <= 5 )
+ return false;
+ boolean progress = false;
+ for( Node n : _ns ) {
+ if( !(n instanceof MachNode mach) ) continue;
+ for( int i=1; i<n.nIns(); i++ ) {
+ RegMask use = mach.regmap(i);
+ if( use != null && use.size1() && lrg(n.in(i))==lrg ) {
+ insertBefore(n,i,"use/emptyN",round,lrg);
+ progress = true;
+ }
+ }
+ }
+ return progress;
+ }
+
+ // Narrowing preserves all earlier users of a class; new classes are disjoint.
+ // Each use contains its first overlapping class, so one pass suffices.
+ private static void putIntoRegClass( Ary<RegMask> rclass, RegMask rmask ) {
+ for( int i=0; i<rclass._len; i++ ) {
+ RegMask omask = rclass.at(i);
+ if( !omask.overlap(rmask) ) continue;
+ if( omask.and(rmask)!=omask )
+ rclass.set(i,new RegMask(omask.copy().and(rmask)));
+ return;
+ }
+ rclass.push(rmask);
+ }
+
+
+ // Self conflicts require Phis (or two-address).
+ // Insert a split after every def.
+ boolean splitSelfConflict( byte round, LRG lrg ) {
+ // Sort conflict set, so we're deterministic
+ Node[] conflicts = lrg._selfConflicts.keySet().toArray(new Node[0]);
+ Arrays.sort(conflicts, (x,y) -> x._nid - y._nid );
+
+ // First encounter with a loop Phi: try only cold splits this round.
+ // Remember the Phi across rounds so a persistent self-conflict forces
+ // the hot def/backedge splits on the next encounter.
+ IdentityHashMap<PhiNode,String> deferred = new IdentityHashMap<>();
+ for( Node def : conflicts ) {
+ if( def instanceof PhiNode phi && phi.region() instanceof LoopNode )
+ deferLoopSelf(phi,deferred);
+ for( Node use : def._outputs )
+ if( use instanceof PhiNode phi && phi.region() instanceof LoopNode &&
+ phi._inputs.find(def)==2 )
+ deferLoopSelf(phi,deferred);
+ }
+
+ // For all conflicts
+ for( Node def : conflicts ) {
+ assert lrg(def)==lrg; // Might be conflict use-side
+ // Split before each use that extends the live range; i.e. is a
+ // Phi or two-address
+ for( int i=0; i<def._outputs._len; i++ ) {
+ Node use = def.out(i);
+ if( (use instanceof PhiNode phi &&
+ !(deferred.containsKey(phi) && phi.region() instanceof LoopNode && phi._inputs.find(def)==2) &&
+ !(phi.region() instanceof LoopNode loop && phi.in(2)==def && def.cfg0().idepth() > loop.idepth() ) ) ||
+ (use instanceof MachNode mach && mach.twoAddress()!=0 && use.in(mach.twoAddress())==def) )
+ insertBefore( use, use._inputs.find(def), "use/self/use",round,lrg );
+ }
+ // Split after the Phi and before its entry. Also isolate inputs
+ // shared with another Phi: otherwise the shared input keeps the
+ // conflicting live ranges joined, even after splitting the entry.
+ if( def instanceof PhiNode phi && !(def instanceof ParmNode) ) {
+ if( !deferred.containsKey(phi) ) {
+ SplitNode split = makeSplit("def/self",round,lrg);
+ insertAfterAndReplace(split,def,false);
+ if( split.nOuts()==0 )
+ split.killOrdered();
+ }
+ for( int i=1; i<(deferred.containsKey(phi) ? 2 : phi.nIns()); i++ )
+ if( i==1 || sharedPhiInput(phi,i) )
+ insertBefore(phi,i,"use/self/phi",round,lrg);
+ }
+ // Split before two-address ops which extend the live range
+ if( def instanceof MachNode mach && mach.twoAddress()!= 0 )
+ insertBefore(def,mach.twoAddress(),"use/self/two",round,lrg);
+ }
+ return true;
+ }
+
+ private void deferLoopSelf(PhiNode phi, IdentityHashMap<PhiNode,String> deferred) {
+ if( !_deferredLoopSelf.containsKey(phi) ) {
+ _deferredLoopSelf.put(phi,"");
+ deferred.put(phi,"");
+ }
+ }
+
+
+ private static boolean sharedPhiInput(PhiNode phi, int i) {
+ for( Node use : phi.in(i)._outputs )
+ if( use instanceof PhiNode && use!=phi ) return true;
+ return false;
+ }
+
+ // Generic: split around the outermost loop with non-split def/uses. This
+ // covers both self-conflicts (once we split deep enough) and register
+ // pressure issues.
+ boolean splitByLoop( byte round, LRG lrg ) {
+ findAllLRG(lrg);
+
+ // Find min loop depth for all non-split defs and uses.
+ long ld = (-1L<<32) | 9999;
+ for( Node n : _ns ) {
+ if( lrg(n)==lrg ) // This is a LRG def
+ ld = ldepth(ld,n,n.cfg0());
+ // PhiNodes check all CFG inputs
+ if( n instanceof PhiNode phi ) {
+ for( int i=1; i<n.nIns(); i++ )
+ if( phi.in(i)._type!=Type.TOP )
+ ld = ldepth(ld, phi.in(i), phi.region().cfg(i));
+ } else {
+ // Others check uses
+ for( int i=1; i<n.nIns(); i++ )
+ if( lrgSame(n.in(i),lrg) ) // This is a LRG use
+ ld = ldepth(ld,n,n.cfg0());
+ }
+ }
+ int min = (int)ld;
+ int max = (int)(ld>>32);
+
+ boolean emptyMulti = lrg._mask.isEmpty() && lrg._multiDef && lrg._1regDefCnt <= 1 &&
+ (lrg._1regUseCnt > 1 || _ns._len > 5);
+ // Multi-def empty-mask conflicts need the hot/deep side isolated from
+ // fixed register uses. Ordinary pressure splits keep the old shallow
+ // boundary behavior.
+ for( Node n : _ns ) {
+ if( n instanceof SplitNode split && !split.isNonStackSplit() && min!=max ) continue; // Ordinary splits already reach stack storage.
+ if( n.isDead() ) continue; // Some Cloneable went dead by other spill changes
+ // If this is a 2-address commutable op (e.g. AddX86, MulX86) and the rhs has only a single user,
+ // commute the inputs... which chops the LHS live ranges' upper bound to just the RHS.
+ if( n instanceof MachNode mach && lrg(n)==lrg && mach.twoAddress()==1 && mach.commutes() && n.in(2).nOuts()==1 )
+ n.swap12();
+
+ if( lrg(n)==lrg && // This is a LRG def
+ // At loop boundary, or splitting in inner loop
+ (min==max || (emptyMulti ? n.cfg0().loopDepth() >= max : n.cfg0().loopDepth() <= min)) ) {
+ // Cloneable constants will be cloned at uses, not after def
+ if( !(n instanceof MachNode mach && mach.isClone()) &&
+ // Single user is already a split adjacent
+ !(n.nOuts()==1 && n.out(0) instanceof SplitNode split && sameBlockNoClobber(split) ) )
+ // Split after def in min loop nest
+ insertAfterAndReplace( makeSplit("def/loop",round,lrg), n,true);
+ }
+
+ // PhiNodes check all CFG inputs
+ if( n instanceof PhiNode phi && !(n instanceof ParmNode)) {
+ for( int i=1; i<n.nIns(); i++ )
+ // No split in front of a split
+ if( !(n.in(i) instanceof SplitNode) &&
+ // splitting in inner loop or at loop border
+ (min==max || (emptyMulti ? phi.region().cfg(i).loopDepth() >= max : phi.region().cfg(i).loopDepth() <= min)) &&
+ // and not around the backedge of a loop (bad place to force a split, hard to remove)
+ !(phi.region() instanceof LoopNode && i==2 && (phi.in(i) instanceof PhiNode pp && pp.region()==phi.region())) )
+ // Split before phi-use in prior block
+ insertBefore(phi,i, "use/loop/phi",round,lrg);
+
+ } else {
+ // Others check uses
+ for( int i=1; i<n.nIns(); i++ ) {
+ // This is a LRG use
+ // splitting in inner loop or at loop border
+ if( lrgSame( n.in( i ), lrg ) &&
+ (min == max || (n.in(i) instanceof MachNode mach && mach.isClone()) ||
+ (emptyMulti ? n.cfg0().loopDepth() >= max : n.cfg0().loopDepth() <= min)) )
+ // Split before in this block
+ insertBefore( n, i, "use/loop/use", round,lrg, false );
+ }
+ }
+ }
+ return true;
+ }
+
+ private boolean lrgSame(Node x, LRG lrg) {
+ LRG xlrg = lrg(x);
+ while( xlrg==null && x instanceof SplitNode )
+ xlrg=lrg(x = x.in(1));
+ return xlrg==lrg;
+ }
+
+ private static long ldepth( long ld, Node n, CFGNode cfg ) {
+ // Ordinary splits already reach stack storage; register-only transfers do not.
+ if( n instanceof SplitNode split && !split.isNonStackSplit() ) return ld;
+ // Collect min/max loop depth
+ int min = (int)ld;
+ int max = (int)(ld>>32);
+ int d = cfg.loopDepth();
+ // if n will lower the min loop and is in the tail end of the loop
+ // header, splitting "around" the loop will not help. Treat n as being
+ // in the loop.
+ if( d < min ) {
+ if( cfg.uctrl() instanceof LoopNode loop && loop.entry()==cfg ) {
+ for( int i=cfg.nOuts()-2; i>=0; i-- ) {
+ Node out = cfg.out(i);
+ if( n==out )
+ { d = loop.loopDepth(); break; } // Treat n as being "in the loop"
+ if( !((out instanceof MachNode mach && mach.isClone()) || out instanceof SplitNode ) )
+ break; // Treat b as "normal", out of loop
+ }
+ }
+ }
+
+ // lower min, raise max, and re-fold
+ min = Math.min(min,d);
+ max = Math.max(max,d);
+ return ((long)max<<32) | min;
+ }
+
+ // Find all members of a live range, both defs and uses
+ private final Ary<Node> _ns = new Ary<>(Node.class);
+ void findAllLRG( LRG lrg ) {
+ _ns.clear();
+ int wd = 0;
+ if( lrg._machDef != null ) _ns.push((Node)lrg._machDef);
+ if( lrg._machUse != null ) _ns.push((Node)lrg._machUse);
+ while( wd < _ns._len ) {
+ Node n = _ns.at(wd++);
+ if( lrg(n)!=lrg ) continue;
+ for( Node def : n._inputs )
+ if( lrg(def)==lrg && _ns.find(def)== -1 )
+ _ns.push(def);
+ for( Node use : n._outputs )
+ if( _ns.find(use)== -1 )
+ _ns.push(use);
+ }
+ for( Node n : _ns ) assert !n.isDead();
+ }
+
+ void insertBefore(Node n, int i, String kind, byte round, LRG lrg, boolean skip) {
+ Node def = n.in(i);
+ // Phi splitting may visit the synthetic never-exit arm.
+ if( def._type==Type.TOP ) return;
+ // Effective block for use
+ CFGNode cfg = n instanceof PhiNode phi ? phi.region().cfg(i) : n.cfg0();
+ // Use-side RegMask, if available
+ RegMask umask = n instanceof MachNode mach ? mach.regmap(i) : null;
+ // Def is a split ?
+ if( skip && def instanceof SplitNode ) {
+ boolean singleReg = umask!=null && umask.size1();
+ // Same block, multiple registers, split is only used by n,
+ // assume this is good enough and do not split again.
+ if( cfg==def.cfg0() && def.nOuts()==1 && !singleReg )
+ return;
+ }
+ makeSplit(def,kind,round,lrg,umask).insertBefore(n, i);
+ // Skip split-of-split same block
+ if( skip && def instanceof SplitNode && cfg==def.cfg0() )
+ n.in(i).setDefOrdered(1,def.in(1));
+ }
+ void insertBefore(Node n, int i, String kind, byte round, LRG lrg) {
+ insertBefore(n,i,kind,round,lrg,true);
+ }
+
+ // Replace uses of `def` with `split`, and insert `split` immediately after
+ // `def` in the basic block.
+ public void insertAfterAndReplace( Node split, Node def, boolean must ) {
+ split.insertAfter(def);
+ if( split.nIns()>1 ) split.setDef(1,def);
+ for( int j=def.nOuts()-1; j>=0; j-- ) {
+ Node use = def.out(j);
+ if( use==split ) continue; // Skip self
+ // Can we avoid a split of a split? 'this' split is used by
+ // another split in the same block.
+ if( !must && use instanceof SplitNode split2 && sameBlockNoClobber(split2) )
+ continue;
+ int idx = use._inputs.find(def);
+ use.setDefOrdered(idx,split);
+ if( j < def.nOuts() ) j++;
+ }
+ }
+
+ private Node makeSplit( Node def, String kind, byte round, LRG lrg, RegMask umask ) {
+ // Clone simple constants if possible
+ Node split = def instanceof MachNode mach && mach.isClone() && (umask==null || mach.outregmap().overlap(umask))
+ ? mach.copy()
+ : _code._mach.split(lrg,kind,round);
+ _lrgs.put(split,lrg);
+ return split;
+ }
+ private SplitNode makeSplit( String kind, byte round, LRG lrg ) {
+ SplitNode split = _code._mach.split(lrg,kind,round);
+ _lrgs.put(split,lrg);
+ return split;
+ }
+
+
+ // -----------------------
+ // POST PASS: Remove empty spills that biased-coloring made
+ private void postColor() {
+ int maxReg = -1;
+ for( CFGNode bb : _code._cfg ) { // For all ops
+ if( bb instanceof FunNode )
+ maxReg = -1; // Reset for new function
+ // Compute frame size, based on arguments and largest reg seen
+ if( bb instanceof ReturnNode ret )
+ ret.fun().computeFrameAdjust(_code,maxReg);
+ // Raise frame size by max stack args passed, even if ignored
+ if( bb instanceof CallEndMach cend )
+ maxReg = Math.max(maxReg,cend._xslot);
+
+ for( int j=0; j<bb.nOuts(); j++ ) {
+ Node n = bb.out(j);
+ if( lrg(n)!=null )
+ maxReg = Math.max(maxReg,lrg(n)._reg+1);
+ // Raise frame size by max stack args passed to New
+ if( n instanceof NewNode nnn )
+ maxReg = Math.max(maxReg,nnn._xslot);
+
+ if( !(n instanceof SplitNode ) ) continue;
+ int defreg = lrg(n )._reg;
+ int usereg = lrg(n.in(1))._reg;
+ // Attempt to bypass split
+ if( defreg != usereg && splitBypass(bb,j,n,defreg) )
+ usereg = lrg(n.in(1))._reg;
+
+ // Same register, or the next instruction can use the source directly.
+ if( defreg == usereg || splitForward(bb,j,n,usereg) ) {
+ n.removeSplit();
+ j--;
+ continue;
+ }
+ // Split is kept; count against split score
+ _spills++;
+ _spillScaled += (1<<bb.loopDepth()*3);
+ assert _spillScaled >= 0;
+ }
+ }
+ }
+
+ // A sole, adjacent use can read the source register directly. Nothing can
+ // clobber it in between; only the operand mask and two-address tie remain.
+ private boolean splitForward( CFGNode bb, int j, Node split, int reg ) {
+ if( split.nOuts()!=1 || j+1==bb.nOuts() ) return false;
+ Node use = split.out(0);
+ if( use!=bb.out(j+1) || use instanceof PhiNode || use instanceof CFGNode ||
+ !(use instanceof MachNode mach) ) return false;
+ int i = use._inputs.find(split);
+ RegMask mask = mach.regmap(i);
+ return mask!=null && mask.test(reg) && mach.twoAddress()!=i;
+ }
+
+ private boolean splitBypass( CFGNode bb, int j, Node lo, int defreg ) {
+ // Attempt to bypass split
+ Node hi = lo.in(1);
+ while( true ) {
+ if( !(hi instanceof SplitNode && lo.cfg0() == hi.cfg0()) )
+ return false;
+ if( lrg(hi.in(1))._reg==defreg )
+ break;
+ hi = hi.in(1);
+ }
+ // Walk backward from the later copy, including instructions without results.
+ for( int idx = j-1; bb.out(idx) != hi; idx--) {
+ Node n = bb.out(idx);
+ if( clobbers(n,defreg) )
+ return false; // Clobbered
+ }
+ lo.setDefOrdered(1,hi.in(1));
+ return true;
+ }
+
+
+ private boolean clobbers(Node n, int reg) {
+ LRG lrg = lrg(n);
+ return lrg!=null && (lrg._reg==reg || lrg._reg==-1 && lrg._mask.size1() && lrg._mask.firstReg()==reg) ||
+ n instanceof MachNode mach && mach.killmap()!=null && mach.killmap().test(reg);
+ }
+
+ public boolean sameBlockNoClobber( SplitNode split ) {
+ Node def = split.in(1);
+ CFGNode cfg = def.cfg0();
+ if( cfg != split.cfg0() ) return false; // Not same block
+ // Get multinode head
+ Node def0 = def instanceof ProjNode ? def.in(0) : def;
+ int defreg = lrg(def)._reg;
+ if( defreg == -1 ) defreg = lrg(def)._mask.firstReg();
+ if( defreg == -1 ) return false; // no allowed registers -> clobbered
+ for( int idx = cfg._outputs.find(split) -1; idx >= 0; idx-- ) {
+ Node n = cfg.out(idx);
+ if( n==def0 ) return true; // No clobbers
+ if( lrg(n) == lrg(def) ) return false; // Self conflict
+ if( clobbers(n,defreg) )
+ return false; // Clobbered
+ }
+ throw Utils.TODO("Should not reach here");
+ }
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/RegMask.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/RegMask.java
new file mode 100644
index 00000000..9cb9ec16
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/RegMask.java
@@ -0,0 +1,121 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.util.SB;
+import com.seaofnodes.simple.util.Utils;
+
+/** RegMask
+ * A "register mask" - 1 bit set for each allowed register. In addition,
+ * "stack slot" registers may be allowed, effectively making the set infinite.
+ * <p>
+ * For smaller and simpler machines it suffices to make such masks an i64 or
+ * i128 (64- or 128-bit integers), and this presentation is by far the better
+ * way to go... if all register allocations can fit in this bit limitation.
+ * The allocator will need bits for stack-based parameters and for splits
+ * which cannot get a register. For a 32-register machine like the X86, add 1
+ * for flags - gives 33 registers. Using a Java `long` has 64 bits, leaving
+ * 31 for spills and stack passing. This is adequate for nearly all
+ * allocations; only the largest allocations will run this out. However, if
+ * we move to a chip with 64 registers we'll immediately run out, and need at
+ * least a 128 bit mask. Since you cannot *return* a 128 bit value directly
+ * in Java, Simple will pick up a `RegMask` class object.
+*/
+public class RegMask {
+
+ long _bits0, _bits1;
+
+ private static final RegMask EMPTY = new RegMask(0L);
+ public static final RegMask FULL = new RegMask(-1L);
+
+ public RegMask(int bit) {
+ if( bit < 64 ) _bits0 = 1L<<bit;
+ else _bits1 = 1L<<(bit-64);
+ }
+ public RegMask(long bits ) { _bits0 = bits; }
+ public RegMask(long bits0, long bits1 ) { _bits0 = bits0; _bits1 = bits1; }
+ public RegMask(RegMask r) { _bits0 = r._bits0; _bits1 = r._bits1; }
+
+ // AND, with copy-on-write if changing
+ RegMask and( RegMask mask ) {
+ if( mask==null ) return this;
+ long bits0 = _bits0 & mask._bits0;
+ long bits1 = _bits1 & mask._bits1;
+ if( bits0==_bits0 && bits1==_bits1 ) return this;
+ if( bits0==mask._bits0 && bits1==mask._bits1 ) return mask;
+ if( bits0==0 && bits1==0 ) return EMPTY;
+ // Update-in-place a mutable mask, or make a defensive copy
+ return null;
+ }
+ RegMask sub( RegMask mask ) {
+ if( mask==null ) return this;
+ long bits0 = _bits0 & ~mask._bits0;
+ long bits1 = _bits1 & ~mask._bits1;
+ if( bits0==_bits0 && bits1==_bits1 ) return this;
+ if( bits0==mask._bits0 && bits1==mask._bits1 ) return mask;
+ if( bits0==0 && bits1==0 ) return EMPTY;
+ // Update-in-place a mutable mask, or make a defensive copy
+ return null;
+ }
+
+ // Fails if bit is set, because this is immutable
+ public boolean clr( int reg ) {
+ if( reg < 64 ) return ((_bits0 >> reg)&1)==0;
+ if( reg < 128 ) return ((_bits1 >> reg)&1)==0;
+ return true;
+ }
+
+
+ public short firstReg() {
+ if( isEmpty() ) return -1;
+ return (short)(_bits0 != 0
+ ? Long.numberOfTrailingZeros(_bits0)
+ : Long.numberOfTrailingZeros(_bits1)+64);
+ }
+ public short nextReg(short reg) {
+ if( reg < 63 ) {
+ long bits0 = _bits0 & (-1L << (reg+1));
+ if( bits0!=0 ) return (short)Long.numberOfTrailingZeros(bits0);
+ }
+ long bits1 = reg < 64 ? _bits1 : reg < 127 ? _bits1 & (-1L << (reg-63)) : 0;
+ return bits1==0 ? -1 : (short)(64+Long.numberOfTrailingZeros(bits1));
+ }
+
+ boolean isEmpty() { return _bits0==0 && _bits1==0; }
+
+ public boolean test( int reg ) {
+ return ((reg<64 ? (_bits0 >> reg) : (_bits1 >> (reg-64))) & 1) != 0;
+ }
+
+ // checks if the 2 masks have at least 1 bit in common
+ public boolean overlap( RegMask mask ) { return (_bits0 & mask._bits0)!=0 || (_bits1 & mask._bits1)!=0; }
+
+ // Defensive writable copy
+ public RegMaskRW copy() { return new RegMaskRW( _bits0, _bits1 ); }
+
+ // Has exactly 1 bit set
+ public boolean size1() {
+ return !isEmpty() && (((_bits0 & -_bits0)==_bits0 && _bits1==0) ||
+ ((_bits1 & -_bits1)==_bits1 && _bits0==0));
+ }
+
+ // Cardinality
+ public short size() { return (short)(Long.bitCount(_bits0)+Long.bitCount(_bits1)); }
+
+ @Override public String toString() { return toString(new SB()).toString(); }
+ public SB toString(SB sb) {
+ Machine mach = CodeGen.CODE._mach;
+ if( _bits0==0 && _bits1==0 ) return sb.p("[]");
+ sb.p("[");
+ String[] regs = mach.regs();
+ for( int i=0; i<64; i++ )
+ if( ((_bits0 >> i)&1) != 0 )
+ sb.p(reg(regs,i)).p(",");
+ for( int i=0; i<64; i++ )
+ if( ((_bits1 >> i)&1) != 0 )
+ sb.p(reg(regs,i+64)).p(",");
+ return sb.unchar().p("]");
+ }
+
+ public static String reg(String[] regs, int i) {
+ return i<regs.length ? regs[i] : "[stk#"+i+"]";
+ }
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/RegMaskRW.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/RegMaskRW.java
new file mode 100644
index 00000000..1da9a3b1
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/RegMaskRW.java
@@ -0,0 +1,36 @@
+package com.seaofnodes.simple.codegen;
+
+// Mutable regmask(writable)
+public class RegMaskRW extends RegMask {
+ public RegMaskRW(long x, long y) { super(x,y); }
+ // clears bit at position r. Returns true if the mask is still not empty.
+ public boolean clr(int r) {
+ if( r < 64 ) _bits0 &= ~(1L<<(r ));
+ else _bits1 &= ~(1L<<(r-64));
+ return _bits0!=0 || _bits1!=0;
+ }
+ // sets bit at position r.
+ public void set(int r) {
+ if( r < 64 ) _bits0 |= (1L<<(r ));
+ else _bits1 |= (1L<<(r-64));
+ }
+ @Override RegMaskRW and( RegMask mask ) {
+ if( mask==null ) return this;
+ _bits0 &= mask._bits0;
+ _bits1 &= mask._bits1;
+ return this;
+ }
+ @Override RegMaskRW sub( RegMask mask ) {
+ if( mask==null ) return this;
+ _bits0 &= ~mask._bits0;
+ _bits1 &= ~mask._bits1;
+ return this;
+ }
+ public RegMaskRW or( RegMask mask ) {
+ if( mask!=null ) {
+ _bits0 |= mask._bits0;
+ _bits1 |= mask._bits1;
+ }
+ return this;
+ }
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/Serialize.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Serialize.java
new file mode 100644
index 00000000..76829151
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Serialize.java
@@ -0,0 +1,716 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.parser.Parser;
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.type.*;
+import com.seaofnodes.simple.util.*;
+import java.util.*;
+
+public class Serialize {
+
+ // Shared by all objects in this serialization pass. The graph and its
+ // declarations stay unchanged while each compilation unit is written.
+ private final ClassInitDependencies _init;
+ private final HashSet<TypeStruct> _pubTypes = new HashSet<>();
+ private final HashMap<CompUnit,Ary<TypeStruct>> _classes = new HashMap<>();
+
+ private Serialize(CodeGen code) {
+ _init = code._classInitDependencies;
+ assert _init!=null; // TypeCheck checked this same, still-unmodified graph.
+ // Preserve the canonical declarations, distinct from sharper graph
+ // types with the same names. Gather/sort them once for all objects.
+ TreeMap<String,TypeStruct> named = new TreeMap<>();
+ for( Type type : Parser.TYPES.values() )
+ if( type instanceof TypeStruct ts ) named.put(ts._name,ts);
+ for( TypeStruct ts : named.values() ) {
+ _pubTypes.add(ts);
+ CompUnit cu = classOwner(code,ts);
+ if( cu!=null && cu._src!=null )
+ _classes.computeIfAbsent(cu,k -> new Ary<>(TypeStruct.class)).add(ts);
+ }
+ }
+
+ // Encoding/emulation need only machine code. File export also needs ideal
+ // IR for a later compiler to import; an explicit Serialize request stops here.
+ static void forTarget(CodeGen code, CodeGen.Phase target, boolean inMemory) {
+ if( target==CodeGen.Phase.Serialize || (target.ordinal()>=CodeGen.Phase.Export.ordinal() && !inMemory) )
+ code.serialize();
+ }
+
+ static void serialize( CodeGen code ) {
+ // Anonymous functions also need names when another object refers to them.
+ for( FunNode fun : code._linker )
+ if( fun!=null && !fun.isDead() && fun._name==null )
+ fun._name=code._fidxs.symbol(fun.sig().fidx());
+ ArrayList<CompUnit> units = new ArrayList<>(code._compunits.values());
+ units.sort(Comparator.comparing(cu -> cu._fname));
+ Serialize writer = new Serialize(code);
+ for( CompUnit cu : units ) {
+ if( cu._src==null ) continue;
+ BAOS serial = writer.serialize(code,cu);
+ code._serials.put(cu,serial);
+ if( cu._fname.equals(code._srcName==null ? "Test" : code._srcName) ) code._serial=serial;
+ }
+ }
+
+ private BAOS serialize(CodeGen code, CompUnit top) {
+ Ary<Node> nodes = nodeOrder(code,top);
+ HashSet<String> xdeps = new HashSet<>();
+ // Retain build dependencies even when their calls were inlined away.
+ if( top._deps!=null ) for( CompUnit dep : top._deps )
+ if( dep!=top ) xdeps.add(dep._fname);
+ if( top._par!=null ) xdeps.add(top._par._fname);
+ String[] deps = xdeps.toArray(new String[0]);
+ Arrays.sort(deps);
+ String[] units = {top._fname};
+ TypeStruct clz = top._clz;
+
+ // Compress into bytes
+ Ary<TypeStruct> classes = _classes.get(top);
+ BAOS baos = write(nodes, clz, classes, deps, units, code._aliases, code._fidxs, code._rpcs, code._externFunc);
+
+ // --- Expensive bijection assert
+ if( false ) {
+ // Inflate into POJOs; renumbers everything
+ ElfReader elf = new ElfReader(new BAOS(baos.toByteArray()));
+ readAll(code,elf, code._aliases, code._fidxs, code._rpcs);
+ BAOS baos2 = write(elf._nodes,elf._clz,classes,elf._deps,elf._units, code._aliases, code._fidxs, code._rpcs, code._externFunc);
+
+ // Bi-jection
+ for( int i=0; i<baos.size(); i++ )
+ assert baos.buf()[i]==baos2.buf()[i] : "bijection mismatch at "+i+": "+(baos.buf()[i]&0xFF)+" != "+(baos2.buf()[i]&0xFF);
+ assert baos.size()==baos2.size() : "bijection size mismatch: "+baos.size()+" != "+baos2.size();
+ }
+ // --- Expensive bijection assert
+
+ // Record serialized IR for later ELF writing
+ return baos;
+ }
+
+ // --------------------------------------------------
+ private BAOS write(Ary<Node> nodes, TypeStruct clz, Ary<TypeStruct> classes, String[] depobjs, String[] units, GlobalBits aliases, GlobalBits fidxs, GlobalBits rpcs, HashMap<Integer,String> externFunc) {
+ // Initialize the mapping from bits/tags to types
+ Type.TAGOFFS();
+
+ IdentityHashMap<Node,Integer> anodes = new IdentityHashMap<>();
+ for( Node n : nodes ) anodes.put(n,anodes.size()+1); // 0 is null.
+ // The saved wrappers summarize this object's units only. Keep the
+ // live whole-program graph intact for subsequent lowering.
+ TypeTuple stopType = TypeTuple.STOP_HIGH;
+ for( Node n : nodes.last()._inputs )
+ if( anodes.containsKey(n) ) stopType = (TypeTuple)stopType.meet(n._type);
+ Type startType = TypeTuple.make(Type.CONTROL,stopType.mem(),((StartNode)nodes.at(0)).arg());
+
+ BAOS baos = new BAOS();
+ // A - Print the C0DE header. The format is not versioned yet.
+ baos.write('C').write('0').write('D').write('E');
+
+ // Count unique Types
+ var types = new HashMap<Type,Integer>();
+ IdentityHashMap<FunNode,TypeTuple> summaries = new IdentityHashMap<>();
+ // Count published types *first*, so can align with published strings
+ types.put(Type.TOP,0);
+ types.put( clz, types.size() );
+ // Have to visit the innards specially, since they did not get the
+ // normal recursive visit when first touching a type, because the
+ // published types are packed early to align with their string names so
+ // lookup in ElfReader can be fast.
+ for( int i=0; i<clz.nkids(); i++ )
+ clz.at(i).gather(types);
+ // Gather types from all sources
+ for( Node n : nodes ) {
+ n._type.gather(types);
+ if( n instanceof CallEndNode cend ) cend._rpc.gather(types);
+ if( n instanceof FunNode fun ) {
+ fun.sig().gather(types);
+ TypeTuple state = (TypeTuple)fun.ret()._type;
+ TypeMem mem = (TypeMem)state.mem();
+ // Retention can happen on a path that never returns. The
+ // unit's escape summary includes those native-call effects.
+ TypeMem escaped = (TypeMem)((TypeTuple)fun._compunit._stop._type).mem();
+ mem=mem.makeFrom(XInt.meet(mem._escFs,escaped._escFs),XInt.meet(mem._escAs,escaped._escAs));
+ state=state.makeFrom(1,mem);
+ summaries.put(fun,state);
+ state.gather(types);
+ }
+ if( n instanceof ParmNode parm ) parm._declaredType.gather(types);
+ if( n instanceof TypeNode con ) con._con .gather(types);
+ }
+ stopType.gather(types);
+ startType.gather(types);
+ // A defined class remains importable even when its initializer body dies.
+ // Nested structs also need their class declaration: <new> and <ctor>
+ // can be absent from graph types even while their function bodies live.
+ // Keep the nearest file owner, so an imported child is not re-exported
+ // merely because its parent was compiled here.
+ if( classes!=null )
+ for( TypeStruct ts : classes )
+ ts.gather(types);
+
+ // Only record canonical declarations already reachable from this
+ // object's graph. Pulling unrelated parser types into this closure
+ // can include open or partially built types from other units.
+ Ary<TypeStruct> writtenPubs = new Ary<>(TypeStruct.class);
+ for( Type type : types.keySet() )
+ if( type instanceof TypeStruct ts && _pubTypes.contains(ts) )
+ writtenPubs.add(ts);
+ Arrays.sort(writtenPubs._es,0,writtenPubs._len,Comparator.comparing(ts -> ts._name));
+ // Radix sort by Type ID#
+ Type[] atypes = new Type[types.size()];
+ for( Type t : types.keySet() )
+ atypes[types.get(t)] = t;
+
+ // Count all unique Strings
+ var strs = new HashMap<String,Integer>();
+ strs.put("",0); // Null string is always 0
+ // Find and count *published* Strings first
+ gather(strs,clz._name);
+ // Find and count dependent object file string names
+ if( depobjs != null )
+ for( String fname : depobjs )
+ gather(strs,fname);
+ for( String fname : units ) gather(strs,fname);
+ // Count strings from all types
+ for( Type t : atypes ) {
+ if( t instanceof Field fld ) gather(strs,fld._fname);
+ if( t instanceof TypeStruct ts ) gather(strs,ts . _name);
+ }
+ // Count strings used by local->global bit mappings.
+ aliases.gather(strs);
+ fidxs .gather(strs);
+ rpcs .gather(strs);
+ for( String extern : externFunc.values() )
+ gather(strs,extern);
+ // Count strings from nodes
+ for( Node n : nodes )
+ n.gather(strs);
+ // Radix sort by string ID#
+ String[] astrs = new String[strs.size()];
+ for( String s : strs.keySet() )
+ astrs[strs.get(s)] = s;
+
+ // Check that published strings and types align
+ assert astrs[1] == ((TypeStruct)atypes[1])._name;
+ if( depobjs != null )
+ for( int i=0; i<depobjs.length; i++ )
+ assert astrs[1+i+1] == depobjs[i];
+
+
+ // B - Write unique strings
+ baos.packed4(depobjs==null ? 0 : depobjs.length);
+ baos.packed4(strs.size());
+ // Write in ID# order; the first depobjs.length of these are dependent strings
+ for( String s : astrs )
+ baos.packedS(s);
+
+ // Defined units are loader metadata, not initialization dependencies.
+ baos.packed4(units.length);
+ for( String fname : units ) baos.packed4(strs.get(fname));
+
+ // B.2 - Write the local->global mappings
+ aliases.packed(baos,strs);
+ fidxs .packed(baos,strs);
+ rpcs .packed(baos,strs);
+ ArrayList<Integer> externFidxs = new ArrayList<>(externFunc.keySet());
+ Collections.sort(externFidxs);
+ baos.packed2(externFidxs.size());
+ for( int fidx : externFidxs ) {
+ baos.packed2(fidx);
+ baos.packed2(strs.get(externFunc.get(fidx)));
+ }
+
+ // C - Write unique Types
+ baos.packed4(atypes.length);
+ baos.packed2(writtenPubs._len);
+ for( TypeStruct ts : writtenPubs )
+ baos.packed4(types.get(ts));
+ // Write Types in ID# order, no children
+ for( Type t : atypes )
+ t.packed(baos,strs);
+ // Write Types in ID# order, only child IDs
+ for( Type t : atypes ) {
+ int nkids = t.nkids();
+ for( int i=0; i<nkids; i++ )
+ baos.packed4(types.get(t.at(i)));
+ }
+
+
+ // Function summaries precede the graph, so declarations and call
+ // behavior are available without allocating the implementation nodes.
+ int nfuncs = 0;
+ for( Node n : nodes ) if( n instanceof FunNode ) nfuncs++;
+ baos.packed4(nfuncs);
+ for( Node n : nodes ) if( n instanceof FunNode fun ) {
+ baos.packed2(types.get(fun.sig()));
+ baos.packed2(types.get(summaries.get(fun)));
+ baos.packed2(fun._name==null ? 0 : strs.get(fun._name));
+ baos.packed4(fun._approxUIDs);
+ int[] edges = _init._edgess[fun.sig().fidx()];
+ baos.packed4(edges==null ? 0 : edges.length);
+ if( edges!=null ) for( int edge : edges ) baos.packed4(edge);
+ }
+
+ // D - Write the nodes. Since this is expected to be the bulk of the
+ // data, we might want to explore various options.
+
+ // How many nodes?
+ baos.packed4(nodes._len);
+
+ // First cut: 1 byte opcode, optional per-node info, includes variable nIns
+ for( Node n : nodes ) {
+ baos.packed1(n.serialTag().ordinal());
+ baos.packed2(types.get(n==nodes.at(0) ? startType : n==nodes.last() ? stopType : n._type));
+ n.packed(baos,strs,types,anodes);
+ }
+ // Write out the input indices packed.
+ for( Node n : nodes ) {
+ for( int j=0; j<n.nIns(); j++ ) {
+ if( !writeInput(n,j,anodes) ) continue;
+ Node def = n.in(j);
+ Integer idx = def==null ? Integer.valueOf(0) : anodes.get(def);
+ // Imported function constants retain their symbol, not a body.
+ if( idx==null && n instanceof FunPtrNode && j==1 ) idx=0;
+ // Shared escape analysis belongs to this compilation only.
+ if( def instanceof GlobalEscapeNode ) idx=0;
+ assert idx != null : "Missing serialized input for "+n+" input "+j+" def "+def;
+ baos.packed2(idx);
+ }
+ if( n instanceof FunNode fun )
+ baos.packed2(anodes.get(fun.ret()));
+ }
+
+ return baos;
+ }
+
+ // Omit cross-object call-graph links without unlinking the live IR.
+ // A parameter follows exactly the same retained callers as its function.
+ private static boolean writeInput(Node n, int i, IdentityHashMap<Node,Integer> nodes) {
+ if( n instanceof ParmNode parm ) n=parm.fun();
+ if( (n instanceof FunNode || n instanceof CallEndNode) && i>0 )
+ return nodes.containsKey(n.in(i));
+ if( n instanceof StopNode ) return nodes.containsKey(n.in(i));
+ return true;
+ }
+
+ public static int nIns(Node n, IdentityHashMap<Node,Integer> nodes) {
+ int len=0;
+ for( int i=0; i<n.nIns(); i++ ) if( writeInput(n,i,nodes) ) len++;
+ return len;
+ }
+ public static <T> void gather( HashMap<T,Integer> strs, T s) {
+ if( s!=null && !strs.containsKey(s) )
+ strs.put(s,strs.size());
+ }
+ public static void gather( AryInt is, int s) {
+ if( is.atX(s)==0 ) {
+ int cnt=0;
+ for( int i=1; i<is._len; i++ )
+ if( is._es[i] != 0 )
+ cnt++;
+ is.setX(s,cnt+1);
+ }
+ }
+
+ // A nested declaration belongs to its nearest enclosing file.
+ private static CompUnit classOwner(CodeGen code, TypeStruct ts) {
+ if( !Parser.startsClzPrefix(ts._name) ) return null;
+ String name = ts._name.substring(Parser.CLZ.length()).replace('.','/');
+ while( true ) {
+ CompUnit cu = code._compunits.get(name);
+ if( cu!=null ) return cu;
+ int slash = name.lastIndexOf('/');
+ if( slash<0 ) return null;
+ name = name.substring(0,slash);
+ }
+ }
+
+ static void read_public_strs(ElfReader elf) {
+ BAOS bais = elf._bais;
+ Type.TAGOFFS();
+
+ // A - Read a header
+ if( bais.read()!='C' || bais.read()!='0' || bais.read()!='D' || bais.read()!='E' )
+ throw new IllegalArgumentException("Unsupported serialized IR format; rebuild the object file");
+
+ // B - Packed read of #strings, then strings
+ int ndependents = bais.packed4(); // Number of dependent object files
+ int nstrs = bais.packed4(); // Total number of strings
+ String[] strs = new String[nstrs];
+ for( int i=0; i<nstrs; i++ )
+ strs[i] = bais.packedS();
+ elf._strs = strs;
+
+ // Collect dependent file names
+ elf._deps = new String[ndependents];
+ for( int i=0; i<ndependents; i++ )
+ elf._deps[i] = strs[1+i+1];
+ elf._units = new String[bais.packed4()];
+ for( int i=0; i<elf._units.length; i++ )
+ elf._units[i] = strs[bais.packed4()];
+ }
+
+ // --------------------------------------------------
+ record TypeRead(Type[] types, String[] strs, GlobalBits fileAliases, GlobalBits fileFidxs, GlobalBits fileRpcs) {}
+
+ static TypeRead readTypes( CodeGen code, ElfReader elf, GlobalBits aliases, GlobalBits fidxs, GlobalBits rpcs ) {
+ if( elf._types!=null ) return elf._types;
+ BAOS bais = elf._bais;
+ // Initialize the mapping from bits/tags to types
+ Type.TAGOFFS();
+
+ // If not read the strings already, read them now
+ String[] strs = elf._strs;
+ if( strs == null ) {
+ // A - Read a header
+ if( bais.read() != 'C' || bais.read() != '0' || bais.read() != 'D' || bais.read() != 'E' )
+ throw new IllegalArgumentException( "Unsupported serialized IR format; rebuild the object file" );
+
+ // B - Packed read of #strings, then strings
+ int ndependents = bais.packed4(); // Number of dependent object files
+ int nstrs = bais.packed4(); // Total number of strings
+ elf._strs = strs = new String[nstrs];
+ for( int i = 0; i < nstrs; i++ )
+ strs[i] = bais.packedS();
+
+ // Collect dependent file names
+ elf._deps = new String[ndependents];
+ for( int i = 0; i < ndependents; i++ )
+ elf._deps[i] = strs[1 + i + 1];
+ elf._units = new String[bais.packed4()];
+ for( int i=0; i<elf._units.length; i++ )
+ elf._units[i] = strs[bais.packed4()];
+ }
+
+ // B.2 - Read the local->global mappings. Further down we will map
+ // Elf file-local -> global, then global -> CodeGen.CODE local
+ GlobalBits fileAliases = GlobalBits.packed(bais,strs);
+ GlobalBits fileFidxs = GlobalBits.packed(bais,strs);
+ GlobalBits fileRpcs = GlobalBits.packed(bais,strs);
+ int nexterns = bais.packed2();
+ for( int i=0; i<nexterns; i++ ) {
+ int fileFidx = bais.packed2();
+ String extern = strs[bais.packed2()];
+ int fidx = fileFidx < GlobalBits.RESERVED ? fileFidx : fidxs.map(fileFidxs,fileFidx);
+ String old = code.externFunc(fidx);
+ if( old == null )
+ code.externFunc(fidx,extern);
+ else
+ assert old.equals(extern);
+ }
+
+
+ // C - Packed read of #types, then types
+ int ntypes = bais.packed4();
+ int npubTypes = bais.packed2();
+ int[] pubTypeIds = new int[npubTypes];
+ for( int i=0; i<npubTypes; i++ )
+ pubTypeIds[i] = bais.packed4();
+ Type[] types = Type.packed(bais,strs,ntypes, Parser.TYPES,
+ fileAliases,aliases,
+ fileFidxs ,fidxs,
+ fileRpcs ,rpcs);
+
+ // aliases.set(0,0);
+ // Check and collect first published symbols map to TypeStructs
+ elf._clz = (TypeStruct)types[1/*skip zero*/];
+ assert elf._clz._name==strs[1/*skip null ptr*/];
+ for( Type type : types )
+ if( type instanceof TypeStruct ts )
+ Parser.TYPES.put(ts._name,ts);
+ for( Type type : types ) {
+ if( !(type instanceof TypeStruct ts) || !Parser.startsClzPrefix(ts._name) )
+ continue;
+ String cname = ts._name.substring(Parser.CLZ.length());
+ int dot = cname.lastIndexOf('.');
+ String name = dot == -1 ? cname : cname.substring(dot+1);
+ Field inst = ts.field(name.intern());
+ if( inst != null && inst._t instanceof TypeFunPtr tfp && tfp._ret instanceof TypeMem mem && mem._t instanceof TypeStruct its )
+ Parser.TYPES.put(cname,its);
+ }
+ elf._pubTypes = new TypeStruct[npubTypes];
+ for( int i=0; i<npubTypes; i++ ) {
+ TypeStruct ts = elf._pubTypes[i] = (TypeStruct)types[pubTypeIds[i]];
+ Parser.TYPES.put(ts._name,ts);
+ }
+ elf._functions = new FunctionSummary[bais.packed4()];
+ for( int i = 0; i<elf._functions.length; i++ ) {
+ TypeFunPtr sig = (TypeFunPtr)types[bais.packed2()];
+ TypeTuple state = (TypeTuple)types[bais.packed2()];
+ String name = strs[bais.packed2()];
+ int size = bais.packed4();
+ int[] edges = new int[bais.packed4()];
+ for( int j = 0; j<edges.length; j++ ) edges[j]=fidxs.map(fileFidxs,bais.packed4());
+ elf._functions[i]=new FunctionSummary(sig,state,name.isEmpty() ? fidxs.symbol(sig.fidx()) : name,size,edges);
+ }
+ return elf._types = new TypeRead(types,strs,fileAliases,fileFidxs,fileRpcs);
+ }
+
+ static void readAll( CodeGen code, ElfReader elf, GlobalBits aliases, GlobalBits fidxs, GlobalBits rpcs ) {
+ BAOS bais = elf._bais;
+ TypeRead tread = readTypes(code, elf, aliases, fidxs, rpcs);
+ Type[] types = tread.types();
+ String[] strs = tread.strs();
+ GlobalBits fileAliases = tread.fileAliases();
+
+ // D - Read the nodes
+
+ // Number of nodes
+ int num = bais.packed4();
+ // Packed array of nodes
+ Ary<Node> nodes = new Ary<>(Node.class);
+ CompUnit cu = null;
+ for( int i = 0; i<num; i++ ) {
+ Node.Tag tag = Node.Tag.VALS[bais.packed1()];
+ Type t = types[bais.packed2()];
+ Node n = tag.make(bais,strs,types,fileAliases,aliases);
+ n._type = t;
+ nodes.push(n);
+ // A little extra for FunNodes
+ if( n instanceof StartCUNode startcu ) {
+ cu = code._compunits.get(startcu._fname==null ? (code._srcName==null ? "Test" : code._srcName) : startcu._fname);
+ assert cu != null;
+ }
+ if( n instanceof StopCUNode stopcu ) {
+ cu = null;
+ }
+ if( n instanceof FunNode fun ) {
+ assert cu != null;
+ fun._compunit = cu;
+ if( fun._name==null || fun._name.isEmpty() ) fun._name=fidxs.symbol(fun.sig().fidx());
+ }
+ if( n instanceof ExternNode ext && ext._con instanceof TypeFunPtr tfp && tfp.isConstant() ) {
+ String old = code.externFunc(tfp.fidx());
+ if( old == null )
+ code.externFunc(tfp.fidx(), ext._extern);
+ else
+ assert old.equals(ext._extern);
+ }
+ }
+
+ // Node edges
+ for( Node n : nodes ) {
+ int len = n.nIns();
+ for( int i = 0; i<len; i++ ) {
+ int idx = bais.packed2();
+ Node def = idx==0 ? null : nodes.at(idx-1);
+ // MemMerge precise aliases are encoded by their input slot.
+ // Remap populated file-local slots into this compilation's
+ // local alias numbering; null holes carry no identity.
+ int local = i;
+ if( n instanceof MemMergeNode && i >= GlobalBits.RESERVED ) {
+ if( def == null ) continue;
+ local = aliases.map(fileAliases,i);
+ }
+ n.setDefX(local,def);
+ // Call's output[0] is always the CallEnd
+ if( n instanceof CallEndNode cend && i==0 ) {
+ var outs = cend.call()._outputs;
+ outs.swap(0,outs._len-1);
+ }
+ }
+ // A little extra for FunNodes
+ if( n instanceof FunNode fun ) {
+ // Return is hooked like an edge
+ ReturnNode ret = (ReturnNode)nodes.at(bais.packed2()-1);
+ fun.setRet(ret);
+ ret._fun = fun;
+ }
+ }
+ // Add edge from Start to Stop
+ StartNode start = (StartNode)nodes.at(0);
+ StopNode stop = (StopNode)nodes.last();
+ start.setDef(1,stop);
+
+ elf._nodes = nodes;
+ }
+
+ // --------------------------------------------------
+ public static Ary<Node> nodeOrder( CodeGen code, CompUnit cu ) {
+ BitSet visit = code.visit();
+ Ary<Node> nodes = new Ary<>(Node.class);
+ visit.set(code._stop._nid);
+ // FinalizeIR attached every surviving function to its unit's Stop.
+ for( Node ret : cu._stop._inputs )
+ _funWalk(((ReturnNode)ret).fun(),nodes,visit);
+
+ // Gather constants and shared expressions through this unit's inputs.
+ // Walking their users instead would search other units and miss new
+ // kinds of shared expressions unless we special-cased each one.
+ Ary<Node> cons = new Ary<>(Node.class);
+ cons.add(code._start);
+ visit.set(code._start._nid);
+ visit.set(cu._start._nid);
+ for( Node n : cu._start._outputs )
+ if( n instanceof ProjNode ) visit.set(n._nid);
+ for( Node n : nodes )
+ for( int i = 0; i<n.nIns(); i++ ) {
+ // Parameters omit arguments from callers in other objects,
+ // just as writeInput omits those call-graph edges.
+ if( n instanceof ParmNode parm && i>0 && !visit.get(parm.fun().in(i)._nid) ) continue;
+ constantInputs(n.in(i),cons,visit);
+ }
+ // StartCU and memory projection
+ cons.add(cu._start);
+ for( Node n : cu._start._outputs )
+ if( n instanceof ProjNode ) cons.add(n);
+ // All the innards of functions
+ cons.addAll(nodes);
+ // StopCU, Stop
+ cons.add(cu._stop);
+ cons.add(code._stop);
+ visit.clear();
+ return cons;
+ }
+
+ private static void constantInputs(Node n, Ary<Node> cons, BitSet visit) {
+ if( n==null || visit.get(n._nid) ) return;
+ // Call graph edges lead to other functions, not shared values.
+ if( n instanceof CFGNode && !n.isConst() ) return;
+ visit.set(n._nid);
+ cons.add(n);
+ // A FunPtr's Return edge must not pull in the callee's body.
+ if( !n.isConst() )
+ for( Node def : n._inputs ) constantInputs(def,cons,visit);
+ }
+
+ // --------------------------------------------------
+ public static Ary<Node> nodeOrder( CodeGen code ) {
+ Ary<Node> nodes = new Ary<>(Node.class);
+ BitSet visit = code.visit();
+ nodes.add(code._start);
+
+ // All the global constants
+ for( Node n : code._start._outputs ) {
+ if( n.isConst() ) {
+ visit.set( n._nid );
+ nodes.add( n );
+ // Special for Cast:BOT
+ for( Node cast : n._outputs )
+ if( cast != null && cast.nIns() == 2 && cast.in( 0 ) == null ) {
+ visit.set( cast._nid );
+ nodes.add( cast );
+ // Array pointer conversion adds the body offset after
+ // the zero-code PtrToInt. Both nodes are global-value
+ // descendants of the constant and otherwise have no
+ // function-local root from which to walk them.
+ if( cast instanceof PtrToIntNode )
+ for( Node add : cast.outs() )
+ if( add instanceof AddNode && !visit.get(add._nid) ) {
+ visit.set(add._nid);
+ nodes.add(add);
+ }
+ }
+ }
+ }
+
+ // All the CompUnits
+ for( CompUnit cu : code._compunits.values() ) {
+ // Whole CompUnits have no users, went dead
+ if( cu._src!=null && cu._start != null && !cu._start.isDead() ) {
+ nodes.add(cu._start);
+ // A memory-free, nonreturning unit can lose its memory projection.
+ ProjNode mem=cu._start.proj(1);
+ if( mem!=null ) nodes.add(mem);
+ // All the functions, including internal ones, in comp-unit order
+ for( FunNode fun : code._linker )
+ if( fun!=null && !fun.isDead() && fun._compunit == cu )
+ _funWalk(fun,nodes,visit);
+ nodes.add(cu._stop);
+ }
+ }
+
+ nodes.add(code._stop);
+ visit.clear();
+ return nodes;
+ }
+
+ // Walk and print whole functions at a time, in CG order
+ private static void _funWalk( FunNode fun, Ary<Node> rpo, BitSet visit ) {
+ int start = rpo._len;
+ _funRPO(fun,rpo,visit);
+ // A nonreturning function still owns a Return, even before LoopTree
+ // inserts synthetic exits. No loop-depth ordering is needed on disk.
+ _funRPO(fun.ret(),rpo,visit);
+ int len = rpo._len - start;
+ for( int i=0; i< len>>1; i++ )
+ rpo.swap(start+i,start+len-1-i);
+ }
+
+ // Walk and gather RPO nodes.
+ private static void _funRPO(Node n, Ary<Node> rpo, BitSet visit ) {
+ // Return-linked constants can have users in other compilation units.
+ if( n.isConst() ) return;
+ if( visit.get(n._nid) ) return; // Been there, done that
+ visit.set(n._nid); // Stop recursion
+ // Walk outputs ordered
+ if( n instanceof CFGNode ncfg ) {
+ if( n instanceof IfNode iff ) {
+ CProjNode c0 = iff.cproj(0);
+ CProjNode c1 = iff.cproj(1);
+ _funRPO(c0,rpo,visit);
+ _funRPO(c1,rpo,visit);
+ } else if( n instanceof StopNode ) {
+ // nothing
+ } else {
+ // CFG to CFG before CFG to data
+ for( Node use : n._outputs )
+ if( use instanceof CFGNode usecfg && !ncfg.skip(usecfg) ) // Do not walk from Call to Fun
+ _funRPO(use,rpo,visit);
+ }
+ // Walk CFG to data eventually
+ for( Node use : n._outputs )
+ if( !(use instanceof CFGNode) )
+ _funRPO(use,rpo,visit);
+ } else {
+ // Do not walk from a non-CFG to a CFG; CFGs walk to CFGs to
+ // preserve CFG order. Do not walk *into* Phis; wait for the
+ // Region to walk the Phis (otherwise you visit some of the
+ // Phis ahead of others forcing the Phis to be spread around).
+ for( Node use : n._outputs ) {
+ if( use!=null && !(use instanceof CFGNode) && !(use instanceof PhiNode) )
+ _funRPO(use,rpo,visit);
+ }
+ }
+
+ // If parent is a Multi, do not add (yet).
+ // If self is a Multi, add self and children.
+ if( multiChild(n) || n instanceof CallEndNode ) {
+ // nothing; delay until Multi-head or CallNode
+ } else if( n instanceof MultiNode ) {
+ // Now lump all multi/projections together
+ printMulti(n,rpo);
+ } else if( n instanceof CallNode call ) {
+ printMulti(call.cend(),rpo);
+ rpo.add(call);
+ } else if( n instanceof RegionNode ) {
+ // Now lump all Phis together
+ int old = rpo._len;
+ for( Node use : n._outputs )
+ if( use instanceof PhiNode )
+ rpo.add(use);
+ // Sort by label? Want Phis before control outputs
+ Arrays.sort( rpo._es, old, rpo._len, (x,y) -> y.label().compareTo(x.label()) );
+ rpo.add(n);
+ } else if( n instanceof StopNode ) {
+ // nothing
+ } else {
+ rpo.add(n); // Post-order add
+ }
+ }
+
+ private static boolean multiChild(Node n) {
+ return n instanceof Proj || n instanceof PhiNode;
+ }
+
+ private static void printMulti(Node n, Ary<Node> rpo) {
+ // Now lump all multi/projections together
+ for( Node use : n._outputs )
+ rpo.add(use);
+ // Sort by projection order
+ if( !(n instanceof StartNode) )
+ Arrays.sort( rpo._es,rpo._len-n.nOuts(),rpo._len,
+ (x,y) -> x instanceof ProjNode px && y instanceof ProjNode py ? (py.idx() - px.idx()) : (y._nid - x._nid) );
+ rpo.add(n);
+ }
+
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/SourceError.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/SourceError.java
new file mode 100644
index 00000000..f2f46146
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/SourceError.java
@@ -0,0 +1,49 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.util.SB;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/** Format one parser byte offset. Source is read and text built only on error. */
+public abstract class SourceError {
+ public static String format(CompUnit cu, int pos, String message) {
+ String file=cu.sourceFile();
+ String name=file==null ? cu._fname+".smp" : file;
+ byte[] source;
+ try {
+ source=cu._src!=null ? cu._src.getBytes(StandardCharsets.UTF_8)
+ : file==null ? null : Files.readAllBytes(Path.of(file));
+ } catch( IOException ex ) { source=null; }
+ if( source==null || pos<0 || pos>source.length )
+ return name+": error: "+message+" (source unavailable at byte offset "+pos+")";
+
+ int line=1, start=0;
+ for( int i=0; i<pos; i++ )
+ if( source[i]=='\n' ) { line++; start=i+1; }
+ int end=pos;
+ while( end<source.length && source[end]!='\n' ) end++;
+ if( end>start && source[end-1]=='\r' ) end--;
+ int point=Math.min(pos,end);
+ String before=new String(source,start,point-start,StandardCharsets.UTF_8);
+ String text=new String(source,start,end-start,StandardCharsets.UTF_8);
+ int column=before.codePointCount(0,before.length())+1;
+ SB sb=new SB().p(name).p(':').p(line).p(':').p(column).p(": error: ").p(message).p("\n ");
+ int width=0, caret=0;
+ for( int i=0; i<text.length(); ) {
+ if( i==before.length() ) caret=width;
+ int ch=text.codePointAt(i);
+ if( ch=='\t' ) {
+ int spaces=8-(width%8);
+ for( int j=0; j<spaces; j++ ) sb.p(' ');
+ width+=spaces;
+ } else { sb.p(new String(Character.toChars(ch))); width++; }
+ i+=Character.charCount(ch);
+ }
+ if( before.length()==text.length() ) caret=width;
+ sb.p("\n ");
+ for( int i=0; i<caret; i++ ) sb.p('-');
+ return sb.p('^').toString();
+ }
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/StaticData.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/StaticData.java
new file mode 100644
index 00000000..f813fab5
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/StaticData.java
@@ -0,0 +1,177 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.parser.Parser;
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.type.*;
+import com.seaofnodes.simple.util.BAOS;
+import java.util.*;
+
+// Pool objects have identity independently of their initial field values.
+// References in their contents use the same symbols in ELF and emulator images.
+public final class StaticData {
+ public static final class ObjectData {
+ public final Type type;
+ public final String symbol;
+ public final boolean readOnly, external;
+ public final int alignment, size;
+ public int offset;
+ ObjectData(Type t, String name, boolean ro, boolean ext) {
+ type=t; symbol=name; readOnly=ro; external=ext;
+ alignment=t.alignment();
+ int align=1<<alignment;
+ size=t instanceof TypeStruct ts ? (Math.max(1,ts.size())+align-1)&-align : align;
+ }
+ }
+ public record Relocation(ObjectData object, int offset, int logSize, Type value) {}
+
+ final Encoding enc;
+ final LinkedHashMap<Object,ObjectData> objects = new LinkedHashMap<>();
+ final ArrayList<Relocation> relocations = new ArrayList<>();
+ final String prefix;
+
+ public ArrayList<ObjectData> entries() {
+ ArrayList<ObjectData> entries=new ArrayList<>(objects.values());
+ entries.sort(Comparator.comparingInt((ObjectData obj) -> obj.alignment).reversed());
+ return entries;
+ }
+
+ StaticData(Encoding enc) {
+ this.enc=enc;
+ prefix=(enc._code._emitUnit==null ? enc._code.entryClinitName() : enc._code._emitUnit._cname)+"$DATA$";
+ }
+
+ static boolean isClass(Type t) {
+ return t instanceof TypeStruct ts && Parser.startsClzPrefix(ts._name);
+ }
+
+ static CompUnit owner(CodeGen code,TypeStruct ts) {
+ return owner(code,ts._name.substring(Parser.CLZ.length()));
+ }
+ static CompUnit owner(CodeGen code,String name) {
+ if(name==null) return null;
+ CompUnit owner=null;
+ for( CompUnit cu : code._compunits.values() )
+ if( cu._cname!=null && (name.equals(cu._cname) || name.startsWith(cu._cname+".")) &&
+ (owner==null || cu._cname.length()>owner._cname.length()) ) owner=cu;
+ return owner;
+ }
+
+ ObjectData object(Type t) {
+ Object key=isClass(t) ? ((TypeStruct)t)._name
+ : t instanceof TypeFunPtr fp ? functionSymbol(fp) : t;
+ ObjectData old=objects.get(key);
+ if( old!=null ) return old;
+ boolean clz=isClass(t);
+ if( clz && Parser.TYPES.get(((TypeStruct)t)._name) instanceof TypeStruct canonical ) t=canonical;
+ CompUnit owner=clz ? owner(enc._code,(TypeStruct)t)
+ : t instanceof TypeFunPtr fp ? owner(enc._code,enc._code._fidxs.owner(fp.fidx())) : null;
+ String symbol=clz ? ((TypeStruct)t)._name.substring(Parser.CLZ.length())+".$class"
+ : t instanceof TypeFunPtr fp ? functionSymbol(fp) : prefix+objects.size();
+ ObjectData obj=new ObjectData(t,symbol,!clz && (!(t instanceof TypeStruct ts) || ts.isConstant()),
+ owner!=null && !enc._code.emits(owner));
+ objects.put(key,obj);
+ // Register before following fields: class objects may refer to each other.
+ if( !obj.external && t instanceof TypeStruct ts )
+ for( Field f : ts._fields )
+ if( !f._extern && f._t.isConstant() ) target(f._t);
+ return obj;
+ }
+
+ private static boolean pointerConstant(TypeMemPtr ptr) {
+ return ptr.notNull() && (isClass(ptr._obj) || (ptr._obj.isConstant() && (!ptr._obj.isAry() || ptr._obj.isConAry())));
+ }
+
+ private void target(Type t) {
+ if( t instanceof TypeMemPtr ptr && pointerConstant(ptr) ) object(ptr._obj);
+ if( t instanceof TypeFunPtr fp && fp.notNull() &&
+ enc._code.link(fp)==null && !enc._code._externFunc.containsKey(fp.fidx()) )
+ // A dead function still needs a unique non-null identity for comparisons.
+ object(fp);
+ }
+
+ void write() {
+ ArrayList<Encoding.Relo> refs=new ArrayList<>(enc._bigCons.values());
+ refs.sort(Comparator.comparingInt(r -> r._op._nid));
+ for( Encoding.Relo ref : refs ) object(ref._t);
+ // Definitions belong to their compilation unit, even if every local use folded.
+ ArrayList<CompUnit> units=new ArrayList<>(enc._code._compunits.values());
+ units.sort(Comparator.comparing(cu -> cu._cname==null ? "" : cu._cname));
+ for( CompUnit cu : units ) if( enc._code.emits(cu) && cu._clz!=null ) object(cu._clz);
+ for( ObjectData obj : entries() ) {
+ if( obj.external ) continue;
+ BAOS bits=obj.readOnly ? enc._cpool : enc._sdata;
+ Encoding.padN(1<<obj.alignment,bits);
+ obj.offset=bits.size();
+ if( obj.type instanceof TypeStruct ts ) writeStruct(obj,ts,bits);
+ else Encoding.addN(obj.alignment,obj.type,bits);
+ while( bits.size()<obj.offset+obj.size ) bits.write(0);
+ }
+ for( Encoding.Relo ref : refs ) {
+ ObjectData obj=object(ref._t);
+ ref._target=obj.offset;
+ ref._opStart=enc.opStart(ref._op);
+ }
+ }
+
+ private void writeStruct(ObjectData obj, TypeStruct ts, BAOS bits) {
+ int[] layout=ts.layout();
+ for( int idx : layout ) {
+ if( idx==ts._fields.length ) continue;
+ Field f=ts._fields[idx];
+ if( f._extern ) continue;
+ int off=ts.offset(idx);
+ while( bits.size()<obj.offset+off ) bits.write(0);
+ if( f._fname.equals("[]") ) ((TypeConAry)f._t).write(bits);
+ else {
+ Type value=f._t.isConstant() ? f._t : f._t.makeZero();
+ int log=f._t.log_size();
+ Encoding.addN(log,value,bits);
+ if( (value instanceof TypeFunPtr fp && fp.notNull()) ||
+ (value instanceof TypeMemPtr ptr && pointerConstant(ptr)) )
+ relocations.add(new Relocation(obj,off,log,value));
+ }
+ }
+ while( bits.size()<obj.offset+ts.size() ) bits.write(0);
+ }
+
+ String functionSymbol(TypeFunPtr fp) {
+ String ext=enc._code._externFunc.get(fp.fidx());
+ return ext!=null ? ext : enc._code._fidxs.symbol(fp.fidx());
+ }
+
+ String targetSymbol(Type value) {
+ if( value instanceof TypeMemPtr ptr ) return object(ptr._obj).symbol;
+ TypeFunPtr fp=(TypeFunPtr)value;
+ return enc._code.link(fp)==null && !enc._code._externFunc.containsKey(fp.fidx())
+ ? object(fp).symbol : functionSymbol(fp);
+ }
+
+ void link(int cpool, int sdata) {
+ for( Relocation rel : relocations ) {
+ long address;
+ if( rel.value instanceof TypeMemPtr ptr ) address=address(object(ptr._obj),cpool,sdata);
+ else {
+ TypeFunPtr fp=(TypeFunPtr)rel.value;
+ String ext=enc._code._externFunc.get(fp.fidx());
+ FunNode fun=enc._code.link(fp);
+ if( ext!=null ) address=enc.externalAddress(ext);
+ else if( fun==null ) address=address(object(fp),cpool,sdata);
+ else {
+ if( !fun.isFromSource() ) throw new IllegalArgumentException("Unlinked function "+functionSymbol(fp));
+ address=enc.opStart(fun);
+ }
+ }
+ // Emulator native-call sentinels are signed 32-bit addresses.
+ if( rel.logSize==2 && (address<Integer.MIN_VALUE || address>0xffff_ffffL) )
+ throw new IllegalArgumentException("Function address outside 32 bits: "+address);
+ BAOS bits=rel.object.readOnly ? enc._cpool : enc._sdata;
+ int off=rel.object.offset+rel.offset;
+ for( int i=0; i<(1<<rel.logSize); i++ ) bits.buf()[off+i]=(byte)(address>>>(8*i));
+ }
+ }
+
+ int address(ObjectData obj,int cpool,int sdata) {
+ if( obj.external ) throw new IllegalArgumentException("Unlinked class object "+obj.symbol);
+ return obj.offset+(obj.readOnly ? cpool : sdata);
+ }
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/TypeCheck.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/TypeCheck.java
new file mode 100644
index 00000000..c9e53311
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/TypeCheck.java
@@ -0,0 +1,59 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.parser.Parser;
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.type.Type;
+import com.seaofnodes.simple.util.Ary;
+
+// Reject invalid optimized programs before lowering erases their source-level
+// obligations. Queue surviving CLInits during the same walk for Lower's Iter.
+abstract class TypeCheck {
+ static void run(CodeGen code) {
+ // Check for class-init cycles first; such a cycle can prevent other
+ // things from resolving types. Use the graph and summaries left by
+ // optimization; do not load more bodies to disprove a possible cycle.
+ ClassInitDependencies deps = new ClassInitDependencies(code);
+ String cycle = deps.check();
+ if( cycle!=null ) throw Parser.error(cycle,null);
+ code._classInitDependencies = deps; // Serialize reuses this checked graph.
+
+ Ary<Node> errs = new Ary<>(Node.class);
+ code._stop.walk( n -> {
+ if( n.err() != null )
+ errs.add(n);
+ if( n instanceof CLInitNode ) code.add(n);
+ return null;
+ });
+
+ if( !errs.isEmpty() ) {
+ // Prefer an originating error over a secondary error caused by a
+ // BOTTOM input. If all errors are downstream-poisoned, retain
+ // the old first-error behavior.
+ Node min = null;
+ for( Node n : errs )
+ if( noBottomInputs(n) )
+ { min = n; break; }
+ if( min==null )
+ min = errs.at(0);
+ for( Node n : errs )
+ if( n!=min &&
+ noBottomInputs(n) &&
+ ((n instanceof MemOpNode nmop &&
+ min instanceof MemOpNode mmop &&
+ nmop._loc.before(mmop._loc)) ||
+ (n instanceof CFGNode ncfg &&
+ min instanceof CFGNode mincfg &&
+ ncfg.idepth() < mincfg.idepth())) )
+ min = n;
+ if( code._obs != null ) code._obs.error(min, min.err().getMessage());
+ throw min.err();
+ }
+ }
+
+ private static boolean noBottomInputs(Node n) {
+ for( Node def : n._inputs )
+ if( def!=null && !(def instanceof FldOffNode) && def._type==Type.BOTTOM )
+ return false;
+ return true;
+ }
+}
diff --git a/chapter29/src/main/java/com/seaofnodes/simple/codegen/Unlink.java b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Unlink.java
new file mode 100644
index 00000000..bb9a6196
--- /dev/null
+++ b/chapter29/src/main/java/com/seaofnodes/simple/codegen/Unlink.java
@@ -0,0 +1,174 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.util.Ary;
+
+// Discard call-graph edges after LoopTree. Machine selection keeps only this
+// object's bodies; ideal evaluation keeps imports so their calls can execute.
+abstract class Unlink {
+ // Disconnect compiler-only interprocedural edges before scheduling. Keep
+ // signatures in the linker; runtime calls resolve FIDXs, not graph edges.
+ static Ary<FunNode> forEval(CodeGen code, StartNode start, StopNode stop) {
+ Ary<Node> nodes = new Ary<>(Node.class);
+ start.walk(n -> { nodes.add(n); return null; });
+ Ary<FunNode> funs = new Ary<>(FunNode.class);
+ for( Node n : nodes )
+ if( n instanceof FunNode fun ) {
+ fun.ret().keep();
+ if( fun.summary()==null ) funs.add(fun);
+ }
+ for( Node n : nodes ) {
+ if( n instanceof CallNode call ) call.unlink_all();
+ if( n instanceof CLInitNode init ) init.unlink();
+ if( n instanceof FunPtrNode ptr && ptr.ret()!=null ) ptr.setDef(1,null);
+ }
+ for( FunNode fun : funs ) {
+ // Keep bodies rooted for scheduling after their callers detach.
+ if( fun.nIns()==1 ) {
+ fun.addDef(fun._compunit._start);
+ for( Node use : fun._outputs )
+ if( use instanceof ParmNode parm ) {
+ ConstantNode con = ConstantNode.raw(parm._type);
+ con.setDef(0,start);
+ parm.addDef(con);
+ }
+ }
+ restoreRPC(fun);
+ }
+ code.releaseUnlinkedNodes();
+ for( Node n : nodes )
+ if( n instanceof StartCUNode unitStart ) detachEmptyStart(unitStart);
+ // A summary is a dispatch target, not a schedulable implementation.
+ // Retain its own wrapper graph without including it in this schedule.
+ for( int i = 0; i<stop.nIns(); i++ ) {
+ StopCUNode unit = (StopCUNode)stop.in(i);
+ boolean summary = false;
+ for( Node r : unit._inputs )
+ summary |= ((ReturnNode)r).fun().summary()!=null;
+ if( summary || unit.nIns()==0 ) {
+ unit.start().setDef(0,null);
+ stop.delDef(i--);
+ }
+ }
+ for( Node n : nodes )
+ if( n instanceof FunNode fun ) fun.ret().unkeep();
+ return funs;
+ }
+
+ private static void unlinkImports(CodeGen code) {
+ // Imported function constants keep their symbolic identity, not an
+ // embedded copy of the other object's Return and body.
+ for( Node n : code._start._outputs.asAry() )
+ if( n instanceof FunPtrNode ptr && ptr.ret()!=null && !ptr.fun().isFromSource() ) {
+ ptr.liftType(ptr._type);
+ ptr.setDef(1,null);
+ }
+ for( FunNode fun : code._linker ) {
+ if( fun==null || fun.isDead() || fun.isFromSource() )
+ continue;
+ for( int i=1; i<fun.nIns(); i++ )
+ if( fun.in(i) instanceof CallNode call )
+ call.unlink(fun,i--);
+ }
+ for( int i=0; i<code._stop.nIns(); i++ ) {
+ StopCUNode stop=(StopCUNode)code._stop.in(i);
+ CompUnit cu=code._compunits.get(stop.start()._fname);
+ if( cu._src==null ) code._stop.delDef(i--);
+ }
+ // The whole-object wrappers now summarize only the emitted units.
+ code._stop ._type=code._stop .compute();
+ code._start._type=code._start.compute();
+ }
+
+ static void run(CodeGen code, boolean removeImports) {
+ if( removeImports ) unlinkImports(code);
+
+ // The remaining passes assume all calls are unlinked; i.e. we are throwing
+ // away the Call Graph here. Functions only reachable from internal calls
+ // need to be re-hooked to stop/start less they go dead.
+ for( FunNode fun : code._linker ) {
+ // Already linked to start, not going dead
+ if( fun==null || fun.isDead() )
+ continue;
+
+ // Imported functions are not emitted by this object. Ideal
+ // evaluation retains them so their calls can actually execute.
+ if( removeImports && !fun.isFromSource() ) {
+ while( fun.nIns() > 1 ) {
+ if( fun.in(1) instanceof CallNode call )
+ call.unlink(fun,1);
+ else
+ fun.removeDeadPath(1);
+ }
+ StartCUNode start = fun._compunit._start;
+ StopCUNode stop = fun._compunit._stop;
+ // Unhook the Stop->Return edge also
+ ReturnNode ret = fun.ret();
+ int idx = stop._inputs.find(ret);
+ if( idx != -1 ) {
+ stop._inputs.del(idx);
+ ret.delUse(stop);
+ }
+ //
+ if( stop.nIns()==0 ) {
+ idx = code._stop._inputs.find(stop);
+ if( idx != -1 )
+ code._stop.delDef(idx);
+ if( start.nIns() > 0 && start.in(0) != null )
+ start.setDef(0,null);
+ if( start.nIns() > 1 && start.in(1) != null )
+ start.setDef(1,null);
+ }
+ continue;
+ }
+
+ // Insert the post-Opto code-generation hook. Unlike an Opto-time
+ // Start input, this no longer carries unknown-caller semantics.
+ ReturnNode ret = fun.ret();
+ StartNode start = fun._compunit._start;
+ assert start!=null;
+ if( fun.nIns() <= 1 || fun.in(1) != start ) {
+ fun.insertDef(1,start);
+ for( Node use : fun._outputs )
+ if( use instanceof ParmNode parm )
+ parm.insertDef(1,ConstantNode.raw(parm._type));
+ if( fun._compunit._stop._inputs.find(ret) == -1 )
+ fun._compunit._stop.addDef(ret);
+ }
+ // Unlink from Call
+ for( int i=2; i<fun.nIns(); i++ )
+ ((CallNode)fun.in(i)).unlink(fun,i--);
+ restoreRPC(fun);
+ }
+ code.releaseUnlinkedNodes();
+ for( CompUnit cu : code._compunits.values() )
+ if( cu._start!=null && !cu._start.isDead() ) detachEmptyStart(cu._start);
+
+ code._stop.walk(n -> {
+ if( n instanceof CallEndNode cend )
+ for( int i=1; i<cend.nIns(); i++ )
+ if( cend.in(i) instanceof ReturnNode )
+ cend.delDef(i--);
+ if( n instanceof CallNode call )
+ call.unlink_all();
+ return null;
+ });
+ }
+
+ private static void detachEmptyStart(StartCUNode start) {
+ // An unused unit can lose its outer Stop edge during optimization.
+ // Detach its empty wrapper from the scheduling root as well.
+ if( start.in(1) instanceof StopCUNode stop && stop.nIns()==0 ) start.setDef(0,null);
+ }
+
+ private static void restoreRPC(FunNode fun) {
+ if( fun.rpc()!=null ) return;
+ // A single-call target can have a folded RPC even when a multi-target
+ // call cannot inline it. Restore the parameter for codegen and for
+ // Eval2's continuation, now that calls no longer supply graph edges.
+ ReturnNode ret = fun.ret();
+ ParmNode rpc = new ParmNode("$rpc",0,ret.rpc()._type,fun);
+ rpc.addDef(ret.rpc());
+ ret.setDef(3,rpc.init());
+ }
+}