Cleanup & backportclinit

The phase rework, made new phases explicit and add them to the CodeGen driver.  Fix the flag-spilling on X86 and ARM.  Fix the GCM conservative anti-deps and backport.

15 files changed, 313 insertions, 236 deletions

diff --git a/chapter22/src/main/java/com/seaofnodes/simple/codegen/CodeGen.java b/chapter22/src/main/java/com/seaofnodes/simple/codegen/CodeGen.java
index 8a3d6322..e5ad596d 100644
--- a/chapter22/src/main/java/com/seaofnodes/simple/codegen/CodeGen.java
+++ b/chapter22/src/main/java/com/seaofnodes/simple/codegen/CodeGen.java
@@ -9,7 +9,6 @@ import com.seaofnodes.simple.type.*;
 import java.io.IOException;
 import java.util.BitSet;
 import java.util.HashMap;
-import java.util.IdentityHashMap;
 
 @SuppressWarnings("unchecked")
 public class CodeGen {
@@ -20,19 +19,63 @@ public class CodeGen {
     public static CodeGen CODE;
 
     public enum Phase {
-        Parse,                  // Parse ASCII text into Sea-of-Nodes IR
-        Opto,                   // Run ideal optimizations
-        TypeCheck,              // Last check for bad programs
-        LoopTree,               // Build a loop tree; break infinite loops
-        Select,                 // Convert to target hardware nodes
-        Schedule,               // Global schedule (code motion) nodes
-        LocalSched,             // Local schedule
-        RegAlloc,               // Register allocation
-        Encoding,               // Encoding
-        Export,                 // Export
+        Parse,                  // Build ideal IR from source
+        Iter,                   // Pessimistic peepholes
+        FinalizeIR,             // Remove unused internal function entries
+        TypeCheck,              // Check the surviving ideal graph
+        LoopTree,               // Discover loops and connect nonreturning paths
+        Select,                 // Select machine instructions
+        Schedule,               // Global code motion
+        LocalSched,             // Order instructions within blocks
+        RegAlloc,               // Assign registers and stack slots
+        Encoding,               // Encode instructions
+        Export;                 // Link in memory or write ELF
+
     }
     public Phase _phase;
 
+    // Run through target, skipping phases already completed.
+    public CodeGen driver(Phase target) { return driver(target,null,null); }
+    public CodeGen driver(Phase target, String cpu, String callingConv) { return driver(target,cpu,callingConv,0); }
+    public CodeGen driver(Phase target, String cpu, String callingConv, int dump) {
+        if( target.ordinal()>=Phase.RegAlloc.ordinal() && cpu==null && _mach==null )
+            throw new IllegalArgumentException("Machine code requires a CPU target");
+        if( target==Phase.Export ) throw new IllegalArgumentException("Use exportELF with an output file or null for memory linking");
+        step(target, Phase.Parse, this::parse, dump);
+        step(target, Phase.Iter, this::iter, dump);
+        step(target, Phase.FinalizeIR, this::finalizeIR, dump);
+        step(target, Phase.TypeCheck, this::typeCheck, dump);
+        step(target, Phase.LoopTree, this::loopTree, dump);
+        step(target, Phase.Select, () -> instSelect(cpu,callingConv), dump);
+        step(target, Phase.Schedule, this::GCM, dump);
+        step(target, Phase.LocalSched, this::localSched, dump);
+        step(target, Phase.RegAlloc, this::regAlloc, dump);
+        step(target, Phase.Encoding, this::encode, dump);
+        return this;
+    }
+
+    private void step(Phase target, Phase next, Runnable work, int dump) {
+        if( next.ordinal()>target.ordinal() || (_phase!=null && _phase.ordinal()>=next.ordinal()) ) return;
+        Phase prior = _phase;
+        work.run();
+        if( _phase!=prior && (dump & (1<<_phase.ordinal()))!=0 ) {
+            if( (dump & (1<<30))!=0 ) System.err.println("After "+_phase+":");
+            System.err.println(this);
+        }
+    }
+
+    public final long[] _times = new long[Phase.values().length];
+    private CodeGen phase(Phase next, boolean ordered, Runnable work) {
+        assert ordered : "Cannot run "+next+" after "+_phase;
+        _phase = next;
+        long t0 = System.currentTimeMillis();
+        try { work.run(); }
+        finally { _times[next.ordinal()] = System.currentTimeMillis()-t0; }
+        if( _obs!=null ) _obs.phase(next.name());
+        return this;
+    }
+
+
     // ---------------------------
     // Compilation source code
     public final String _src;
@@ -54,22 +97,6 @@ public class CodeGen {
     }
 
 
-    // All passes up to Phase, except ELF
-    public CodeGen driver( Phase phase ) { return driver(phase,null,null); }
-    public CodeGen driver( Phase phase, String cpu, String callingConv ) {
-        if( _phase==null )                       parse();
-        int p1 = phase.ordinal();
-        if( _phase.ordinal() < p1 && _phase.ordinal() < Phase.Opto      .ordinal() ) opto();
-        if( _phase.ordinal() < p1 && _phase.ordinal() < Phase.TypeCheck .ordinal() ) typeCheck();
-        if( _phase.ordinal() < p1 && _phase.ordinal() < Phase.LoopTree  .ordinal() ) loopTree();
-        if( _phase.ordinal() < p1 && _phase.ordinal() < Phase.Select    .ordinal() && cpu != null ) instSelect(cpu,callingConv);
-        if( _phase.ordinal() < p1 && _phase.ordinal() < Phase.Schedule  .ordinal() ) GCM();
-        if( _phase.ordinal() < p1 && _phase.ordinal() < Phase.LocalSched.ordinal() ) localSched();
-        if( _phase.ordinal() < p1 && _phase.ordinal() < Phase.RegAlloc  .ordinal() ) regAlloc();
-        if( _phase.ordinal() < p1 && _phase.ordinal() < Phase.Encoding  .ordinal() ) encode();
-        return this;
-    }
-
     // Run all the phases through final ELF emission
     public CodeGen driver( String cpu, String callingConv, String obj ) throws IOException {
         return driver(Phase.Encoding,cpu,callingConv).exportELF(obj);
@@ -145,16 +172,10 @@ public class CodeGen {
     public final Parser P;
 
     // Parse ASCII text into Sea-of-Nodes IR
-    public int _tParse;
     public CodeGen parse() {
-        assert _phase == null;
-        _phase = Phase.Parse;
-        long t0 = System.currentTimeMillis();
-
-        P.parse();
-        _tParse = (int)(System.currentTimeMillis() - t0);
-        if( _obs != null ) _obs.phase("Parse");
-        return this;
+        return phase(Phase.Parse, _phase==null, () -> {
+            P.parse();
+        });
     }
 
 
@@ -169,80 +190,25 @@ public class CodeGen {
     public void iterCnt() { if( !_midAssert ) _iter_cnt++; }
     public void iterNop() { if( !_midAssert ) _iter_nop_cnt++; }
 
-    // Run ideal optimizations
-    public int _tOpto;
-    public CodeGen opto() {
-        assert _phase == Phase.Parse;
-        _phase = Phase.Opto;
-        long t0 = System.currentTimeMillis();
+    // Pessimistic peepholes reach a fixed point after parsing.
+    public CodeGen iter() { return phase(Phase.Iter, _phase==Phase.Parse, () -> _iter.iterate(this)); }
 
-        // Pessimistic peephole optimization on a worklist
-        _iter.iterate(this);
-
-        // Not really a true optimistic pass, but look for unlinked functions.
-        // This can be removed, which may trigger another round of pessimistic.
-        // This is the point where we flip from a virtual Call Graph (any call
-        // can call any function) to having a correct (but conservative) CG.
-
-        FunNode main = link(_main);
-        for( int i=0; i<_start.nOuts(); i++ ) {
-            Node use = _start.out(i);
-            if( use instanceof FunNode fun &&
-                fun.nIns()==2 && fun.in(1)==_start && fun != main && !fun.hasPtr() &&
-                    (fun._name==null || fun._name.startsWith("sys.")) ) {
-                add(fun).setDef(1,Parser.XCTRL);
-                addAll(fun._outputs);
-                i--;
-            }
-        }
-        _iter.iterate(this);
-
-        _tOpto = (int)(System.currentTimeMillis() - t0);
-
-        // TODO:
-        // Optimistic
-        // TODO:
-        // loop unroll, peel, RCE, etc
-        if( _obs != null ) _obs.phase("Opto");
-        return this;
-    }
+    public CodeGen finalizeIR() { return phase(Phase.FinalizeIR, _phase==Phase.Iter, () -> FinalizeIR.run(this)); }
     public <N extends Node> N add( N n ) { return (N)_iter.add(n); }
     public void addAll( Ary<Node> ary ) { _iter.addAll(ary); }
 
 
     // ---------------------------
     // Last check for bad programs
-    public int _tTypeCheck;
-    public CodeGen typeCheck() {
-        // Demand phase Opto for cleaning up dead control flow at least,
-        // required for the following GCM.
-        assert _phase.ordinal() <= Phase.Opto.ordinal();
-        _phase = Phase.TypeCheck;
-        long t0 = System.currentTimeMillis();
-
-        Parser.ParseException err = _stop.walk(n -> {
-            Parser.ParseException msg = n.err();
-            if( msg != null && _obs != null ) _obs.error(n, msg.getMessage());
-            return msg;
-        });
-        _tTypeCheck = (int)(System.currentTimeMillis() - t0);
-        if( err != null )
-            throw err;
-        if( _obs != null ) _obs.phase("TypeCheck");
-        return this;
-    }
+    public CodeGen typeCheck() { return phase(Phase.TypeCheck, _phase==Phase.FinalizeIR, () -> TypeCheck.run(this)); }
 
     // ---------------------------
     // Build the loop tree; break never-exit loops
-    public int _tLoopTree;
     public CodeGen loopTree() {
-        assert _phase.ordinal() <= Phase.TypeCheck.ordinal();
-        _phase = Phase.LoopTree;
-        long t0 = System.currentTimeMillis();
-        // Build the loop tree, fix never-exit loops
-        _start.buildLoopTree(_start,_stop);
-        _tLoopTree = (int)(System.currentTimeMillis() - t0);
-        return this;
+        return phase(Phase.LoopTree, _phase==Phase.TypeCheck, () -> {
+            // Build the loop tree, fix never-exit loops
+            _start.buildLoopTree(_start,_stop);
+        });
     }
 
     // ---------------------------
@@ -264,138 +230,41 @@ public class CodeGen {
     public RegMask _rpcMask;
 
     // Convert to target hardware nodes
-    public int _tInsSel;
     public CodeGen instSelect( String cpu, String callingConv ) { return instSelect(cpu,callingConv,PORTS); }
     public CodeGen instSelect( String cpu, String callingConv, String base ) {
-        assert _phase.ordinal() == Phase.LoopTree.ordinal();
-        _phase = Phase.Select;
-
-        _callingConv = callingConv;
-
-        // Not timing the class load...
-        // Look for CPU in fixed named place:
-        //   com.seaofnodes.simple.node.cpus."cpu"."cpu.class"
-        String clzFile = base+"."+cpu+"."+cpu;
-        try { _mach = ((Class<Machine>) Class.forName( clzFile )).getDeclaredConstructor(new Class[]{CodeGen.class}).newInstance(this); }
-        catch( Exception e ) { throw new RuntimeException(e); }
-
-        // Build global copies of common register masks.
-        long callerSave = _mach.callerSave();
-        long  neverSave = _mach. neverSave();
-        int maxReg = Math.min(64,_mach.regs().length);
-        assert maxReg>=64 || (-1L << maxReg & callerSave)==0; // No stack slots in callerSave
-        _callerSave = new RegMask(callerSave);
-
-        // Build a Return RegMask array.  All returns have the following inputs:
-        // 0 - ctrl
-        // 1 - memory
-        // 2 - varies with returning GPR,FPR
-        // 3 - RPC
-        // 4+- Caller save registers
-        _retMasks = new RegMask[(maxReg-_callerSave.size()-Long.bitCount( neverSave ))+4];
-        for( int reg=0, i=4; reg<maxReg; reg++ )
-            if( !_callerSave.test(reg) && ((1L<<reg)&neverSave)==0 )
-                _retMasks[i++] = new RegMask(reg);
-        _rpcMask = new RegMask(_mach.rpc());
-        _retMasks[3] = _rpcMask;
-
-        // Convert to machine ops
-        long t0 = System.currentTimeMillis();
-        _uid = 1;               // All new machine nodes reset numbering
-        var map = new IdentityHashMap<Node,Node>();
-        _instSelect( _stop, map );
-        _stop  = ( StopNode)map.get(_stop );
-        StartNode start = (StartNode)map.get(_start);
-        _start = start==null ? new StartNode(_start) : start;
-        _instOuts(_stop,visit());
-        _visit.clear();
-        _tInsSel = (int)(System.currentTimeMillis() - t0);
-        return this;
-    }
-
-    // Walk all ideal nodes, recursively mapping ideal to machine nodes, then
-    // make a machine node for "this".
-    private Node _instSelect( Node n, IdentityHashMap<Node,Node> map ) {
-        if( n==null ) return null;
-        Node x = map.get(n);
-        if( x !=null ) return x; // Been there, done that
-
-        // If n is a MachNode already, then its part of a multi-node expansion.
-        // It does not need instruction selection (already selected!)
-        // but it does need its inputs walked.
-        if( n instanceof MachNode ) {
-            for( int i=0; i < n.nIns(); i++ )
-                n._inputs.set(i, _instSelect(n.in(i),map) );
-            return n;
-        }
-
-        // Produce a machine node from n; map it to flag as done so stops cycles.
-        map.put(n, x=_mach.instSelect(n) );
-        // Stop retains the callable body. Its lifetime edge is not a machine operand.
-        if( n instanceof FunPtrNode ) x._inputs.pop();
-        // Walk machine op and replace inputs with mapped inputs
-        for( int i=0; i < x.nIns(); i++ )
-            x._inputs.set(i, _instSelect(x.in(i),map) );
-        if( x instanceof MachNode mach )
-            mach.postSelect(this);  // Post selection action
-
-        return x;
-    }
-
-    // Walk all machine Nodes, and set their output edges
-    private void _instOuts( Node n, BitSet visit ) {
-        if( visit.get(n._nid) ) return;
-        visit.set(n._nid);
-        for( Node in : n._inputs )
-            if( in!=null ) {
-                in._outputs.push(n);
-                _instOuts(in,visit);
-            }
+        if( cpu==null ) return this;
+        return phase(Phase.Select, _phase==Phase.LoopTree,
+                     () -> new InstSelect(this).run(cpu,callingConv,base));
     }
 
 
     // ---------------------------
     // Control Flow Graph in Reverse Post Order.
-    public int _tGCM;
     public Ary<CFGNode> _cfg = new Ary<>(CFGNode.class);
 
     // Global schedule (code motion) nodes
     public CodeGen GCM() {
-        assert _phase.ordinal() <= Phase.Select.ordinal();
-        _phase = Phase.Schedule;
-        long t0 = System.currentTimeMillis();
-
-        GlobalCodeMotion.buildCFG(this);
-        _tGCM = (int)(System.currentTimeMillis() - t0);
+        return phase(Phase.Schedule, (_phase==Phase.LoopTree || _phase==Phase.Select), () -> {
+            GlobalCodeMotion.buildCFG(this);
 
-        return this;
+        });
     }
 
     // ---------------------------
     // Local (basic block) scheduler phase, a classic list scheduler
-    public int _tLocal;
     public CodeGen localSched() {
-        assert _phase == Phase.Schedule;
-        _phase = Phase.LocalSched;
-        long t0 = System.currentTimeMillis();
-        ListScheduler.sched(this);
-        _tLocal = (int)(System.currentTimeMillis() - t0);
-        return this;
-     }
+        return phase(Phase.LocalSched, _phase==Phase.Schedule, () -> ListScheduler.sched(this));
+    }
 
 
     // ---------------------------
     // Register Allocation
-    public int _tRegAlloc;
     public RegAlloc _regAlloc;
     public CodeGen regAlloc() {
-        assert _phase == Phase.LocalSched;
-        _phase = Phase.RegAlloc;
-        long t0 = System.currentTimeMillis();
-        _regAlloc = new RegAlloc(this);
-        _regAlloc.regAlloc();
-        _tRegAlloc = (int)(System.currentTimeMillis() - t0);
-        return this;
+        return phase(Phase.RegAlloc, _phase==Phase.LocalSched, () -> {
+            _regAlloc = new RegAlloc(this);
+            _regAlloc.regAlloc();
+        });
     }
 
     // Human readable register name
@@ -411,27 +280,27 @@ public class CodeGen {
 
     // ---------------------------
     // Encoding
-    public int _tEncode;
     public Encoding _encoding;   // Encoding object
     public void preEncode() {  } // overridden by alternative ports
     public CodeGen encode() {
-        assert _phase == Phase.RegAlloc;
-        _phase = Phase.Encoding;
-        long t0 = System.currentTimeMillis();
-        _encoding = new Encoding(this);
-        preEncode();
-        _encoding.encode();
-        _tEncode = (int)(System.currentTimeMillis() - t0);
-        return this;
+        return phase(Phase.Encoding, _phase==Phase.RegAlloc, () -> {
+            _encoding = new Encoding(this);
+            preEncode();
+            _encoding.encode();
+        });
     }
 
     // ---------------------------
     // Exporting to external formats
     public CodeGen exportELF(String fname) throws IOException {
-        assert _phase == Phase.Encoding;
-        _phase = Phase.Export;
-        if( fname == null ) new LinkMem(this).link(); // In memory patching
-        else new ElfFile(this).export(fname); // External ELF file
+        assert _phase==Phase.Encoding;
+        _phase=Phase.Export;
+        long t0=System.currentTimeMillis();
+        try {
+            if( fname == null ) new LinkMem(this).link(); // In memory patching
+            else new ElfFile(this).export(fname); // External ELF file
+        } finally { _times[Phase.Export.ordinal()]=System.currentTimeMillis()-t0; }
+        if( _obs!=null ) _obs.phase(Phase.Export.name());
         return this;
     }
 
diff --git a/chapter22/src/main/java/com/seaofnodes/simple/codegen/FinalizeIR.java b/chapter22/src/main/java/com/seaofnodes/simple/codegen/FinalizeIR.java
new file mode 100644
index 00000000..0e7f3060
--- /dev/null
+++ b/chapter22/src/main/java/com/seaofnodes/simple/codegen/FinalizeIR.java
@@ -0,0 +1,30 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.Parser;
+import com.seaofnodes.simple.node.*;
+import com.seaofnodes.simple.type.*;
+import java.util.Arrays;
+
+// Finish the ideal graph before type checking and scheduling.
+abstract class FinalizeIR {
+    static void run(CodeGen code) {
+        // Not really a true optimistic pass, but look for unlinked functions.
+        // This can be removed, which may trigger another round of pessimistic.
+        // This is the point where we flip from a virtual Call Graph (any call
+        // can call any function) to having a correct (but conservative) CG.
+
+        FunNode main = code.link(code._main);
+        for( int i=0; i<code._start.nOuts(); i++ ) {
+            Node use = code._start.out(i);
+            if( use instanceof FunNode fun &&
+                fun.nIns()==2 && fun.in(1)==code._start && fun != main && !fun.hasPtr() &&
+                    (fun._name==null || fun._name.startsWith("sys.")) ) {
+                code.add(fun).setDef(1,Parser.XCTRL);
+                code.addAll(fun._outputs);
+                i--;
+            }
+        }
+        code._iter.iterate(code);
+
+    }
+}
diff --git a/chapter22/src/main/java/com/seaofnodes/simple/codegen/GlobalCodeMotion.java b/chapter22/src/main/java/com/seaofnodes/simple/codegen/GlobalCodeMotion.java
index 530336d6..1818be73 100644
--- a/chapter22/src/main/java/com/seaofnodes/simple/codegen/GlobalCodeMotion.java
+++ b/chapter22/src/main/java/com/seaofnodes/simple/codegen/GlobalCodeMotion.java
@@ -273,10 +273,11 @@ public abstract class GlobalCodeMotion {
                 lca = anti_dep(load,late[st._nid],st.cfg0(),lca,st,anti);
                 break;
             case PhiNode phi:
-                // Repeat anti-dep for matching Phi inputs.
-                // No anti-dep edges but may raise the LCA.
+                // Inputs carrying different memory might overwrite this alias.
+                // Raise the LCA for those paths; unchanged memory does not
+                // clobber the load. No scheduling edge to the Phi.
                 for( int i=1; i<phi.nIns(); i++ )
-                    if( phi.in(i)==load.mem() )
+                    if( phi.in(i)!=load.mem() )
                         lca = anti_dep(load,phi.region().cfg(i),load.mem().cfg0(),lca,null,anti);
                 break;
             case NewNode st:
@@ -293,7 +294,9 @@ public abstract class GlobalCodeMotion {
     private static CFGNode anti_dep( MemOpNode load, CFGNode stblk, CFGNode defblk, CFGNode lca, Node st, int[] anti ) {
         // A conditional writer can precede the load without dominating its late block.
         // Walk back to the first overlap with the load's placement range.
-        for( ; stblk != defblk.idom(); stblk = stblk.idom() ) {
+        // Another Phi input need not be dominated by this memory definition.
+        // Include its depth, and retain the sentinel for an entry's null idom.
+        for( ; stblk != defblk.idom() && stblk.idepth() >= defblk.idepth(); stblk = stblk.idom() ) {
             // Store and Load overlap, need anti-dependence
             if( anti[stblk._nid]==load._nid ) {
                 lca = stblk.domLCA(lca,null); // Raise Loads LCA
diff --git a/chapter22/src/main/java/com/seaofnodes/simple/codegen/InstSelect.java b/chapter22/src/main/java/com/seaofnodes/simple/codegen/InstSelect.java
new file mode 100644
index 00000000..e8e87785
--- /dev/null
+++ b/chapter22/src/main/java/com/seaofnodes/simple/codegen/InstSelect.java
@@ -0,0 +1,90 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.node.*;
+import java.util.BitSet;
+import java.util.IdentityHashMap;
+
+// Replace ideal nodes with machine nodes, then rebuild the copied graph's uses.
+class InstSelect {
+    private final CodeGen _code;
+    InstSelect(CodeGen code) { _code=code; }
+
+    @SuppressWarnings("unchecked")
+    void run(String cpu, String callingConv, String base) {
+        _code._callingConv = callingConv;
+
+        // Look for CPU in fixed named place:
+        //   com.seaofnodes.simple.node.cpus."cpu"."cpu.class"
+        String clzFile = base+"."+cpu+"."+cpu;
+        try { _code._mach = ((Class<Machine>) Class.forName( clzFile )).getDeclaredConstructor(new Class[]{CodeGen.class}).newInstance(_code); }
+        catch( Exception e ) { throw new RuntimeException(e); }
+
+        // Build global copies of common register masks.
+        long callerSave = _code._mach.callerSave();
+        long  neverSave = _code._mach. neverSave();
+        int maxReg = Math.min(64,_code._mach.regs().length);
+        assert maxReg>=64 || (-1L << maxReg & callerSave)==0; // No stack slots in callerSave
+        _code._callerSave = new RegMask(callerSave);
+
+        // Build a Return RegMask array.  All returns have the following inputs:
+        // 0 - ctrl
+        // 1 - memory
+        // 2 - varies with returning GPR,FPR
+        // 3 - RPC
+        // 4+- Caller save registers
+        _code._retMasks = new RegMask[(maxReg-_code._callerSave.size()-Long.bitCount( neverSave ))+4];
+        for( int reg=0, i=4; reg<maxReg; reg++ )
+            if( !_code._callerSave.test(reg) && ((1L<<reg)&neverSave)==0 )
+                _code._retMasks[i++] = new RegMask(reg);
+        _code._rpcMask = new RegMask(_code._mach.rpc());
+        _code._retMasks[3] = _code._rpcMask;
+
+        // Convert to machine ops
+        _code._uid = 1;               // All new machine nodes reset numbering
+        var map = new IdentityHashMap<Node,Node>();
+        _instSelect( _code._stop, map );
+        _code._stop  = ( StopNode)map.get(_code._stop );
+        StartNode start = (StartNode)map.get(_code._start);
+        _code._start = start==null ? new StartNode(_code._start) : start;
+        _instOuts(_code._stop,_code.visit());
+        _code._visit.clear();
+    }
+
+    private Node _instSelect( Node n, IdentityHashMap<Node,Node> map ) {
+        if( n==null ) return null;
+        Node x = map.get(n);
+        if( x !=null ) return x; // Been there, done that
+
+        // If n is a MachNode already, then its part of a multi-node expansion.
+        // It does not need instruction selection (already selected!)
+        // but it does need its inputs walked.
+        if( n instanceof MachNode ) {
+            for( int i=0; i < n.nIns(); i++ )
+                n._inputs.set(i, _instSelect(n.in(i),map) );
+            return n;
+        }
+
+        // Produce a machine node from n; map it to flag as done so stops cycles.
+        map.put(n, x=_code._mach.instSelect(n) );
+        // Stop retains the callable body. Its lifetime edge is not a machine operand.
+        if( n instanceof FunPtrNode ) x._inputs.pop();
+        // Walk machine op and replace inputs with mapped inputs
+        for( int i=0; i < x.nIns(); i++ )
+            x._inputs.set(i, _instSelect(x.in(i),map) );
+        if( x instanceof MachNode mach )
+            mach.postSelect(_code);  // Post selection action
+
+        return x;
+    }
+
+    private void _instOuts( Node n, BitSet visit ) {
+        if( visit.get(n._nid) ) return;
+        visit.set(n._nid);
+        for( Node in : n._inputs )
+            if( in!=null ) {
+                in._outputs.push(n);
+                _instOuts(in,visit);
+            }
+    }
+
+}
diff --git a/chapter22/src/main/java/com/seaofnodes/simple/codegen/LRG.java b/chapter22/src/main/java/com/seaofnodes/simple/codegen/LRG.java
index ec254a2e..6ece27a2 100644
--- a/chapter22/src/main/java/com/seaofnodes/simple/codegen/LRG.java
+++ b/chapter22/src/main/java/com/seaofnodes/simple/codegen/LRG.java
@@ -59,6 +59,8 @@ public class LRG {
 
     public int nadj() { return _adj==null ? 0 : _adj._len; }
 
+    public RegMask useMask() { return _machUse==null ? null : _machUse.regmap(_uidx); }
+
     void addNeighbor(LRG lrg) {
         if( _adj==null ) _adj = new Ary<>(LRG.class);
         _adj.push(lrg);
diff --git a/chapter22/src/main/java/com/seaofnodes/simple/codegen/RegAlloc.java b/chapter22/src/main/java/com/seaofnodes/simple/codegen/RegAlloc.java
index 5a641490..39c6dd88 100644
--- a/chapter22/src/main/java/com/seaofnodes/simple/codegen/RegAlloc.java
+++ b/chapter22/src/main/java/com/seaofnodes/simple/codegen/RegAlloc.java
@@ -361,7 +361,7 @@ public class RegAlloc {
         // If the minLoopDepth is less than the maxLoopDepth: for-all defs and
         // uses, if at minLoopDepth or lower, split after def and before use.
         for( Node n : _ns ) {
-            if( n instanceof SplitNode ) continue; // Ignoring splits; since spilling need to split in a deeper loop
+            if( n instanceof SplitNode split && !split.isNonStackSplit() ) continue; // Ordinary splits already reach stack storage.
             if( n.isDead() ) continue; // Some Clonable went dead by other spill changes
             // If this is a 2-address commutable op (e.g. AddX86, MulX86) and the rhs has only a single user,
             // commute the inputs... which chops the LHS live ranges' upper bound to just the RHS.
@@ -414,8 +414,8 @@ public class RegAlloc {
     }
 
     private static long ldepth( long ld, Node n, CFGNode cfg ) {
-        // Do not count splits
-        if( n instanceof SplitNode ) return ld;
+        // Ordinary splits already reach stack storage; register-only transfers do not.
+        if( n instanceof SplitNode split && !split.isNonStackSplit() ) return ld;
         // Collect min/max loop depth
         int min = (int)ld;
         int max = (int)(ld>>32);
diff --git a/chapter22/src/main/java/com/seaofnodes/simple/codegen/TypeCheck.java b/chapter22/src/main/java/com/seaofnodes/simple/codegen/TypeCheck.java
new file mode 100644
index 00000000..0997fd2f
--- /dev/null
+++ b/chapter22/src/main/java/com/seaofnodes/simple/codegen/TypeCheck.java
@@ -0,0 +1,15 @@
+package com.seaofnodes.simple.codegen;
+
+import com.seaofnodes.simple.Parser;
+
+abstract class TypeCheck {
+    static void run(CodeGen code) {
+        Parser.ParseException err = code._stop.walk(n -> {
+            Parser.ParseException msg = n.err();
+            if( msg != null && code._obs != null ) code._obs.error(n, msg.getMessage());
+            return msg;
+        });
+        if( err != null )
+            throw err;
+    }
+}
diff --git a/chapter22/src/main/java/com/seaofnodes/simple/node/SplitNode.java b/chapter22/src/main/java/com/seaofnodes/simple/node/SplitNode.java
index ff18b292..437a700a 100644
--- a/chapter22/src/main/java/com/seaofnodes/simple/node/SplitNode.java
+++ b/chapter22/src/main/java/com/seaofnodes/simple/node/SplitNode.java
@@ -10,6 +10,8 @@ public abstract class SplitNode extends MachConcreteNode {
     public final String _kind;  // Kind of split
     public final byte _round;
     public SplitNode(String kind, byte round, Node[] nodes) { super(nodes); _kind = kind; _round = round; }
+    // A register-only transfer may need another split to reach stack storage.
+    public boolean isNonStackSplit() { return false; }
     @Override public String op() { return "mov"; }
     @Override protected ExprPrinter<Node> _print1(ExprPrinter<Node> p) {
         p.p("mov(");
diff --git a/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/arm/SplitFlagsARM.java b/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/arm/SplitFlagsARM.java
new file mode 100644
index 00000000..3e045a6f
--- /dev/null
+++ b/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/arm/SplitFlagsARM.java
@@ -0,0 +1,21 @@
+package com.seaofnodes.simple.node.cpus.arm;
+
+import com.seaofnodes.isa.Arm64;
+import com.seaofnodes.simple.codegen.*;
+
+// NZCV transfers require a GPR. Further splits can spill that GPR to the stack.
+public class SplitFlagsARM extends SplitARM {
+    private static final RegMask MASK = new RegMask(arm.WR_BITS,1L);
+    SplitFlagsARM(String kind,byte round) { super(kind,round); }
+    @Override public boolean isNonStackSplit() { return true; }
+    @Override public RegMask regmap(int i) { return MASK; }
+    @Override public RegMask outregmap() { return MASK; }
+
+    @Override public void encoding(Encoding enc) {
+        short dst=enc.reg(this), src=enc.reg(in(1));
+        if( dst==src ) return;
+        if( src==arm.FLAGS ) enc.add4(Arm64.flags(dst,false));
+        else if( dst==arm.FLAGS ) enc.add4(Arm64.flags(src,true));
+        else super.encoding(enc);
+    }
+}
diff --git a/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/arm/SplitGPRARM.java b/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/arm/SplitGPRARM.java
new file mode 100644
index 00000000..46eff4ca
--- /dev/null
+++ b/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/arm/SplitGPRARM.java
@@ -0,0 +1,11 @@
+package com.seaofnodes.simple.node.cpus.arm;
+
+import com.seaofnodes.simple.codegen.*;
+
+// Keep a saved NZCV value in GPRs and stack slots through subsequent spills.
+public class SplitGPRARM extends SplitARM {
+    private static final RegMask MASK = new RegMask(arm.WR_BITS,-2L);
+    SplitGPRARM(String kind,byte round) { super(kind,round); }
+    @Override public RegMask regmap(int i) { return MASK; }
+    @Override public RegMask outregmap() { return MASK; }
+}
diff --git a/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/arm/arm.java b/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/arm/arm.java
index ddd9797b..be77eba9 100644
--- a/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/arm/arm.java
+++ b/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/arm/arm.java
@@ -269,7 +269,16 @@ public class arm extends Machine {
     @Override public int rpc() { return X30; }
 
     // Create a split op; any register to any register, including stack slots
-    @Override public SplitNode split(String kind, byte round, LRG lrg) {  return new SplitARM(kind,round);  }
+    @Override public SplitNode split(String kind, byte round, LRG lrg) {
+        RegMask def = lrg._machDef==null ? null : lrg._machDef.outregmap(), use = lrg.useMask();
+        if( (def!=null && def.size1() && def.test(FLAGS)) ||
+            (use!=null && use.size1() && use.test(FLAGS)) )
+            return new SplitFlagsARM(kind,round);
+        // Spilling the saved flags requires a GPR between NZCV and the stack.
+        if( lrg._machDef instanceof SplitFlagsARM || lrg._machDef instanceof SplitGPRARM )
+            return new SplitGPRARM(kind,round);
+        return new SplitARM(kind,round);
+    }
 
     // Return a MachNode unconditional branch
     @Override public CFGNode jump() {
diff --git a/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/x86_64_v2/SplitFlagsX86.java b/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/x86_64_v2/SplitFlagsX86.java
new file mode 100644
index 00000000..4650b535
--- /dev/null
+++ b/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/x86_64_v2/SplitFlagsX86.java
@@ -0,0 +1,25 @@
+package com.seaofnodes.simple.node.cpus.x86_64_v2;
+
+import com.seaofnodes.isa.X86;
+import com.seaofnodes.simple.codegen.*;
+
+// Flag snapshots can use GPRs or stack slots, but never XMM registers.
+public class SplitFlagsX86 extends SplitX86 {
+    private static final RegMask MASK = new RegMask(x86_64_v2.WR_BITS | (1L<<x86_64_v2.FLAGS) | x86_64_v2.SPILLS,-1L);
+    SplitFlagsX86(String kind,byte round) { super(kind,round); }
+    @Override public RegMask regmap(int i) { return MASK; }
+    @Override public RegMask outregmap() { return MASK; }
+
+    @Override public void encoding(Encoding enc) {
+        short dst=enc.reg(this), src=enc.reg(in(1));
+        if( dst==src ) return;
+        if( dst==x86_64_v2.FLAGS || src==x86_64_v2.FLAGS ) {
+            boolean toFlags = dst==x86_64_v2.FLAGS;
+            int reg = toFlags ? src : dst;
+            if( reg>=x86_64_v2.MAX_REG )
+                X86.flagsStack(enc,enc._fun.computeStackOffset(enc._code,reg),toFlags);
+            else
+                X86.flags(enc,reg,toFlags);
+        } else super.encoding(enc);
+    }
+}
diff --git a/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/x86_64_v2/SplitX86.java b/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/x86_64_v2/SplitX86.java
index 5cf668d1..3c4dde17 100644
--- a/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/x86_64_v2/SplitX86.java
+++ b/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/x86_64_v2/SplitX86.java
@@ -17,12 +17,10 @@ public class SplitX86 extends SplitNode {
     @Override public RegMask regmap(int i) { return x86_64_v2.SPLIT_MASK; }
     @Override public RegMask outregmap() { return x86_64_v2.SPLIT_MASK; }
 
-    // Need to handle 8 cases: . reg->reg, reg->xmm, reg->flags, xmm->reg, xmm->xmm, xmm->flags, flags->reg, flags->xmm,
-    // flags->flags.
+    // Ordinary GPR, XMM, and stack moves. Flags use SplitFlagsX86.
     @Override public void encoding( Encoding enc ) {
         short dst=enc.reg(this), src=enc.reg(in(1));
-        if( dst==x86_64_v2.FLAGS ) { X86.flags(enc,src,true); return; }
-        if( src==x86_64_v2.FLAGS ) { X86.flags(enc,dst,false); return; }
+        assert dst!=x86_64_v2.FLAGS && src!=x86_64_v2.FLAGS;
         boolean dstX=dst>=x86_64_v2.XMM_OFFSET, srcX=src>=x86_64_v2.XMM_OFFSET;
         // Stack spills
         if( dst >= x86_64_v2.MAX_REG ) {
diff --git a/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/x86_64_v2/x86_64_v2.java b/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/x86_64_v2/x86_64_v2.java
index 7ddd17d4..dc3f813e 100644
--- a/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/x86_64_v2/x86_64_v2.java
+++ b/chapter22/src/main/java/com/seaofnodes/simple/node/cpus/x86_64_v2/x86_64_v2.java
@@ -47,7 +47,7 @@ public class x86_64_v2 extends Machine {
     static RegMask DIV_MASK = new RegMask(RD_BITS & ~(1L<<RAX) & ~(1L<<RDX));
 
     static final long SPILLS = -(1L << MAX_REG);
-    static final RegMask SPLIT_MASK = new RegMask(WR_BITS | FP_BITS /* | (1L<<FLAGS)*/ | SPILLS, -1L );
+    static final RegMask SPLIT_MASK = new RegMask(WR_BITS | FP_BITS | SPILLS, -1L );
 
     // Load/store mask; both GPR and FPR
     static RegMask MEM_MASK = new RegMask(WR_BITS | FP_BITS);
@@ -238,7 +238,9 @@ public class x86_64_v2 extends Machine {
 
     // Create a split op; any register to any register, including stack slots
     @Override public SplitNode split(String kind, byte round, LRG lrg) {
-        return new SplitX86(kind, round);
+        return lrg._machDef!=null && lrg._machDef.outregmap().test(FLAGS)
+            ? new SplitFlagsX86(kind,round)
+            : new SplitX86(kind,round);
     }
 
     // Return a MachNode unconditional branch
diff --git a/chapter22/src/main/java/com/seaofnodes/simple/print/SimpleGraphObserver.java b/chapter22/src/main/java/com/seaofnodes/simple/print/SimpleGraphObserver.java
index 22b6beba..b6c09d6b 100644
--- a/chapter22/src/main/java/com/seaofnodes/simple/print/SimpleGraphObserver.java
+++ b/chapter22/src/main/java/com/seaofnodes/simple/print/SimpleGraphObserver.java
@@ -17,7 +17,7 @@ public class SimpleGraphObserver extends GraphCapture<Node> {
     @Override protected void compile(String src) {
         _code = new CodeGen(src);
         _code._obs = this;
-        _code.parse().opto().typeCheck();
+        _code.driver(CodeGen.Phase.TypeCheck);
     }
     @Override protected String phase() { return _code._phase.name(); }
     @Override protected int pos() { return _code._phase == CodeGen.Phase.Parse ? _code.P.pos() : -1; }