From 7914681413f3e3562733786aa02b1fc7adfbe9b0 Mon Sep 17 00:00:00 2001 From: Cliff Click Date: Mon, 10 Aug 2026 11:11:18 -0700 Subject: Chapter 28b: Separate Compilation Co-authored-by: Hsiang-Ying Fu <66062782+fuhsnn@users.noreply.github.com> Co-authored-by: fuhsnn <66062782+fuhsnn@users.noreply.github.com> Co-authored-by: Symious <14933944+symious@users.noreply.github.com> Co-authored-by: Jun Kawasaki Co-authored-by: Claude Opus 5 --- .../com/seaofnodes/simple/codegen/CodeGen.java | 193 +++++- .../com/seaofnodes/simple/codegen/CompUnit.java | 74 ++- .../com/seaofnodes/simple/codegen/ElfReader.java | 236 +++++++ .../com/seaofnodes/simple/codegen/ElfWriter.java | 9 +- .../com/seaofnodes/simple/codegen/FinishCalls.java | 57 -- .../com/seaofnodes/simple/codegen/GlobalBits.java | 177 ++++++ .../simple/codegen/GlobalCodeMotion.java | 2 +- .../com/seaofnodes/simple/codegen/InstSelect.java | 2 +- .../java/com/seaofnodes/simple/codegen/Iter.java | 32 +- .../com/seaofnodes/simple/codegen/LocalIds.java | 33 - .../java/com/seaofnodes/simple/codegen/Module.java | 98 +++ .../java/com/seaofnodes/simple/codegen/Opto.java | 2 +- .../com/seaofnodes/simple/codegen/ParseAll.java | 181 +++++- .../com/seaofnodes/simple/codegen/Serialize.java | 687 +++++++++++++++++++++ .../java/com/seaofnodes/simple/codegen/Unlink.java | 112 ++++ 15 files changed, 1734 insertions(+), 161 deletions(-) create mode 100644 src/main/java/com/seaofnodes/simple/codegen/ElfReader.java delete mode 100644 src/main/java/com/seaofnodes/simple/codegen/FinishCalls.java create mode 100644 src/main/java/com/seaofnodes/simple/codegen/GlobalBits.java delete mode 100644 src/main/java/com/seaofnodes/simple/codegen/LocalIds.java create mode 100644 src/main/java/com/seaofnodes/simple/codegen/Module.java create mode 100644 src/main/java/com/seaofnodes/simple/codegen/Serialize.java create mode 100644 src/main/java/com/seaofnodes/simple/codegen/Unlink.java (limited to 'src/main/java/com/seaofnodes/simple/codegen') diff --git a/src/main/java/com/seaofnodes/simple/codegen/CodeGen.java b/src/main/java/com/seaofnodes/simple/codegen/CodeGen.java index e6bace8c..8302ede6 100644 --- a/src/main/java/com/seaofnodes/simple/codegen/CodeGen.java +++ b/src/main/java/com/seaofnodes/simple/codegen/CodeGen.java @@ -29,8 +29,10 @@ public class CodeGen { IterPostOpto, // Peepholes and inlining using SCCP results FinalizeIR, // Field layout, offset cleanup, function/CU roots TypeCheck, // Last check for bad programs + Serialize, // Save ideal IR, including opaque class initialization Lower, // Expand checked class initialization to ordinary IR LoopTree, // Build a loop tree; break infinite loops + Unlink, // Unlink call sites before machine code generation Select, // Convert to target hardware nodes Schedule, // Global schedule (code motion) nodes LocalSched, // Local schedule @@ -64,7 +66,12 @@ public class CodeGen { public final String _modDir; // Module Build Root public final String _buildDir; - // Slashed path from source root to the entry file, without ".smp". + // Search path from CWD for `.o` files containing external symbols and types. + // This can contain archives, zips and other .o file containers. + public final Ary _externPaths; + + // Dotted path from module root to file containing the source, sans ".smp". + // e.g. module root: "sys", nested source name "sys.io". public final String _srcName; // Only available for tests, otherwise source comes from the CompUnit @@ -73,8 +80,12 @@ public class CodeGen { // Current Working Directory; default module base private final String _cwd; - // Source files discovered in this invocation. All live initializers - // participate in cycle checking, including fully inlined bodies. + // Compilation Units used in this compile; typically one per source/object + // file, although you can nest structs and get more than one. These are all + // required to compile, and are expected to have their source timestamps + // ahead of their obj file. Some of these may entirely be dead or inlined + // in this CompUnit and thus not needed in any linking stage. All the + // alive ones (inlined away or not) participate in class init cycle checks. public HashMap _compunits; // Test shortcut for only one compilation unit @@ -88,7 +99,7 @@ public class CodeGen { // True when fidx is published through a public class or instance field. public boolean publicFIDX(int fidx) { - if( fidx < LocalIds.RESERVED ) return false; + if( fidx < GlobalBits.RESERVED ) return false; // A public class initializer is itself an external root; unlike // constructors and methods, it is not reached through a class field. FunNode fun = _linker.atX(fidx); @@ -142,7 +153,7 @@ public class CodeGen { void freezePublicInterface() { assert _publicFIDXs==null && _publicAliases==null; int[] fidxs = XInt.EMPTY; - for( int fidx=LocalIds.RESERVED; fidx<_linker._len; fidx++ ) + for( int fidx=GlobalBits.RESERVED; fidx<_linker._len; fidx++ ) if( publicFIDX(fidx) ) fidxs = XInt.make(fidxs,fidx); int[] aliases = XInt.EMPTY; @@ -185,15 +196,15 @@ public class CodeGen { // Test setup; no module nor file; can alter seed & argument; can re-run same CodeGen public CodeGen( String src, Type arg ) { - this(null,null,null, src, 126L, true, arg); + this(null,null,null,null, src, 126L, true, arg); } // Test setup; no module nor file; can alter seed & argument; can re-run same CodeGen public CodeGen( String src, long workListSeed, boolean resetTypes ) { - this(null,null,null, src, workListSeed, resetTypes, TypeInteger.BOT); + this(null,null,null,null, src, workListSeed, resetTypes, TypeInteger.BOT); } // Generic CodeGen, including full module setup - public CodeGen( String modDir, String buildDir, + public CodeGen( String modDir, String buildDir, Ary externPaths, String srcName, String src, long workListSeed, boolean resetTypes, Type arg ) { // Public singleton to avoid passing about this state to a huge count // of places. Probably becomes a TLS at some point. @@ -203,6 +214,7 @@ public class CodeGen { _cwd = System.getProperty("user.dir")+"/"; _modDir = modDir == null ? _cwd : modDir; _buildDir = buildDir == null ? _cwd : buildDir; + _externPaths = externPaths; _srcName = srcName; _callingConv = null; // Calling convention // Source code from test strings, not files @@ -225,7 +237,9 @@ public class CodeGen { } - // Run the requested phases. ELF output contains machine code and data. + // Run requested phases. File export must request Serialize before Lower; + // requesting Export up front does that automatically. Encoding alone does + // not retain reloadable ideal IR. // No code emission, just IR generation public CodeGen driver( Phase phase ) { return driver(phase,null,null,false,false,0); } @@ -245,7 +259,8 @@ public class CodeGen { // The Generic driver public CodeGen driver( Phase target, String cpu, String callingConv, boolean inMemory, boolean emitEntrySymbol, int dump ) { - // Parse the entry source and discover all referenced source files. + // Parse a Simple file, and for mentioned external symbols either load + // from .o or recursively parse step(target, Phase.Parse, this::parse, dump); // Run peepholes to a fixed point step(target, Phase.Iter, this::iter, dump); @@ -257,12 +272,16 @@ public class CodeGen { step(target, Phase.FinalizeIR, this::finalizeIR, dump); // Final error check - bad programs fail here step(target, Phase.TypeCheck, this::typeCheck, dump); + // Save reloadable ideal IR for a later seperate compilation + step(target, Phase.Serialize, () -> Serialize.forTarget(this,target,inMemory), dump); // Replace "fat" nodes with their exploded explicit IR; this can // introduce CFG diamonds step(target, Phase.Lower, this::lower, dump); // Compute the loop-tree, used for approximating exec frequencies for // scheduling and register allocation step(target, Phase.LoopTree, this::loopTree, dump); + // Drop imports for machine selection, retain them for ideal evaluation. + step(target, Phase.Unlink, () -> unlink(cpu!=null), dump); // Instruction selection step(target, Phase.Select, () -> instSelect(cpu,callingConv), dump); // Global Code Motion @@ -273,7 +292,7 @@ public class CodeGen { step(target, Phase.RegAlloc, this::regAlloc, dump); // Encode machine instructions into bits step(target, Phase.Encoding, this::encode, dump); - // Export machine code and data into one ELF file + // Export bits (and serialized IR) into ELF files step(target, Phase.Export, () -> exportELF(inMemory,emitEntrySymbol), dump); return this; } @@ -329,13 +348,28 @@ public class CodeGen { } - // Allocate dense alias/function/call IDs within this invocation. - // Class slots let a forward reference reserve an initializer's identity. - public final LocalIds _aliases = new LocalIds(); + // These next fields all 2-way map some *global* program feature to a + // local dense index, suitable for packing in BitSets. The mapping takes a + // global value - always a {source file/class} name, and an order number + // counting up per feature in the same file. The dense local index can be + // different in different compilations. + + // The global info allows loading the same remote class info from different + // paths and aligning them. Example: Compiling A loads pre-compiled B and + // C, both of which load pre-compiled D. The 2 different D loads unify via + // this global info. + + // These mappings are all trivial identities when compiling one file; they + // only become complex when loading separately compiled code. + + // Compute local function index (FIDX) from global function info. This is + // called *in order* during parsing, and that order is part of the global + // unique mapping + public final GlobalBits _aliases = new GlobalBits(); public int alias(String clz) { return _aliases.next(clz); } // C storage has one global alias per symbol, including across object files. - public int externAlias(String name) { return _aliases.next("C:"+name,LocalIds.RESERVED); } + public int externAlias(String name) { return _aliases.next("C:"+name,GlobalBits.RESERVED); } // External data addresses supplied by an in-memory linker (image offsets). public final HashMap _externDataAddresses = new HashMap<>(); @@ -343,14 +377,14 @@ public class CodeGen { // Compute local function index (FIDX) from global function info. This is // called *in order* during parsing, and that order is part of the global // unique mapping - public final LocalIds _fidxs = new LocalIds(); + public final GlobalBits _fidxs = new GlobalBits(); // Return local index for specific global file & order. Used to find fidxs for e.g. FREF class fcns public int fidx( String clz, int order ) { return _fidxs.next(clz, order); } // New fidx in the current file public int fidx( String clz ) { return fidx(clz, -1); } // Compute local RPC index from global RPC info, one per call - public final LocalIds _rpcs = new LocalIds(); + public final GlobalBits _rpcs = new GlobalBits(); public int rpc( String clz ) { return _rpcs.next(clz); } @@ -407,7 +441,6 @@ public class CodeGen { // TFPs all have exact single fidxs and their return is wiped to BOTTOM (so // the return is not part of the match). public final Ary _linker = new Ary<>(FunNode.class); - public final Ary _calls = new Ary<>(CallNode.class); // Extern function declarations; input is the FIDX assigned to the name. final HashMap _externFunc = new HashMap<>(); @@ -430,6 +463,8 @@ public class CodeGen { // --------------------------- // Parser object + public final Ary _calls = new Ary<>(CallNode.class); + public final Parser P; // Parse ASCII text into Sea-of-Nodes IR @@ -483,17 +518,20 @@ public class CodeGen { // --------------------------- // TypeCheck queued surviving CLInits during its existing error walk. public CodeGen lower() { - return phase(Phase.Lower, _phase==Phase.TypeCheck, () -> _iter.iterate(this)); + return phase(Phase.Lower, (_phase==Phase.TypeCheck || _phase==Phase.Serialize), () -> _iter.iterate(this)); } // --------------------------- // Build the loop tree; break never-exit loops public CodeGen loopTree() { - return phase(Phase.LoopTree, _phase==Phase.Lower, () -> { _start.buildLoopTree(_linker,_stop); FinishCalls.run(this); }); + return phase(Phase.LoopTree, _phase==Phase.Lower, () -> _start.buildLoopTree(_linker,_stop)); } // --------------------------- - + public BAOS _serial; + public void serialize() { + phase(Phase.Serialize, _phase==Phase.TypeCheck, () -> Serialize.serialize(this)); + } // --------------------------- // Code generation CPU target @@ -522,7 +560,12 @@ public class CodeGen { } // --------------------------- - + public void unlink() { + unlink(true); + } + private void unlink(boolean removeImports) { + phase(Phase.Unlink, _phase==Phase.LoopTree, () -> Unlink.run(this,removeImports)); + } // --------------------------- // Control Flow Graph in Reverse Post Order. @@ -530,7 +573,7 @@ public class CodeGen { // Global schedule (code motion) nodes public CodeGen GCM() { - return phase(Phase.Schedule, (_phase==Phase.LoopTree || _phase==Phase.Select), () -> GlobalCodeMotion.buildCFG(this)); + return phase(Phase.Schedule, (_phase==Phase.Unlink || _phase==Phase.Select), () -> GlobalCodeMotion.buildCFG(this)); } // --------------------------- @@ -576,6 +619,110 @@ public class CodeGen { // --------------------------- + // Search a external path list, each path is recursively searched for .o + // files, which are partially loaded and searched for public symbols. + + // This is a state machine which lazily searches down the set of search + // paths until the requested module is class is found. + + // Map from external Strings to either partially read Simple ElfFile or ExternNode + public final HashMap _externSymbols = new HashMap<>(); + + // State machine elements; the outermost element is _externPaths + private int _extPathIdx; // Search index into the extern path list + private final Ary _files = new Ary<>(File.class); // Files in the current extern path being searched + private int _extFileIdx; // Index into _files + + public ElfReader findExternalSimple( String name ) { + return findExternalSymbol(name) instanceof ElfReader elf ? elf : null; + } + + public ExternNode findExternal( String name ) { + return findExternalSymbol(name) instanceof ExternNode extern ? extern : null; + } + + // Search the search-path for the name; return an ElfReader if found in a + // Simple-made ELF; return an ExternNode if from another ELF, or null if + // not found. + private Object findExternalSymbol( String name ) { + // State Machine! + + while( true ) { + // Check for an immediate hit + switch( _externSymbols.get(name) ) { + + case ExternNode extern: + // Name maps to an ExternNode + return extern; + + case ElfReader elf: + return elf; + + case null: + // Name is unknown. Advance the file-system search, pulling + // out new ELF files from the search directory list and loading + // their published symbols. + + // The last directory was mined out, so get the next search + // directory and find all ELF-like files. + if( _extFileIdx >= _files._len ) { + if( _externPaths==null || _extPathIdx >= _externPaths._len ) + return null; // A true miss in the whole search path + loadExtPath(); // Load next batch file of files + break; + } + // Load and parse an ELF header + ElfReader elf = getElf(_files.at(_extFileIdx++)); + if( elf == null ) break; // Not an ELF + // Load public symbols from the ELF + elf.loadPublicSymbols(); + if( elf._strs==null ) break; // No .simple section with strings + // Put public symbol in global table; first names shadow all others + // Map from symbol string to ELFReader; most lookups + // will miss and no need to unpack ElfReader types + _externSymbols.putIfAbsent(elf._strs[1],elf); + + // Publish definitions only; referenced classes belong to other objects. + for( String fname : elf._units ) + _externSymbols.putIfAbsent(Parser.addClzPrefix(fname.replace('/','.')),elf); + break; + + default: throw Utils.TODO("should not reach here"); + } + } + } + + // Load all the .o files in the next external path. + private void loadExtPath() { + _files.clear(); + _extFileIdx = 0; + String extpath = _externPaths.at(_extPathIdx++); + File f = new File(extpath); + if( !f.isAbsolute() ) + f = new File(_cwd+extpath); + fillExtFiles(f); + } + + // Recursive search (TODO: gzip, archives) and gather all .o files. + private void fillExtFiles(File dir) { + if( dir.isDirectory() ) + for( File f : dir.listFiles() ) + fillExtFiles(f); + else if( dir.getName().endsWith(".o") ) + _files.add(dir); + } + + // Map from object file names to ElfReaders + public final HashMap _elfs = new HashMap<>(); + ElfReader getElf(File f) { + ElfReader elf = _elfs.get(f.getPath()); + if( elf != null ) + return elf; + _elfs.put(f.getPath(), elf = ElfReader.load(f,null)); + return elf; + } + + // --------------------------- public boolean _asmLittle=true; public String asm() { return asm(new SB()).toString(); } SB asm(SB sb) { diff --git a/src/main/java/com/seaofnodes/simple/codegen/CompUnit.java b/src/main/java/com/seaofnodes/simple/codegen/CompUnit.java index c8b0a82c..71126795 100644 --- a/src/main/java/com/seaofnodes/simple/codegen/CompUnit.java +++ b/src/main/java/com/seaofnodes/simple/codegen/CompUnit.java @@ -12,23 +12,30 @@ import java.nio.file.Path; import java.util.*; import static com.seaofnodes.simple.codegen.CodeGen.CODE; -// A source file and its class. Parent links follow the source directory tree. +// A compilation-unit, which is an external reference and source and object +// files. These are mirrored over a file system tree (so form a tree), both +// the source tree and build/object tree. public class CompUnit { public final CompUnit _par; // Parent, e.g. A/B final String _fname; // Full slashed file name starting from module root, e.g. A/B/C final public String _cname; // Full dotted class name starting from module root, e.g. A.B.C final String _name; // Base name, e.g. C + final ExternNode _ext; // Found in external libs; e.g. libc final File _smp; // Simple source file, e.g. module_root/A/B/C.smp + public final File _obj; // ELF output file, e.g. build/A/B/C.o final public String _src; // Source code, from file or test case + // Original source path from an imported diagnostic location. Does not + // affect discovery/rebuild decisions, which use _smp and _src. + public String _sourceFile; public String sourceFile() { - return _smp!=null ? _smp.getAbsolutePath() : null; + return _smp!=null ? _smp.getAbsolutePath() : _sourceFile; } public TypeStruct _clz; // The one TypeStruct being published // _deps records units required for types, code, or class references, - // without requiring their initializers to run. These source + // without requiring their initializers to run. These discovery/rebuild // dependencies may form a cycle. Ary _deps; @@ -45,17 +52,16 @@ public class CompUnit { _fname= fname; _cname= cname; _name = name; - _smp = source(fname); + _ext = null; + _smp = new File( CODE. _modDir + "/" + fname + ".smp"); + _obj = new File( CODE._buildDir + "/" + fname + ".o" ); - _src = _smp.exists() ? Files.readString(_smp.toPath()) : null; - } - - // The bundled library is ordinary source. A local sys tree can replace it; - // this chapter never loads Simple code from an object file. - static File source(String fname) { - File file=new File(CODE._modDir,fname+".smp"); - if( file.exists() || !(fname.equals("sys") || fname.startsWith("sys/")) ) return file; - return new File("src/main/smp",fname+".smp"); + boolean mustCompile = _smp.exists() && // Has source to compile + ( fname.equals(CODE._srcName) || // Explicit input is a compilation request + !_obj.exists() || // No object file + _smp.lastModified() > _obj.lastModified() || // Out-of-date with source? + checkDependentObjs() ); // Out-of-date with other objs? + _src = mustCompile ? new String(Files.readAllBytes(_smp.toPath())) : null; } // Test case. No source file. Source code passed in as a String. Fake @@ -65,11 +71,49 @@ public class CompUnit { _name = fname; _fname= fname; _cname= fname; + _ext = null; _smp = null; + _obj = null; _src = src; } - // Record source discovery without adding an initialization edge. + // External CompUnit, pulled in from an external library like libc + CompUnit( ExternNode ext, String symbol ) { + _par = null; + _name = symbol; + _fname= _name; + _cname= null; + _ext = ext; + _smp = null; + _obj = null; + _src = null; + } + + // External Simple object file, or namespace parent, found on an external search path. + CompUnit( File obj, CompUnit par, String fname, String cname, String name ) { + _par = par; + _name = name; + _fname= fname; + _cname= cname; + _ext = null; + _smp = null; + _obj = obj; + _src = null; + } + + // Return true if _obj is out-of-date relative to dependents. + private boolean checkDependentObjs() { + ElfReader elf = CODE.getElf(_obj); + elf.loadPublicSymbols(); + for( String dep : elf._deps ) { + File obj = new File(CODE._buildDir+"/"+dep+".o"); + if( obj.lastModified() > _obj.lastModified() ) // Out-of-date with deps? + return true; + } + return false; + } + + // Add a discovered dependency, for later writing out in an ELF file, ".simple" section void addDep(CompUnit dep) { if( _deps == null ) _deps = new Ary<>(CompUnit.class); _deps.add(dep); @@ -91,7 +135,9 @@ public class CompUnit { @Override public String toString() { String x = "{"+_cname; if( _smp!=null && _smp.exists()) x += ",hasSMP"; + if( _obj!=null && _obj.exists()) x += ",hasObj"; if( _src!=null ) x += ",hasSrc"; + if( _ext != null ) x += ",isExt"; x += "}"; return x; } diff --git a/src/main/java/com/seaofnodes/simple/codegen/ElfReader.java b/src/main/java/com/seaofnodes/simple/codegen/ElfReader.java new file mode 100644 index 00000000..10720300 --- /dev/null +++ b/src/main/java/com/seaofnodes/simple/codegen/ElfReader.java @@ -0,0 +1,236 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.Node; +import com.seaofnodes.simple.type.TypeMemPtr; +import com.seaofnodes.simple.type.TypeStruct; +import com.seaofnodes.simple.util.Ary; +import com.seaofnodes.simple.util.BAOS; +import com.seaofnodes.simple.util.Utils; +import java.io.*; +import java.nio.file.Files; +import java.util.HashMap; + +public class ElfReader { + + public final File _file; + public final byte[] _buf; + int _x; // Cursor into buffer + // ELF Header + public final ElfHeader _header; + // All sections + public final Section[] _sections; + // Simple section, if available + public final Section _simple; + // Byte array in/out stream - used as a reader + public final BAOS _bais; + // Compilation unit; written into FunNodes + public final CompUnit _compunit; + + // Strings from the Simple section, either small public set or whole set + public String[] _strs; + // Dependent file names + public String[] _deps; + // Compilation units defined by this object, independent of init analysis. + public String[] _units; + // Only published symbol this file + public TypeStruct _clz; + // Canonical named types, distinct from optimized same-named graph states. + public TypeStruct[] _pubTypes; + // Nodes read in this Elf + public Ary _nodes; + + // Load an Elf file (mostly lazily) + public static ElfReader load( File f, CompUnit cu ) { + try { return new ElfReader(f,cu); } + catch( IOException ioe ) { return null; } + } + + // Load the Simple section from the ELF file + private ElfReader( File f, CompUnit cu ) throws IOException { + _file = f; + _compunit = cu; + _buf = Files.readAllBytes(f.toPath()); + _header = new ElfHeader(); + // No attempt is made to parse the program header table. + _sections = new Section[_header._shnum]; + for( int i=0; i<_header._shnum; i++ ) + _sections[i] = new Section(i); + + // Load section names + Section secstr = _sections[_header._shstrndx]; + for( Section sec : _sections ) + if( sec._type != 0 ) + sec._name = secstr.str(sec._noff); + + Section simple = null; + for( Section sec : _sections ) + if( ".simple".equals(sec._name) ) + simple = sec; + _simple = simple; + _bais = new BAOS(_buf, (int)_simple._offset); + } + + // For Serialize self-test + ElfReader( BAOS bais ) { + _file = null; + _compunit = null; + _buf = bais.buf(); + _header = null; + _sections = null; + _simple = null; + _bais = bais; + } + + // Load the public symbols and dependent objs, but not the whole IR + void loadPublicSymbols() { + // Check already loaded and cached + if( _strs == null && _simple != null ) + // Read only the published public strings, for speed purposes when + // scanning a large count of ELF files but getting hits in only a + // few of them. i.e. optimizing (prematurely?) for the traditional + // case of linking a small file against a large library + Serialize.read_public_strs(this); + } + + // Loads the entire IR + public TypeStruct loadSimple(CodeGen code) { + Serialize.readAll(code, this, code._aliases, code._fidxs, code._rpcs); + return _clz; + } + + // Loads strings, mappings, and types, but not Nodes. Used after the + // public-string scan finds a matching Simple ELF, so parsing can resolve + // imported field/array aliases before the full graph is linked in later. + public TypeStruct loadPublicTypes(CodeGen code) { + Serialize.readTypes(code, this, code._aliases, code._fidxs, code._rpcs); + return _clz; + } + + int u1() { return _buf[_x++]&0xFF; } + int magic() { // Big endian + return (u1() << 24) | (u1() << 16) | (u1() << 8) | u1(); + } + int u2() { return u1() | (u1()<<8); } // Little endian + int u4() { // Little endian + return u1() | (u1() << 8) | (u1() << 16) | (u1() << 24); + } + long u8() { return u4() | ((long)u4()<<32); } + void skip(int skip) { _x += skip; } + + + public class ElfHeader { + final int _magic; + final boolean _format32; // Format32 or format64 + final byte _osabi; + final boolean _little; // True if little endian, false if big endian + final char _type; + final char _machine; + final long _entry; // Process entry point + final long _phoff; // Program header table offset, from file start + final long _shoff; // Section header table offset, from file start + final int _flags; // Flags + final int _phnum; // Number of program header entries + final int _shentsize; + final int _shnum; // Number of section header entries + final int _shstrndx; // Section header index contains section names + + + ElfHeader() throws IOException { + _x = 0; + if( (_magic = magic()) != 0x7F454C46 ) bad("magic number",_magic); + byte EI_CLASS = (byte)u1(); + if( EI_CLASS != 1 && EI_CLASS != 2 ) bad("EI_CLASS",EI_CLASS); + _format32 = (EI_CLASS==1); + byte EI_DATA = (byte)u1(); + if( EI_DATA != 1 && EI_DATA != 2 ) bad("EI_DATA",EI_DATA); + _little = EI_DATA == 1; + if( u1() != 1 ) bad("version",1); + if( (_osabi=(byte)u1()) != 0 ) bad("EI_VERSION",_osabi); + byte abiversion = (byte)u1(); + if( abiversion != 0 ) bad("EI_ABIVERSION",abiversion); + skip(7); // Pad + + // Endian matters now + if( !_little ) throw new IOException("TODO: Big endian"); + if( (_type = (char)u2()) != 1 ) bad("ET_REL",_type); + if( (_machine = (char)u2()) != 0x3E ) bad("AMD x86-64",_machine); + int ver = u4(); + if( ver != 1 ) bad("Version",ver); + // Format32 matters now + if( _format32 ) bad("Format",1); + _entry = u8(); // Process entry point + _phoff = u8(); // Program header table offset + _shoff = u8(); // Section header table offset + _flags = u4(); // Flags + int ehsize = u2(); + if( ehsize != 0x40 ) bad("EHSIZE",ehsize); + int phentsize = u2(); + if( phentsize != 0x38 && phentsize != 0 ) bad("PHENTSIZE",phentsize); + _phnum = u2(); + _shentsize = u2(); + if( _shentsize != 0x40 ) bad("SHENTSIZE",_shentsize); + _shnum = u2(); + // String section + _shstrndx = u2(); + assert _x == ehsize; + } + } + + public class Section { + final int _noff; // Offset in string section for name + final int _type; // Section type + final long _flags; + final long _addr; // Offset in virtual mem, if loaded + final long _offset; // Offset in file + final long _size; // Size in bytes + final int _link; // Associated section index, varies + final int _info; // Varies + final byte _align; // Log2 alignment + final int _esize; // Element size, if non-zero + String _name; + + Section(int sidx) throws IOException { + _x = (int)(_header._shoff + (sidx * _header._shentsize)); + _noff = u4(); + _type = u4(); + _flags = u8(); + _addr = u8(); + _offset= u8(); + _size = u8(); + _link = u4(); + _info = u4(); + long align = u8(); + if( align==0 ) align=1; + _align = (byte)Long.numberOfTrailingZeros(align); + if( (1L << _align ) != align ) + throw new IOException("Bad section alignment: "+align); + _esize = (int)u8(); + } + + String str(int off) { + assert _type == 3/*SHT_STRTAB*/; // String table section header + int start = (int)(_offset+off); + int x = start; + while( _buf[x++] !=0 ) ; + return new String(_buf,start,x-start-1); + } + } + + private static void bad(String msg, int val) throws IOException { + throw new IOException("Bad "+msg+": "+val); + } + + + + //public TypeMemPtr lookup( String name ) { + // if( _simple == null ) return null; + // BAOS baos = new BAOS(_buf, (int)_simple._offset); + // Ary strs = Serialize.read_strs(baos); + // + // + // + // Ary nodes = Serialize.read_types(baos); + // + // throw Utils.TODO("Elf load symbol "+name); + //} +} diff --git a/src/main/java/com/seaofnodes/simple/codegen/ElfWriter.java b/src/main/java/com/seaofnodes/simple/codegen/ElfWriter.java index c52209dd..5a280723 100644 --- a/src/main/java/com/seaofnodes/simple/codegen/ElfWriter.java +++ b/src/main/java/com/seaofnodes/simple/codegen/ElfWriter.java @@ -27,6 +27,7 @@ public class ElfWriter { static final int SHT_SYMTAB = 2; static final int SHT_STRTAB = 3; static final int SHT_RELA = 4; + static final int SHT_SIMPLE = 10; // Simple section final String _name; final int _type, _entsize, _index, _name_pos, _flags; @@ -275,6 +276,9 @@ public class ElfWriter { public static final int SECTION_HDR_SIZE = 64; // ------------------------------------------------------------------------ + public class SimpleSection extends DataSection { + SimpleSection(BAOS serial) { super(".simple", SHT_SIMPLE, serial, 0); } + } // ------------------------------------------------------------------------ @@ -300,6 +304,8 @@ public class ElfWriter { } public ElfWriter export(boolean emitEntrySymbol) { + if( _code._serial==null ) + throw new IllegalStateException("ELF export needs ideal IR: request Export from the driver, or Serialize before Lower"); // Sections are created in the order they are emitted. _sections = new Ary<>(Section.class); @@ -311,7 +317,7 @@ public class ElfWriter { _strtab = new StringSection(); // place all the symbols - int nlocals = 1/*null*/+9/*number of sections*/; + int nlocals = 1/*null*/+9/*number of sections*/+1/*Simple section*/; SymbolSection symbols = new SymbolSection(nlocals); // we've already constructed these entire sections in the encoding phase @@ -392,6 +398,7 @@ public class ElfWriter { } // Create a "Simple" section for Simple types and ir + SimpleSection simple = new SimpleSection(_code._serial); // Write the local section header symbols assert nlocals == 1+_sections._len; diff --git a/src/main/java/com/seaofnodes/simple/codegen/FinishCalls.java b/src/main/java/com/seaofnodes/simple/codegen/FinishCalls.java deleted file mode 100644 index 7ba696d3..00000000 --- a/src/main/java/com/seaofnodes/simple/codegen/FinishCalls.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.seaofnodes.simple.codegen; - -import com.seaofnodes.simple.node.*; - -// Detach analysis call edges after building the loop tree. Every live body -// belongs to this invocation; preserve it for scheduling and emission. -abstract class FinishCalls { - - - static void run(CodeGen code) { - - // The remaining passes assume all calls are unlinked; i.e. we are throwing - // away the Call Graph here. Functions only reachable from internal calls - // need to be re-hooked to stop/start less they go dead. - for( FunNode fun : code._linker ) { - // Already linked to start, not going dead - if( fun==null || fun.isDead() ) - continue; - - // Insert the post-Opto code-generation hook. Unlike an Opto-time - // Start input, this no longer carries unknown-caller semantics. - ReturnNode ret = fun.ret(); - StartNode start = fun._compunit._start; - assert start!=null; - if( fun.nIns() <= 1 || fun.in(1) != start ) { - fun.insertDef(1,start); - for( Node use : fun._outputs ) - if( use instanceof ParmNode parm ) - parm.insertDef(1,ConstantNode.raw(parm._type)); - if( fun._compunit._stop._inputs.find(ret) == -1 ) - fun._compunit._stop.addDef(ret); - } - // Unlink from Call - for( int i=2; i { - if( n instanceof CallEndNode cend ) - for( int i=1; i _clzOrder2Local = new HashMap<>(); + private final Ary _local2Clz = new Ary<>(String.class); + private final AryInt _local2Order = new AryInt(); + // Local dense index + private int _local; + + GlobalBits() { + _local = RESERVED; + // Reserved null-class bits are global identities and are never + // remapped. + int[] xs = new int[RESERVED]; + xs[0] = RESERVED; // Per-class next order counter + for( int i=0; i 0 ) xs[i] = i; + _local2Clz .setX(i,null); + _local2Order.setX(i,i); + } + _clzOrder2Local.put(null,xs); + } + + // Get a next local index for this clz + int next( String clz ) { return next(clz,-1); } + + // Given a clz and order, return an existing local index or make a new + // mapping and return that. + int next(String clz, int order ) { + // Get the per-clz Order->Local mapping + int[] xs = _clzOrder2Local.get(clz); + if( xs == null ) + xs = new int[]{RESERVED,1}; // Make, if it does not exist + // Caller does not have an order in mind, just take the next one stored + // in xs[0] + if( order == -1 ) + order = xs[0]++; + // Order exists from prior compilation, but not recorded locally + else if( order >= xs[0] ) + xs[0] = order+1; + // Grow as needed + while( xs.length <= order ) + xs = Arrays.copyOf(xs,xs.length*2); + // Have a local mapping already? + if( xs[order] != 0 ) + return xs[order]; + int local = _local++; + bind(local,clz,order); + return local; // The dense local index + } + + // Bind a specific dense local index to {clz,order}. Used by both fresh + // allocation and deserialization, where local indexes must not compact + // across holes because XInt bitsets are serialized against this space. + private void bind( int local, String clz, int order ) { + if( _local <= local ) + _local = local+1; + _local2Clz .setX(local,clz ); + _local2Order.setX(local,order); + + // Inline-only fidx holes are represented as {null,0}. They need the + // reverse mapping above, but no forward map because xs[0] is the + // per-class next-order counter. + if( clz==null && order==0 ) + return; + + int[] xs = _clzOrder2Local.get(clz); + if( xs == null ) + xs = new int[]{RESERVED,1}; + if( order >= xs[0] ) + xs[0] = order+1; + while( xs.length <= order ) + xs = Arrays.copyOf(xs,xs.length*2); + _clzOrder2Local.put(clz,xs); + xs[order] = local; + } + + // A file-local function that will inline, and never write to disk + public int nextInline( ) { + int local = _local++; + _local2Clz .setX(local,null); + _local2Order.setX(local,0); + return local; + } + // Get a next local index for the same clz as this old index + public int next( int old ) { return next(_local2Clz.at(old)); } + + // Map a file-local dense index to this compilation unit's local dense index. + public int map( GlobalBits fileBits, int fileLocal ) { + if( fileLocal >= fileBits._local2Clz.size() ) + throw new ArrayIndexOutOfBoundsException(""+fileLocal+" >= "+fileBits._local2Clz.size()); + return findOrNext(fileBits._local2Clz.at(fileLocal), fileBits._local2Order.at(fileLocal)); + } + + // Map a global {clz,order} to a local one; keep any existing mapping or + // make a new one as needed. + private int findOrNext(String clz, int order ) { + // Reserved null-class identities are not represented in xs[order], + // because xs[0] is the next-order counter. + if( clz==null && order < _local2Clz.size() && _local2Order.at(order)==order && _local2Clz.at(order)==null ) + return order; + int[] xs = _clzOrder2Local.get(clz); + if( xs != null && order < xs.length ) { + int local = xs[order]; + if( local < _local2Clz.size() && _local2Order.at(local)==order && Objects.equals(_local2Clz.at(local),clz) ) + return local; + } + return next(clz,order); + } + + // Stable native symbol identity, independent of this compilation's dense IDs. + String owner(int local) { return _local2Clz.at(local); } + String symbol(int local) { + return _local2Clz.at(local)+"$F$"+_local2Order.at(local); + } + + // Write out to BAOS enough bits to unwind the local back to global index + void packed( BAOS baos, HashMap strs ) { + // Unwind a local index to the clz. + baos.packed2(_local2Clz.size()); + for( int i=RESERVED; i<_local2Clz.size(); i++ ) { + String clz = _local2Clz.at(i); + baos.packed2(clz==null ? 0 : strs.get(clz)); + baos.packed2( _local2Order.at(i) ); + } + } + + void gather( HashMap strs ) { + for( int i=RESERVED; i<_local2Clz.size(); i++ ) + Serialize.gather(strs,_local2Clz.at(i)); + } + + // Read in the BAIS bits needed to make a file-local -> global mapping + static GlobalBits packed( BAOS bais, String[] strs ) { + var gb = new GlobalBits(); + int n = bais.packed2(); + for( int i=RESERVED; i 0 ) sb.p(','); + sb.p(i).p(" <-> ").p(_local2Clz.at(i)).p('#').p(_local2Order.at(i)); + } + return sb.p(']').toString(); + } + +} diff --git a/src/main/java/com/seaofnodes/simple/codegen/GlobalCodeMotion.java b/src/main/java/com/seaofnodes/simple/codegen/GlobalCodeMotion.java index 4477025b..09eb8d33 100644 --- a/src/main/java/com/seaofnodes/simple/codegen/GlobalCodeMotion.java +++ b/src/main/java/com/seaofnodes/simple/codegen/GlobalCodeMotion.java @@ -60,7 +60,7 @@ public abstract class GlobalCodeMotion { if( use==null || globals.containsKey(use) ) continue; FunNode fun = useFun(use); if( fun==null ) continue; // Global metadata has no function owner. - assert CodeGen.CODE._mach==null || CodeGen.CODE.owns(fun); + assert CodeGen.CODE._mach==null || CodeGen.CODE.owns(fun); // Ideal evaluation includes imported bodies. var local = copies.computeIfAbsent(fun,f -> new IdentityHashMap<>()); Node copy = cloneGlobal(con,fun,globals,local); for( int i=0; i new InstSelect(code).run(cpu,callingConv,base)); } diff --git a/src/main/java/com/seaofnodes/simple/codegen/Iter.java b/src/main/java/com/seaofnodes/simple/codegen/Iter.java index 1ffd9737..c5120370 100644 --- a/src/main/java/com/seaofnodes/simple/codegen/Iter.java +++ b/src/main/java/com/seaofnodes/simple/codegen/Iter.java @@ -3,15 +3,19 @@ package com.seaofnodes.simple.codegen; import com.seaofnodes.simple.node.*; import com.seaofnodes.simple.type.*; -// Establish a pessimistic fixed point after parsing. Later Iter runs +// Establish a pessimistic fixed point after parsing/loading. Later Iter runs // reuse the worklist and types produced by the preceding phase. abstract class Iter { static void run(CodeGen code) { - // Close parser-in-progress states. Seed a fresh pessimistic solve; - // exposure sets start at FULL and can only shrink during Iter. + // The phase shift closes all parser-in-progress states and freezes + // unknown external inputs. Imported nodes carry post-Opto escape + // precision, but Iter is a fresh pessimistic solve: memory escape + // summaries must restart at FULL so newly discovered private escapes + // can only be removed, never added. Seed the complete graph once. + boolean hasImports = code._externPaths != null; code._start.walk(n -> { if( !(n instanceof ConstantNode) ) - n._type = iterEscapeReset(n._type); + n._type = hasImports ? iterImportReset(n._type) : iterEscapeReset(n._type); code.add(n); return null; }); @@ -34,5 +38,23 @@ abstract class Iter { return t; } - + private static Type iterImportReset(Type t) { + if( t.isHigh() ) + return t.dual(); + if( t instanceof TypeMem mem ) + return TypeMem.make(mem._alias,iterImportReset(mem._t),mem._one,mem._clz,mem._final,XInt.FULL,XInt.FULL); + if( t instanceof TypeMemPtr ) + return t.makeStorage(); + if( t instanceof TypeStruct ) + return t.makeStorage(); + if( t instanceof TypeTuple tt ) { + TypeTuple rez = tt; + for( int i=0; i _owners=new com.seaofnodes.simple.util.Ary<>(String.class); - private final HashMap _slots=new HashMap<>(); - - public int next(String owner) { return next(owner,-1); } - public int next(String owner, int slot) { - int[] ids=_slots.get(owner); - if( ids==null ) ids=new int[]{RESERVED,1}; - if( slot==-1 ) slot=ids[0]++; - else if( slot>=ids[0] ) ids[0]=slot+1; - if( slot>=ids.length ) ids=Arrays.copyOf(ids,Math.max(slot+1,ids.length*2)); - if( ids[slot]==0 ) { ids[slot]=_next++; _owners.setX(ids[slot],owner); } - _slots.put(owner,ids); - return ids[slot]; - } - public String owner(int id) { return _owners.atX(id); } - public String symbol(int id) { return owner(id)+".$fun"+id; } - public int nextInline() { return _next++; } - public int next(int old) { - int id=_next++; - _owners.setX(id,owner(old)); - return id; - } -} diff --git a/src/main/java/com/seaofnodes/simple/codegen/Module.java b/src/main/java/com/seaofnodes/simple/codegen/Module.java new file mode 100644 index 00000000..a5d85dbc --- /dev/null +++ b/src/main/java/com/seaofnodes/simple/codegen/Module.java @@ -0,0 +1,98 @@ +package com.seaofnodes.simple.codegen; + + +/*** + Modules!?!?!? + + Goal: under no circumstances can you load from an uninitialized variable + (although you can see default inits for fields that have defaults). + Goal: nested tree-structured name spaces, to control complexity. + Goal: field decl order within a file is fairly flexible. + + A Module a root-level SMP file, and the full file path must be provided on + the command line. All other Simple structs are relative to this file. + + Example: "simple /user/Dave/hello.smp" + + Defines a module "hello" in directory "/user/Dave", compiles and produces + a `hello` or `hello.exe` output file, and runs it. + + A SMP file has some file-init code which is static-global. + This code is executed on "first touch" of the file (defined below). + + Example: hello.smp contains: + sys.io.p("Hello World!"); + On first-touch, will print "Hello World!". + + A SMP file is a lexical scope, and code inside it can reference those names. + + Example: hello.smp contains: + val greet = "Hello World!"; // Declare file-scoped variable + sys.io.p(greet); // Reference file-scoped variable + + Static code is NOT allowed to have forward-references AND be executed + before the finishes initializing all variables. After var + initialization, a singleton instance of the implied file struct is made, + and then code can call anything. + + Example: + sys.io.p(greet); // Illegal forward reference + val greet = "Oops!"; + + Example: + val delay = { -> greet }; // Legal forward ref, but cannot execute (yet) + val greet = "Hello World!"; // Ref is resolved + sys.io.p(delay()); // Allowed to call delay() + val nprimes = { N -> + // compute number of primes < N + } + // Can otherwise execute any code in a + sys.io.p(nprimes(100)); + + Inside a directory, you can have sibling SMP files: + File hello.smp contains: + sys.io.p(I9A.GREETING); // File hello references sibling file I9A static globals + File I9A.smp contains: + val GREETING = translate("Hello World!"); + val translate = { str msg -> ... }; + + code cannot reference sibling files (nor child files, see below) + until all static variables are initialized. This prevents cyclic + initialization bugs, where file A uses vars from file B and vice-versa. + + File Chicken.smp contains: + val lay = Egg.hatch; // Illegal, defining var 'lay' after calling sibling Egg + File Egg.smp contains: + val hatch = Chicken.lay; // Illegal + + You can make cross references, if they get delayed until after all fields + are declared. + + File Chicken.smp contains: + int lay = 2; + lay = Egg.egg; + + File Egg.smp contains: + int egg = 3; + egg = Chicken.lay; + + The problem here, of course, is that you need to touch one of Chicken or + Egg first. + + + Inside a SMP file, you can define structs; structs are also a lexical + scope. Bare code in the struct is part of the struct's and only + runs when an instance is created. Structs do not have static (nor + ). Structs cannot have any forward references, and locally + declared functions cannot have both forward references and be executed (or + passed along to a possibly-executing function). + + In the same directory as a SMP file, Simple allows other directories with + names matching SMP files. Inside these dirs can be more SMP files that are + lexically nested, similar to embedded structs. + + +*/ + +public abstract class Module { +} diff --git a/src/main/java/com/seaofnodes/simple/codegen/Opto.java b/src/main/java/com/seaofnodes/simple/codegen/Opto.java index 4db277df..3b3f9f19 100644 --- a/src/main/java/com/seaofnodes/simple/codegen/Opto.java +++ b/src/main/java/com/seaofnodes/simple/codegen/Opto.java @@ -74,7 +74,7 @@ abstract public class Opto { // Start is a semantic unknown-caller edge. Public functions // keep it; closed-world functions must be reached by a // concrete Call edge discovered during SCCP. Initializers, - // including private ones, are called by opaque CLInit + // including private/imported ones, are called by opaque CLInit // obligations whose Call edges are introduced only in Lower. if( !fun.isClz() && !code.publicFIDX(fun.sig().fidx()) && diff --git a/src/main/java/com/seaofnodes/simple/codegen/ParseAll.java b/src/main/java/com/seaofnodes/simple/codegen/ParseAll.java index 171d9923..c55f9a90 100644 --- a/src/main/java/com/seaofnodes/simple/codegen/ParseAll.java +++ b/src/main/java/com/seaofnodes/simple/codegen/ParseAll.java @@ -28,21 +28,34 @@ public abstract class ParseAll { // Worklist of symbols to search and files to parse private static final Ary WORK = new Ary<>(CompUnit.class); + private static final Ary NEEDS_LOAD = new Ary<>(CompUnit.class); // Return a compilation unit (cached or new). If new, it does not necessarily // need to be compiled. public static CompUnit makeCUnit( CodeGen code, CompUnit par, String name ) throws IOException { - int idx=name.indexOf('/'); - if( idx>=0 ) return makeCUnit(code,makeCUnit(code,par,name.substring(0,idx)),name.substring(idx+1)); - String fname=par==null ? name : par._fname+"/"+name; - CompUnit cu=code._compunits.get(fname); - if( cu==null ) { - String cname=par==null ? name : par._cname+"."+name; - code._compunits.put(fname,cu=new CompUnit(par,fname,cname,name)); - } - return cu; + return makeCUnit(code,par,name,null,false); + } + + private static CompUnit makeCUnit( CodeGen code, CompUnit par, String name, File obj ) throws IOException { + return makeCUnit(code,par,name,obj,obj!=null); } + private static CompUnit makeCUnit( CodeGen code, CompUnit par, String name, File obj, boolean external ) throws IOException { + // Slashed file name? Move leading parts into parent CUs. + int idx = name.indexOf("/"); + if( idx >= 0 ) { + String head = name.substring(0,idx); + CompUnit sub = makeCUnit(code,par,head,external ? externalObj(code,par,head) : null,external); + return makeCUnit(code,sub,name.substring(idx+1).intern(),obj,external); + } + + String fname = par==null ? name : (par._fname + "/" + name).intern(); + CompUnit cunit = code._compunits.get(fname); + if( cunit != null ) return cunit; + String cname = par==null ? name : (par._cname + "." + name).intern(); + code._compunits.put(fname, cunit = external ? new CompUnit(obj,par,fname,cname,name) : new CompUnit(par,fname,cname,name)); + return cunit; + } // Parse a top-level string, used for e.g. testing static void parseSource( CodeGen code, String src ) { @@ -69,22 +82,29 @@ public abstract class ParseAll { } private static void enqueue(CompUnit cu) { - if( cu._par!=null ) enqueue(cu._par); + if( cu._par!=null && (cu._par._src!=null || (cu._par._obj!=null && cu._par._obj.exists())) ) enqueue(cu._par); if( cu._start==null && WORK.find(cu)==-1 ) WORK.add(cu); } private static void parseAll(CodeGen code, CompUnit cunit) { WORK.clear(); // Cleanup from prior tests + NEEDS_LOAD.clear(); // Prime the pump with the first CompUnit enqueue(cunit); // Work through all unparsed compilation units and parse while( !WORK.isEmpty() ) { CompUnit cu = WORK.remove(0); - if( cu._src==null ) - throw new RuntimeException("Missing source file '"+cu._fname+".smp'"); - parseOne(code,cu); + // No source code? Means we do not need to parse, but instead need to load the symbols as-if we parsed + if( cu._src==null || cu._src.isEmpty() ) + loadDeps(code,cu); // Load recursive dependents + else + parseOne( code, cu ); // Parse } + // Load all pre-compiled CompUnits + for( CompUnit cu : NEEDS_LOAD ) + loadCompUnit(code,cu); + // Many types will have been declared cyclically either internally or // across multiple compilation units. @@ -97,7 +117,8 @@ public abstract class ParseAll { for( TypeStruct ts : tss ) Parser.TYPES.put(ts._name,ts); for( CompUnit cu : code._compunits.values() ) - cu._clz = (TypeStruct)Parser.TYPES.get(Parser.addClzPrefix(cu._cname)); + if( cu._ext==null ) + cu._clz = (TypeStruct)Parser.TYPES.get(Parser.addClzPrefix(cu._cname)); // Walk over all Nodes, and upgrade the internal constants to the // closed-over types. @@ -135,6 +156,9 @@ public abstract class ParseAll { // Adds sort by NIDs, which just got shuffled if( n instanceof AddNode add ) code.add(add); + // All imported bodies are loaded; link them before Iter checks types. + if( n instanceof CallNode call && !NEEDS_LOAD.isEmpty() ) + code.add(call).idealize(); return null; } ); @@ -157,6 +181,8 @@ public abstract class ParseAll { // Search the module for the fref try { CompUnit xcunit = findCUnitModule(code,cunit,fref._name); + if( xcunit == null ) + xcunit = findCUnitExternalSimple(code,fref._name); if( xcunit != null ) { // Replace the FRef with the discovered class name. fref._name = Parser.addClzPrefix(xcunit._cname); @@ -200,9 +226,88 @@ public abstract class ParseAll { if( xcunit != null ) return xcunit; } - return null; + return findCUnitExternalSimple(code,typeName); + } + + // Load the public symbols for this CompUnit, and add dependent CompUnits to the NEEDS_LOAD list. + private static void loadDeps(CodeGen code, CompUnit cunit) { + // Load IR after parsing + if( NEEDS_LOAD.find(cunit) != -1 ) + return; // Already on the to-be-loaded list? + for( CompUnit loaded : NEEDS_LOAD ) + if( loaded._obj.equals(cunit._obj) ) return; // Another class in the same object. + try { + NEEDS_LOAD.add(cunit); + // Load dependent classes + ElfReader elf = ElfReader.load(cunit._obj,cunit); + elf.loadPublicSymbols(); + // Register bundled definitions before resolving dependencies or IR. + for( String fname : elf._units ) + makeCUnit(code,null,fname,cunit._obj); + for( String fname : elf._deps ) { + // Check CompUnit for being up to date + CompUnit sub = code._compunits.get(fname); + if( sub==null ) sub=makeCUnit(code,null,fname,depObj(code,cunit,fname)); + // If not seen before or out of date, parse the new nested cunit + if( sub._clz == null && WORK.find(sub)==-1 && NEEDS_LOAD.find(sub)==-1 ) + enqueue(sub); + } + } catch( IOException ioe ) { + throw new RuntimeException(ioe); + } + } + + private static File depObj(CodeGen code, CompUnit cunit, String fname) { + if( cunit._smp != null ) + return null; + ElfReader elf = code.findExternalSimple(Parser.addClzPrefix(fname.replace('/','.'))); + if( elf == null ) + throw new RuntimeException("Cannot find external Simple dependency '"+fname+"'"); + return elf._file; + } + + private static File externalObj(CodeGen code, CompUnit par, String name) { + String cname = par==null ? name : par._cname + "." + name; + ElfReader elf = code.findExternalSimple(Parser.addClzPrefix(cname)); + return elf == null ? null : elf._file; } + private static void loadCompUnit(CodeGen code, CompUnit cunit) { + // Expecting a previously compiled object file + assert cunit._obj != null; + // Extract published symbols and put them in the internal tables; + // cross-references will pick them up there. + ElfReader elf = ElfReader.load(cunit._obj,cunit); + elf.loadSimple(code); + cunit._clz = (TypeStruct)Parser.TYPES.get(Parser.addClzPrefix(cunit._cname)); + StartNode lstart = (StartNode)elf._nodes.at(0); + StopNode lstop = ( StopNode)elf._nodes.last(); + + // Add all new loaded functions to the linker table + for( Node n : lstop._inputs ) { + StopCUNode stop = (StopCUNode)n; + StartCUNode start = stop.start(); + CompUnit cu = code._compunits.get(start._fname); + cu._start = start; + cu._stop = stop ; + + for( Node ret : stop._inputs ) { + FunNode fun = ((ReturnNode)ret).fun(); + code.link(fun); + } + } + + // Hook the new code into the existing code graph + for( Node lstopcu : lstop._inputs ) + code._stop.addDef(lstopcu); + lstart.subsume(code._start); + + // Add the new top-level loaded class + Parser.TYPES.put(cunit._clz._name,cunit._clz); + Field inst = cunit._clz.field(cunit._name); + if( inst != null && inst._t instanceof TypeFunPtr tfp && tfp._ret instanceof TypeMem mem ) + Parser.TYPES.put(cunit._cname,mem._t); + } // Symbol not found in source code (or we would not be here). // Search the module for the fref @@ -219,7 +324,7 @@ public abstract class ParseAll { CompUnit local = findCUnitModule(code,cu,symbol); if( local != null ) return local; - return null; + return findCUnitExternalSimple(code,symbol); } catch( IOException ioe ) { throw new RuntimeException(ioe); } @@ -235,12 +340,8 @@ public abstract class ParseAll { // However if A/B contains "Y", the search would be for A.B.Y then A.Y both of // which do not exist. private static CompUnit findCUnitModule( CodeGen code, CompUnit cunit, String symbol ) throws IOException { - if( cunit == null ) { - if( !symbol.equals("sys") || !CompUnit.source("sys").exists() ) return null; - CompUnit sys=makeCUnit(code,null,"sys"); - enqueue(sys); - return sys; - } + if( cunit == null ) // Ran off top of module scope? + return null; // Failed search // Check self if( cunit._name.equals(symbol) ) @@ -256,10 +357,11 @@ public abstract class ParseAll { String pre = symbol.substring(0,dotIdx); String rest = symbol.substring(dotIdx+1 ); String fname = cunit._fname + "/" + pre; - File f = CompUnit.source(fname); + File f = new File( code._modDir + "/" + fname + ".smp" ); if( f.exists() ) { CompUnit sub = makeCUnit(code,cunit,pre); - // enqueue also parses parents: their initialization must precede a child. + // Note that "sub" itself is not put on the to-be-parsed-list. We're + // referring to a child class and do not need to parse the middles. CompUnit dcunit = findCUnitModule(code,sub,rest); if( dcunit != null ) return dcunit; // Nested dot search found cunit @@ -268,7 +370,7 @@ public abstract class ParseAll { // Look in mod / fname / name / symbol.smp String fname = cunit._fname + "/" + symbol; - File f = CompUnit.source(fname); + File f = new File( code._modDir + "/" + fname + ".smp" ); if( f.exists() ) { CompUnit sub = makeCUnit(code,cunit,symbol); // If not seen before or out of date, parse the new nested cunit @@ -281,5 +383,34 @@ public abstract class ParseAll { return findCUnitModule( code, cunit._par, symbol ); } + // Search external Simple objects only. C linkage uses explicit "C" + // declarations, which produce ExternNodes and are resolved by the linker. + public static CompUnit findCUnitExternalSimple( CodeGen code, String symbol ) { + String clzName = symbol.startsWith(Parser.CLZ) ? symbol : Parser.addClzPrefix(symbol); + String cname = clzName.substring(Parser.CLZ.length()); + String fname = cname.replace('.','/'); + CompUnit cunit = code._compunits.get(fname); + if( cunit != null ) + return cunit; + + ElfReader elf = code.findExternalSimple(clzName); + if( elf == null ) + return null; + + try { + cunit = makeCUnit(code,null,fname,elf._file); + } catch( IOException ioe ) { + throw new RuntimeException(ioe); + } + ElfReader typedElf = ElfReader.load(cunit._obj,cunit); + if( typedElf != null ) typedElf.loadPublicTypes(code); + TypeStruct clz = (TypeStruct)Parser.TYPES.get(clzName); + if( clz == null ) + Parser.TYPES.put(clzName,clz = TypeStruct.make(clzName,true)); + cunit._clz = clz; + if( WORK.find(cunit) == -1 && NEEDS_LOAD.find(cunit) == -1 ) + enqueue(cunit); + return cunit; + } } diff --git a/src/main/java/com/seaofnodes/simple/codegen/Serialize.java b/src/main/java/com/seaofnodes/simple/codegen/Serialize.java new file mode 100644 index 00000000..5a74ded2 --- /dev/null +++ b/src/main/java/com/seaofnodes/simple/codegen/Serialize.java @@ -0,0 +1,687 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.Parser; +import com.seaofnodes.simple.node.*; +import com.seaofnodes.simple.type.*; +import com.seaofnodes.simple.util.*; +import java.util.*; + +abstract public class Serialize { + + // Encoding/emulation need only machine code. File export also needs ideal + // IR for a later compiler to import; an explicit Serialize request stops here. + static void forTarget(CodeGen code, CodeGen.Phase target, boolean inMemory) { + if( target==CodeGen.Phase.Serialize || (target.ordinal()>=CodeGen.Phase.Export.ordinal() && !inMemory) ) + code.serialize(); + } + + static void serialize( CodeGen code ) { + + // Get all Nodes in a sane order + Ary nodes = nodeOrder(code); + + // Array of dependent file names + HashSet xdeps = new HashSet<>(); + for( CompUnit cu : code._compunits.values() ) { + if( cu._deps != null ) + for( CompUnit cudep : cu._deps ) + xdeps.add(cudep._fname); + } + String[] deps = xdeps.toArray(new String[xdeps.size()]); + Arrays.sort(deps); + Ary owned = new Ary<>(String.class); + for( CompUnit cu : code._compunits.values() ) + if( cu._src!=null ) owned.add(cu._fname); + String[] units = owned.asAry(); + Arrays.sort(units); + + // Top-level clazz being reported + String topName = code._srcName == null ? "Test" : code._srcName; + CompUnit top = code._compunits.get(topName); + TypeStruct clz = top._clz; + + // Preserve which same-named structs are the source-level declarations. + // Optimized graph types can carry sharper field states (e.g. len==0), + // and cannot be used to reconstruct this mapping during loading. + TreeMap named = new TreeMap<>(); + for( Type type : Parser.TYPES.values() ) + if( type instanceof TypeStruct ts ) + named.put(ts._name,ts); + TypeStruct[] pubTypes = named.values().toArray(new TypeStruct[0]); + + // Compress into bytes + BAOS baos = write(nodes, clz, pubTypes, deps, units, code._aliases, code._fidxs, code._rpcs, code._externFunc); + + // --- Expensive bijection assert + if( false ) { + // Inflate into POJOs; renumbers everything + ElfReader elf = new ElfReader(new BAOS(baos.toByteArray())); + readAll(code,elf, code._aliases, code._fidxs, code._rpcs); + BAOS baos2 = write(elf._nodes,elf._clz,elf._pubTypes,elf._deps,elf._units, code._aliases, code._fidxs, code._rpcs, code._externFunc); + + // Bi-jection + for( int i=0; iglobal bit mappings. + aliases.gather(strs); + fidxs .gather(strs); + rpcs .gather(strs); + for( String extern : externFunc.values() ) + gather(strs,extern); + // Count strings from nodes + for( Node n : nodes ) + n.gather(strs); + // Radix sort by string ID# + String[] astrs = new String[strs.size()]; + for( String s : strs.keySet() ) + astrs[strs.get(s)] = s; + + // Check that published strings and types align + assert astrs[1] == ((TypeStruct)atypes[1])._name; + if( depobjs != null ) + for( int i=0; iglobal mappings + aliases.packed(baos,strs); + fidxs .packed(baos,strs); + rpcs .packed(baos,strs); + ArrayList externFidxs = new ArrayList<>(externFunc.keySet()); + Collections.sort(externFidxs); + baos.packed2(externFidxs.size()); + for( int fidx : externFidxs ) { + baos.packed2(fidx); + baos.packed2(strs.get(externFunc.get(fidx))); + } + + // C - Write unique Types + baos.packed4(atypes.length); + baos.packed2(writtenPubs._len); + for( TypeStruct ts : writtenPubs ) + baos.packed4(types.get(ts)); + // Write Types in ID# order, no children + for( Type t : atypes ) + t.packed(baos,strs); + // Write Types in ID# order, only child IDs + for( Type t : atypes ) { + int nkids = t.nkids(); + for( int i=0; i nodes) { + if( n instanceof ParmNode parm ) n=parm.fun(); + if( (n instanceof FunNode || n instanceof CallEndNode) && i>0 ) + return nodes.containsKey(n.in(i)); + if( n instanceof StopNode ) return nodes.containsKey(n.in(i)); + return true; + } + + public static int nIns(Node n, IdentityHashMap nodes) { + int len=0; + for( int i=0; i void gather( HashMap strs, T s) { + if( s!=null && !strs.containsKey(s) ) + strs.put(s,strs.size()); + } + public static void gather( AryInt is, int s) { + if( is.atX(s)==0 ) { + int cnt=0; + for( int i=1; i=0; + int slash=name.lastIndexOf('/'); + if( slash<0 ) return false; + name=name.substring(0,slash); + } + } + + static void read_public_strs(ElfReader elf) { + BAOS bais = elf._bais; + Type.TAGOFFS(); + + // A - Read a header + if( bais.read()!='C' || bais.read()!='0' || bais.read()!='D' || bais.read()!='E' ) + throw new IllegalArgumentException("Unsupported serialized IR format; rebuild the object file"); + + // B - Packed read of #strings, then strings + int ndependents = bais.packed4(); // Number of dependent object files + int nstrs = bais.packed4(); // Total number of strings + String[] strs = new String[nstrs]; + for( int i=0; iglobal mappings. Further down we will map + // Elf file-local -> global, then global -> CodeGen.CODE local + GlobalBits fileAliases = GlobalBits.packed(bais,strs); + GlobalBits fileFidxs = GlobalBits.packed(bais,strs); + GlobalBits fileRpcs = GlobalBits.packed(bais,strs); + int nexterns = bais.packed2(); + for( int i=0; i nodes = new Ary<>(Node.class); + CompUnit cu = null; + for( int i=0; i= GlobalBits.RESERVED ) { + if( def == null ) continue; + local = aliases.map(fileAliases,i); + } + n.setDefX(local,def); + // Call's output[0] is always the CallEnd + if( n instanceof CallEndNode cend && i==0 ) { + var outs = cend.call()._outputs; + outs.swap(0,outs._len-1); + } + } + // A little extra for FunNodes + if( n instanceof FunNode fun ) { + // Return is hooked like an edge + ReturnNode ret = (ReturnNode)nodes.at(bais.packed2()-1); + fun.setRet(ret); + ret._fun = fun; + } + } + // Add edge from Start to Stop + StartNode start = (StartNode)nodes.at(0); + StopNode stop = (StopNode)nodes.last(); + start.setDef(1,stop); + + assert check(code,nodes); + elf._nodes = nodes; + } + + // Check loaded types and nodes are consistent + private static boolean check( CodeGen code, Ary nodes ) { + // Loaded nodes are complete, unlike nodes still being built by the + // parser. Recompute them with post-parse semantics. The compilation- + // unit Stop is the exception: its escape type depends on the complete + // program's public interface, which ParseAll freezes only after every + // compilation unit has been loaded. + boolean good = true; + CodeGen.Phase phase = code._phase; + code._phase = CodeGen.Phase.Opto; + try { + for( Node n : nodes ) { + if( n instanceof StopCUNode && !code.publicInterfaceFrozen() ) + continue; + // Cross-object calls are relinked only after every body has + // loaded. Their saved return types cannot be recomputed yet. + if( n instanceof CallEndNode end && end.call().fptr()._type instanceof TypeFunPtr fp && + end.nIns()<=fp.nfcns() ) continue; + Type nval = n.compute(); + if( nval != n._type ) { + System.err.println("Node "+n+" read in as "+n._type+" but computes as "+nval); + good = false; + } + } + } finally { + code._phase = phase; + } + return good; + } + + // -------------------------------------------------- + public static Ary nodeOrder( CodeGen code, CompUnit cu ) { + BitSet visit = code.visit(); + Ary nodes = new Ary<>(Node.class); + visit.set(code._stop._nid); + // Walk all listed functions, in this single compilation unit + for( FunNode fun : code._linker ) + if( fun!=null && !fun.isDead() && fun._compunit == cu ) + _funWalk(fun,nodes,visit); + + // Just constants used by the listed functions + Ary cons = new Ary<>(Node.class); + cons.add(code._start); + for( Node n : code._start._outputs ) { + if( n instanceof ConstantNode con && !visit.get(con._nid) ) { + for( Node use : con.outs() ) { + // PtrToInt is another zero-code value rooted at a global + // constant. Its array-body Add is the value used by the + // function, so include that extra link in the ownership + // test as well. + if( use instanceof PtrToIntNode ptr ) + for( Node add : ptr.outs() ) { + for( Node addUse : add.outs() ) + conUsed(add,addUse,visit,cons); + conUsed(ptr,add,visit,cons); + } + // Constant used by somebody, so flag for output + conUsed(con,use,visit,cons); + } + } + } + // StartCU and memory projection + cons.add(cu._start); + for( Node n : cu._start._outputs ) + if( n instanceof ProjNode ) cons.add(n); + // All the innards of functions + cons.addAll(nodes); + // StopCU, Stop + cons.add(cu._stop); + cons.add(code._stop); + visit.clear(); + return cons; + } + + private static void conUsed( Node con, Node use, BitSet visit, Ary cons ) { + if( use == null || !visit.get(use._nid) || visit.get(con._nid) ) return; + cons.add(con); + visit.set(con._nid); + } + + // -------------------------------------------------- + public static Ary nodeOrder( CodeGen code ) { + Ary nodes = new Ary<>(Node.class); + BitSet visit = code.visit(); + nodes.add(code._start); + + // All the global constants + for( Node n : code._start._outputs ) { + if( n.isConst() ) { + visit.set( n._nid ); + nodes.add( n ); + // Special for Cast:BOT + for( Node cast : n._outputs ) + if( cast != null && cast.nIns() == 2 && cast.in( 0 ) == null ) { + visit.set( cast._nid ); + nodes.add( cast ); + // Array pointer conversion adds the body offset after + // the zero-code PtrToInt. Both nodes are global-value + // descendants of the constant and otherwise have no + // function-local root from which to walk them. + if( cast instanceof PtrToIntNode ) + for( Node add : cast.outs() ) + if( add instanceof AddNode && !visit.get(add._nid) ) { + visit.set(add._nid); + nodes.add(add); + } + } + } + } + + // All the CompUnits + for( CompUnit cu : code._compunits.values() ) { + // Whole CompUnits have no users, went dead + if( cu._src!=null && cu._start != null && !cu._start.isDead() ) { + nodes.add(cu._start); + // A memory-free, nonreturning unit can lose its memory projection. + ProjNode mem=cu._start.proj(1); + if( mem!=null ) nodes.add(mem); + // All the functions, including internal ones, in comp-unit order + for( FunNode fun : code._linker ) + if( fun!=null && !fun.isDead() && fun._compunit == cu ) + _funWalk(fun,nodes,visit); + nodes.add(cu._stop); + } + } + + nodes.add(code._stop); + visit.clear(); + return nodes; + } + + // Walk and print whole functions at a time, in CG order + private static void _funWalk( FunNode fun, Ary rpo, BitSet visit ) { + int start = rpo._len; + _funRPO(fun,rpo,visit); + // A nonreturning function still owns a Return, even before LoopTree + // inserts synthetic exits. No loop-depth ordering is needed on disk. + _funRPO(fun.ret(),rpo,visit); + int len = rpo._len - start; + for( int i=0; i< len>>1; i++ ) + rpo.swap(start+i,start+len-1-i); + } + + // Walk and gather RPO nodes. + private static void _funRPO(Node n, Ary rpo, BitSet visit ) { + if( visit.get(n._nid) ) return; // Been there, done that + visit.set(n._nid); // Stop recursion + // Walk outputs ordered + if( n instanceof CFGNode ncfg ) { + if( n instanceof IfNode iff ) { + CProjNode c0 = iff.cproj(0); + CProjNode c1 = iff.cproj(1); + _funRPO(c0,rpo,visit); + _funRPO(c1,rpo,visit); + } else if( n instanceof StopNode ) { + // nothing + } else { + // CFG to CFG before CFG to data + for( Node use : n._outputs ) + if( use instanceof CFGNode usecfg && !ncfg.skip(usecfg) ) // Do not walk from Call to Fun + _funRPO(use,rpo,visit); + } + // Walk CFG to data eventually + for( Node use : n._outputs ) + if( !(use instanceof CFGNode) ) + _funRPO(use,rpo,visit); + } else { + // Do not walk from a non-CFG to a CFG; CFGs walk to CFGs to + // preserve CFG order. Do not walk *into* Phis; wait for the + // Region to walk the Phis (otherwise you visit some of the + // Phis ahead of others forcing the Phis to be spread around). + for( Node use : n._outputs ) { + if( use!=null && !(use instanceof CFGNode) && !(use instanceof PhiNode) ) + _funRPO(use,rpo,visit); + } + } + + // If parent is a Multi, do not add (yet). + // If self is a Multi, add self and children. + if( multiChild(n) || n instanceof CallEndNode ) { + // nothing; delay until Multi-head or CallNode + } else if( n instanceof MultiNode ) { + // Now lump all multi/projections together + printMulti(n,rpo); + } else if( n instanceof CallNode call ) { + printMulti(call.cend(),rpo); + rpo.add(call); + } else if( n instanceof RegionNode ) { + // Now lump all Phis together + int old = rpo._len; + for( Node use : n._outputs ) + if( use instanceof PhiNode ) + rpo.add(use); + // Sort by label? Want Phis before control outputs + Arrays.sort( rpo._es, old, rpo._len, (x,y) -> y.label().compareTo(x.label()) ); + rpo.add(n); + } else if( n instanceof StopNode ) { + // nothing + } else { + rpo.add(n); // Post-order add + } + } + + private static boolean multiChild(Node n) { + return n instanceof Proj || n instanceof PhiNode; + } + + private static void printMulti(Node n, Ary rpo) { + // Now lump all multi/projections together + for( Node use : n._outputs ) + rpo.add(use); + // Sort by projection order + if( !(n instanceof StartNode) ) + Arrays.sort( rpo._es,rpo._len-n.nOuts(),rpo._len, + (x,y) -> x instanceof ProjNode px && y instanceof ProjNode py ? (py.idx() - px.idx()) : (y._nid - x._nid) ); + rpo.add(n); + } + +} diff --git a/src/main/java/com/seaofnodes/simple/codegen/Unlink.java b/src/main/java/com/seaofnodes/simple/codegen/Unlink.java new file mode 100644 index 00000000..fb1c4a6f --- /dev/null +++ b/src/main/java/com/seaofnodes/simple/codegen/Unlink.java @@ -0,0 +1,112 @@ +package com.seaofnodes.simple.codegen; + +import com.seaofnodes.simple.node.*; + +// Discard call-graph edges after LoopTree. Machine selection keeps only this +// object's bodies; ideal evaluation keeps imports so their calls can execute. +abstract class Unlink { + private static void unlinkImports(CodeGen code) { + // Imported function constants keep their symbolic identity, not an + // embedded copy of the other object's Return and body. + for( Node n : code._start._outputs.asAry() ) + if( n instanceof FunPtrNode ptr && ptr.ret()!=null && !code.owns(ptr.fun()) ) { + ptr.liftType(ptr._type); + ptr.setDef(1,null); + } + for( FunNode fun : code._linker ) { + if( fun==null || fun.isDead() || code.owns(fun) ) + continue; + for( int i=1; i 1 ) { + if( fun.in(1) instanceof CallNode call ) + call.unlink(fun,1); + else + fun.removeDeadPath(1); + } + StartCUNode start = fun._compunit._start; + StopCUNode stop = fun._compunit._stop; + // Unhook the Stop->Return edge also + ReturnNode ret = fun.ret(); + int idx = stop._inputs.find(ret); + if( idx != -1 ) { + stop._inputs.del(idx); + ret.delUse(stop); + } + // + if( stop.nIns()==0 ) { + idx = code._stop._inputs.find(stop); + if( idx != -1 ) + code._stop.delDef(idx); + if( start.nIns() > 0 && start.in(0) != null ) + start.setDef(0,null); + if( start.nIns() > 1 && start.in(1) != null ) + start.setDef(1,null); + } + continue; + } + + // Insert the post-Opto code-generation hook. Unlike an Opto-time + // Start input, this no longer carries unknown-caller semantics. + ReturnNode ret = fun.ret(); + StartNode start = fun._compunit._start; + assert start!=null; + if( fun.nIns() <= 1 || fun.in(1) != start ) { + fun.insertDef(1,start); + for( Node use : fun._outputs ) + if( use instanceof ParmNode parm ) + parm.insertDef(1,ConstantNode.raw(parm._type)); + if( fun._compunit._stop._inputs.find(ret) == -1 ) + fun._compunit._stop.addDef(ret); + } + // Unlink from Call + for( int i=2; i { + if( n instanceof CallEndNode cend ) + for( int i=1; i