diff options
Diffstat (limited to 'chapter28b')
9 files changed, 333 insertions, 23 deletions
diff --git a/chapter28b/README.ja.md b/chapter28b/README.ja.md index d04e9ccb..a3221da5 100644 --- a/chapter28b/README.ja.md +++ b/chapter28b/README.ja.md @@ -85,10 +85,20 @@ java -ea -cp "build/classes/main:build/classes/test:lib/*" \ コンストラクタのフィールド更新はローカル値と Phi のままで、ヘルパーの出口に Store を 置きます。未完成のオブジェクトでも初期化済みフィールドは読めます。 アンダースコアは公開インターフェースの選択に使いますが、ソースのアクセス制限は -[レビュー待ち](../docs/chapter-backports.md)です。 +まだ実装されていません。 ## 検証と今後 +明示的に指定したソースは常にコンパイルします。他のローカルオブジェクトを再利用する前に、 +記録された依存関係をたどり、各オブジェクトにまとめられた全ソースを検査します。 +入力の欠落や、より新しいソース・依存オブジェクトがあれば、保存した型や IR を使わず +再解析します。A が B をインライン化していれば、B のオブジェクトが未更新でも、 +B のソース変更で A は無効になります。循環は訪問済み集合で処理します。 + +ライブラリは検査と読み込みで同じ検索パスを使います。ソースのないライブラリは整合した +組み合わせで提供する必要があります。時刻だけでは互換性は判断できません。入力ハッシュ、 +コンパイラ・ABI のバージョン、自動再リンクを含むビルドドライバは今後の課題です。 + リポジトリのルートから次を実行します。 ```sh @@ -100,9 +110,12 @@ make tests CHAPTERS=chapter28b オブジェクトにします。`make release` は jar と標準ライブラリを作り、`make view` は 共通のグラフビューアーを起動します。 +`IncrementalBuildTest` は時刻を明示的に設定し、再利用、間接的な変更、まとめられた +循環、入力の欠落、ライブラリ更新を検査します。変更後のネイティブ実行結果も +クリーンビルドと比較します。 + 保存形式は `C0DE` 以外のバージョン番号をまだ持ちません。コンパイラや形式を変えたら オブジェクトを再構築してください。高コストの往復一致検査は無効のままで、古い増分 ビルドの実験 `testModule0` も実行対象外です。 -[分割計画](../docs/chapter-split.md)の主要な七つのスナップショットはこれで揃います。 ネイティブライブラリの構築と最適化の調整は[付録](../docs/appendices/README.md)にまとめています。 diff --git a/chapter28b/README.md b/chapter28b/README.md index 189f2b7b..0a7baa68 100644 --- a/chapter28b/README.md +++ b/chapter28b/README.md @@ -116,10 +116,18 @@ normal inlining heuristics. The combined graph goes through the normal compilation phases, including peepholes, Opto and TypeCheck. Dependencies between `CompUnit`s are recorded. An explicitly requested source -file is always compiled. Other source files are recompiled if their object is -missing, their source is newer, or a recorded dependency's object in the build -directory is newer. These timestamp checks are not yet a complete incremental -build system, there is some modest future work needed to finish this. +file is always compiled. Before reusing another local object, `CompUnit` walks +its recorded dependencies and checks every source bundled into each local +object. A missing input or a newer source or dependency object forces parsing +again, before saved types or IR are imported. This includes transitive +dependencies: if A has inlined B, editing B's source invalidates A even while +B's object is unchanged. A visited set lets this check follow cycles. + +Library dependencies use the same search paths for checking and loading. +Object-only libraries must be supplied as a coherent build; timestamps alone +cannot establish their compatibility. These checks are not a complete build +driver: explicit requests still compile, and input hashes, compiler/ABI +versions, and automatic relinking remain future work. Loading currently works at the object granularity. Unrelated objects can stay unloaded, but selecting an object loads its entire saved graph and follows its @@ -163,6 +171,9 @@ constructors, initialization cycles across objects, shared static data, native function thunks, and native library/client execution. Make also builds the modular `sys` object used by the library tests. Clients can import that object with `-L`; they do not need to parse the standard library's source again. +`IncrementalBuildTest` checks reuse, transitive edits, bundled cycles, missing +inputs, and changed library objects using controlled timestamps. It also +compares native execution after an edit with a clean build. `make release` builds the launcher jar and standard library; `make view` starts the shared graph viewer. @@ -172,6 +183,5 @@ serialization bijection check remains disabled, and `testModule0`, the older incremental-rebuild experiment, remains ignored. They are not validation guarantees for this snapshot. -This completes the seven main snapshots in the [approved split](../docs/chapter-split.md). See the appendices on [native libraries](../docs/appendices/native-libraries.md) and [optimizer/backend tuning](../docs/appendices/optimizer-tuning.md). diff --git a/chapter28b/ainotes.txt b/chapter28b/ainotes.txt index c0b03a4d..5678e686 100644 --- a/chapter28b/ainotes.txt +++ b/chapter28b/ainotes.txt @@ -1,7 +1,7 @@ Historical design notes from the former combined Chapter 25, now Chapter 28b. -Current status and remaining work are in ../docs/stage-audit.md and -../docs/chapter-backports.md. The original intent below is preserved. +The cleanup record is in ../skills/stage-audit.md; remaining work is in +../docs/roadmap.md. The original intent below is preserved. Meta-info: I intend to keep extending and re-editing this "ainotes.txt" file so I can reprompt the you (the AI) between AI restarts, As such this file will diff --git a/chapter28b/src/main/java/com/seaofnodes/simple/codegen/CompUnit.java b/chapter28b/src/main/java/com/seaofnodes/simple/codegen/CompUnit.java index 71126795..8b303008 100644 --- a/chapter28b/src/main/java/com/seaofnodes/simple/codegen/CompUnit.java +++ b/chapter28b/src/main/java/com/seaofnodes/simple/codegen/CompUnit.java @@ -60,7 +60,7 @@ public class CompUnit { ( fname.equals(CODE._srcName) || // Explicit input is a compilation request !_obj.exists() || // No object file _smp.lastModified() > _obj.lastModified() || // Out-of-date with source? - checkDependentObjs() ); // Out-of-date with other objs? + !current(_obj,new HashSet<>()) ); // Out-of-date anywhere in its inputs? _src = mustCompile ? new String(Files.readAllBytes(_smp.toPath())) : null; } @@ -101,16 +101,42 @@ public class CompUnit { _src = null; } - // Return true if _obj is out-of-date relative to dependents. - private boolean checkDependentObjs() { - ElfReader elf = CODE.getElf(_obj); + // Check source-backed inputs before importing any saved types or IR. + // A dependency's source can be newer while its object is unchanged. + // Use a fresh visited set for each root: encountering a cycle is safe, but + // caching a provisional "current" answer from inside it would not be. + private static boolean current(File obj, HashSet<File> visit) { + if( obj==null || !obj.isFile() ) return false; + if( !visit.add(obj.getAbsoluteFile()) ) return true; + ElfReader elf = CODE.getElf(obj); + if( elf==null ) return false; elf.loadPublicSymbols(); + if( elf._units==null ) return false; + long time = obj.lastModified(); + for( String unit : elf._units ) { + // Never import another copy of a unit explicitly being compiled. + CompUnit cu = CODE._compunits.get(unit); + if( unit.equals(CODE._srcName) || (cu!=null && cu._src!=null) ) return false; + File src = new File(CODE._modDir+"/"+unit+".smp"); + if( !src.isFile() || src.lastModified()>time ) return false; + } for( String dep : elf._deps ) { - File obj = new File(CODE._buildDir+"/"+dep+".o"); - if( obj.lastModified() > _obj.lastModified() ) // Out-of-date with deps? - return true; + if( Arrays.asList(elf._units).contains(dep) ) continue; // Owned by this object. + File file = dependencyObj(dep,true); + if( file==null || !file.isFile() || file.lastModified()>time ) return false; + // Object-only libraries are supplied as a coherent build. Their + // input versions cannot be inferred from installation timestamps. + if( file.equals(new File(CODE._buildDir+"/"+dep+".o")) && !current(file,visit) ) return false; } - return false; + return true; + } + + // Resolve dependencies the same way for freshness checks and actual loads. + static File dependencyObj(String fname, boolean local) { + File obj = new File(CODE._buildDir+"/"+fname+".o"); + if( local && (obj.isFile() || new File(CODE._modDir+"/"+fname+".smp").isFile()) ) return obj; + ElfReader elf = CODE.findExternalSimple(Parser.addClzPrefix(fname.replace('/','.'))); + return elf==null ? null : elf._file; } // Add a discovered dependency, for later writing out in an ELF file, ".simple" section diff --git a/chapter28b/src/main/java/com/seaofnodes/simple/codegen/ParseAll.java b/chapter28b/src/main/java/com/seaofnodes/simple/codegen/ParseAll.java index c55f9a90..d7c0f7e2 100644 --- a/chapter28b/src/main/java/com/seaofnodes/simple/codegen/ParseAll.java +++ b/chapter28b/src/main/java/com/seaofnodes/simple/codegen/ParseAll.java @@ -258,12 +258,10 @@ public abstract class ParseAll { } private static File depObj(CodeGen code, CompUnit cunit, String fname) { - if( cunit._smp != null ) - return null; - ElfReader elf = code.findExternalSimple(Parser.addClzPrefix(fname.replace('/','.'))); - if( elf == null ) - throw new RuntimeException("Cannot find external Simple dependency '"+fname+"'"); - return elf._file; + File obj = CompUnit.dependencyObj(fname,cunit._smp!=null); + if( obj == null ) + throw new RuntimeException("Cannot find Simple dependency '"+fname+"'"); + return cunit._smp!=null && obj.equals(new File(code._buildDir+"/"+fname+".o")) ? null : obj; } private static File externalObj(CodeGen code, CompUnit par, String name) { diff --git a/chapter28b/src/test/java/com/seaofnodes/simple/Chapter19Test.java b/chapter28b/src/test/java/com/seaofnodes/simple/Chapter19Test.java index 6f290279..d5fe207a 100644 --- a/chapter28b/src/test/java/com/seaofnodes/simple/Chapter19Test.java +++ b/chapter28b/src/test/java/com/seaofnodes/simple/Chapter19Test.java @@ -11,6 +11,42 @@ import org.junit.Test; import static org.junit.Assert.*; public class Chapter19Test { + @Test public void testNarrowStores() { + // Return the allocation so Load forwarding cannot hide a bad Store. + for( String type : new String[]{"u8","i8","u16","i16","u32","i32","int","f32","flt"} ) { + boolean flt = type.equals("f32") || type.equals("flt"); + String init = flt ? "flt v=arg; " : ""; + if( type.equals("f32") ) init+="f32 w=v; "; + String value = type.equals("f32") ? "w" : flt ? "v" : "arg"; + for( boolean array : new boolean[]{false,true} ) { + String src = array + ? "var a=new "+type+"[1]; a[0]="+value+"; return a;" + : "struct S { "+type+" x; }; var s=new S; s.x="+value+"; return s;"; + CodeGen code = new CodeGen(init+src).driver(CodeGen.Phase.TypeCheck); + long[] args = flt ? new long[]{0,1,-1,16777217,-16777217} + : new long[]{0,1,-1,0x180,0x18000,0x180000000L,Long.MIN_VALUE,Long.MAX_VALUE}; + for( long arg : args ) { + Object expected = switch( type ) { + case "u8" -> arg & 0xffL; + case "i8" -> (long)(byte)arg; + case "u16" -> arg & 0xffffL; + case "i16" -> (long)(short)arg; + case "u32" -> arg & 0xffffffffL; + case "i32" -> (long)(int)arg; + case "f32" -> (double)(float)arg; + case "flt" -> (double)arg; + default -> arg; + }; + String msg = type+" array="+array+" arg="+arg; + String printed = array + ? (type.equals("u8") ? ""+(char)(long)(Long)expected : (type.equals("int") ? "i64" : type)+"[ "+expected+"]") + : "Test.S{x="+expected+"}"; + assertEquals(msg,printed,Eval2.eval(code,arg)); + } + } + } + } + @Test public void testPrintingFunctionLookup() { var code = new CodeGen("return 0;").parse(); var tfp = TypeFunPtr.make1((byte)2,false,new Type[]{TypeInteger.constant(987654)},TypeInteger.constant(123456),30); diff --git a/chapter28b/src/test/java/com/seaofnodes/simple/Chapter28bTest.java b/chapter28b/src/test/java/com/seaofnodes/simple/Chapter28bTest.java new file mode 100644 index 00000000..fb418b9a --- /dev/null +++ b/chapter28b/src/test/java/com/seaofnodes/simple/Chapter28bTest.java @@ -0,0 +1,47 @@ +package com.seaofnodes.simple; + +// +//import com.seaofnodes.simple.codegen.CodeGen; +//import com.seaofnodes.simple.codegen.RegAllocTestSupport.CheckedCodeGen; +//import com.seaofnodes.simple.codegen.ElfReader; +//import com.seaofnodes.simple.codegen.ParseAll; +//import com.seaofnodes.simple.node.CallNode; +//import com.seaofnodes.simple.node.FunNode; +//import com.seaofnodes.simple.node.LoopNode; +//import com.seaofnodes.simple.type.TypeInteger; +//import com.seaofnodes.simple.util.Ary; +//import java.io.File; +//import java.io.FileWriter; +//import java.io.IOException; +//import java.io.ByteArrayOutputStream; +//import java.io.PrintStream; +//import java.nio.charset.StandardCharsets; +//import java.nio.file.Files; +//import java.nio.file.Path; +import com.seaofnodes.simple.codegen.CodeGen; +import org.junit.Ignore; +import org.junit.Test; +// +import static org.junit.Assert.*; + +public class Chapter28bTest { + + @Ignore + @Test + public void testJig() { + String src = """ +struct C { + int x; + new C = { int n -> + while(n--) + if(n) x=1; + }; +}; +return new C(arg).x; +"""; + CodeGen code = new CodeGen(src).driver(CodeGen.Phase.Encoding,"riscv","SystemV"); + assertEquals("return (arg+3);", code.print()); + assertEquals("3",Eval2.eval(code,0)); + } + +} diff --git a/chapter28b/src/test/java/com/seaofnodes/simple/Eval2.java b/chapter28b/src/test/java/com/seaofnodes/simple/Eval2.java index 366bd7f4..901a8990 100644 --- a/chapter28b/src/test/java/com/seaofnodes/simple/Eval2.java +++ b/chapter28b/src/test/java/com/seaofnodes/simple/Eval2.java @@ -460,6 +460,14 @@ public abstract class Eval2 { ? offToIdx(x(st.off()),tmp._obj) : tmp._obj.find(st._name); Object val = val(st.val()); + // Idealization can remove conversions already performed by the store. + Type type = st.declaredType(); + if( type instanceof TypeInteger ti ) { + int shift = 64-(8 << st.log_size()); + long x = (Long)val << shift; + val = ti._min < 0 ? x >> shift : x >>> shift; + } else if( type instanceof TypeFloat && st.log_size()==2 ) + val = (double)(float)(double)(Double)val; fs[idx] = val; return "$mem"; } diff --git a/chapter28b/src/test/java/com/seaofnodes/simple/IncrementalBuildTest.java b/chapter28b/src/test/java/com/seaofnodes/simple/IncrementalBuildTest.java new file mode 100644 index 00000000..501392f3 --- /dev/null +++ b/chapter28b/src/test/java/com/seaofnodes/simple/IncrementalBuildTest.java @@ -0,0 +1,172 @@ +package com.seaofnodes.simple; + +import com.seaofnodes.simple.codegen.*; +import com.seaofnodes.simple.type.TypeInteger; +import com.seaofnodes.simple.util.Ary; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import org.junit.Test; +import static org.junit.Assert.*; + +public class IncrementalBuildTest { + @Test public void testTransitiveSourceChange() throws Exception { + Path base=Path.of("build/objs/incremental"); + Files.createDirectories(base); + Path root=Files.createTempDirectory(base,"chain-"); + Path src=Files.createDirectory(root.resolve("src")); + Path obj=Files.createDirectory(root.resolve("obj")); + write(src,"App","val marker=0;",1000); + write(src,"App/B","val value={ -> 5; };",1000); + write(src,"App/A","val value={ -> B.value()+1; };",1000); + write(src,"App/C","val value={ -> A.value()+1; };",1000); + write(src,"App/Main","val run={ -> C.value(); }; return run();",1000); + compile(src,obj,"App"); + stamp(obj.resolve("App.o"),1500); + compile(src,obj,"App/B"); + stamp(obj.resolve("App/B.o"),2000); + compile(src,obj,"App/A"); + stamp(obj.resolve("App/A.o"),3000); + compile(src,obj,"App/C"); + stamp(obj.resolve("App/C.o"),3500); + CodeGen code=client(src,obj); + assertNull("Reuse unchanged A",code._compunits.get("App/A")._src); + assertNull("Reuse unchanged B",code._compunits.get("App/B")._src); + assertNull("Reuse unchanged C",code._compunits.get("App/C")._src); + assertEquals("7",Eval2.eval(code,0)); + + write(src,"App/B","val value={ -> 7; };",4000); + code=client(src,obj); + assertEquals("9",Eval2.eval(code,0)); + assertNotNull(code._compunits.get("App/C")._src); + assertNotNull("Reparse A before loading its stale inlined body",code._compunits.get("App/A")._src); + assertNotNull(code._compunits.get("App/B")._src); + compile(src,obj,"App/Main"); + checkNative(obj,new Ary<>(new String[]{obj.resolve("App.o").toString()})); + + Path clean=Files.createDirectory(root.resolve("clean")); + assertEquals("9",Eval2.eval(client(src,clean),0)); + compile(src,clean,"App/Main"); + checkNative(clean,null); + } + + @Test public void testBundledSourceChangeAndCycle() throws Exception { + Path root=fixture("bundle-"); + Path src=root.resolve("src"), obj=root.resolve("obj"); + write(src,"App/A","val value={int n -> n ? B.value(n-1) : 1; };",1000); + write(src,"App/B","val value={int n -> n ? A.value(n-1) : 5; };",1000); + write(src,"App/Main","return A.value(1);",1000); + compile(src,obj,"App/A"); + stamp(obj.resolve("App/A.o"),3000); + assertFalse("A owns both sides of the cycle",Files.exists(obj.resolve("App/B.o"))); + CodeGen code=client(src,obj); + assertEquals("5",Eval2.eval(code,0)); + assertNull(code._compunits.get("App/A")._src); + assertNull(code._compunits.get("App/B")._src); + + write(src,"App/B","val value={int n -> n ? A.value(n-1) : 7; };",4000); + code=client(src,obj); + assertEquals("7",Eval2.eval(code,0)); + assertNotNull(code._compunits.get("App/A")._src); + assertNotNull(code._compunits.get("App/B")._src); + } + + @Test public void testMissingDependency() throws Exception { + Path root=fixture("missing-"); + Path src=root.resolve("src"), obj=root.resolve("obj"); + write(src,"App/A","val value={ -> B.value()+1; };",1000); + write(src,"App/B","val value={ -> 5; };",1000); + write(src,"App/Main","return A.value();",1000); + compile(src,obj,"App/B"); + stamp(obj.resolve("App/B.o"),2000); + compile(src,obj,"App/A"); + stamp(obj.resolve("App/A.o"),3000); + Files.delete(obj.resolve("App/B.o")); + CodeGen code=client(src,obj); + assertEquals("6",Eval2.eval(code,0)); + assertNotNull(code._compunits.get("App/A")._src); + assertNotNull(code._compunits.get("App/B")._src); + Files.delete(src.resolve("App/B.smp")); + try { + client(src,obj); + fail("Missing source and object must not reuse A's inlined body"); + } catch( RuntimeException e ) { + assertTrue(e.getMessage(),e.getMessage().contains("Undefined name 'B'")); + } + } + + @Test public void testLibraryDependency() throws Exception { + Path root=fixture("library-"); + Path src=root.resolve("src"), obj=root.resolve("obj"); + Path lib=Files.createDirectory(root.resolve("lib")); + var paths=new Ary<>(new String[]{lib.toString(),obj.toString()}); + library(lib,5); + stamp(lib.resolve("LibB.o"),2000); + write(src,"App/A","val value={ -> LibB.value()+1; };",1000); + write(src,"App/Main","return A.value();",1000); + new CodeGen(src.toString(),obj.toString(),paths,"App/A",null,123,true,TypeInteger.BOT) + .driver(TestC.CPU_PORT,TestC.CALL_CONVENTION,false,false); + stamp(obj.resolve("App/A.o"),3000); + CodeGen code=new CodeGen(src.toString(),obj.toString(),paths,"App/Main",null,123,true,TypeInteger.BOT) + .driver(CodeGen.Phase.TypeCheck); + assertEquals("6",Eval2.eval(code,0)); + assertNull("Unchanged library inputs permit reuse",code._compunits.get("App/A")._src); + + library(lib,7); + stamp(lib.resolve("LibB.o"),4000); + code=new CodeGen(src.toString(),obj.toString(),paths,"App/Main",null,123,true,TypeInteger.BOT) + .driver(CodeGen.Phase.TypeCheck); + assertEquals("8",Eval2.eval(code,0)); + assertNotNull(code._compunits.get("App/A")._src); + } + + private static void library(Path lib, int value) { + new CodeGen(null,lib.toString(),null,"LibB","val value={ -> "+value+"; };",123,true,TypeInteger.BOT) + .driver(TestC.CPU_PORT,TestC.CALL_CONVENTION,false,false); + } + + private static void checkNative(Path obj, Ary<String> libs) throws Exception { + Path driver=obj.resolve("driver.c"); + Files.writeString(driver,""" + __attribute__((CALL_CONV)) extern long long run(void) __asm__("App.Main.run"); + int main(void) { return run()==9 ? 0 : 1; } + """); + String exe=obj.resolve(TestC.OS.startsWith("Windows")?"test.exe":"test").toString(); + TestC.linkExe(obj.resolve("App/Main.o").toString(), + TestC.CALL_CONVENTION.equals("win64")?"ms_abi":"sysv_abi",driver.toString(),libs,exe); + assertEquals("",TestC.exec(TestC.TEST_TIMEOUT_SECONDS,exe)); + } + + private static Path fixture(String prefix) throws Exception { + Path base=Path.of("build/objs/incremental"); + Files.createDirectories(base); + Path root=Files.createTempDirectory(base,prefix); + Path src=Files.createDirectory(root.resolve("src")); + Path obj=Files.createDirectory(root.resolve("obj")); + write(src,"App","val marker=0;",1000); + compile(src,obj,"App"); + stamp(obj.resolve("App.o"),1500); + return root; + } + + private static void write(Path src, String name, String text, long time) throws Exception { + Path path=src.resolve(name+".smp"); + Files.createDirectories(path.getParent()); + Files.writeString(path,text); + stamp(path,time); + } + + private static void stamp(Path path, long time) throws Exception { + Files.setLastModifiedTime(path,FileTime.fromMillis(time)); + } + + private static void compile(Path src, Path obj, String name) { + new CodeGen(src.toString(),obj.toString(),null,name,null,123,true,TypeInteger.BOT) + .driver(TestC.CPU_PORT,TestC.CALL_CONVENTION,false,false); + } + + private static CodeGen client(Path src, Path obj) { + return new CodeGen(src.toString(),obj.toString(),null,"App/Main",null,123,true,TypeInteger.BOT) + .driver(CodeGen.Phase.TypeCheck); + } +} |