Chapter 25: Modules, Separate Compilation, and SSA Constructionlinear

Co-authored-by: Hsiang-Ying Fu <66062782+fuhsnn@users.noreply.github.com>
Co-authored-by: fuhsnn <66062782+fuhsnn@users.noreply.github.com>
Co-authored-by: Symious <14933944+symious@users.noreply.github.com>
Co-authored-by: Jun Kawasaki <jun784@gmail.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>

1 files changed, 202 insertions, 66 deletions

diff --git a/src/test/java/com/seaofnodes/simple/TestC.java b/src/test/java/com/seaofnodes/simple/TestC.java
index 3b3aabc0..b08be824 100644
--- a/src/test/java/com/seaofnodes/simple/TestC.java
+++ b/src/test/java/com/seaofnodes/simple/TestC.java
@@ -5,6 +5,8 @@ import com.seaofnodes.simple.codegen.CodeGen;
 import com.seaofnodes.simple.type.TypeInteger;
 import com.seaofnodes.simple.util.Ary;
 import com.seaofnodes.simple.util.Utils;
+
+import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -14,93 +16,171 @@ import static org.junit.Assert.*;
 // Call a C driver program, which calls a Simple-generated .o, and self-checks.
 // Expects GCC from the default CLI, compiles C, compiles Simple, links and runs.
 public abstract class TestC {
+    public static final int TEST_TIMEOUT_SECONDS = 60;
 
     public static final String OS  = System.getProperty("os.name");
     public static final String CPU = System.getProperty("os.arch");
 
-    public static final String CALL_CONVENTION = OS.startsWith("Windows") ? "Win64" : "SystemV";
+    public static final String CALL_CONVENTION = OS.startsWith("Windows") ? "win64" : "SystemV";
     public static final String CPU_PORT = switch( CPU ) {
         case "amd64" -> "x86_64_v2";
         default -> throw Utils.TODO("Map Yer CPU Port Here");
     };
+    public static final String CPU_ABI = CPU_PORT+"_"+CALL_CONVENTION;
+    public static final String RELEASE_SYS_DIR = "build/release/"+CPU_ABI;
 
-    public static void run( String file, int spills ) throws IOException { run(file,"",spills); }
-
-    public static void run( String file, String expected, int spills ) throws IOException {
-        run("src/test/java/com/seaofnodes/simple/progs",file,expected,spills);
+    public static String runtimeObject() {
+        String home = System.getenv("SIMPLE_HOME");
+        String root = home==null || home.isEmpty() ? "build/release" : home;
+        return Path.of(root,CPU_ABI,"simple_crt.o").toString();
     }
 
-    // Compile and run a simple program
-    public static void run( String dir, String file, TypeInteger arg, String expected, int spills, boolean standalone) throws IOException {
-        // Files
-        String  cfile = dir+"/"+file+".c"  ;
-        String  sfile = dir+"/"+file+".smp";
-        String  efile = "build/objs/"+file;
+    public static final String C_DRIVERS_DIR = "src/test/java/com/seaofnodes/simple/progs/";
 
-        // Compile and export Simple
-        String src = Files.readString(Path.of(sfile));
-        run(src,CALL_CONVENTION,arg, "",standalone? null: cfile,efile,"S",expected,spills);
+    /**
+     * Compile, link and run
+     * - WITH a C driver
+     * - using the default OS/CPU calling convention.
+     * - no lib sys
+     */
+    public static void runC( String src, String base, String expected, int spills) throws IOException {
+        String cfile = C_DRIVERS_DIR+base+".c";
+        run(src, base, null, CALL_CONVENTION, "", cfile, expected, spills);
     }
 
-    // link with c and also inline
-    public static void run(String src, String file, TypeInteger arg, String expected, int spills) throws IOException {
-            String dir = "src/test/java/com/seaofnodes/simple/progs";
-            String cfile = dir+"/"+file+".c";
-            String efile = "build/objs/"+file;
-            run(src, CALL_CONVENTION, arg, "", cfile, efile, "S", expected, spills);
+    /**
+     * Compile, link and run
+     * - withOUT a C driver
+     * - using the default OS/CPU calling convention.
+     * - no lib sys
+     */
+    public static void runSF( String src, String base, String expected, int spills ) throws IOException {
+        run(src, base, null, CALL_CONVENTION, null,null, expected,spills);
     }
 
-    public static void run(String dir, String file, String expected, int spills ) throws IOException {
-        run(dir, file,null, expected, spills, false);
+    /**
+     * Compile, link and run
+     * - withOUT a C driver
+     * - using the default OS/CPU calling convention.
+     * - WITH a test lib sys
+     */
+    public static void runSYS( String src, String base, String expected, int spills ) throws IOException {
+        Ary<String> externPaths = new Ary<>(new String[]{RELEASE_SYS_DIR});
+        run(src, base, externPaths, CALL_CONVENTION, null,null, expected,spills);
     }
 
-    // Do not link with c file - no inline.
-    public static void runS(String file, String expected, int spills ) throws IOException {
-        run("src/test/java/com/seaofnodes/simple/progs", file, null, expected, spills, true);
-    }
 
+    /**
+     * Compile, link and run - general case
+     *
+     * @param src         Program source to compile
+     * @param base        Compiled binary final base name
+     * @param externPaths External symbol search path
+     * @param simple_conv Argument convention when calling Simple code
+     * @param c_conv      Argument convention when calling C code, or null if not
+     *                    linking against a C driver program
+     * @param cfile       Associated C program which will drive the Simple program,
+     *                    or null if not linking against a C driver program
+     * @param expected    Expected stdout string
+     * @param spills      Expected limit on spill code, used to validate the
+     *                    register allocation is reasonable
+     */
+    public static void run( String src, String base, Ary<String> externPaths, String simple_conv, String c_conv, String cfile, String expected, int spills ) throws IOException {
+        run(src, base, externPaths, simple_conv, c_conv, cfile, null, expected, spills);
+    }
 
-    // Do not link with c file - just inline with source.
-    public static void runSF(String name, String src, TypeInteger arg, String expected, int spills ) throws IOException {
-        String efile = "build/objs/"+name;
-        run(src,CALL_CONVENTION,arg, "",null,efile,"S",expected,spills);
+    public static void runArgs( String src, String base, Ary<String> externPaths, String simple_conv,
+                                String expected, int spills, String... programArgs ) throws IOException {
+        run0(src,base,externPaths,simple_conv,null,null,null,expected,spills,programArgs);
     }
 
-    public static void runS(String file, TypeInteger arg, String expected, int spills ) throws IOException {
-        run("src/test/java/com/seaofnodes/simple/progs", file, arg, expected, spills, true);
+    public static void run( String src, String base, Ary<String> externPaths, String simple_conv, String c_conv, String cfile, String stdin, String expected, int spills ) throws IOException {
+        run0(src,base,externPaths,simple_conv,c_conv,cfile,stdin,expected,spills,new String[0]);
     }
 
+    private static void run0( String src, String base, Ary<String> externPaths, String simple_conv, String c_conv,
+                              String cfile, String stdin, String expected, int spills, String[] programArgs ) throws IOException {
+        String exe = compile(src,base,externPaths,simple_conv,c_conv,cfile,spills);
 
-    public static void run( String src, String simple_conv, TypeInteger arg, String c_conv, String cfile, String efile, String xtn, String expected, int spills ) throws IOException {
-        String bin = efile+xtn;
-        String obj = bin+".o";
-        String exe = OS.startsWith("Windows") ? bin+".exe" : bin;
+        String[] execArgs = new String[programArgs.length+1];
+        execArgs[0] = exe;
+        System.arraycopy(programArgs,0,execArgs,1,programArgs.length);
+        String result = execStdin(stdin, 0, execArgs );
+        assertEquals(expected,result);
+    }
+
+    public static String compile( String src, String base, Ary<String> externPaths, String simple_conv, String c_conv,
+                                  String cfile, int spills ) throws IOException {
+        // Simple file base-name example:
+        // foo.smp ->
+        //   build/objs/foo.o   - object file
+        //   build/objs/foo.exe - linked executable Windows
+        //   build/objs/foo     - linked executable Linux
+        String pathBase = "build/objs/"+base;
+        String obj = pathBase+".o";
+        String exe = pathBase+(OS.startsWith("Windows") ? ".exe" : "");
         // Compile simple, emit ELF
-        CodeGen code = new CheckedCodeGen(src, arg).driver( CPU_PORT, simple_conv, obj);
-        SpillStats.recordNative(code,CPU_PORT,simple_conv);
+        CodeGen code = new CheckedCodeGen(null,"build/objs",externPaths,base,src,126L,true,TypeInteger.BOT);
+        code.driver( CPU_PORT, simple_conv, false, cfile==null );
 
-        String result = gcc(obj, c_conv, cfile, false, exe );
-        assertEquals(expected,result);
+        linkExe(obj, c_conv, cfile, linkObjs(externPaths), exe);
 
         // Allocation quality not degraded
         SpillStats.checkSpills(spills,code._regAlloc._spillScaled);
+        return exe;
     }
 
+    // Link with gcc, and execute the resulting binary, returning stdout as a
+    // String.  Any errors *assert* instead of returning some error code; this
+    // utility is meant to execute inside a JUnit test which will catch the
+    // assert and flag the test as failed.
+    public static String gcc( String main, double ignore, String... objs ) throws IOException {
+        String BLDDIR = "build/objs/test0/";
+        String exe = BLDDIR+main+(OS.startsWith("Windows") ? ".exe" : "");
+        linkExe(objs[0],null,null,null,exe);
+        // Module tests intentionally use the program exit code as their result.
+        ExecResult result = execute(null,0,exe);
+        return result.exit==0 ? result.out : "exec exit code: "+result.exit;
+    }
+
+    // Link with gcc, and execute the resulting binary, returning stdout as a
+    // String.  Any errors *assert* instead of returning some error code; this
+    // utility is meant to execute inside a JUnit test which will catch the
+    // assert and flag the test as failed.
     public static String gcc( String obj, String c_conv, String cfile, boolean stdin, String... args ) throws IOException {
+        return gcc(obj,c_conv,cfile,stdin ? "" : null,null,args);
+    }
+
+    public static String gcc( String obj, String c_conv, String cfile, String stdin, String... args ) throws IOException {
+        return gcc(obj,c_conv,cfile,stdin,null,args);
+    }
+
+    public static String gcc( String obj, String c_conv, String cfile, String stdin, Ary<String> linkObjs, String... args ) throws IOException {
+        linkExe(obj,c_conv,cfile,linkObjs,args[0]);
+        return execStdin(stdin,0,args);
+    }
 
+    public static void linkExe( String obj, String c_conv, String cfile, Ary<String> linkObjs, String exe ) throws IOException {
         // Compile the C program.  Compiling code and constants in the low
         // 2Gig.  Pointers are 64b BUT since always in the low 2G all the
         // high bits are zero - and Simple code can be emitted treating
         // pointers as 4bytes.
         var params = new Ary<>(String.class);
         params.add("gcc");
+        params.add(OS.startsWith("Windows") ? "-Wl,--image-base,0x400000" : "-no-pie");
         if( cfile!=null ) params.add(cfile); // Associated C driver, usually has a `main`
+        params.add(obj);
+        // Standalone Simple programs export `simple_main`; the small C
+        // runtime owns the native main symbol and converts argc/argv into a
+        // normal Simple array before making the ABI-qualified call.
+        if( cfile==null ) params.add(runtimeObject());
+        if( linkObjs != null )
+            params.addAll(linkObjs.asAry());
         params.addAll(new String[] {
-                obj,
-                "-lm", // Picks up 'sqrt' for newtonFloat tests to compare
-                "-g",
-                "-o",
-                args[0],
+            "-lm", // Picks up 'sqrt' for newtonFloat tests to compare
+            "-g",
+            "-o",
+            exe,
         });
         // Calling convention for C calls, if any
         if( cfile!=null ) {
@@ -109,29 +189,85 @@ public abstract class TestC {
         }
 
         // Run GCC to link (optionally compile C driver code)
-        Process gcc = new ProcessBuilder(params.asAry()).redirectErrorStream(true).start();
-        int exit;
-        try {
-            boolean normal = gcc.waitFor(2, TimeUnit.SECONDS);
-            exit = normal ? gcc.exitValue() : -1; // no exit???
-        }  catch( InterruptedException e ) {
-            throw new IOException("interrupted");
-        }
-        String result = new String(gcc.getInputStream().readAllBytes());
-        if( exit!=0 ) {
-            System.err.println("gcc error code: "+exit);
-            System.err.println(result);
-        }
-        assertEquals( 0, exit );
-        //assertTrue(result.isEmpty()); // No data in error stream
+        exec(TEST_TIMEOUT_SECONDS,params.asAry());
+    }
+
+    public static String exec( String... args ) throws IOException {
+        return execStdin(null,0,args);
+    }
+
+    // Tests can bound execution without imposing a deadline on the CLI driver.
+    public static String exec( int seconds, String... args ) throws IOException {
+        return execStdin(null,seconds,args);
+    }
 
+    private static String execStdin( String stdin, int seconds, String... args ) throws IOException {
+        ExecResult result = execute(stdin,seconds,args);
+        if( result.exit!=0 )
+            throw new IOException(String.join(" ",args)+"\nexec exit code: "+result.exit+
+                                  "\nstdout:\n"+result.out+"\nstderr:\n"+result.err);
+        return result.out;
+    }
+
+    private record ExecResult(int exit, String out, String err) {}
+
+    private static ExecResult execute( String stdin, int seconds, String... args ) throws IOException {
         // Execute results
         ProcessBuilder smp = new ProcessBuilder(args);
-        if( stdin ) smp.redirectInput(ProcessBuilder.Redirect.INHERIT);
-        Process p = smp.start();
-        try { exit = p.waitFor(); } catch( InterruptedException e ) { throw new IOException("interrupted"); }
-        result = new String(p.getInputStream().readAllBytes());
-        assertEquals("Program exit status",0,exit);
-        return result;
+        Path stdinFile = null;
+        Path stdoutFile = Files.createTempFile("simple-stdout",".txt");
+        Path stderrFile = Files.createTempFile("simple-stderr",".txt");
+        Process p = null;
+        try {
+            smp.redirectOutput(stdoutFile.toFile());
+            smp.redirectError(stderrFile.toFile());
+            if( stdin!=null && stdin.isEmpty() ) smp.redirectInput(ProcessBuilder.Redirect.INHERIT);
+            else if( stdin!=null ) {
+                stdinFile = Files.createTempFile("simple-stdin",".txt");
+                Files.writeString(stdinFile,stdin);
+                smp.redirectInput(stdinFile.toFile());
+            }
+            p = smp.start();
+            if( stdin==null )
+                p.getOutputStream().close();
+            if( seconds>0 && !p.waitFor(seconds,TimeUnit.SECONDS) ) {
+                String process = "pid="+p.pid()+", cpu="+
+                    p.info().totalCpuDuration().map(Object::toString).orElse("unknown");
+                p.destroyForcibly();
+                boolean stopped = p.waitFor(5,TimeUnit.SECONDS);
+                throw new IOException(String.join(" ",args)+"\ntimed out after "+seconds+" seconds ("+process+")"+
+                                      (stopped ? "" : "; process did not terminate")+
+                                      "\nstdout:\n"+Files.readString(stdoutFile)+
+                                      "\nstderr:\n"+Files.readString(stderrFile));
+            }
+            // Windows statuses have 32 bits; narrowing can turn a failure into 0.
+            int exit = seconds>0 ? p.exitValue() : p.waitFor();
+            return new ExecResult(exit,Files.readString(stdoutFile),Files.readString(stderrFile));
+        } catch( InterruptedException e ) {
+            if( p!=null ) p.destroyForcibly();
+            Thread.currentThread().interrupt();
+            throw new IOException("interrupted",e);
+        } finally {
+            if( stdinFile!=null ) Files.deleteIfExists(stdinFile);
+            Files.deleteIfExists(stdoutFile);
+            Files.deleteIfExists(stderrFile);
+        }
+    }
+
+    static Ary<String> linkObjs(Ary<String> externPaths) {
+        if( externPaths == null )
+            return null;
+        Ary<String> objs = new Ary<>(String.class);
+        for( String path : externPaths ) {
+            File file = new File(path);
+            if( file.isFile() && file.getName().endsWith(".o") )
+                objs.add(file.toString());
+            else if( file.isDirectory() ) {
+                File sys = new File(file,"sys.o");
+                if( sys.isFile() )
+                    objs.add(sys.toString());
+            }
+        }
+        return objs.isEmpty() ? null : objs;
     }
 }