Add transpiler for JUnit tests

10 files changed, 472 insertions, 0 deletions

diff --git a/transpile-tests/Makefile b/transpile-tests/Makefile
new file mode 100644
index 00000000..b2a6e74b
--- /dev/null
+++ b/transpile-tests/Makefile
@@ -0,0 +1,93 @@
+SHELL := /bin/bash
+.DELETE_ON_ERROR:
+
+# for printing variable values
+# usage: make print-VARIABLE
+#        > VARIABLE = value_of_variable
+print-%  : ; @echo $* = $($*)
+
+# literal space
+space := $() $()
+
+# Decide OS-specific questions
+# jar-file seperator
+ifeq ($(OS),Windows_NT)
+	SEP = ;
+else
+	SEP = :
+endif
+# Find a reasonable ctags.
+CTAGS = $(shell which ctags)
+# Hack for MacOS: /usr/bin/ctags is unfriendly, so look for ctags from brew
+ifeq ($(UNAME),Darwin)
+	CTAGS = $(shell brew list ctags 2> /dev/null | grep bin/ctags)
+endif
+
+# Fun Args to javac.  Mostly limit to java8 source definitions, and fairly
+# aggressive lint warnings.
+JAVAC_ARGS = -g
+
+# Source code
+SIMPLE := com/seaofnodes/simple/transpile
+SRC := src/main/java
+TST := src/test/java
+CLZDIR:= build/classes
+main_javas   := $(wildcard $(SRC)/$(SIMPLE)/*java $(SRC)/$(SIMPLE)/*/*java)
+test_javas   := $(wildcard $(TST)/$(SIMPLE)/*java $(TST)/$(SIMPLE)/*/*java)
+main_classes := $(patsubst $(SRC)/%java,$(CLZDIR)/main/%class,$(main_javas))
+test_classes := $(patsubst $(TST)/%java,$(CLZDIR)/test/%class,$(test_javas))
+test_cp      := $(patsubst $(TST)/$(SIMPLE)/%.java,com.seaofnodes.simple.transpile.%,$(wildcard $(TST)/$(SIMPLE)/*Test.java))
+classes = $(main_classes) $(test_classes)
+# All the libraries
+libs = $(wildcard lib/*jar)
+jars = $(subst $(space),$(SEP),$(libs))
+
+
+default_targets := $(main_classes) $(test_classes)
+# Optionally add ctags to the default target if a reasonable one was found.
+ifneq ($(CTAGS),)
+default_targets += tags
+endif
+
+default: $(default_targets)
+
+# Compile just the out-of-date files
+$(main_classes): build/classes/main/%class: $(SRC)/%java
+	@echo "compiling " $@ " because " $?
+	@[ -d $(CLZDIR)/main ] || mkdir -p $(CLZDIR)/main
+	@javac $(JAVAC_ARGS) -cp "$(CLZDIR)/main$(SEP)$(jars)" -sourcepath $(SRC) -d $(CLZDIR)/main $(main_javas)
+
+$(test_classes): $(CLZDIR)/test/%class: $(TST)/%java $(main_classes)
+	@echo "compiling " $@ " because " $?
+	@[ -d $(CLZDIR)/test ] || mkdir -p $(CLZDIR)/test
+	@javac $(JAVAC_ARGS) -cp "$(CLZDIR)/test$(SEP)$(CLZDIR)/main$(SEP)$(jars)" -sourcepath $(TST) -d $(CLZDIR)/test $(test_javas)
+
+# Base launch line for JVM tests
+JVM=nice java -ea -cp "build/classes/main${SEP}${jars}${SEP}$(CLZDIR)/test"
+
+tests:	$(default_targets)
+	@echo "testing " $(test_cp)
+	@$(JVM) org.junit.runner.JUnitCore $(test_cp)
+
+.PHONY: clean
+clean:
+	rm -rf build
+	rm -f TAGS
+	(find . -name "*~" -exec rm {} \; 2>/dev/null; exit 0)
+
+# Download libs from maven
+lib:	lib/junit-4.12.jar  lib/hamcrest-core-1.3.jar
+
+# Jars
+lib/junit-4.12.jar:
+	@[ -d lib ] || mkdir -p lib
+	@(cd lib; wget https://repo1.maven.org/maven2/junit/junit/4.12/junit-4.12.jar)
+
+lib/hamcrest-core-1.3.jar:
+	@[ -d lib ] || mkdir -p lib
+	@(cd lib; wget https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar)
+
+# Build emacs tags (part of a tasty emacs ide experience)
+tags:	$(main_javas) $(test_javas)
+	@rm -f TAGS
+	@$(CTAGS) -e --recurse=yes --extra=+q --fields=+fksaiS $(SRC) $(TST)
diff --git a/transpile-tests/README.md b/transpile-tests/README.md
new file mode 100644
index 00000000..0f13756a
--- /dev/null
+++ b/transpile-tests/README.md
@@ -0,0 +1,18 @@
+# Transpile Tests
+
+This module may be helpful if you want to create an implementation of Simple in another language.
+It can parse the JUnit tests to automatically translate most of the boilerplate.
+Some manual tweaks will still be required though, because only common patterns are detected.
+
+## Usage
+
+Just call `JUnitParser.parseRepository` to get a list of parsed test classes for each chapter,
+and implement a pretty printer for your favourite language.
+
+For an example you can refer to `printers.RustPrinter`,
+which produces tests similar to the ones used by
+[Simple-Rust](https://github.com/SeaOfNodes/Simple-Rust).
+
+Note that although all tests of all chapters are parsed,
+you probably only want to transpile the latest test of each chapter.
+For the rest you can just manually port the small changes between chapters.
diff --git a/transpile-tests/pom.xml b/transpile-tests/pom.xml
new file mode 100644
index 00000000..092e26ee
--- /dev/null
+++ b/transpile-tests/pom.xml
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.seaofnodes</groupId>
+        <artifactId>simple</artifactId>
+        <version>1.0</version>
+    </parent>
+    <artifactId>transpile-tests</artifactId>
+    <packaging>jar</packaging>
+    <name>Transpile Tests</name>
+</project>
diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/Arg.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/Arg.java
new file mode 100644
index 00000000..4334641f
--- /dev/null
+++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/Arg.java
@@ -0,0 +1,23 @@
+package com.seaofnodes.simple.transpile;
+
+public sealed interface Arg {
+    record IntBot() implements Arg {
+    }
+
+    record IntTop() implements Arg {
+    }
+
+    record IntConstant(long value) implements Arg {
+    }
+
+    static Arg parse(String s) {
+        if (s.equals("TypeInteger.BOT"))
+            return new IntBot();
+        if (s.equals("TypeInteger.TOP"))
+            return new IntTop();
+        var parts = s.split("\\(|\\)");
+        if (parts.length != 2 | !parts[0].equals("TypeInteger.constant"))
+            throw new RuntimeException("Unexpected Argument " + s);
+        return new IntConstant(Long.parseLong(parts[1]));
+    }
+}
diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/AstVisitor.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/AstVisitor.java
new file mode 100644
index 00000000..581969c0
--- /dev/null
+++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/AstVisitor.java
@@ -0,0 +1,103 @@
+package com.seaofnodes.simple.transpile;
+
+import com.sun.source.tree.*;
+import com.sun.source.util.TreeScanner;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+
+class AstVisitor extends TreeScanner<Void, Void> {
+    final HashMap<String, TestClass> results = new HashMap<>();
+
+    private String className;
+    private TestMethod current;
+    private boolean inCatch;
+
+    @Override
+    public Void visitClass(ClassTree node, Void unused) {
+        var name = node.getSimpleName().toString();
+        if (name.matches("Chapter\\d+Test")) {
+            var backup = className;
+            className = name;
+            super.visitClass(node, unused);
+            className = backup;
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitMethod(MethodTree node, Void unused) {
+        if (className != null) {
+            current = new TestMethod();
+            current.name = node.getName().toString();
+            super.visitMethod(node, unused);
+            results.compute(className, (k, v) -> {
+                if (v == null)
+                    v = new TestClass(className, new ArrayList<>());
+                v.methods().add(current);
+                return v;
+            });
+            current = null;
+        }
+        return null;
+    }
+
+    @Override
+    public Void visitMethodInvocation(MethodInvocationTree node, Void unused) {
+        if (current != null) {
+            if (node.getMethodSelect().toString().endsWith(".parse")) {
+                current.showAfterParse = switch (node.getArguments().size()) {
+                    case 0 -> false;
+                    case 1 -> (Boolean) ((LiteralTree) node.getArguments().getFirst()).getValue();
+                    default -> throw new RuntimeException("unexpected parse arguments" + node);
+                };
+            }
+            if (node.getMethodSelect().toString().endsWith("assertEquals")) {
+                var args = node.getArguments();
+                if (inCatch) {
+                    if (args.size() != 2)
+                        throw new RuntimeException("Unexpected number of arguments " + node);
+                    if (args.get(0).toString().contains("e.getMessage()")) {
+                        current.parseErrorMessage = args.get(1).toString();
+                    } else if (args.get(1).toString().contains("e.getMessage()")) {
+                        current.parseErrorMessage = args.get(0).toString();
+                    }
+                } else {
+                    if (args.size() != 2)
+                        throw new RuntimeException("Unexpected number of arguments " + node);
+                    if (args.get(0).toString().contains("stop.toString()")) {
+                        current.assertStopEquals = args.get(1).toString();
+                    } else if (args.get(1).toString().contains("stop.toString()")) {
+                        current.assertStopEquals = args.get(0).toString();
+                    }
+                }
+            }
+        }
+        return super.visitMethodInvocation(node, unused); // otherwise we would miss the first part of new Parser("").parse()
+    }
+
+    @Override
+    public Void visitNewClass(NewClassTree node, Void unused) {
+        if (current != null) {
+            if (node.getIdentifier().toString().equals("Parser")) {
+                var args = node.getArguments();
+                switch (args.size()) {
+                    case 1 -> {
+                    }
+                    case 2 -> current.parserArg = Arg.parse(args.get(1).toString());
+                    default -> throw new RuntimeException("Parser constructor with unexpected arguments: " + node);
+                }
+                current.parserInput = args.get(0).toString();
+            }
+        }
+        return super.visitNewClass(node, unused);
+    }
+
+    @Override
+    public Void visitCatch(CatchTree node, Void unused) {
+        inCatch = true;
+        super.visitCatch(node, unused);
+        inCatch = false;
+        return null;
+    }
+}
diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/JUnitParser.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/JUnitParser.java
new file mode 100644
index 00000000..e396d56f
--- /dev/null
+++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/JUnitParser.java
@@ -0,0 +1,71 @@
+package com.seaofnodes.simple.transpile;
+
+import com.sun.source.util.JavacTask;
+
+import javax.tools.ToolProvider;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.*;
+
+import static java.util.Comparator.comparing;
+
+public class JUnitParser {
+
+    /**
+     * @param simpleRepo The path to the project root
+     * @return A sorted map from chapter name to a sorted list of parsed test classes.
+     */
+    public static TreeMap<String, List<TestClass>> parseRepository(Path simpleRepo) throws IOException {
+        var chapters = new TreeMap<String, List<TestClass>>();
+        var inputs = simpleRepo.toFile().listFiles(File::isDirectory);
+        if (inputs == null)
+            throw new IOException("failed to read Simple project root");
+        Arrays.sort(inputs);
+        for (File file : inputs) {
+            var chapter = file.getName();
+            if (chapter.matches("chapter\\d+")) {
+                chapters.put(chapter, parseChapter(file.toPath()));
+            }
+        }
+        return chapters;
+    }
+
+    /**
+     * @param input The path to a chapter in the repository.
+     * @return a sorted list of parsed test classes
+     */
+    public static List<TestClass> parseChapter(Path input) throws IOException {
+        var tests = new ArrayList<Path>();
+        Files.walkFileTree(input, Set.of(), 20, new SimpleFileVisitor<>() {
+            @Override
+            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+                if (file.getFileName().toString().matches("Chapter\\d+Test\\.java")) {
+                    tests.add(file);
+                }
+                return FileVisitResult.CONTINUE;
+            }
+        });
+
+        var compiler = ToolProvider.getSystemJavaCompiler();
+        var compilationUnits = compiler
+                .getStandardFileManager(null, null, null)
+                .getJavaFileObjectsFromPaths(tests);
+
+        var task = (JavacTask) compiler.getTask(null, null, null, null, null, compilationUnits);
+
+        var visitor = new AstVisitor();
+        for (var tree : task.parse()) {
+            tree.accept(visitor, null);
+        }
+
+        var testClasses = new ArrayList<>(visitor.results.values());
+        testClasses.sort(comparing(TestClass::name));
+        return testClasses;
+    }
+}
+
diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestClass.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestClass.java
new file mode 100644
index 00000000..5bb5609c
--- /dev/null
+++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestClass.java
@@ -0,0 +1,6 @@
+package com.seaofnodes.simple.transpile;
+
+import java.util.ArrayList;
+
+public record TestClass(String name, ArrayList<TestMethod> methods) {
+}
diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestMethod.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestMethod.java
new file mode 100644
index 00000000..56357c4a
--- /dev/null
+++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestMethod.java
@@ -0,0 +1,10 @@
+package com.seaofnodes.simple.transpile;
+
+public class TestMethod {
+    public String name;
+    public String parserInput;
+    public Arg parserArg;
+    public String parseErrorMessage;
+    public boolean showAfterParse;
+    public String assertStopEquals;
+}
diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/printers/RustPrinter.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/printers/RustPrinter.java
new file mode 100644
index 00000000..7362ef60
--- /dev/null
+++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/printers/RustPrinter.java
@@ -0,0 +1,108 @@
+package com.seaofnodes.simple.transpile.printers;
+
+import com.seaofnodes.simple.transpile.Arg;
+import com.seaofnodes.simple.transpile.JUnitParser;
+import com.seaofnodes.simple.transpile.TestClass;
+import com.seaofnodes.simple.transpile.TestMethod;
+
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+public class RustPrinter {
+
+    public static void main(String[] args) throws Exception {
+        if (args.length != 2) {
+            System.out.println("Usage: <path-to-simple-repo> <output-dir>");
+            System.exit(1);
+        }
+        var printer = new RustPrinter();
+        var output = Path.of(args[1]);
+
+        for (var e : JUnitParser.parseRepository(Path.of(args[0])).entrySet()) {
+            printer.printToFile(e.getKey(), e.getValue(), output);
+        }
+    }
+
+    private void printToFile(String chapter, List<TestClass> testClasses, Path outputDirectory) throws IOException, InterruptedException {
+        var output = outputDirectory.resolve(chapter);
+        Files.createDirectories(output);
+        for (var testClass : testClasses) {
+            var outputFile = output.resolve(testClass.name().replace("Test", "").toLowerCase() + ".rs");
+            System.out.println(outputFile);
+            Files.write(outputFile, printAndRustfmt(testClass.methods()));
+        }
+    }
+
+    private byte[] printAndRustfmt(ArrayList<TestMethod> tests) throws IOException, InterruptedException {
+        var p = Runtime.getRuntime().exec(new String[]{"rustfmt"});
+        try (var print = new PrintStream(new BufferedOutputStream(p.getOutputStream()))) {
+            printFile(tests, print);
+        }
+        p.waitFor();
+
+        var errors = new String(p.getErrorStream().readAllBytes(), StandardCharsets.UTF_8);
+        if (!errors.isEmpty() || p.exitValue() != 0) {
+            throw new RuntimeException("rustfmt exit=" + p.exitValue() + " stderr=" + errors);
+        }
+
+        return p.getInputStream().readAllBytes();
+    }
+
+    private void printFile(ArrayList<TestMethod> tests, PrintStream out) {
+        out.println("""
+                use crate::datastructures::arena::DroplessArena;
+                use crate::sea_of_nodes::parser::Parser;
+                use crate::sea_of_nodes::types::Types;
+                """);
+
+        for (TestMethod test : tests) {
+            if (test.parseErrorMessage != null) {
+                out.println("use crate::sea_of_nodes::tests::test_error;");
+                break;
+            }
+        }
+
+        for (TestMethod test : tests) {
+            printTest(test, out);
+        }
+    }
+
+    private void printTest(TestMethod test, PrintStream out) {
+        out.println();
+        out.println("#[test]");
+        out.println("fn " + test.name + "() {");
+        if (test.parseErrorMessage != null) {
+            out.println("test_error(" + test.parserInput + ", " + test.parseErrorMessage + ")");
+        } else if (test.parserInput != null) {
+            out.println("let arena = DroplessArena::new();");
+            out.println("let types = Types::new(&arena);");
+            out.printf("let mut parser = Parser::new(%s%s, &types);\n", multiline(test.parserInput), switch (test.parserArg) {
+                case null -> "";
+                case Arg.IntBot ignored -> ", types.ty_int_bot";
+                case Arg.IntTop ignored -> ", types.ty_int_top";
+                case Arg.IntConstant c -> ", types.get_int(" + c.value() + ")";
+            });
+            out.println("let stop = parser.parse().unwrap();");
+            if (test.showAfterParse) {
+                out.println("parser.show_graph();");
+            }
+            if (test.assertStopEquals != null) {
+                out.printf("assert_eq!(parser.print(stop), %s);\n", test.assertStopEquals);
+            }
+        }
+        out.println("}");
+    }
+
+    private String multiline(String string) {
+        if (!string.contains("\\n")) {
+            return string;
+        }
+        return string.replaceFirst("^\"", "\"\\\\\\\\n").replace("\\n", "\n");
+    }
+}
diff --git a/transpile-tests/src/test/java/com/seaofnodes/simple/transpile/TranspilerTest.java b/transpile-tests/src/test/java/com/seaofnodes/simple/transpile/TranspilerTest.java
new file mode 100644
index 00000000..a2c3a782
--- /dev/null
+++ b/transpile-tests/src/test/java/com/seaofnodes/simple/transpile/TranspilerTest.java
@@ -0,0 +1,26 @@
+package com.seaofnodes.simple.transpile;
+
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TranspilerTest {
+    @Test
+    public void testParser() throws IOException {
+        var cwd = Path.of(".").toAbsolutePath().normalize();
+        assertEquals("transpile-tests", cwd.getFileName().toString());
+        var all = JUnitParser.parseRepository(cwd.getParent());
+        assertTrue(all.size() >= 14);
+
+        var chapter09 = all.get("chapter09");
+        assertEquals(9, chapter09.size());
+
+        var class6 = chapter09.get(5);
+        assertEquals("Chapter06Test", class6.name());
+        assertEquals(1, class6.methods().stream().filter(m -> m.name.equals("testPeepholeReturn")).count());
+    }
+}