From c28653712f232300a66f7ca9d89fb327cb2d5513 Mon Sep 17 00:00:00 2001 From: Robert Obkircher Date: Sun, 8 Sep 2024 19:37:37 +0200 Subject: Add transpiler for JUnit tests --- pom.xml | 1 + transpile-tests/Makefile | 93 ++++++++++++++++++ transpile-tests/README.md | 18 ++++ transpile-tests/pom.xml | 14 +++ .../java/com/seaofnodes/simple/transpile/Arg.java | 23 +++++ .../seaofnodes/simple/transpile/AstVisitor.java | 103 ++++++++++++++++++++ .../seaofnodes/simple/transpile/JUnitParser.java | 71 ++++++++++++++ .../com/seaofnodes/simple/transpile/TestClass.java | 6 ++ .../seaofnodes/simple/transpile/TestMethod.java | 10 ++ .../simple/transpile/printers/RustPrinter.java | 108 +++++++++++++++++++++ .../simple/transpile/TranspilerTest.java | 26 +++++ 11 files changed, 473 insertions(+) create mode 100644 transpile-tests/Makefile create mode 100644 transpile-tests/README.md create mode 100644 transpile-tests/pom.xml create mode 100644 transpile-tests/src/main/java/com/seaofnodes/simple/transpile/Arg.java create mode 100644 transpile-tests/src/main/java/com/seaofnodes/simple/transpile/AstVisitor.java create mode 100644 transpile-tests/src/main/java/com/seaofnodes/simple/transpile/JUnitParser.java create mode 100644 transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestClass.java create mode 100644 transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestMethod.java create mode 100644 transpile-tests/src/main/java/com/seaofnodes/simple/transpile/printers/RustPrinter.java create mode 100644 transpile-tests/src/test/java/com/seaofnodes/simple/transpile/TranspilerTest.java diff --git a/pom.xml b/pom.xml index 0d1d4906..ce7640ec 100644 --- a/pom.xml +++ b/pom.xml @@ -33,6 +33,7 @@ chapter12 chapter13 chapter14 + transpile-tests diff --git a/transpile-tests/Makefile b/transpile-tests/Makefile new file mode 100644 index 00000000..b2a6e74b --- /dev/null +++ b/transpile-tests/Makefile @@ -0,0 +1,93 @@ +SHELL := /bin/bash +.DELETE_ON_ERROR: + +# for printing variable values +# usage: make print-VARIABLE +# > VARIABLE = value_of_variable +print-% : ; @echo $* = $($*) + +# literal space +space := $() $() + +# Decide OS-specific questions +# jar-file seperator +ifeq ($(OS),Windows_NT) + SEP = ; +else + SEP = : +endif +# Find a reasonable ctags. +CTAGS = $(shell which ctags) +# Hack for MacOS: /usr/bin/ctags is unfriendly, so look for ctags from brew +ifeq ($(UNAME),Darwin) + CTAGS = $(shell brew list ctags 2> /dev/null | grep bin/ctags) +endif + +# Fun Args to javac. Mostly limit to java8 source definitions, and fairly +# aggressive lint warnings. +JAVAC_ARGS = -g + +# Source code +SIMPLE := com/seaofnodes/simple/transpile +SRC := src/main/java +TST := src/test/java +CLZDIR:= build/classes +main_javas := $(wildcard $(SRC)/$(SIMPLE)/*java $(SRC)/$(SIMPLE)/*/*java) +test_javas := $(wildcard $(TST)/$(SIMPLE)/*java $(TST)/$(SIMPLE)/*/*java) +main_classes := $(patsubst $(SRC)/%java,$(CLZDIR)/main/%class,$(main_javas)) +test_classes := $(patsubst $(TST)/%java,$(CLZDIR)/test/%class,$(test_javas)) +test_cp := $(patsubst $(TST)/$(SIMPLE)/%.java,com.seaofnodes.simple.transpile.%,$(wildcard $(TST)/$(SIMPLE)/*Test.java)) +classes = $(main_classes) $(test_classes) +# All the libraries +libs = $(wildcard lib/*jar) +jars = $(subst $(space),$(SEP),$(libs)) + + +default_targets := $(main_classes) $(test_classes) +# Optionally add ctags to the default target if a reasonable one was found. +ifneq ($(CTAGS),) +default_targets += tags +endif + +default: $(default_targets) + +# Compile just the out-of-date files +$(main_classes): build/classes/main/%class: $(SRC)/%java + @echo "compiling " $@ " because " $? + @[ -d $(CLZDIR)/main ] || mkdir -p $(CLZDIR)/main + @javac $(JAVAC_ARGS) -cp "$(CLZDIR)/main$(SEP)$(jars)" -sourcepath $(SRC) -d $(CLZDIR)/main $(main_javas) + +$(test_classes): $(CLZDIR)/test/%class: $(TST)/%java $(main_classes) + @echo "compiling " $@ " because " $? + @[ -d $(CLZDIR)/test ] || mkdir -p $(CLZDIR)/test + @javac $(JAVAC_ARGS) -cp "$(CLZDIR)/test$(SEP)$(CLZDIR)/main$(SEP)$(jars)" -sourcepath $(TST) -d $(CLZDIR)/test $(test_javas) + +# Base launch line for JVM tests +JVM=nice java -ea -cp "build/classes/main${SEP}${jars}${SEP}$(CLZDIR)/test" + +tests: $(default_targets) + @echo "testing " $(test_cp) + @$(JVM) org.junit.runner.JUnitCore $(test_cp) + +.PHONY: clean +clean: + rm -rf build + rm -f TAGS + (find . -name "*~" -exec rm {} \; 2>/dev/null; exit 0) + +# Download libs from maven +lib: lib/junit-4.12.jar lib/hamcrest-core-1.3.jar + +# Jars +lib/junit-4.12.jar: + @[ -d lib ] || mkdir -p lib + @(cd lib; wget https://repo1.maven.org/maven2/junit/junit/4.12/junit-4.12.jar) + +lib/hamcrest-core-1.3.jar: + @[ -d lib ] || mkdir -p lib + @(cd lib; wget https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar) + +# Build emacs tags (part of a tasty emacs ide experience) +tags: $(main_javas) $(test_javas) + @rm -f TAGS + @$(CTAGS) -e --recurse=yes --extra=+q --fields=+fksaiS $(SRC) $(TST) diff --git a/transpile-tests/README.md b/transpile-tests/README.md new file mode 100644 index 00000000..0f13756a --- /dev/null +++ b/transpile-tests/README.md @@ -0,0 +1,18 @@ +# Transpile Tests + +This module may be helpful if you want to create an implementation of Simple in another language. +It can parse the JUnit tests to automatically translate most of the boilerplate. +Some manual tweaks will still be required though, because only common patterns are detected. + +## Usage + +Just call `JUnitParser.parseRepository` to get a list of parsed test classes for each chapter, +and implement a pretty printer for your favourite language. + +For an example you can refer to `printers.RustPrinter`, +which produces tests similar to the ones used by +[Simple-Rust](https://github.com/SeaOfNodes/Simple-Rust). + +Note that although all tests of all chapters are parsed, +you probably only want to transpile the latest test of each chapter. +For the rest you can just manually port the small changes between chapters. diff --git a/transpile-tests/pom.xml b/transpile-tests/pom.xml new file mode 100644 index 00000000..092e26ee --- /dev/null +++ b/transpile-tests/pom.xml @@ -0,0 +1,14 @@ + + + 4.0.0 + + com.seaofnodes + simple + 1.0 + + transpile-tests + jar + Transpile Tests + diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/Arg.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/Arg.java new file mode 100644 index 00000000..4334641f --- /dev/null +++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/Arg.java @@ -0,0 +1,23 @@ +package com.seaofnodes.simple.transpile; + +public sealed interface Arg { + record IntBot() implements Arg { + } + + record IntTop() implements Arg { + } + + record IntConstant(long value) implements Arg { + } + + static Arg parse(String s) { + if (s.equals("TypeInteger.BOT")) + return new IntBot(); + if (s.equals("TypeInteger.TOP")) + return new IntTop(); + var parts = s.split("\\(|\\)"); + if (parts.length != 2 | !parts[0].equals("TypeInteger.constant")) + throw new RuntimeException("Unexpected Argument " + s); + return new IntConstant(Long.parseLong(parts[1])); + } +} diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/AstVisitor.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/AstVisitor.java new file mode 100644 index 00000000..581969c0 --- /dev/null +++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/AstVisitor.java @@ -0,0 +1,103 @@ +package com.seaofnodes.simple.transpile; + +import com.sun.source.tree.*; +import com.sun.source.util.TreeScanner; + +import java.util.ArrayList; +import java.util.HashMap; + +class AstVisitor extends TreeScanner { + final HashMap results = new HashMap<>(); + + private String className; + private TestMethod current; + private boolean inCatch; + + @Override + public Void visitClass(ClassTree node, Void unused) { + var name = node.getSimpleName().toString(); + if (name.matches("Chapter\\d+Test")) { + var backup = className; + className = name; + super.visitClass(node, unused); + className = backup; + } + return null; + } + + @Override + public Void visitMethod(MethodTree node, Void unused) { + if (className != null) { + current = new TestMethod(); + current.name = node.getName().toString(); + super.visitMethod(node, unused); + results.compute(className, (k, v) -> { + if (v == null) + v = new TestClass(className, new ArrayList<>()); + v.methods().add(current); + return v; + }); + current = null; + } + return null; + } + + @Override + public Void visitMethodInvocation(MethodInvocationTree node, Void unused) { + if (current != null) { + if (node.getMethodSelect().toString().endsWith(".parse")) { + current.showAfterParse = switch (node.getArguments().size()) { + case 0 -> false; + case 1 -> (Boolean) ((LiteralTree) node.getArguments().getFirst()).getValue(); + default -> throw new RuntimeException("unexpected parse arguments" + node); + }; + } + if (node.getMethodSelect().toString().endsWith("assertEquals")) { + var args = node.getArguments(); + if (inCatch) { + if (args.size() != 2) + throw new RuntimeException("Unexpected number of arguments " + node); + if (args.get(0).toString().contains("e.getMessage()")) { + current.parseErrorMessage = args.get(1).toString(); + } else if (args.get(1).toString().contains("e.getMessage()")) { + current.parseErrorMessage = args.get(0).toString(); + } + } else { + if (args.size() != 2) + throw new RuntimeException("Unexpected number of arguments " + node); + if (args.get(0).toString().contains("stop.toString()")) { + current.assertStopEquals = args.get(1).toString(); + } else if (args.get(1).toString().contains("stop.toString()")) { + current.assertStopEquals = args.get(0).toString(); + } + } + } + } + return super.visitMethodInvocation(node, unused); // otherwise we would miss the first part of new Parser("").parse() + } + + @Override + public Void visitNewClass(NewClassTree node, Void unused) { + if (current != null) { + if (node.getIdentifier().toString().equals("Parser")) { + var args = node.getArguments(); + switch (args.size()) { + case 1 -> { + } + case 2 -> current.parserArg = Arg.parse(args.get(1).toString()); + default -> throw new RuntimeException("Parser constructor with unexpected arguments: " + node); + } + current.parserInput = args.get(0).toString(); + } + } + return super.visitNewClass(node, unused); + } + + @Override + public Void visitCatch(CatchTree node, Void unused) { + inCatch = true; + super.visitCatch(node, unused); + inCatch = false; + return null; + } +} diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/JUnitParser.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/JUnitParser.java new file mode 100644 index 00000000..e396d56f --- /dev/null +++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/JUnitParser.java @@ -0,0 +1,71 @@ +package com.seaofnodes.simple.transpile; + +import com.sun.source.util.JavacTask; + +import javax.tools.ToolProvider; +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.*; + +import static java.util.Comparator.comparing; + +public class JUnitParser { + + /** + * @param simpleRepo The path to the project root + * @return A sorted map from chapter name to a sorted list of parsed test classes. + */ + public static TreeMap> parseRepository(Path simpleRepo) throws IOException { + var chapters = new TreeMap>(); + var inputs = simpleRepo.toFile().listFiles(File::isDirectory); + if (inputs == null) + throw new IOException("failed to read Simple project root"); + Arrays.sort(inputs); + for (File file : inputs) { + var chapter = file.getName(); + if (chapter.matches("chapter\\d+")) { + chapters.put(chapter, parseChapter(file.toPath())); + } + } + return chapters; + } + + /** + * @param input The path to a chapter in the repository. + * @return a sorted list of parsed test classes + */ + public static List parseChapter(Path input) throws IOException { + var tests = new ArrayList(); + Files.walkFileTree(input, Set.of(), 20, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + if (file.getFileName().toString().matches("Chapter\\d+Test\\.java")) { + tests.add(file); + } + return FileVisitResult.CONTINUE; + } + }); + + var compiler = ToolProvider.getSystemJavaCompiler(); + var compilationUnits = compiler + .getStandardFileManager(null, null, null) + .getJavaFileObjectsFromPaths(tests); + + var task = (JavacTask) compiler.getTask(null, null, null, null, null, compilationUnits); + + var visitor = new AstVisitor(); + for (var tree : task.parse()) { + tree.accept(visitor, null); + } + + var testClasses = new ArrayList<>(visitor.results.values()); + testClasses.sort(comparing(TestClass::name)); + return testClasses; + } +} + diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestClass.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestClass.java new file mode 100644 index 00000000..5bb5609c --- /dev/null +++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestClass.java @@ -0,0 +1,6 @@ +package com.seaofnodes.simple.transpile; + +import java.util.ArrayList; + +public record TestClass(String name, ArrayList methods) { +} diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestMethod.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestMethod.java new file mode 100644 index 00000000..56357c4a --- /dev/null +++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestMethod.java @@ -0,0 +1,10 @@ +package com.seaofnodes.simple.transpile; + +public class TestMethod { + public String name; + public String parserInput; + public Arg parserArg; + public String parseErrorMessage; + public boolean showAfterParse; + public String assertStopEquals; +} diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/printers/RustPrinter.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/printers/RustPrinter.java new file mode 100644 index 00000000..7362ef60 --- /dev/null +++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/printers/RustPrinter.java @@ -0,0 +1,108 @@ +package com.seaofnodes.simple.transpile.printers; + +import com.seaofnodes.simple.transpile.Arg; +import com.seaofnodes.simple.transpile.JUnitParser; +import com.seaofnodes.simple.transpile.TestClass; +import com.seaofnodes.simple.transpile.TestMethod; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +public class RustPrinter { + + public static void main(String[] args) throws Exception { + if (args.length != 2) { + System.out.println("Usage: "); + System.exit(1); + } + var printer = new RustPrinter(); + var output = Path.of(args[1]); + + for (var e : JUnitParser.parseRepository(Path.of(args[0])).entrySet()) { + printer.printToFile(e.getKey(), e.getValue(), output); + } + } + + private void printToFile(String chapter, List testClasses, Path outputDirectory) throws IOException, InterruptedException { + var output = outputDirectory.resolve(chapter); + Files.createDirectories(output); + for (var testClass : testClasses) { + var outputFile = output.resolve(testClass.name().replace("Test", "").toLowerCase() + ".rs"); + System.out.println(outputFile); + Files.write(outputFile, printAndRustfmt(testClass.methods())); + } + } + + private byte[] printAndRustfmt(ArrayList tests) throws IOException, InterruptedException { + var p = Runtime.getRuntime().exec(new String[]{"rustfmt"}); + try (var print = new PrintStream(new BufferedOutputStream(p.getOutputStream()))) { + printFile(tests, print); + } + p.waitFor(); + + var errors = new String(p.getErrorStream().readAllBytes(), StandardCharsets.UTF_8); + if (!errors.isEmpty() || p.exitValue() != 0) { + throw new RuntimeException("rustfmt exit=" + p.exitValue() + " stderr=" + errors); + } + + return p.getInputStream().readAllBytes(); + } + + private void printFile(ArrayList tests, PrintStream out) { + out.println(""" + use crate::datastructures::arena::DroplessArena; + use crate::sea_of_nodes::parser::Parser; + use crate::sea_of_nodes::types::Types; + """); + + for (TestMethod test : tests) { + if (test.parseErrorMessage != null) { + out.println("use crate::sea_of_nodes::tests::test_error;"); + break; + } + } + + for (TestMethod test : tests) { + printTest(test, out); + } + } + + private void printTest(TestMethod test, PrintStream out) { + out.println(); + out.println("#[test]"); + out.println("fn " + test.name + "() {"); + if (test.parseErrorMessage != null) { + out.println("test_error(" + test.parserInput + ", " + test.parseErrorMessage + ")"); + } else if (test.parserInput != null) { + out.println("let arena = DroplessArena::new();"); + out.println("let types = Types::new(&arena);"); + out.printf("let mut parser = Parser::new(%s%s, &types);\n", multiline(test.parserInput), switch (test.parserArg) { + case null -> ""; + case Arg.IntBot ignored -> ", types.ty_int_bot"; + case Arg.IntTop ignored -> ", types.ty_int_top"; + case Arg.IntConstant c -> ", types.get_int(" + c.value() + ")"; + }); + out.println("let stop = parser.parse().unwrap();"); + if (test.showAfterParse) { + out.println("parser.show_graph();"); + } + if (test.assertStopEquals != null) { + out.printf("assert_eq!(parser.print(stop), %s);\n", test.assertStopEquals); + } + } + out.println("}"); + } + + private String multiline(String string) { + if (!string.contains("\\n")) { + return string; + } + return string.replaceFirst("^\"", "\"\\\\\\\\n").replace("\\n", "\n"); + } +} diff --git a/transpile-tests/src/test/java/com/seaofnodes/simple/transpile/TranspilerTest.java b/transpile-tests/src/test/java/com/seaofnodes/simple/transpile/TranspilerTest.java new file mode 100644 index 00000000..a2c3a782 --- /dev/null +++ b/transpile-tests/src/test/java/com/seaofnodes/simple/transpile/TranspilerTest.java @@ -0,0 +1,26 @@ +package com.seaofnodes.simple.transpile; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Path; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TranspilerTest { + @Test + public void testParser() throws IOException { + var cwd = Path.of(".").toAbsolutePath().normalize(); + assertEquals("transpile-tests", cwd.getFileName().toString()); + var all = JUnitParser.parseRepository(cwd.getParent()); + assertTrue(all.size() >= 14); + + var chapter09 = all.get("chapter09"); + assertEquals(9, chapter09.size()); + + var class6 = chapter09.get(5); + assertEquals("Chapter06Test", class6.name()); + assertEquals(1, class6.methods().stream().filter(m -> m.name.equals("testPeepholeReturn")).count()); + } +} -- cgit v1.2.3