diff options
Diffstat (limited to 'transpile-tests')
9 files changed, 573 insertions, 0 deletions
diff --git a/transpile-tests/Makefile b/transpile-tests/Makefile new file mode 100644 index 00000000..b2a6e74b --- /dev/null +++ b/transpile-tests/Makefile @@ -0,0 +1,93 @@ +SHELL := /bin/bash +.DELETE_ON_ERROR: + +# for printing variable values +# usage: make print-VARIABLE +# > VARIABLE = value_of_variable +print-% : ; @echo $* = $($*) + +# literal space +space := $() $() + +# Decide OS-specific questions +# jar-file seperator +ifeq ($(OS),Windows_NT) + SEP = ; +else + SEP = : +endif +# Find a reasonable ctags. +CTAGS = $(shell which ctags) +# Hack for MacOS: /usr/bin/ctags is unfriendly, so look for ctags from brew +ifeq ($(UNAME),Darwin) + CTAGS = $(shell brew list ctags 2> /dev/null | grep bin/ctags) +endif + +# Fun Args to javac. Mostly limit to java8 source definitions, and fairly +# aggressive lint warnings. +JAVAC_ARGS = -g + +# Source code +SIMPLE := com/seaofnodes/simple/transpile +SRC := src/main/java +TST := src/test/java +CLZDIR:= build/classes +main_javas := $(wildcard $(SRC)/$(SIMPLE)/*java $(SRC)/$(SIMPLE)/*/*java) +test_javas := $(wildcard $(TST)/$(SIMPLE)/*java $(TST)/$(SIMPLE)/*/*java) +main_classes := $(patsubst $(SRC)/%java,$(CLZDIR)/main/%class,$(main_javas)) +test_classes := $(patsubst $(TST)/%java,$(CLZDIR)/test/%class,$(test_javas)) +test_cp := $(patsubst $(TST)/$(SIMPLE)/%.java,com.seaofnodes.simple.transpile.%,$(wildcard $(TST)/$(SIMPLE)/*Test.java)) +classes = $(main_classes) $(test_classes) +# All the libraries +libs = $(wildcard lib/*jar) +jars = $(subst $(space),$(SEP),$(libs)) + + +default_targets := $(main_classes) $(test_classes) +# Optionally add ctags to the default target if a reasonable one was found. +ifneq ($(CTAGS),) +default_targets += tags +endif + +default: $(default_targets) + +# Compile just the out-of-date files +$(main_classes): build/classes/main/%class: $(SRC)/%java + @echo "compiling " $@ " because " $? + @[ -d $(CLZDIR)/main ] || mkdir -p $(CLZDIR)/main + @javac $(JAVAC_ARGS) -cp "$(CLZDIR)/main$(SEP)$(jars)" -sourcepath $(SRC) -d $(CLZDIR)/main $(main_javas) + +$(test_classes): $(CLZDIR)/test/%class: $(TST)/%java $(main_classes) + @echo "compiling " $@ " because " $? + @[ -d $(CLZDIR)/test ] || mkdir -p $(CLZDIR)/test + @javac $(JAVAC_ARGS) -cp "$(CLZDIR)/test$(SEP)$(CLZDIR)/main$(SEP)$(jars)" -sourcepath $(TST) -d $(CLZDIR)/test $(test_javas) + +# Base launch line for JVM tests +JVM=nice java -ea -cp "build/classes/main${SEP}${jars}${SEP}$(CLZDIR)/test" + +tests: $(default_targets) + @echo "testing " $(test_cp) + @$(JVM) org.junit.runner.JUnitCore $(test_cp) + +.PHONY: clean +clean: + rm -rf build + rm -f TAGS + (find . -name "*~" -exec rm {} \; 2>/dev/null; exit 0) + +# Download libs from maven +lib: lib/junit-4.12.jar lib/hamcrest-core-1.3.jar + +# Jars +lib/junit-4.12.jar: + @[ -d lib ] || mkdir -p lib + @(cd lib; wget https://repo1.maven.org/maven2/junit/junit/4.12/junit-4.12.jar) + +lib/hamcrest-core-1.3.jar: + @[ -d lib ] || mkdir -p lib + @(cd lib; wget https://repo1.maven.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar) + +# Build emacs tags (part of a tasty emacs ide experience) +tags: $(main_javas) $(test_javas) + @rm -f TAGS + @$(CTAGS) -e --recurse=yes --extra=+q --fields=+fksaiS $(SRC) $(TST) diff --git a/transpile-tests/README.md b/transpile-tests/README.md new file mode 100644 index 00000000..0f13756a --- /dev/null +++ b/transpile-tests/README.md @@ -0,0 +1,18 @@ +# Transpile Tests + +This module may be helpful if you want to create an implementation of Simple in another language. +It can parse the JUnit tests to automatically translate most of the boilerplate. +Some manual tweaks will still be required though, because only common patterns are detected. + +## Usage + +Just call `JUnitParser.parseRepository` to get a list of parsed test classes for each chapter, +and implement a pretty printer for your favourite language. + +For an example you can refer to `printers.RustPrinter`, +which produces tests similar to the ones used by +[Simple-Rust](https://github.com/SeaOfNodes/Simple-Rust). + +Note that although all tests of all chapters are parsed, +you probably only want to transpile the latest test of each chapter. +For the rest you can just manually port the small changes between chapters. diff --git a/transpile-tests/pom.xml b/transpile-tests/pom.xml new file mode 100644 index 00000000..092e26ee --- /dev/null +++ b/transpile-tests/pom.xml @@ -0,0 +1,14 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.seaofnodes</groupId> + <artifactId>simple</artifactId> + <version>1.0</version> + </parent> + <artifactId>transpile-tests</artifactId> + <packaging>jar</packaging> + <name>Transpile Tests</name> +</project> diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/AstVisitor.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/AstVisitor.java new file mode 100644 index 00000000..36b3d368 --- /dev/null +++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/AstVisitor.java @@ -0,0 +1,154 @@ +package com.seaofnodes.simple.transpile; + +import com.sun.source.tree.*; +import com.sun.source.util.TreeScanner; + +import java.util.ArrayList; +import java.util.HashMap; + +class AstVisitor extends TreeScanner<Void, Void> { + final HashMap<String, TestClass> results = new HashMap<>(); + + private String className; + private TestMethod current; + private boolean inCatch; + + @Override + public Void visitClass(ClassTree node, Void unused) { + var name = node.getSimpleName().toString(); + if (name.matches("Chapter\\d+Test")) { + var backup = className; + className = name; + super.visitClass(node, unused); + className = backup; + } + return null; + } + + @Override + public Void visitMethod(MethodTree node, Void unused) { + if (className != null) { + current = new TestMethod(); + current.name = node.getName().toString(); + super.visitMethod(node, unused); + results.compute(className, (k, v) -> { + if (v == null) + v = new TestClass(className, new ArrayList<>()); + v.methods().add(current); + return v; + }); + current = null; + } + return null; + } + + @Override + public Void visitMethodInvocation(MethodInvocationTree node, Void unused) { + if (current != null) { + var methodSelect = node.getMethodSelect().toString(); + var args = node.getArguments(); + + if (methodSelect.endsWith(".parse")) { + current.showAfterParse = switch (args.size()) { + case 0 -> false; + case 1 -> (Boolean) literal(args.getFirst()); + default -> throw new RuntimeException("unexpected parse arguments" + node); + }; + } + if (methodSelect.endsWith(".iterate")) { + current.iterate = true; + current.showAfterIterate = switch (args.size()) { + case 0 -> false; + case 1 -> (Boolean) literal(args.getFirst()); + default -> throw new RuntimeException("unexpected parse arguments" + node); + }; + } + if (methodSelect.endsWith("assertEquals")) { + if (inCatch) { + if (args.size() != 2) + throw new RuntimeException("Unexpected number of arguments " + node); + if (args.get(1).toString().contains("e.getMessage()")) { + current.parseErrorMessage = (String) literal(args.get(0)); + } + } else { + if (args.size() != 2) + throw new RuntimeException("Unexpected number of arguments " + node); + if (args.get(1).toString().endsWith(".toString()") || args.get(1).toString().endsWith(".print()")) { + current.assertStopEquals = (String) literal(args.get(0)); + } else if (args.get(1).toString().contains("evaluate(")) { + Number result = (Number) literal(args.get(0)); + var evalArgs = ((MethodInvocationTree) args.get(1)).getArguments(); + if (evalArgs.size() != 2 || !evalArgs.get(0).toString().equals("stop")) + throw new RuntimeException("Unexpected eval arguments " + node); + long parameter = Long.parseLong(literal(evalArgs.get(1)).toString()); + current.evaluations.add(new TestMethod.Evaluation(result, parameter)); + } + } + } + if (methodSelect.endsWith("assertTrue")) { + var arg = args.getFirst().toString(); + current.assertStopRetCtrlIsProj = arg.contains("stop.ret().ctrl() instanceof ProjNode"); + current.assertStopRetCtrlIsCProj = arg.contains("stop.ret().ctrl() instanceof CProjNode"); + current.assertStopRetCtrlIsRegion = arg.contains("stop.ret().ctrl() instanceof RegionNode"); + } + if (methodSelect.endsWith("prettyPrint")) { + if (args.size() < 2 || args.size() > 3 || !args.get(0).toString().equals("stop") || !args.get(1).toString().equals("99")) + throw new RuntimeException("unexpected prettyPrint args " + node); + current.irPrinter = true; + current.irPrinterLLVM = args.size() == 3 && (Boolean) literal(args.get(2)); + } + } + return super.visitMethodInvocation(node, unused); // otherwise we would miss the first part of new Parser("").parse() + } + + @Override + public Void visitNewClass(NewClassTree node, Void unused) { + if (current != null) { + if (node.getIdentifier().toString().equals("Parser")) { + var args = node.getArguments(); + switch (args.size()) { + case 1 -> { + } + case 2 -> current.parserArg = switch (args.get(1).toString()) { + case "TypeInteger.BOT" -> new TestMethod.Arg.IntBot(); + case "TypeInteger.TOP" -> new TestMethod.Arg.IntTop(); + case String s -> { + var parts = s.split("\\(|\\)"); + if (parts.length != 2 | !parts[0].equals("TypeInteger.constant")) + throw new RuntimeException("Unexpected Argument " + s); + yield new TestMethod.Arg.IntConstant(Long.parseLong(parts[1])); + } + }; + default -> throw new RuntimeException("Parser constructor with unexpected arguments: " + node); + } + current.parserInput = (String) literal(args.get(0)); + } else if (node.getIdentifier().toString().equals("GraphVisualizer")) { + // only happens in Chapter02Tests + current.showAfterParse = true; + } + } + return super.visitNewClass(node, unused); + } + + @Override + public Void visitCatch(CatchTree node, Void unused) { + inCatch = true; + super.visitCatch(node, unused); + inCatch = false; + return null; + } + + @Override + public Void visitAssignment(AssignmentTree node, Void unused) { + if (current != null && node.getVariable().toString().equals("Node._disablePeephole")) { + if ((Boolean) literal(node.getExpression())) { + current.disablePeephole = true; + } + } + return super.visitAssignment(node, unused); + } + + private Object literal(ExpressionTree t) { + return ((LiteralTree) t).getValue(); + } +} diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/JUnitParser.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/JUnitParser.java new file mode 100644 index 00000000..e396d56f --- /dev/null +++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/JUnitParser.java @@ -0,0 +1,71 @@ +package com.seaofnodes.simple.transpile; + +import com.sun.source.util.JavacTask; + +import javax.tools.ToolProvider; +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.*; + +import static java.util.Comparator.comparing; + +public class JUnitParser { + + /** + * @param simpleRepo The path to the project root + * @return A sorted map from chapter name to a sorted list of parsed test classes. + */ + public static TreeMap<String, List<TestClass>> parseRepository(Path simpleRepo) throws IOException { + var chapters = new TreeMap<String, List<TestClass>>(); + var inputs = simpleRepo.toFile().listFiles(File::isDirectory); + if (inputs == null) + throw new IOException("failed to read Simple project root"); + Arrays.sort(inputs); + for (File file : inputs) { + var chapter = file.getName(); + if (chapter.matches("chapter\\d+")) { + chapters.put(chapter, parseChapter(file.toPath())); + } + } + return chapters; + } + + /** + * @param input The path to a chapter in the repository. + * @return a sorted list of parsed test classes + */ + public static List<TestClass> parseChapter(Path input) throws IOException { + var tests = new ArrayList<Path>(); + Files.walkFileTree(input, Set.of(), 20, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + if (file.getFileName().toString().matches("Chapter\\d+Test\\.java")) { + tests.add(file); + } + return FileVisitResult.CONTINUE; + } + }); + + var compiler = ToolProvider.getSystemJavaCompiler(); + var compilationUnits = compiler + .getStandardFileManager(null, null, null) + .getJavaFileObjectsFromPaths(tests); + + var task = (JavacTask) compiler.getTask(null, null, null, null, null, compilationUnits); + + var visitor = new AstVisitor(); + for (var tree : task.parse()) { + tree.accept(visitor, null); + } + + var testClasses = new ArrayList<>(visitor.results.values()); + testClasses.sort(comparing(TestClass::name)); + return testClasses; + } +} + diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestClass.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestClass.java new file mode 100644 index 00000000..5bb5609c --- /dev/null +++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestClass.java @@ -0,0 +1,6 @@ +package com.seaofnodes.simple.transpile; + +import java.util.ArrayList; + +public record TestClass(String name, ArrayList<TestMethod> methods) { +} diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestMethod.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestMethod.java new file mode 100644 index 00000000..107b5f56 --- /dev/null +++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/TestMethod.java @@ -0,0 +1,36 @@ +package com.seaofnodes.simple.transpile; + +import java.util.ArrayList; + +public class TestMethod { + public String name; + public String parserInput; + public Arg parserArg; + // if this is set you can ignore the rest + public String parseErrorMessage; + public boolean disablePeephole; + public boolean showAfterParse; + public boolean iterate; + public boolean showAfterIterate; + public boolean irPrinter; + public boolean irPrinterLLVM; + public String assertStopEquals; + public boolean assertStopRetCtrlIsProj; + public boolean assertStopRetCtrlIsCProj; + public boolean assertStopRetCtrlIsRegion; + public final ArrayList<Evaluation> evaluations = new ArrayList<>(); + + public sealed interface Arg { + record IntBot() implements Arg { + } + + record IntTop() implements Arg { + } + + record IntConstant(long value) implements Arg { + } + } + + public record Evaluation(Number result, long parameter) { + } +} diff --git a/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/printers/RustPrinter.java b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/printers/RustPrinter.java new file mode 100644 index 00000000..5397eb5e --- /dev/null +++ b/transpile-tests/src/main/java/com/seaofnodes/simple/transpile/printers/RustPrinter.java @@ -0,0 +1,155 @@ +package com.seaofnodes.simple.transpile.printers; + +import com.seaofnodes.simple.transpile.JUnitParser; +import com.seaofnodes.simple.transpile.TestClass; +import com.seaofnodes.simple.transpile.TestMethod; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +public class RustPrinter { + + public static void main(String[] args) throws Exception { + if (args.length != 2) { + System.out.println("Usage: <path-to-simple-repo> <output-dir>"); + System.exit(1); + } + var printer = new RustPrinter(); + var output = Path.of(args[1]); + + for (var e : JUnitParser.parseRepository(Path.of(args[0])).entrySet()) { + printer.printToFile(e.getKey(), e.getValue(), output); + } + } + + private void printToFile(String chapter, List<TestClass> testClasses, Path outputDirectory) throws IOException, InterruptedException { + var output = outputDirectory.resolve(chapter); + Files.createDirectories(output); + for (var testClass : testClasses) { + var outputFile = output.resolve(testClass.name().replace("Test", "").toLowerCase() + ".rs"); + System.out.println(outputFile); + Files.write(outputFile, printAndRustfmt(testClass.methods())); + } + } + + private byte[] printAndRustfmt(ArrayList<TestMethod> tests) throws IOException, InterruptedException { + var p = Runtime.getRuntime().exec(new String[]{"rustfmt"}); + try (var print = new PrintStream(new BufferedOutputStream(p.getOutputStream()))) { + printFile(tests, print); + } + p.waitFor(); + + var errors = new String(p.getErrorStream().readAllBytes(), StandardCharsets.UTF_8); + if (!errors.isEmpty() || p.exitValue() != 0) { + throw new RuntimeException("rustfmt exit=" + p.exitValue() + " stderr=" + errors); + } + + return p.getInputStream().readAllBytes(); + } + + private void printFile(ArrayList<TestMethod> tests, PrintStream out) { + out.println(""" + use crate::datastructures::arena::DroplessArena; + use crate::sea_of_nodes::parser::Parser; + use crate::sea_of_nodes::types::Types; + """); + + if (tests.stream().anyMatch(t -> t.parseErrorMessage != null)) { + out.println("use crate::sea_of_nodes::tests::test_error;"); + } + if (tests.stream().anyMatch(t -> t.irPrinter && !t.irPrinterLLVM)) { + out.println("use crate::sea_of_nodes::ir_printer::pretty_print;"); + } + if (tests.stream().anyMatch(t -> t.irPrinterLLVM)) { + out.println("use crate::sea_of_nodes::ir_printer::pretty_print_llvm;"); + } + if (tests.stream().anyMatch(t -> !t.evaluations.isEmpty())) { + out.println("use crate::sea_of_nodes::graph_evaluator::evaluate;"); + } + + for (TestMethod test : tests) { + printTest(test, out); + } + } + + private void printTest(TestMethod test, PrintStream out) { + out.println(); + out.println("#[test]"); + out.println("fn " + snakeCase(test.name) + "() {"); + if (test.parseErrorMessage != null) { + out.println("test_error(" + string(test.parserInput) + ", " + string(test.parseErrorMessage) + ");"); + } else if (test.parserInput != null) { + out.println("let arena = DroplessArena::new();"); + out.println("let types = Types::new(&arena);"); + if (test.parserArg == null) { + out.println("let mut parser = Parser::new(" + string(test.parserInput, true) + ", &types);"); + } else { + out.printf("let mut parser = Parser::new_with_arg(%s, &types, %s);\n", string(test.parserInput, true), switch (test.parserArg) { + case TestMethod.Arg.IntBot ignored -> "types.ty_int_bot"; + case TestMethod.Arg.IntTop ignored -> "types.ty_int_top"; + case TestMethod.Arg.IntConstant c -> "types.get_int(" + c.value() + ")"; + }); + } + if (test.disablePeephole) { + out.println("parser.nodes.disable_peephole = true;"); + } + out.println("let stop = parser.parse().unwrap();"); + if (test.showAfterParse) { + out.println("parser.show_graph();"); + } + if (test.iterate) { + out.println("parser.iterate(stop);"); + } + if (test.showAfterIterate) { + out.println("parser.show_graph();"); + } + if (test.irPrinter) { + out.println("println!(\"{}\", pretty_print" + (test.irPrinterLLVM ? "_llvm" : "") + "(&parser.nodes, stop, 99));"); + } + if (test.assertStopEquals != null) { + out.println("assert_eq!(parser.print(stop), " + string(test.assertStopEquals) + ");"); + } + if (test.assertStopRetCtrlIsProj) { + out.println("assert!(matches!(parser.nodes.ret_ctrl(stop), Node::Proj(_)));"); + } + if (test.assertStopRetCtrlIsCProj) { + out.println("assert!(matches!(parser.nodes.ret_ctrl(stop), Node::CProj(_)));"); + } + if (test.assertStopRetCtrlIsRegion) { + out.println("assert!(matches!(parser.nodes.ret_ctrl(stop), Node::Region{..}));"); + } + for (var evaluation : test.evaluations) { + out.println("assert_eq!(evaluate(&parser.nodes, stop, Some(" + evaluation.parameter() + "), None), " + evaluation.result() + ");"); + } + } else { + out.println("todo!();"); + } + out.println("}"); + } + + private String string(String value) { + return string(value, false); + } + + private String string(String value, boolean multiline) { + var escaped = value.replace("\\", "\\\\") + .replace("\r", "\\r") + .replace("\t", "\\t") + .replace("\"", "\\\""); + if (multiline && escaped.contains("\n")) { + return "\"\\\n" + escaped + "\""; + } else { + return "\"" + escaped.replace("\n", "\\n") + "\""; + } + } + + private String snakeCase(String identifier) { + return identifier.replaceAll("[A-Z]+", "_$0").toLowerCase(); + } +} diff --git a/transpile-tests/src/test/java/com/seaofnodes/simple/transpile/TranspilerTest.java b/transpile-tests/src/test/java/com/seaofnodes/simple/transpile/TranspilerTest.java new file mode 100644 index 00000000..a2c3a782 --- /dev/null +++ b/transpile-tests/src/test/java/com/seaofnodes/simple/transpile/TranspilerTest.java @@ -0,0 +1,26 @@ +package com.seaofnodes.simple.transpile; + +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Path; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TranspilerTest { + @Test + public void testParser() throws IOException { + var cwd = Path.of(".").toAbsolutePath().normalize(); + assertEquals("transpile-tests", cwd.getFileName().toString()); + var all = JUnitParser.parseRepository(cwd.getParent()); + assertTrue(all.size() >= 14); + + var chapter09 = all.get("chapter09"); + assertEquals(9, chapter09.size()); + + var class6 = chapter09.get(5); + assertEquals("Chapter06Test", class6.name()); + assertEquals(1, class6.methods().stream().filter(m -> m.name.equals("testPeepholeReturn")).count()); + } +} |