Add fuzzer (#58)

* Add fuzzer

* Add fuzzer

---------

Co-authored-by: Cliff Click <cliffc@acm.org>

3 files changed, 578 insertions, 0 deletions

diff --git a/chapter08/src/test/java/com/seaofnodes/simple/Fuzzer08Test.java b/chapter08/src/test/java/com/seaofnodes/simple/Fuzzer08Test.java
new file mode 100644
index 00000000..d8f10eef
--- /dev/null
+++ b/chapter08/src/test/java/com/seaofnodes/simple/Fuzzer08Test.java
@@ -0,0 +1,154 @@
+package com.seaofnodes.simple;

+

+import com.seaofnodes.simple.node.Node;

+import com.seaofnodes.simple.node.StopNode;

+import org.junit.Ignore;

+import org.junit.Test;

+

+import java.util.ArrayList;

+import java.util.Random;

+

+import static org.junit.Assert.assertTrue;

+

+public class Fuzzer08Test {

+

+    private final ArrayList<Throwable> exceptions = new ArrayList<>();

+

+    public static StopNode parse(String script, boolean disablePeeps) {

+        var parser = new Parser(script);

+        Node._disablePeephole = disablePeeps;

+        return parser.parse();

+    }

+

+    private static boolean sameException(Throwable e1, Throwable e2) {

+        if (e1 == null || e2 == null) return e1 == e2;

+        if (e1.getClass() != e2.getClass()) return false;

+        if (e1.getMessage() != null && e2.getMessage() != null && !e1.getMessage().equals(e2.getMessage())) return false;

+        var s1 = e1.getStackTrace();

+        var s2 = e2.getStackTrace();

+        if (s1 != null && s1.length > 0 && s2 != null && s2.length > 0 && !s1[0].equals(s2[0])) return false;

+        if (!sameException(e1.getCause(), e2.getCause())) return false;

+        var o1 = e1.getSuppressed();

+        var o2 = e2.getSuppressed();

+        if (o1 == null || o2 == null || o1.length != o2.length) return o1 == o2;

+        for (int i=0; i<o1.length; i++) {

+            if (!sameException(o1[i], o2[i])) return false;

+        }

+        return true;

+    }

+

+    private void recordException(Throwable e, String script) {

+        for (var ex : exceptions) {

+            if (sameException(ex, e)) return;

+        }

+        exceptions.add(e);

+        System.out.println("===========================");

+        System.out.println("========== Stack ==========");

+        System.out.flush();

+        e.printStackTrace(System.err);

+        System.err.flush();

+        System.out.println("========== Code ===========");

+        System.out.println(script);

+        System.out.println("========= Reduced =========");

+        System.out.println(Reducer.reduce(script, false));

+        System.out.flush();

+    }

+

+    private void checkGraphs(StopNode stop1, StopNode stop2, long in, String script) {

+        long e1;

+        try {

+            e1 = GraphEvaluator.evaluate(stop1, in);

+        } catch (RuntimeException e) {

+            if (!e.getMessage().equals("Timeout")) recordException(e, script);

+            return;

+        }

+        long e2;

+        try {

+            e2 = GraphEvaluator.evaluate(stop2, in);

+        } catch (RuntimeException e) {

+            if (!e.getMessage().equals("Timeout")) recordException(e, script);

+            return;

+        }

+        if (e1 != e2) {

+            recordException(new RuntimeException("Different calculations " + e1 + " vs " + e2), script);

+        }

+    }

+

+    private void check(String script, boolean valid) {

+        StopNode stop1;

+        try {

+            stop1 = parse(script, true);

+        } catch (Throwable e1) {

+            try {

+                parse(script, false);

+                var rt = new RuntimeException("Exception only without peeps " + e1.getMessage());

+                rt.addSuppressed(e1);

+                recordException(rt, script);

+                return;

+            } catch (Throwable e2) {

+                if (e1.getClass() == e2.getClass() && e1.getMessage().equals(e2.getMessage())) {

+                    if (valid || e1.getClass() != RuntimeException.class) recordException(e1, script);

+                    return;

+                }

+                var rt = new RuntimeException("Bad different exceptions " + e1.getMessage() + " & " + e2.getMessage());

+                rt.addSuppressed(e1);

+                rt.addSuppressed(e2);

+                recordException(rt, script);

+                return;

+            }

+        }

+        StopNode stop2;

+        try {

+            stop2 = parse(script, false);

+        } catch (Throwable e2) {

+            var rt = new RuntimeException("Exception only with peeps " + e2.getMessage());

+            rt.addSuppressed(e2);

+            recordException(rt, script);

+            return;

+        }

+        checkGraphs(stop1, stop2, 0, script);

+        checkGraphs(stop1, stop2, 1, script);

+        checkGraphs(stop1, stop2, 10, script);

+    }

+

+    private void checkParsing(String script) {

+        try {

+            parse(script, false);

+        } catch (Throwable e) {

+            recordException(e, script);

+        }

+    }

+

+    public void fuzzParser(long seed) {

+        var rand = new Random(seed);

+        var sb = new StringBuilder();

+        new ScriptGenerator(rand, sb, true).genProgram();

+        checkParsing(sb.toString());

+    }

+

+    public void fuzzPeeps(long seed) {

+        var rand = new Random(seed);

+        var sb = new StringBuilder();

+        var valid = new ScriptGenerator(rand, sb, false).genProgram();

+        check(sb.toString(), valid);

+    }

+

+    @Test

+    @Ignore

+    public void fuzzParser() {

+        var fuzzer = new Fuzzer08Test();

+        for (int i=0; i<1000000; i++)

+            fuzzer.fuzzParser(i);

+        assertTrue(fuzzer.exceptions.isEmpty());

+    }

+

+    @Test

+    @Ignore

+    public void fuzzPeeps() {

+        var fuzzer = new Fuzzer08Test();

+        for (int i=0; i<1000000; i++)

+            fuzzer.fuzzPeeps(i);

+        assertTrue(fuzzer.exceptions.isEmpty());

+    }

+

+}

diff --git a/chapter08/src/test/java/com/seaofnodes/simple/Reducer.java b/chapter08/src/test/java/com/seaofnodes/simple/Reducer.java
new file mode 100644
index 00000000..38a4502c
--- /dev/null
+++ b/chapter08/src/test/java/com/seaofnodes/simple/Reducer.java
@@ -0,0 +1,119 @@
+package com.seaofnodes.simple;

+

+import com.seaofnodes.simple.node.Node;

+

+import java.util.regex.Pattern;

+

+public class Reducer {

+

+    private record Replace(Pattern p, String r) {}

+

+    private static final Replace[] REPLACEMENTS = {

+            new Replace(Pattern.compile("(?<=[+\\-*/<>=(])--"), ""),

+            new Replace(Pattern.compile("\\b--"), "+"),

+            new Replace(Pattern.compile("\\b\\+-"), "-"),

+            new Replace(Pattern.compile("\\bfalse\\b"), "0"),

+            new Replace(Pattern.compile("\\btrue\\b"), "1"),

+            new Replace(Pattern.compile("-"), ""),

+            new Replace(Pattern.compile("(?<![+\\-*/<>=])(?:-|\\*|/|>=|<=|>|<|!=|==)(?![+\\-*/<>=])"), "+"),

+            new Replace(Pattern.compile("\\{(?:[^{}]|\\{})+}"), "{}"),

+            new Replace(Pattern.compile("\\{((?:[^{}]|\\{})+)}"), "$1"),

+            new Replace(Pattern.compile("\\(([^()]+)\\)"), "$1"),

+            new Replace(Pattern.compile("\\b\\w+\\b"), "1"),

+            new Replace(Pattern.compile("\\b\\w+\\b"), "0"),

+            new Replace(Pattern.compile("\\d+\\+"), ""),

+            new Replace(Pattern.compile("\\+\\d+"), ""),

+            new Replace(Pattern.compile("(?<=\\n|^)[^\\n]*(?=\\n|$)"), "\n"),

+            new Replace(Pattern.compile("\\n+"), "\n"),

+            new Replace(Pattern.compile("^\\n+|\\n+$"), ""),

+            new Replace(Pattern.compile("else "), ""),

+            new Replace(Pattern.compile("\\bif\\([^()]+\\) *"), ""),

+            new Replace(Pattern.compile("\\bwhile\\([^()]+\\) *"), ""),

+    };

+

+    private static boolean sameException(Throwable e1, Throwable e2) {

+        if (e1 == null || e2 == null) return e1 == e2;

+        if (e1.getClass() != e2.getClass()) return false;

+        if (e1.getMessage() != null && e2.getMessage() != null && !e1.getMessage().equals(e2.getMessage())) return false;

+        var s1 = e1.getStackTrace();

+        var s2 = e2.getStackTrace();

+        if (s1 == null || s1.length == 0 || s2 == null || s2.length == 0) return true;

+        return s1[0].equals(s2[0]);

+    }

+

+    private final boolean disablePeeps;

+

+    private Reducer(boolean disablePeeps) {

+        this.disablePeeps = disablePeeps;

+    }

+

+    private Throwable parse(String script) {

+        try {

+            var parser = new Parser(script);

+            Node._disablePeephole = disablePeeps;

+            parser.parse();

+            return null;

+        } catch (Throwable e) {

+            return e;

+        }

+    }

+

+    private String doReplacement(String script, Throwable ex, Pattern pattern, String with) {

+        var matcher = pattern.matcher(script);

+        var sb = new StringBuilder();

+        int last = 0;

+        while (matcher.find()) {

+            var pos = sb.length();

+            matcher.appendReplacement(sb, with);

+            var tail = sb.length();

+            matcher.appendTail(sb);

+            var ex2 = parse(sb.toString());

+            if (sameException(ex, ex2)) {

+                sb.setLength(tail);

+            } else {

+                sb.setLength(pos);

+                sb.append(script, last, matcher.end());

+            }

+            last = matcher.end();

+        }

+        matcher.appendTail(sb);

+        return sb.toString();

+    }

+

+    private String cleanVariables(String script, Throwable ex) {

+        var matcher = Pattern.compile("\\bint ([0-9a-zA-Z_]+)").matcher(script);

+        int num = 0;

+        while (matcher.find()) {

+            var var = matcher.group(1);

+            if (var.matches("v\\d+")) continue;

+            while (script.contains("v"+num)) num++;

+            var n = script.replaceAll("\\b"+var+"\\b", "v" + num);

+            var ex2 = parse(n);

+            if (sameException(ex, ex2)) {

+                script = n;

+            }

+        }

+        return script;

+    }

+

+    private String doAllReplacements(String script, Throwable ex) {

+        for (var replace: REPLACEMENTS) {

+            script = doReplacement(script, ex, replace.p, replace.r);

+        }

+        return script;

+    }

+

+    public static String reduce(String script, boolean disablePeeps) {

+        var reducer = new Reducer(disablePeeps);

+        var ex = reducer.parse(script);

+        if (ex == null) return "return 0;";

+        String old;

+        do {

+            old = script;

+            script = reducer.doAllReplacements(script, ex);

+            script = reducer.cleanVariables(script, ex);

+        } while (!script.equals(old));

+        return script;

+    }

+

+}

diff --git a/chapter08/src/test/java/com/seaofnodes/simple/ScriptGenerator.java b/chapter08/src/test/java/com/seaofnodes/simple/ScriptGenerator.java
new file mode 100644
index 00000000..8c51089c
--- /dev/null
+++ b/chapter08/src/test/java/com/seaofnodes/simple/ScriptGenerator.java
@@ -0,0 +1,305 @@
+package com.seaofnodes.simple;

+

+import com.seaofnodes.simple.node.ScopeNode;

+

+import java.util.ArrayList;

+import java.util.HashSet;

+import java.util.Random;

+

+public class ScriptGenerator {

+

+    @SuppressWarnings("unchecked")

+    private static HashSet<String> getKeywords() {

+        try {

+            var field = Parser.class.getDeclaredField("KEYWORDS");

+            field.setAccessible(true);

+            return (HashSet<String>) field.get(new Parser(""));

+        } catch (Throwable e) {

+            throw new ExceptionInInitializerError(e);

+        }

+    }

+

+    private static final int INDENTATION = 4;

+    private static final String VAR_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789";

+    private static final HashSet<String> KEYWORDS = getKeywords();

+

+    private static final int FLAG_STOP = 0x01;

+    private static final int FLAG_IF_WITHOUT_ELSE = 0x02;

+

+    private final Random random;

+    private int indentation = 0;

+    private final StringBuilder sb;

+    private int loopDepth = 0;

+    private int currScopeStart = 0;

+    private int depth = 0;

+    private int exprDepth = 3;

+    private final ArrayList<String> variables = new ArrayList<>();

+    private final boolean generateValid;

+    private boolean maybeInvalid = false;

+

+    public ScriptGenerator(Random random, StringBuilder sb, boolean generateValid) {

+        this.random = random;

+        this.sb = sb;

+        this.generateValid = generateValid;

+    }

+

+    private int randLog(int max) {

+        max--;

+        int n = 1 << max;

+        var num = random.nextInt(n);

+        for(int i=max-1; i>=0; i--) {

+            if ((num & (1 << i))!=0) return max-i-1;

+        }

+        return max;

+    }

+

+    private StringBuilder printIndentation() {

+        return sb.repeat(' ', indentation);

+    }

+

+    public boolean genProgram() {

+        this.maybeInvalid = false;

+        variables.add(ScopeNode.ARG0);

+        depth = random.nextInt(9) + 1;

+        currScopeStart = variables.size();

+        if ((genStatements() & FLAG_STOP) != 0)

+            genReturn();

+        return !this.maybeInvalid;

+    }

+

+    public int genStatements() {

+        var num = random.nextInt(10);

+        for (int i=0; i<num; i++) {

+            printIndentation();

+            var stop = genStatement();

+            sb.append("\n");

+            if ((stop & FLAG_STOP) != 0) return stop;

+        }

+        return 0;

+    }

+

+    public int genStatement() {

+        return switch (random.nextInt(10)) {

+            case 1 -> genBlock();

+            case 2 -> genIf();

+            case 3 -> genWhile();

+            case 4, 5 -> genDecl();

+            case 6, 7, 8 -> genAssignment();

+            default -> genExit();

+        };

+    }

+

+    public int genStatementBlock() {

+        indentation += INDENTATION;

+        int res;

+        if (depth == 0) {

+            res = random.nextBoolean() ? genAssignment() : genExit();

+        } else {

+            depth--;

+            res = switch (random.nextInt(10)) {

+                case 1, 2, 3, 4, 5 -> {

+                    depth++;

+                    indentation -= INDENTATION;

+                    var ret = genBlock();

+                    indentation += INDENTATION;

+                    depth--;

+                    yield ret;

+                }

+                case 6 -> genIf();

+                case 7 -> genWhile();

+                case 8 -> genAssignment();

+                default -> genExit();

+            };

+            depth++;

+        }

+        indentation -= INDENTATION;

+        return res;

+    }

+

+    public int genExit() {

+        if (loopDepth == 0 && generateValid) return genReturn();

+        return switch (random.nextInt(7)) {

+            case 0,1,2 -> {

+                if (loopDepth == 0) maybeInvalid = true;

+                sb.append("continue;");

+                yield FLAG_STOP;

+            }

+            case 3,4,5 -> {

+                if (loopDepth == 0) maybeInvalid = true;

+                sb.append("break;");

+                yield FLAG_STOP;

+            }

+            default -> genReturn();

+        };

+    }

+

+    public int genBlock() {

+        if (depth == 0) {

+            sb.append("{}");

+            return 0;

+        }

+        depth--;

+        int oldCSS = currScopeStart;

+        currScopeStart = variables.size();

+        sb.append("{\n");

+        indentation += INDENTATION;

+        var stop = genStatements();

+        indentation -= INDENTATION;

+        printIndentation().append("}");

+        while (variables.size() > currScopeStart)

+            variables.removeLast();

+        currScopeStart = oldCSS;

+        depth++;

+        return stop & ~FLAG_IF_WITHOUT_ELSE;

+    }

+

+    public int genIf() {

+        sb.append("if(");

+        genExpression();

+        sb.append(") ");

+        var stop = genStatementBlock();

+        if ((stop & FLAG_IF_WITHOUT_ELSE) == 0 && random.nextInt(10) > 3) {

+            sb.append("\n");

+            printIndentation().append("else ");

+            stop &= genStatementBlock();

+        } else {

+            stop = FLAG_IF_WITHOUT_ELSE;

+        }

+        return stop;

+    }

+

+    public int genWhile() {

+        sb.append("while(");

+        genExpression();

+        sb.append(") ");

+        loopDepth++;

+        var stop = genStatementBlock();

+        loopDepth--;

+        return stop & FLAG_IF_WITHOUT_ELSE;

+    }

+

+    private StringBuilder getRandomName() {

+        int len = random.nextInt(10) + 1;

+        StringBuilder sb = new StringBuilder(len);

+        sb.append(VAR_CHARS.charAt(random.nextInt(VAR_CHARS.length()-10)));

+        for (int i=1; i<len; i++)

+            sb.append(VAR_CHARS.charAt(random.nextInt(VAR_CHARS.length())));

+        return sb;

+    }

+

+    private String getVarName() {

+        if (currScopeStart > 0 && random.nextInt(10) > 7) {

+            var v = variables.get(random.nextInt(currScopeStart));

+            if (variables.lastIndexOf(v) < currScopeStart) return v;

+            if (!generateValid) {

+                maybeInvalid = true;

+                return v;

+            }

+        }

+        StringBuilder sb = getRandomName();

+        var v = sb.toString();

+        if (!generateValid && (KEYWORDS.contains(v) || variables.lastIndexOf(v) >= currScopeStart)) {

+            maybeInvalid = true;

+            return v;

+        }

+        while (KEYWORDS.contains(v) || variables.lastIndexOf(v) >= currScopeStart) {

+            sb.append(VAR_CHARS.charAt(random.nextInt(VAR_CHARS.length())));

+            v = sb.toString();

+        }

+        return v;

+    }

+

+    public int genDecl() {

+        String name = getVarName();

+        sb.append("int ").append(name).append("=");

+        genExpression();

+        sb.append(";");

+        variables.add(name);

+        return 0;

+    }

+

+    public int genAssignment() {

+        if (variables.isEmpty()) return genDecl();

+        genVariable();

+        sb.append("=");

+        genExpression();

+        sb.append(";");

+        return 0;

+    }

+

+    public int genReturn() {

+        sb.append("return ");

+        genExpression();

+        sb.append(";");

+        return FLAG_STOP;

+    }

+

+    private static final String[] EXPRESSIONS = {"==", "!=", "<", "<=", ">", ">=", "+", "-", "*", "/"};

+    private static final String[] UNARY = {"-"};

+

+    public void genExpression() {

+        var num = randLog(10);

+        while(num-->0) {

+            genUnary();

+            sb.append(EXPRESSIONS[random.nextInt(EXPRESSIONS.length)]);

+        }

+        genUnary();

+    }

+

+    public void genUnary() {

+        var num = randLog(6);

+        while(num-->0) {

+            sb.append(UNARY[random.nextInt(UNARY.length)]);

+        }

+        genPrimary();

+    }

+

+    public void genPrimary() {

+        switch (random.nextInt(10)) {

+            case 0:

+                sb.append("true");

+                break;

+            case 1:

+                sb.append("false");

+                break;

+            case 2:

+                if (exprDepth == 0) {

+                    if (random.nextBoolean()) {

+                        genNumber();

+                    } else {

+                        genVariable();

+                    }

+                } else {

+                    sb.append("(");

+                    exprDepth--;

+                    genExpression();

+                    exprDepth++;

+                    sb.append(")");

+                }

+                break;

+            case 3, 4, 5, 6:

+                genNumber();

+                break;

+            default:

+                genVariable();

+        }

+    }

+

+    public void genNumber() {

+        sb.append(random.nextInt(100));

+    }

+

+    public void genVariable() {

+        if (!generateValid && random.nextInt(100)>97) {

+            sb.append(getRandomName());

+            maybeInvalid = true;

+            return;

+        }

+        if (variables.isEmpty()) {

+            genNumber();

+        } else {

+            sb.append(variables.get(random.nextInt(variables.size())));

+        }

+    }

+

+}