Update the lexer, parser and AST to match the language definition.

Add toString() method on AST nodes so that we can easily check the results of parsing.
Remove AA lexer
Move the Ecstasy lexer classes to one package, except Handy

2 files changed, 139 insertions, 63 deletions

diff --git a/chapter01/src/main/java/com/seaofnodes/simple/parser/Ast.java b/chapter01/src/main/java/com/seaofnodes/simple/parser/Ast.java
index a93b230f..b11d1ddb 100644
--- a/chapter01/src/main/java/com/seaofnodes/simple/parser/Ast.java
+++ b/chapter01/src/main/java/com/seaofnodes/simple/parser/Ast.java
@@ -1,48 +1,94 @@
+/*
+ * DO NOT REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ * <p>
+ * Contributor(s):
+ * <p>
+ * The Initial Developer of the Original Software is Dibyendu Majumdar.
+ * <p>
+ * This file is part of the Sea Of Nodes Simple Programming language
+ * implementation. See https://github.com/SeaOfNodes/Simple
+ * <p>
+ * The contents of this file are subject to the terms of the
+ * Apache License Version 2 (the "APL"). You may not use this
+ * file except in compliance with the License. A copy of the
+ * APL may be obtained from:
+ * http://www.apache.org/licenses/LICENSE-2.0
+ */
 package com.seaofnodes.simple.parser;
 
-import com.seaofnodes.simple.common.CompilerException;
 import com.seaofnodes.simple.lexer.ecstasy.Token;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
-public abstract class Ast {
+/**
+ * A simple AST definition.
+ */
+public class Ast {
 
-    public static class Node {
+    private Ast() {
     }
 
-    public static class Expr extends Node {
+    public interface Node {
+    }
+
+    public abstract static class Expr implements Node {
         public final Token op;
 
-        public Expr(Token op) {
+        protected Expr(Token op) {
             this.op = op;
         }
     }
 
+    /**
+     * A name in the language
+     */
     public static class Identifier extends Expr {
         public final String name;
+
         public Identifier(Token op) {
             super(op);
             name = op.getValueText();
         }
+
+        @Override
+        public String toString() {
+            return name;
+        }
     }
 
+    /**
+     * Reference to a name in the language
+     */
     public static class Symbol extends Expr {
         public final String name;
+
         public Symbol(Token op) {
             super(op);
             name = op.getValueText();
         }
+
+        @Override
+        public String toString() {
+            return name;
+        }
     }
 
     public static class Binary extends Expr {
-        public final Expr expr1, expr2;
+        public final Expr expr1;
+        public final Expr expr2;
 
         public Binary(Token op, Expr expr1, Expr expr2) {
             super(op);
             this.expr1 = expr1;
             this.expr2 = expr2;
         }
+
+        @Override
+        public String toString() {
+            return "(" + expr1 + op.getValueText() + expr2 + ")";
+        }
     }
 
     public static class Unary extends Expr {
@@ -52,49 +98,65 @@ public abstract class Ast {
             super(op);
             this.expr = expr;
         }
+
+        @Override
+        public String toString() {
+            return "(" + op.getValueText() + "(" + expr + "))";
+        }
     }
 
     public static class Constant extends Expr {
         public Constant(Token op) {
             super(op);
         }
+
+        @Override
+        public String toString() {
+            return op.getValueText();
+        }
     }
 
-    public static class Statement extends Node {
+    public abstract static class Statement implements Node {
     }
 
     public static class IfElse extends Statement {
         public final Expr expr;
-        public final Statement ifStmt, elseStmt;
+        public final Statement ifStmt;
+        public final Statement elseStmt;
 
         public IfElse(Expr expr, Statement ifStmt, Statement elseStmt) {
             this.expr = expr;
             this.ifStmt = ifStmt;
             this.elseStmt = elseStmt;
         }
+
+        @Override
+        public String toString() {
+            if (elseStmt != null)
+                return "if(" + expr + ") " + ifStmt + " else " + elseStmt;
+            return "if(" + expr + ") " + ifStmt;
+        }
     }
 
     public static class While extends Statement {
-        public Expr expr;
-        public Statement stmt;
+        public final Expr expr;
+        public final Statement stmt;
 
-        public While() {
-        }
-        public void init(Expr expr, Statement stmt) {
+        public While(Expr expr, Statement stmt) {
             this.expr = expr;
             this.stmt = stmt;
         }
+
+        @Override
+        public String toString() {
+            return "while(" + expr.toString() + ") " + stmt;
+        }
     }
 
     public static class Break extends Statement {
-        public final Statement stmt;
-
-        public Break(Statement stmt) {
-            if (stmt == null)
-            {
-                throw new CompilerException("uneclosed break");
-            }
-            this.stmt = stmt;
+        @Override
+        public String toString() {
+            return "break;";
         }
     }
 
@@ -106,14 +168,24 @@ public abstract class Ast {
             this.identifier = identifier;
             this.expr = expr;
         }
+
+        @Override
+        public String toString() {
+            return identifier.toString() + "=" + expr.toString() + ";";
+        }
     }
 
-    public static class Type extends Node {
+    public static class Type implements Node {
         public final Token.Id typeId;
 
         public Type(Token.Id typeId) {
             this.typeId = typeId;
         }
+
+        @Override
+        public String toString() {
+            return typeId.TEXT;
+        }
     }
 
     public static class Declare extends Statement {
@@ -124,9 +196,20 @@ public abstract class Ast {
             this.type = type;
             this.identifier = identifier;
         }
+
+        @Override
+        public String toString() {
+            return type.toString() + " " + identifier.toString() + ";";
+        }
     }
 
     public static class Block extends Statement {
         public final List<Statement> stmtList = new ArrayList<>();
+
+        @Override
+        public String toString() {
+            return "{" +
+                    stmtList.stream().map(Object::toString).collect(Collectors.joining()) + "}";
+        }
     }
 }
diff --git a/chapter01/src/main/java/com/seaofnodes/simple/parser/Parser.java b/chapter01/src/main/java/com/seaofnodes/simple/parser/Parser.java
index b36942d4..263d0b41 100644
--- a/chapter01/src/main/java/com/seaofnodes/simple/parser/Parser.java
+++ b/chapter01/src/main/java/com/seaofnodes/simple/parser/Parser.java
@@ -1,4 +1,4 @@
-/**
+/*
  * DO NOT REMOVE COPYRIGHT NOTICES OR THIS HEADER.
  *
  * Contributor(s):
@@ -16,28 +16,28 @@
  */
 package com.seaofnodes.simple.parser;
 
-import com.seaofnodes.simple.common.CompilerException;
+import com.seaofnodes.simple.lexer.ecstasy.CompilerException;
 import com.seaofnodes.simple.lexer.ecstasy.Lexer;
 import com.seaofnodes.simple.lexer.ecstasy.Token;
-import com.seaofnodes.simple.common.Severity;
+import com.seaofnodes.simple.lexer.ecstasy.Severity;
 
 public class Parser {
     final Lexer lexer;
     Token look;
-    Ast.Statement statementEnclosing;
 
     public Parser(Lexer lexer) {
         this.lexer = lexer;
     }
 
-    Token nextOrEof() {
+    private Token nextOrEof() {
         if (lexer.hasNext())
             return lexer.next();
         return new Token(look.getStartPosition(), look.getEndPosition(), Token.Id.EOF);
     }
 
-    void nextToken() {
+    private void nextToken() {
         look = nextOrEof();
+        // skip comments
         while (look.isComment()) {
             look = nextOrEof();
         }
@@ -47,12 +47,12 @@ public class Parser {
         }
     }
 
-    void error(Token t, String errorMessage) {
+    private void error(Token t, String errorMessage) {
         lexer.log(Severity.ERROR, errorMessage, t.getStartPosition(), t.getEndPosition());
         throw new CompilerException(errorMessage + ": " + t.toDebugString());
     }
 
-    void match(Token.Id tag) {
+    private void match(Token.Id tag) {
         if (look.getId() == tag) {
             nextToken();
         } else {
@@ -65,43 +65,40 @@ public class Parser {
         return parseProgram();
     }
 
-    Ast.Statement parseProgram() {
+    private Ast.Statement parseProgram() {
         var block = new Ast.Block();
         parseDeclarations(block);
         parseStatements(block);
         return block;
     }
 
-    void parseDeclarations(Ast.Block block) {
-        while (look.getId() == Token.Id.VAR) {
+    private boolean isTypeName(Token tok) {
+        return switch (tok.getId()) {
+            case TYPE_INT, TYPE_CHAR, TYPE_FLOAT -> true;
+            default -> false;
+        };
+    }
+
+    private void parseDeclarations(Ast.Block block) {
+        while (isTypeName(look)) {
+            var type = new Ast.Type(look.getId());
             nextToken();
             var tok = look;
             match(Token.Id.IDENTIFIER);
-            match(Token.Id.COLON);
-            var type = parseType();
             match(Token.Id.SEMICOLON);
             var identifier = new Ast.Identifier(tok);
             block.stmtList.add(new Ast.Declare(type, identifier));
         }
     }
 
-    Ast.Type parseType() {
-        if (look.getId() == Token.Id.LIT_INTA) {
-            nextToken();
-            return new Ast.Type(look.getId());
-        }
-        error(look, "syntax error: unsupported type (at present only Int type is allowed)");
-        return null;
-    }
-
-    void parseStatements(Ast.Block block) {
+    private void parseStatements(Ast.Block block) {
         while (look.getId() != Token.Id.EOF &&
                 look.getId() != Token.Id.R_CURLY) {
             block.stmtList.add(parseStatement());
         }
     }
 
-    Ast.Statement parseStatement() {
+    private Ast.Statement parseStatement() {
         Ast.Expr x;
         Ast.Statement s1;
         Ast.Statement s2;
@@ -120,22 +117,17 @@ public class Parser {
                 return new Ast.IfElse(x, s1, s2);
             }
             case WHILE -> {
-                var whilenode = new Ast.While();
-                var savedStmt = statementEnclosing;
-                statementEnclosing = whilenode;
                 match(Token.Id.WHILE);
                 match(Token.Id.L_PAREN);
                 x = parseBool();
                 match(Token.Id.R_PAREN);
                 s1 = parseStatement();
-                whilenode.init(x, s1);
-                statementEnclosing = savedStmt;
-                return whilenode;
+                return new Ast.While(x, s1);
             }
             case BREAK -> {
                 match(Token.Id.BREAK);
                 match(Token.Id.SEMICOLON);
-                return new Ast.Break(statementEnclosing);
+                return new Ast.Break();
             }
             case L_CURLY -> {
                 return parseBlock();
@@ -146,7 +138,7 @@ public class Parser {
         }
     }
 
-    Ast.Statement parseBlock() {
+    private Ast.Statement parseBlock() {
         match(Token.Id.L_CURLY);
         var block = new Ast.Block();
         parseDeclarations(block);
@@ -155,16 +147,17 @@ public class Parser {
         return block;
     }
 
-    Ast.Statement parseAssign() {
+    private Ast.Statement parseAssign() {
+        Token tok = look;
         match(Token.Id.IDENTIFIER);
-        Ast.Identifier identifier = new Ast.Identifier(look);
+        Ast.Identifier identifier = new Ast.Identifier(tok);
         match(Token.Id.ASN);
         var s = new Ast.Assign(identifier, parseBool());
         match(Token.Id.SEMICOLON);
         return s;
     }
 
-    Ast.Expr parseBool() {
+    private Ast.Expr parseBool() {
         var x = parseAnd();
         while (look.getId() == Token.Id.COND_OR) {
             var tok = look;
@@ -174,7 +167,7 @@ public class Parser {
         return x;
     }
 
-    Ast.Expr parseAnd() {
+    private Ast.Expr parseAnd() {
         var x = parseRelational();
         while (look.getId() == Token.Id.COND_AND) {
             var tok = look;
@@ -184,7 +177,7 @@ public class Parser {
         return x;
     }
 
-    Ast.Expr parseRelational() {
+    private Ast.Expr parseRelational() {
         var x = parseAddition();
         while (look.getId() == Token.Id.COMP_EQ ||
                 look.getId() == Token.Id.COMP_NEQ ||
@@ -199,7 +192,7 @@ public class Parser {
         return x;
     }
 
-    Ast.Expr parseAddition() {
+    private Ast.Expr parseAddition() {
         var x = parseMultiplication();
         while (look.getId() == Token.Id.ADD ||
                 look.getId() == Token.Id.SUB) {
@@ -210,7 +203,7 @@ public class Parser {
         return x;
     }
 
-    Ast.Expr parseMultiplication() {
+    private Ast.Expr parseMultiplication() {
         var x = parseUnary();
         while (look.getId() == Token.Id.MUL ||
                 look.getId() == Token.Id.DIV) {
@@ -221,7 +214,7 @@ public class Parser {
         return x;
     }
 
-    Ast.Expr parseUnary() {
+    private Ast.Expr parseUnary() {
         if (look.getId() == Token.Id.SUB ||
                 look.getId() == Token.Id.NOT) {
             var tok = look;
@@ -232,7 +225,7 @@ public class Parser {
         }
     }
 
-    Ast.Expr parsePrimary() {
+    private Ast.Expr parsePrimary() {
         switch (look.getId()) {
             case L_PAREN -> {
                 nextToken();