Update the lexer, parser and AST to match the language definition.

Add toString() method on AST nodes so that we can easily check the results of parsing.
Remove AA lexer
Move the Ecstasy lexer classes to one package, except Handy

11 files changed, 110 insertions, 273 deletions

diff --git a/common/src/main/java/com/seaofnodes/simple/common/CompilerException.java b/common/src/main/java/com/seaofnodes/simple/common/CompilerException.java
deleted file mode 100644
index 518dcde6..00000000
--- a/common/src/main/java/com/seaofnodes/simple/common/CompilerException.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package com.seaofnodes.simple.common;
-
-public class CompilerException extends RuntimeException {
-    public CompilerException(String message) {
-        super(message);
-    }
-
-    public CompilerException(String message, Throwable cause) {
-        super(message, cause);
-    }
-}
diff --git a/common/src/main/java/com/seaofnodes/simple/common/Severity.java b/common/src/main/java/com/seaofnodes/simple/common/Severity.java
deleted file mode 100644
index f3325d52..00000000
--- a/common/src/main/java/com/seaofnodes/simple/common/Severity.java
+++ /dev/null
@@ -1,19 +0,0 @@
-package com.seaofnodes.simple.common;
-
-
-/**
- * Severity levels.
- */
-public enum Severity
-    {
-    NONE, INFO, WARNING, ERROR, FATAL;
-
-    public String desc()
-        {
-        String sName = name();
-        StringBuilder sb = new StringBuilder(sName.length());
-        sb.append(sName.charAt(0));
-        sb.append(sName.substring(1).toLowerCase());
-        return sb.toString();
-        }
-    }
diff --git a/common/src/main/java/com/seaofnodes/simple/lexer/aa/Lexer.java b/common/src/main/java/com/seaofnodes/simple/lexer/aa/Lexer.java
deleted file mode 100644
index 162cdea1..00000000
--- a/common/src/main/java/com/seaofnodes/simple/lexer/aa/Lexer.java
+++ /dev/null
@@ -1,228 +0,0 @@
-/**
- * DO NOT REMOVE COPYRIGHT NOTICES OR THIS HEADER.
- *
- * Contributor(s):
- *
- * The Initial Developer of the Original Software is Cliff Click.
- *
- * This file is part of the Sea Of Nodes Simple Programming language
- * implementation. See https://github.com/SeaOfNodes/Simple
- *
- * The contents of this file are subject to the terms of the
- * Apache License Version 2 (the "APL"). You may not use this
- * file except in compliance with the License. A copy of the
- * APL may be obtained from:
- * http://www.apache.org/licenses/LICENSE-2.0
- */
-package com.seaofnodes.simple.lexer.aa;
-
-import com.seaofnodes.util.AryInt;
-import com.seaofnodes.util.SB;
-
-import java.text.NumberFormat;
-import java.text.ParsePosition;
-
-public class Lexer {
-    private final String _srcName;    // Source for error messages; usually a file name
-    private final byte[] _buf;
-    private int _x;               // Parser index
-    private int _lastNWS;         // Index of last non-white-space char
-    private final AryInt _lines;  // char offset of each line
-
-    private final NumberFormat _nf;
-    private final ParsePosition _pp;
-    private final String _str;
-
-    public Lexer(String srcName, String source)
-    {
-        _srcName = srcName;
-        _buf = source.getBytes();
-        _x   = 0;
-
-        // Set fields strictly for Java number parsing
-        _nf = NumberFormat.getInstance();
-        _nf.setGroupingUsed(false);
-        _pp = new ParsePosition(0);
-        _str = source;           // Keep a complete string copy for java number parsing
-        _lines = new AryInt();//
-        _lines.push(0);       // Line 0 at offset 0
-    }
-
-    // Skip WS, return true&skip if match, false & do not skip if miss.
-    public boolean peek( char c ) { return peek1(skipWS(),c); }
-    public boolean peek_noWS( char c ) { return peek1(_x >= _buf.length ? -1 : _buf[_x],c); }
-    // Already skipped WS & have character;
-    // return true & skip if a match, false& do not skip if a miss.
-    public boolean peek1( byte c0, char c ) {
-        assert c0==-1 || c0== _buf[_x];
-        if( c0!=c ) return false;
-        _x++;                       // Skip peeked character
-        return true;
-    }
-    // Already skipped WS & have character;
-    // return true&skip if match, false & do not skip if miss.
-    public boolean peek2( byte c0, String s2 ) {
-        if( c0 != s2.charAt(0) ) return false;
-        if( _x+1 >= _buf.length || _buf[_x+1] != s2.charAt(1) ) return false;
-        _x+=2;                      // Skip peeked characters
-        return true;
-    }
-    public boolean peek( String s ) {
-        if( !peek(s.charAt(0)) ) return false;
-        if( !peek_noWS(s.charAt(1)) ) {  _x--; return false; }
-        return true;
-    }
-    // Peek 'c' and NOT followed by 'no'
-    public boolean peek_not( char c, char no ) {
-        byte c0 = skipWS();
-        if( c0 != c || (_x+1 < _buf.length && _buf[_x+1] == no) ) return false;
-        _x++;
-        return true;
-    }
-    // Match any of these, and return the match or null
-    public String peek(String[] toks) {
-        for( String tok : toks ) if( peek1(tok) ) return tok;
-        return null;
-    }
-    public boolean peek1(String tok) {
-        for( int i=0; i<tok.length(); i++ )
-            if( _x+i >= _buf.length || _buf[_x+i] != tok.charAt(i) )
-                return false;
-        return true;
-    }
-
-
-    /** Advance parse pointer to the first non-whitespace character, and return
-     *  that character, -1 otherwise.  */
-    public byte skipWS() {
-        int oldx = _x;
-        while( _x < _buf.length ) {
-            byte c = _buf[_x];
-            if( c=='/' && _x+1 < _buf.length && _buf[_x+1]=='/' ) { skipEOL()  ; continue; }
-            if( c=='/' && _x+1 < _buf.length && _buf[_x+1]=='*' ) { skipBlock(); continue; }
-            if( c=='\n' && _x+1 > _lines.last() ) _lines.push(_x+1);
-            if( !isWS(c) ) {
-                if( oldx-1 > _lastNWS && !isWS(_buf[oldx-1]) ) _lastNWS = oldx-1;
-                return c;
-            }
-            _x++;
-        }
-        return -1;
-    }
-    public void skipEOL  () { while( _x < _buf.length && _buf[_x] != '\n' ) _x++; }
-    public void skipBlock() { throw new UnsupportedOperationException("Block comments not yet implemented"); }
-    // Advance parse pointer to next WS.  Used for parser syntax error recovery.
-    public void skipNonWS() {
-        while( _x < _buf.length && !isWS(_buf[_x]) ) _x++;
-    }
-
-    /** Return true if `c` passes a test */
-    public static boolean isWS    (byte c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r'; }
-    public static boolean isAlpha0(byte c) { return ('a'<=c && c <= 'z') || ('A'<=c && c <= 'Z') || (c=='_'); }
-    public static boolean isAlpha1(byte c) { return isAlpha0(c) || ('0'<=c && c <= '9'); }
-    public static boolean isJava  (byte c) { return isAlpha1(c) || (c=='$') || (c=='.'); }
-    public  static boolean isDigit (byte c) { return '0' <= c && c <= '9'; }
-
-    public String token() { skipWS();  return token0(); }
-    // Lexical tokens.  Any alpha, followed by any alphanumerics is a alpha-
-    // token; alpha-tokens end with WS or any operator character.  Any collection
-    // of the classic operator characters are a token, except that they will break
-    // un-ambiguously.
-    public String token0() {
-        if( _x >= _buf.length ) return null;
-        byte c=_buf[_x];  int x = _x;
-        if( isOp0(c) || (c=='_' && _x+1 < _buf.length && isOp0(_buf[_x+1])) )
-            while( _x < _buf.length && isOp1(_buf[_x]) ) _x++;
-        else if( isAlpha0(c) )
-            while( _x < _buf.length && isAlpha1(_buf[_x]) ) _x++;
-        else return null; // Not a token; specifically excludes e.g. all bytes >= 128, or most bytes < 32
-        if( (c==':' || c==',') && _x-x==1 ) // Disallow bare ':' as a token; ambiguous with ?: and type annotations; same for ','
-        { _x=x; return null; } // Unwind, not a token
-        if( c=='-' && _x-x>2 && _buf[x+1]=='>' ) // Disallow leading "->", confusing with function parameter list end; eg "not={x->!x}"
-            _x=x+2;                                // Just return the "->"
-        return new String(_buf,x,_x-x);
-    }
-    public boolean isOp(String s) {
-        if( s==null || s.isEmpty() ) return false;
-        byte c = (byte)s.charAt(0);
-        if( !isOp0(c) && (c!='_' || !isOp0((byte)s.charAt(1))) ) return false;
-        for( int i=1; i<s.length(); i++ )
-            if( !isOp1((byte)s.charAt(i)) ) return false;
-        return true;
-    }
-
-    // Parse a number; WS already skipped and sitting at a digit.  Relies on
-    // Javas number parsing.
-    public Number number() {
-        _pp.setIndex(_x);
-        Number n = _nf.parse(_str,_pp);
-        _x = _pp.getIndex();
-        if( n instanceof Long   ) {
-            if( _buf[_x-1]=='.' ) _x--; // Java default parser allows "1." as an integer; pushback the '.'
-        }
-        return n;
-    }
-
-    public int field_number() {
-        byte c = _buf[_x];
-        if( !isDigit(c) ) return -1;
-        _x++;
-        int sum = c-'0';
-        while( _x < _buf.length && isDigit(c=_buf[_x]) ) {
-            _x++;
-            sum = sum*10+c-'0';
-        }
-        return sum;
-    }
-
-    /** Parse a String; _x is at '"'.
-     *  str  = [.\%]*               // String contents; \t\n\r\% standard escapes
-     *  str  = %[num]?[.num]?fact   // Percent escape embeds a 'fact' in a string; "name=%name\n"
-     */
-    public String string() {
-        int oldx = ++_x;
-        byte c;
-        while( (c=_buf[_x++]) != '"' ) {
-            if( c=='%' ) throw new UnsupportedOperationException("Not implemented");
-            if( c=='\\' ) throw new UnsupportedOperationException("Not implemented");
-            if( _x == _buf.length ) return null;
-        }
-        return new String(_buf,oldx,_x-oldx-1).intern();
-    }
-
-    public static boolean isOp0(byte c) { return "!#$%*+-=<>^[]~/&|".indexOf(c) != -1; }
-    public static boolean isOp1(byte c) { return isOp0(c) || ":?_{}".indexOf(c) != -1; }
-
-    public String errLocMsg(String s) {
-        if( s.charAt(0)=='\n' ) return s;
-        // find line start
-        int a=_x;
-        while( a > 0 && _buf[a-1] != '\n' ) --a;
-        if( _buf[a]=='\r' ) a++; // do not include leading \n or \n\r
-        // find line end
-        int b=_x;
-        while( b < _buf.length && _buf[b] != '\n' ) b++;
-        if( b < _buf.length ) b--; // do not include trailing \n or \n\r
-        // Find line number.  Bin-search returns the insertion-point, which is the NEXT
-        // line unless _x is exactly a line start.
-        int line = _lines.binary_search(_x); // Find zero-based line insertion point
-        if( line == _lines._len ||  _lines.at(line)>_x ) line--;
-        // error message using 1-based line
-        SB sb = new SB().p(_srcName).p(':').p(line+1).p(':').p(s).nl();
-        sb.p(new String(_buf,a,b-a)).nl();
-        int line_start = a;
-        for( int i=line_start; i<_x; i++ )
-            sb.p(' ');
-        return sb.p('^').nl().toString();
-    }
-
-    @Override public String toString() { return new String(_buf,_x,_buf.length-_x); }
-    @Override public boolean equals(Object loc) {
-        if( this==loc ) return true;
-        if( !(loc instanceof Lexer p) ) return false;
-        return _x==p._x && _srcName.equals(p._srcName);
-    }
-    @Override public int hashCode() {
-        return _srcName.hashCode()+_x;
-    }
-}
diff --git a/common/src/main/java/com/seaofnodes/simple/lexer/ecstasy/CompilerException.java b/common/src/main/java/com/seaofnodes/simple/lexer/ecstasy/CompilerException.java
new file mode 100644
index 00000000..c23d3672
--- /dev/null
+++ b/common/src/main/java/com/seaofnodes/simple/lexer/ecstasy/CompilerException.java
@@ -0,0 +1,28 @@
+/*
+ * DO NOT REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ *
+ * Contributor(s):
+ *
+ * The Initial Developer of the Original Software is
+ * The Ecstasy Project (https://github.com/xtclang/).
+ *
+ * This file is part of the Sea Of Nodes Simple Programming language
+ * implementation. See https://github.com/SeaOfNodes/Simple
+ *
+ * The contents of this file are subject to the terms of the
+ * Apache License Version 2 (the "APL"). You may not use this
+ * file except in compliance with the License. A copy of the
+ * APL may be obtained from:
+ * http://www.apache.org/licenses/LICENSE-2.0
+ */
+package com.seaofnodes.simple.lexer.ecstasy;
+
+public class CompilerException extends RuntimeException {
+    public CompilerException(String message) {
+        super(message);
+    }
+
+    public CompilerException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}
diff --git a/common/src/main/java/com/seaofnodes/simple/common/ErrorListener.java b/common/src/main/java/com/seaofnodes/simple/lexer/ecstasy/ErrorListener.java
index 29ca46b6..6a29f315 100644
--- a/common/src/main/java/com/seaofnodes/simple/common/ErrorListener.java
+++ b/common/src/main/java/com/seaofnodes/simple/lexer/ecstasy/ErrorListener.java
@@ -1,11 +1,26 @@
-package com.seaofnodes.simple.common;
+/*
+ * DO NOT REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ *
+ * Contributor(s):
+ *
+ * The Initial Developer of the Original Software is
+ * The Ecstasy Project (https://github.com/xtclang/).
+ *
+ * This file is part of the Sea Of Nodes Simple Programming language
+ * implementation. See https://github.com/SeaOfNodes/Simple
+ *
+ * The contents of this file are subject to the terms of the
+ * Apache License Version 2 (the "APL"). You may not use this
+ * file except in compliance with the License. A copy of the
+ * APL may be obtained from:
+ * http://www.apache.org/licenses/LICENSE-2.0
+ */
+package com.seaofnodes.simple.lexer.ecstasy;
 
 import java.text.MessageFormat;
 
 import java.util.Arrays;
 
-import com.seaofnodes.simple.lexer.ecstasy.Source;
-
 import static com.seaofnodes.util.Handy.quotedString;
 
 
diff --git a/common/src/main/java/com/seaofnodes/simple/lexer/ecstasy/Lexer.java b/common/src/main/java/com/seaofnodes/simple/lexer/ecstasy/Lexer.java
index 90571ffd..86a7b77f 100644
--- a/common/src/main/java/com/seaofnodes/simple/lexer/ecstasy/Lexer.java
+++ b/common/src/main/java/com/seaofnodes/simple/lexer/ecstasy/Lexer.java
@@ -1,4 +1,4 @@
-/**
+/*
  * DO NOT REMOVE COPYRIGHT NOTICES OR THIS HEADER.
  *
  * Contributor(s):
@@ -28,10 +28,8 @@ import java.util.NoSuchElementException;
 
 import java.util.function.Consumer;
 
-import com.seaofnodes.simple.common.ErrorListener;
 import com.seaofnodes.simple.lexer.ecstasy.Token.Id;
 import com.seaofnodes.util.PackedInteger;
-import com.seaofnodes.simple.common.Severity;
 
 import static com.seaofnodes.util.Handy.*;
 
@@ -1076,6 +1074,17 @@ public class Lexer
                                 {
                                 switch (name)
                                     {
+                                    case "int":
+                                        idNum = Id.TYPE_INT;
+                                        break;
+                                    case "float":
+                                        idNum = Id.TYPE_FLOAT;
+                                        break;
+                                    case "char":
+                                        idNum = Id.TYPE_CHAR;
+                                        break;
+
+                                    // Following are Ecstasy ids
                                     case "Bit":
                                         idNum = Id.LIT_BIT;
                                         break;
diff --git a/common/src/main/java/com/seaofnodes/simple/lexer/ecstasy/Severity.java b/common/src/main/java/com/seaofnodes/simple/lexer/ecstasy/Severity.java
new file mode 100644
index 00000000..9b01a7e4
--- /dev/null
+++ b/common/src/main/java/com/seaofnodes/simple/lexer/ecstasy/Severity.java
@@ -0,0 +1,36 @@
+/*
+ * DO NOT REMOVE COPYRIGHT NOTICES OR THIS HEADER.
+ *
+ * Contributor(s):
+ *
+ * The Initial Developer of the Original Software is
+ * The Ecstasy Project (https://github.com/xtclang/).
+ *
+ * This file is part of the Sea Of Nodes Simple Programming language
+ * implementation. See https://github.com/SeaOfNodes/Simple
+ *
+ * The contents of this file are subject to the terms of the
+ * Apache License Version 2 (the "APL"). You may not use this
+ * file except in compliance with the License. A copy of the
+ * APL may be obtained from:
+ * http://www.apache.org/licenses/LICENSE-2.0
+ */
+package com.seaofnodes.simple.lexer.ecstasy;
+
+
+/**
+ * Severity levels.
+ */
+public enum Severity
+    {
+    NONE, INFO, WARNING, ERROR, FATAL;
+
+    public String desc()
+        {
+        String sName = name();
+        StringBuilder sb = new StringBuilder(sName.length());
+        sb.append(sName.charAt(0));
+        sb.append(sName.substring(1).toLowerCase());
+        return sb.toString();
+        }
+    }
diff --git a/common/src/main/java/com/seaofnodes/simple/lexer/ecstasy/Source.java b/common/src/main/java/com/seaofnodes/simple/lexer/ecstasy/Source.java
index 6800138a..c0da5d5f 100644
--- a/common/src/main/java/com/seaofnodes/simple/lexer/ecstasy/Source.java
+++ b/common/src/main/java/com/seaofnodes/simple/lexer/ecstasy/Source.java
@@ -1,4 +1,4 @@
-/**
+/*
  * DO NOT REMOVE COPYRIGHT NOTICES OR THIS HEADER.
  *
  * Contributor(s):
diff --git a/common/src/main/java/com/seaofnodes/simple/lexer/ecstasy/Token.java b/common/src/main/java/com/seaofnodes/simple/lexer/ecstasy/Token.java
index 271d8fbc..18cb5759 100644
--- a/common/src/main/java/com/seaofnodes/simple/lexer/ecstasy/Token.java
+++ b/common/src/main/java/com/seaofnodes/simple/lexer/ecstasy/Token.java
@@ -1,4 +1,4 @@
-/**
+/*
  * DO NOT REMOVE COPYRIGHT NOTICES OR THIS HEADER.
  *
  * Contributor(s):
@@ -19,9 +19,6 @@
 package com.seaofnodes.simple.lexer.ecstasy;
 
 
-import com.seaofnodes.simple.common.ErrorListener;
-import com.seaofnodes.simple.common.Severity;
-
 import java.util.HashMap;
 import java.util.Map;
 
@@ -432,6 +429,9 @@ public class Token
             case LIT_FLOAT128:
             case LIT_FLOATN:
             case LIT_BFLOAT16:
+            case TYPE_INT:
+            case TYPE_FLOAT:
+            case TYPE_CHAR:
                 sb.append(m_id.TEXT)
                   .append(':')
                   .append(m_oValue);
@@ -637,6 +637,13 @@ public class Token
         IDENTIFIER   (null             ),
         EOL_COMMENT  (null             ),
         ENC_COMMENT  (null             ),
+
+        // Simple language type keywords
+        TYPE_INT     ("int"            , false, true),
+        TYPE_FLOAT   ("float"          , false, true),
+        TYPE_CHAR    ("char"           , false, true),
+        // End Simple language type keywords
+
         LIT_BIT      (null             ),
         LIT_NIBBLE   (null             ),
         LIT_CHAR     (null             ),
diff --git a/common/src/main/java/com/seaofnodes/util/Handy.java b/common/src/main/java/com/seaofnodes/util/Handy.java
index f3496f8c..62e34ab9 100644
--- a/common/src/main/java/com/seaofnodes/util/Handy.java
+++ b/common/src/main/java/com/seaofnodes/util/Handy.java
@@ -518,7 +518,7 @@ public class Handy
      */
     public static boolean isDigit(char ch)
         {
-        return ch >= '0' & ch <= '9';
+        return ch >= '0' && ch <= '9';
         }
 
     /**
@@ -547,7 +547,7 @@ public class Handy
         //   {..., 0, ..., 9, ..., A, ..., F, ..., a, ..., f, ...}
         // so a hexit has to be in the range 0..f, and the entire character
         // range from 0..f is only 0x36 (so it fits inside a long mask)
-        return ch >= '0' & ch <= 'f' & ((1L << ('f' - ch)) &
+        return ch >= '0' && ch <= 'f' && ((1L << ('f' - ch)) &
                 // 3               4               5               6
                 // 0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456
                 // 0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[/]^_`abcdef
@@ -579,7 +579,7 @@ public class Handy
         {
         // ASCII letters in the unicode range appear in this order:
         //   {..., A, ..., Z, ..., a, ..., z, ...}
-        return ch >= 'A' & ch <= 'z' & ((1L << ('z' - ch)) &
+        return ch >= 'A' && ch <= 'z' && ((1L << ('z' - ch)) &
                 // 4              5               6               7
                 // 123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789A
                 // ABCDEFGHIJKLMNOPQRSTUVWXYZ[/]^_`abcdefghijklmnopqrstuvwxyz
diff --git a/common/src/test/java/com/seaofnodes/simple/lexer/TestLexer.java b/common/src/test/java/com/seaofnodes/simple/lexer/TestLexer.java
index ef29d963..aec5d171 100644
--- a/common/src/test/java/com/seaofnodes/simple/lexer/TestLexer.java
+++ b/common/src/test/java/com/seaofnodes/simple/lexer/TestLexer.java
@@ -15,7 +15,7 @@
 
 package com.seaofnodes.simple.lexer;
 
-import com.seaofnodes.simple.common.ErrorListener;
+import com.seaofnodes.simple.lexer.ecstasy.ErrorListener;
 import com.seaofnodes.simple.lexer.ecstasy.Lexer;
 import com.seaofnodes.simple.lexer.ecstasy.Source;
 import com.seaofnodes.simple.lexer.ecstasy.Token;