Overhaul Lexer

Switched to bytes from chars, since we'll be parsing files eventually and typically compilers parse bytes from files (and not Java chars).

Added a 'peek' to handle bad leading `--123` numbers.
Used simpler number parsing; doesn't do doubles (yet).
Added a filter for punctuation.  Prior setup didn't handle multi-char punctuation like relation tests "<=".
Switched the "EOF" marker char from 0 to Char.MAX_VALUE; harder to make mistakes, and simplifies the WS test.

Switch javadoc comments to plain comments on private functions.
Added tests for broken keyword, broken numbers, negative numbers.

1 files changed, 67 insertions, 78 deletions

diff --git a/chapter01/src/main/java/com/seaofnodes/simple/Parser.java b/chapter01/src/main/java/com/seaofnodes/simple/Parser.java
index 4da758b2..e9cae52f 100644
--- a/chapter01/src/main/java/com/seaofnodes/simple/Parser.java
+++ b/chapter01/src/main/java/com/seaofnodes/simple/Parser.java
@@ -183,123 +183,112 @@ public class Parser {
         public static Token EOF = new Token(Kind.EOZ, "", null);
 
         @Override
-        public String toString() {
-            return _str;
-        }
+        public String toString() { return _str; }
     }
 
     static class Lexer {
 
-        /**
-         * Input buffer
-         */
-        private final char[] _input;
-        /**
-         * Tracks current position in input buffer
-         */
+        // Input buffer; an array of text bytes read from a file or a string
+        private final byte[] _input;
+        // Tracks current position in input buffer
         private int _position = 0;
+
         /**
-         * Current character
+         * Record the source text for lexing
          */
-        private char _cur = ' ';
+        public Lexer(String source) { this(source.getBytes());  }
+        public Lexer(byte[] buf) {  _input = buf;  }
 
-        private final NumberFormat numberFormat;
+        // True if at EOF
+        private boolean isEOF() {  return _position >= _input.length;  }
 
-        public Lexer(String source) {
-            _input = source.toCharArray();
-            numberFormat = NumberFormat.getInstance();
-            numberFormat.setGroupingUsed(false);
+        private char nextChar() {
+            char ch = peek();
+            _position++;
+            return ch;
         }
-
-        private boolean isEOF() {
-            return _position >= _input.length;
-        }
-
-        private char advance() {
-            if (isEOF())
-                return 0;   // Special value that causes parsing to terminate
-            return _input[_position++];
+        // Peek next character, or report EOF
+        private char peek() {
+            return isEOF() ? Character.MAX_VALUE   // Special value that causes parsing to terminate
+                : (char)_input[_position];
         }
 
-        private boolean nextToken() {
-            _cur = advance();
-            return true;
+        // True if a white space
+        private boolean isWhiteSpace(char c) {
+            return c <= ' '; // Includes all the use space, tab, newline, CR
         }
 
         /**
-         * Note that EOF is not whitespace!
+         * Return the next non-white-space character
          */
-        private boolean isWhiteSpace(char c) {
-            return c == ' '  ||
-                   c == '\t' ||
-                   c == '\n' ||
-                   c == '\r';
+        private char skipWhiteSpace() {
+            char c = nextChar();
+            while( isWhiteSpace(c) )
+                c = nextChar();
+            return c;
         }
 
-        private void skipWhiteSpace() {
-            while (isWhiteSpace(_cur))
-                nextToken();
+        // First digit of a number.
+        // Allows [-][1-9] but not a leading 0
+        private boolean isFirstNumber(char ch) {
+            char ch2 = peek();
+            return ('1'<=ch && ch<='9') ||
+                (ch=='-' && ('1'<=ch2 && ch2<='9'));
         }
-
-        /**
-         * Parses number in format nnn[.nnn]
-         * where n is a digit
-         */
         private Token parseNumber() {
-            assert Character.isDigit(_cur);
-            StringBuilder sb = new StringBuilder();
-            sb.append(_cur);
-            while (nextToken() && Character.isDigit(_cur))
-                sb.append(_cur);
-            if (_cur == '.') {
-                sb.append(_cur);
-                while (nextToken() && Character.isDigit(_cur))
-                    sb.append(_cur);
-            }
-            String str = sb.toString();
-            Number number = parseNumber(str);
-            return Token.newNum(number, str);
-        }
-
-        private Number parseNumber(String str) {
-            try {
-                return numberFormat.parse(str);
-            } catch (ParseException e) {
-                throw new RuntimeException("Failed to parse number " + str, e);
-            }
+            int start = _position-1;
+            while( Character.isDigit( nextChar() ) ) ;
+            String snum = new String(_input,start,--_position-start);
+            return Token.newNum(Integer.parseInt(snum),snum);
         }
 
+        // First letter of an identifier 
         private boolean isIdentifierStart(char ch) {
             return Character.isAlphabetic(ch) || ch == '_';
         }
 
+        // All characters of an identifier, e.g. "_x123"
         private boolean isIdentifierLetter(char ch) {
             return Character.isLetterOrDigit(ch) || ch == '_';
         }
 
         private Token parseIdentifier() {
-            assert isIdentifierStart(_cur);
-            StringBuilder sb = new StringBuilder();
-            sb.append(_cur);
-            while (nextToken() && isIdentifierLetter(_cur))
-                sb.append(_cur);
-            return Token.newIdent(sb.toString());
+            int start = _position-1;
+            while( isIdentifierLetter(nextChar()) );
+            return Token.newIdent(new String(_input,start,--_position-start));
         }
 
+        // 
+        private boolean isPunctuation(char ch) {
+            return "=;[]<>()+-/*".indexOf(ch) != -1;
+        }
+
+        private Token parsePuncuation() {
+            int start = _position-1;
+            while( isPunctuation(nextChar()) );
+            return Token.newPunct(new String(_input,start,--_position-start));            
+        }
+            
         /**
-         * Gets the next token
+         * Gets the next Token
          */
         public Token next() {
-            skipWhiteSpace();
-            if (_cur == 0)
+            char ch = skipWhiteSpace();
+            if (ch == Character.MAX_VALUE)
                 return Token.EOF;
-            if (Character.isDigit(_cur))
+            if (isFirstNumber(ch))
                 return parseNumber();
-            if (isIdentifierLetter(_cur))
+            if (isIdentifierLetter(ch))
                 return parseIdentifier();
-            Token token = Token.newPunct(Character.toString(_cur));
-            _cur = ' ';
-            return token;
+            if (isPunctuation(ch))
+                return parsePuncuation();
+            
+            throw error("Unknown token "+ch);
+        }
+
+        private RuntimeException error(String msg) {
+            throw new RuntimeException(msg);
         }
     }
+
 }