Overhaul Lexer

Switched to bytes from chars, since we'll be parsing files eventually and typically compilers parse bytes from files (and not Java chars).

Added a 'peek' to handle bad leading `--123` numbers.
Used simpler number parsing; doesn't do doubles (yet).
Added a filter for punctuation.  Prior setup didn't handle multi-char punctuation like relation tests "<=".
Switched the "EOF" marker char from 0 to Char.MAX_VALUE; harder to make mistakes, and simplifies the WS test.

Switch javadoc comments to plain comments on private functions.
Added tests for broken keyword, broken numbers, negative numbers.

3 files changed, 107 insertions, 278 deletions

diff --git a/.gitignore b/.gitignore
index a8c78987..acbde437 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,197 +5,10 @@
 # Log file
 *.log
 
-# BlueJ files
-*.ctxt
-
-# Mobile Tools for Java (J2ME)
-.mtj.tmp/
-
-# Package Files #
-*.jar
-*.war
-*.nar
-*.ear
-*.zip
-*.tar.gz
-*.rar
-
-# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
-hs_err_pid*
-replay_pid*
-
-# Gradle stuff
-.gradle
-**/build/
-!src/**/build/
-
-# Ignore Gradle GUI config
-gradle-app.setting
-
-# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored)
-!gradle-wrapper.jar
-
-# Avoid ignore Gradle wrappper properties
-!gradle-wrapper.properties
-
-# Cache of project
-.gradletasknamecache
-
-
-# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
-# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
-
-# User-specific stuff
-.idea/**/workspace.xml
-.idea/**/tasks.xml
-.idea/**/usage.statistics.xml
-.idea/**/dictionaries
-.idea/**/shelf
-
-# AWS User-specific
-.idea/**/aws.xml
-
-# Generated files
-.idea/**/contentModel.xml
-
-# Sensitive or high-churn files
-.idea/**/dataSources/
-.idea/**/dataSources.ids
-.idea/**/dataSources.local.xml
-.idea/**/sqlDataSources.xml
-.idea/**/dynamic.xml
-.idea/**/uiDesigner.xml
-.idea/**/dbnavigator.xml
-
-# Gradle
-.idea/**/gradle.xml
-.idea/**/libraries
-
-# Gradle and Maven with auto-import
-# When using Gradle or Maven with auto-import, you should exclude module files,
-# since they will be recreated, and may cause churn.  Uncomment if using
-# auto-import.
-# .idea/artifacts
-# .idea/compiler.xml
-# .idea/jarRepositories.xml
-# .idea/modules.xml
-# .idea/*.iml
-# .idea/modules
-# *.iml
-# *.ipr
-
-# CMake
-cmake-build-*/
-
-# Mongo Explorer plugin
-.idea/**/mongoSettings.xml
-
-# File-based project format
-*.iws
-
-# IntelliJ
-out/
-
-# mpeltonen/sbt-idea plugin
-.idea_modules/
-
-# JIRA plugin
-atlassian-ide-plugin.xml
-
-# Cursive Clojure plugin
-.idea/replstate.xml
-
-# SonarLint plugin
-.idea/sonarlint/
-
-# Crashlytics plugin (for Android Studio and IntelliJ)
-com_crashlytics_export_strings.xml
-crashlytics.properties
-crashlytics-build.properties
-fabric.properties
-
-# Editor-based Rest Client
-.idea/httpRequests
-
-# Android studio 3.1+ serialized cache file
-.idea/caches/build_file_checksums.ser
-
-
+# Idea
+.idea/
 target/
-pom.xml.tag
-pom.xml.releaseBackup
-pom.xml.versionsBackup
-pom.xml.next
-release.properties
-dependency-reduced-pom.xml
-buildNumber.properties
-.mvn/timing.properties
-# https://github.com/takari/maven-wrapper#usage-without-binary-jar
-.mvn/wrapper/maven-wrapper.jar
-
-# Eclipse stuff
-.metadata
-bin/
-tmp/
-*.tmp
-*.bak
-*.swp
-*~.nib
-local.properties
-.settings/
-.loadpath
-.recommenders
-
-# External tool builders
-.externalToolBuilders/
-
-# Locally stored "Eclipse launch configurations"
-*.launch
-
-# PyDev specific (Python IDE for Eclipse)
-*.pydevproject
-
-# CDT-specific (C/C++ Development Tooling)
-.cproject
-
-# CDT- autotools
-.autotools
-
-# Java annotation processor (APT)
-.factorypath
-
-# PDT-specific (PHP Development Tools)
-.buildpath
-
-# sbteclipse plugin
-.target
-
-# Tern plugin
-.tern-project
-
-# TeXlipse plugin
-.texlipse
-
-# STS (Spring Tool Suite)
-.springBeans
-
-# Code Recommenders
-.recommenders/
-
-# Annotation Processing
-.apt_generated/
-.apt_generated_test/
-
-# Scala IDE specific (Scala & Java development for Eclipse)
-.cache-main
-.scala_dependencies
-.worksheet
 
-# Eclipse Gradle plugin generated files
-# Eclipse Core
-.project
-# JDT-specific (Eclipse Java Development Tools)
-.classpath
+# Editors
+.#*
 
-.idea/*
-*.iml
\ No newline at end of file
diff --git a/chapter01/src/main/java/com/seaofnodes/simple/Parser.java b/chapter01/src/main/java/com/seaofnodes/simple/Parser.java
index 4da758b2..e9cae52f 100644
--- a/chapter01/src/main/java/com/seaofnodes/simple/Parser.java
+++ b/chapter01/src/main/java/com/seaofnodes/simple/Parser.java
@@ -183,123 +183,112 @@ public class Parser {
         public static Token EOF = new Token(Kind.EOZ, "", null);
 
         @Override
-        public String toString() {
-            return _str;
-        }
+        public String toString() { return _str; }
     }
 
     static class Lexer {
 
-        /**
-         * Input buffer
-         */
-        private final char[] _input;
-        /**
-         * Tracks current position in input buffer
-         */
+        // Input buffer; an array of text bytes read from a file or a string
+        private final byte[] _input;
+        // Tracks current position in input buffer
         private int _position = 0;
+
         /**
-         * Current character
+         * Record the source text for lexing
          */
-        private char _cur = ' ';
+        public Lexer(String source) { this(source.getBytes());  }
+        public Lexer(byte[] buf) {  _input = buf;  }
 
-        private final NumberFormat numberFormat;
+        // True if at EOF
+        private boolean isEOF() {  return _position >= _input.length;  }
 
-        public Lexer(String source) {
-            _input = source.toCharArray();
-            numberFormat = NumberFormat.getInstance();
-            numberFormat.setGroupingUsed(false);
+        private char nextChar() {
+            char ch = peek();
+            _position++;
+            return ch;
         }
-
-        private boolean isEOF() {
-            return _position >= _input.length;
-        }
-
-        private char advance() {
-            if (isEOF())
-                return 0;   // Special value that causes parsing to terminate
-            return _input[_position++];
+        // Peek next character, or report EOF
+        private char peek() {
+            return isEOF() ? Character.MAX_VALUE   // Special value that causes parsing to terminate
+                : (char)_input[_position];
         }
 
-        private boolean nextToken() {
-            _cur = advance();
-            return true;
+        // True if a white space
+        private boolean isWhiteSpace(char c) {
+            return c <= ' '; // Includes all the use space, tab, newline, CR
         }
 
         /**
-         * Note that EOF is not whitespace!
+         * Return the next non-white-space character
          */
-        private boolean isWhiteSpace(char c) {
-            return c == ' '  ||
-                   c == '\t' ||
-                   c == '\n' ||
-                   c == '\r';
+        private char skipWhiteSpace() {
+            char c = nextChar();
+            while( isWhiteSpace(c) )
+                c = nextChar();
+            return c;
         }
 
-        private void skipWhiteSpace() {
-            while (isWhiteSpace(_cur))
-                nextToken();
+        // First digit of a number.
+        // Allows [-][1-9] but not a leading 0
+        private boolean isFirstNumber(char ch) {
+            char ch2 = peek();
+            return ('1'<=ch && ch<='9') ||
+                (ch=='-' && ('1'<=ch2 && ch2<='9'));
         }
-
-        /**
-         * Parses number in format nnn[.nnn]
-         * where n is a digit
-         */
         private Token parseNumber() {
-            assert Character.isDigit(_cur);
-            StringBuilder sb = new StringBuilder();
-            sb.append(_cur);
-            while (nextToken() && Character.isDigit(_cur))
-                sb.append(_cur);
-            if (_cur == '.') {
-                sb.append(_cur);
-                while (nextToken() && Character.isDigit(_cur))
-                    sb.append(_cur);
-            }
-            String str = sb.toString();
-            Number number = parseNumber(str);
-            return Token.newNum(number, str);
-        }
-
-        private Number parseNumber(String str) {
-            try {
-                return numberFormat.parse(str);
-            } catch (ParseException e) {
-                throw new RuntimeException("Failed to parse number " + str, e);
-            }
+            int start = _position-1;
+            while( Character.isDigit( nextChar() ) ) ;
+            String snum = new String(_input,start,--_position-start);
+            return Token.newNum(Integer.parseInt(snum),snum);
         }
 
+        // First letter of an identifier 
         private boolean isIdentifierStart(char ch) {
             return Character.isAlphabetic(ch) || ch == '_';
         }
 
+        // All characters of an identifier, e.g. "_x123"
         private boolean isIdentifierLetter(char ch) {
             return Character.isLetterOrDigit(ch) || ch == '_';
         }
 
         private Token parseIdentifier() {
-            assert isIdentifierStart(_cur);
-            StringBuilder sb = new StringBuilder();
-            sb.append(_cur);
-            while (nextToken() && isIdentifierLetter(_cur))
-                sb.append(_cur);
-            return Token.newIdent(sb.toString());
+            int start = _position-1;
+            while( isIdentifierLetter(nextChar()) );
+            return Token.newIdent(new String(_input,start,--_position-start));
         }
 
+        // 
+        private boolean isPunctuation(char ch) {
+            return "=;[]<>()+-/*".indexOf(ch) != -1;
+        }
+
+        private Token parsePuncuation() {
+            int start = _position-1;
+            while( isPunctuation(nextChar()) );
+            return Token.newPunct(new String(_input,start,--_position-start));            
+        }
+            
         /**
-         * Gets the next token
+         * Gets the next Token
          */
         public Token next() {
-            skipWhiteSpace();
-            if (_cur == 0)
+            char ch = skipWhiteSpace();
+            if (ch == Character.MAX_VALUE)
                 return Token.EOF;
-            if (Character.isDigit(_cur))
+            if (isFirstNumber(ch))
                 return parseNumber();
-            if (isIdentifierLetter(_cur))
+            if (isIdentifierLetter(ch))
                 return parseIdentifier();
-            Token token = Token.newPunct(Character.toString(_cur));
-            _cur = ' ';
-            return token;
+            if (isPunctuation(ch))
+                return parsePuncuation();
+            
+            throw error("Unknown token "+ch);
+        }
+
+        private RuntimeException error(String msg) {
+            throw new RuntimeException(msg);
         }
     }
+
 }
diff --git a/chapter01/src/test/java/com/seaofnodes/simple/ParserTest.java b/chapter01/src/test/java/com/seaofnodes/simple/ParserTest.java
index 9bc2fc1c..586c53de 100644
--- a/chapter01/src/test/java/com/seaofnodes/simple/ParserTest.java
+++ b/chapter01/src/test/java/com/seaofnodes/simple/ParserTest.java
@@ -1,8 +1,8 @@
 package com.seaofnodes.simple;
 
 import com.seaofnodes.simple.node.*;
-import org.junit.Assert;
 import org.junit.Test;
+import static org.junit.Assert.*;
 
 public class ParserTest {
 
@@ -10,23 +10,50 @@ public class ParserTest {
     public void testSimpleProgram() {
         Parser parser = new Parser();
         StartNode startNode = parser.parse("return 1;");
-        Assert.assertNotNull(startNode);
-        Assert.assertEquals(2, startNode.nOuts());
+        assertNotNull(startNode);
+        assertEquals(2, startNode.nOuts());
         for (int i = 0; i < startNode.nOuts(); i++) {
             switch( startNode.out(i) ) {
             case ReturnNode ret:
-                Assert.assertEquals(2, ret.nIns());
-                Assert.assertEquals(startNode, ret.in(0));
-                Assert.assertTrue(ret.in(1) instanceof ConstantNode);
+                assertEquals(2, ret.nIns());
+                assertEquals(startNode, ret.in(0));
+                assertTrue(ret.in(1) instanceof ConstantNode);
                 break;
             case ConstantNode con:
-                Assert.assertEquals(1, con._value);
-                Assert.assertEquals(1, con.nIns());
-                Assert.assertEquals(startNode, con.in(0));
+                assertEquals(1, con._value);
+                assertEquals(1, con.nIns());
+                assertEquals(startNode, con.in(0));
                 break;
             default:
                 throw new IllegalStateException();
             }
         }
     }
+
+    @Test
+    public void testNegative() {
+        Parser parser = new Parser();
+        StartNode start = parser.parse("return -123;");
+        for( Node use : start._outputs )
+            if( use instanceof ConstantNode con )
+                assertEquals(-123,con._value);
+    }
+
+    @Test
+    public void testBad1() {
+        try { new Parser().parse("ret"); }
+        catch( RuntimeException e ) { assertEquals("syntax error, expected return: ret",e.getMessage()); }
+    }
+
+    @Test
+    public void testBad2() {
+        try { new Parser().parse("return 0123;"); }
+        catch( RuntimeException e ) { assertEquals("syntax error, expected nested expr or integer literal: 0123",e.getMessage()); }
+    }
+
+    @Test
+    public void testBad3() {
+        try { new Parser().parse("return --12;"); }
+        catch( RuntimeException e ) { assertEquals("syntax error, expected ( got --: --",e.getMessage()); }
+    }
 }