From 02f839e3eaab921aecd9c13b59047dd0551e132a Mon Sep 17 00:00:00 2001 From: Cliff Click Date: Tue, 19 Sep 2023 09:19:37 -0700 Subject: Overhaul Lexer Switched to bytes from chars, since we'll be parsing files eventually and typically compilers parse bytes from files (and not Java chars). Added a 'peek' to handle bad leading `--123` numbers. Used simpler number parsing; doesn't do doubles (yet). Added a filter for punctuation. Prior setup didn't handle multi-char punctuation like relation tests "<=". Switched the "EOF" marker char from 0 to Char.MAX_VALUE; harder to make mistakes, and simplifies the WS test. Switch javadoc comments to plain comments on private functions. Added tests for broken keyword, broken numbers, negative numbers. --- .gitignore | 195 +-------------------- .../main/java/com/seaofnodes/simple/Parser.java | 145 +++++++-------- .../java/com/seaofnodes/simple/ParserTest.java | 45 ++++- 3 files changed, 107 insertions(+), 278 deletions(-) diff --git a/.gitignore b/.gitignore index a8c78987..acbde437 100644 --- a/.gitignore +++ b/.gitignore @@ -5,197 +5,10 @@ # Log file *.log -# BlueJ files -*.ctxt - -# Mobile Tools for Java (J2ME) -.mtj.tmp/ - -# Package Files # -*.jar -*.war -*.nar -*.ear -*.zip -*.tar.gz -*.rar - -# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml -hs_err_pid* -replay_pid* - -# Gradle stuff -.gradle -**/build/ -!src/**/build/ - -# Ignore Gradle GUI config -gradle-app.setting - -# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) -!gradle-wrapper.jar - -# Avoid ignore Gradle wrappper properties -!gradle-wrapper.properties - -# Cache of project -.gradletasknamecache - - -# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider -# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 - -# User-specific stuff -.idea/**/workspace.xml -.idea/**/tasks.xml -.idea/**/usage.statistics.xml -.idea/**/dictionaries -.idea/**/shelf - -# AWS User-specific -.idea/**/aws.xml - -# Generated files -.idea/**/contentModel.xml - -# Sensitive or high-churn files -.idea/**/dataSources/ -.idea/**/dataSources.ids -.idea/**/dataSources.local.xml -.idea/**/sqlDataSources.xml -.idea/**/dynamic.xml -.idea/**/uiDesigner.xml -.idea/**/dbnavigator.xml - -# Gradle -.idea/**/gradle.xml -.idea/**/libraries - -# Gradle and Maven with auto-import -# When using Gradle or Maven with auto-import, you should exclude module files, -# since they will be recreated, and may cause churn. Uncomment if using -# auto-import. -# .idea/artifacts -# .idea/compiler.xml -# .idea/jarRepositories.xml -# .idea/modules.xml -# .idea/*.iml -# .idea/modules -# *.iml -# *.ipr - -# CMake -cmake-build-*/ - -# Mongo Explorer plugin -.idea/**/mongoSettings.xml - -# File-based project format -*.iws - -# IntelliJ -out/ - -# mpeltonen/sbt-idea plugin -.idea_modules/ - -# JIRA plugin -atlassian-ide-plugin.xml - -# Cursive Clojure plugin -.idea/replstate.xml - -# SonarLint plugin -.idea/sonarlint/ - -# Crashlytics plugin (for Android Studio and IntelliJ) -com_crashlytics_export_strings.xml -crashlytics.properties -crashlytics-build.properties -fabric.properties - -# Editor-based Rest Client -.idea/httpRequests - -# Android studio 3.1+ serialized cache file -.idea/caches/build_file_checksums.ser - - +# Idea +.idea/ target/ -pom.xml.tag -pom.xml.releaseBackup -pom.xml.versionsBackup -pom.xml.next -release.properties -dependency-reduced-pom.xml -buildNumber.properties -.mvn/timing.properties -# https://github.com/takari/maven-wrapper#usage-without-binary-jar -.mvn/wrapper/maven-wrapper.jar - -# Eclipse stuff -.metadata -bin/ -tmp/ -*.tmp -*.bak -*.swp -*~.nib -local.properties -.settings/ -.loadpath -.recommenders - -# External tool builders -.externalToolBuilders/ - -# Locally stored "Eclipse launch configurations" -*.launch - -# PyDev specific (Python IDE for Eclipse) -*.pydevproject - -# CDT-specific (C/C++ Development Tooling) -.cproject - -# CDT- autotools -.autotools - -# Java annotation processor (APT) -.factorypath - -# PDT-specific (PHP Development Tools) -.buildpath - -# sbteclipse plugin -.target - -# Tern plugin -.tern-project - -# TeXlipse plugin -.texlipse - -# STS (Spring Tool Suite) -.springBeans - -# Code Recommenders -.recommenders/ - -# Annotation Processing -.apt_generated/ -.apt_generated_test/ - -# Scala IDE specific (Scala & Java development for Eclipse) -.cache-main -.scala_dependencies -.worksheet -# Eclipse Gradle plugin generated files -# Eclipse Core -.project -# JDT-specific (Eclipse Java Development Tools) -.classpath +# Editors +.#* -.idea/* -*.iml \ No newline at end of file diff --git a/chapter01/src/main/java/com/seaofnodes/simple/Parser.java b/chapter01/src/main/java/com/seaofnodes/simple/Parser.java index 4da758b2..e9cae52f 100644 --- a/chapter01/src/main/java/com/seaofnodes/simple/Parser.java +++ b/chapter01/src/main/java/com/seaofnodes/simple/Parser.java @@ -183,123 +183,112 @@ public class Parser { public static Token EOF = new Token(Kind.EOZ, "", null); @Override - public String toString() { - return _str; - } + public String toString() { return _str; } } static class Lexer { - /** - * Input buffer - */ - private final char[] _input; - /** - * Tracks current position in input buffer - */ + // Input buffer; an array of text bytes read from a file or a string + private final byte[] _input; + // Tracks current position in input buffer private int _position = 0; + /** - * Current character + * Record the source text for lexing */ - private char _cur = ' '; + public Lexer(String source) { this(source.getBytes()); } + public Lexer(byte[] buf) { _input = buf; } - private final NumberFormat numberFormat; + // True if at EOF + private boolean isEOF() { return _position >= _input.length; } - public Lexer(String source) { - _input = source.toCharArray(); - numberFormat = NumberFormat.getInstance(); - numberFormat.setGroupingUsed(false); + private char nextChar() { + char ch = peek(); + _position++; + return ch; } - - private boolean isEOF() { - return _position >= _input.length; - } - - private char advance() { - if (isEOF()) - return 0; // Special value that causes parsing to terminate - return _input[_position++]; + // Peek next character, or report EOF + private char peek() { + return isEOF() ? Character.MAX_VALUE // Special value that causes parsing to terminate + : (char)_input[_position]; } - private boolean nextToken() { - _cur = advance(); - return true; + // True if a white space + private boolean isWhiteSpace(char c) { + return c <= ' '; // Includes all the use space, tab, newline, CR } /** - * Note that EOF is not whitespace! + * Return the next non-white-space character */ - private boolean isWhiteSpace(char c) { - return c == ' ' || - c == '\t' || - c == '\n' || - c == '\r'; + private char skipWhiteSpace() { + char c = nextChar(); + while( isWhiteSpace(c) ) + c = nextChar(); + return c; } - private void skipWhiteSpace() { - while (isWhiteSpace(_cur)) - nextToken(); + // First digit of a number. + // Allows [-][1-9] but not a leading 0 + private boolean isFirstNumber(char ch) { + char ch2 = peek(); + return ('1'<=ch && ch<='9') || + (ch=='-' && ('1'<=ch2 && ch2<='9')); } - - /** - * Parses number in format nnn[.nnn] - * where n is a digit - */ private Token parseNumber() { - assert Character.isDigit(_cur); - StringBuilder sb = new StringBuilder(); - sb.append(_cur); - while (nextToken() && Character.isDigit(_cur)) - sb.append(_cur); - if (_cur == '.') { - sb.append(_cur); - while (nextToken() && Character.isDigit(_cur)) - sb.append(_cur); - } - String str = sb.toString(); - Number number = parseNumber(str); - return Token.newNum(number, str); - } - - private Number parseNumber(String str) { - try { - return numberFormat.parse(str); - } catch (ParseException e) { - throw new RuntimeException("Failed to parse number " + str, e); - } + int start = _position-1; + while( Character.isDigit( nextChar() ) ) ; + String snum = new String(_input,start,--_position-start); + return Token.newNum(Integer.parseInt(snum),snum); } + // First letter of an identifier private boolean isIdentifierStart(char ch) { return Character.isAlphabetic(ch) || ch == '_'; } + // All characters of an identifier, e.g. "_x123" private boolean isIdentifierLetter(char ch) { return Character.isLetterOrDigit(ch) || ch == '_'; } private Token parseIdentifier() { - assert isIdentifierStart(_cur); - StringBuilder sb = new StringBuilder(); - sb.append(_cur); - while (nextToken() && isIdentifierLetter(_cur)) - sb.append(_cur); - return Token.newIdent(sb.toString()); + int start = _position-1; + while( isIdentifierLetter(nextChar()) ); + return Token.newIdent(new String(_input,start,--_position-start)); } + // + private boolean isPunctuation(char ch) { + return "=;[]<>()+-/*".indexOf(ch) != -1; + } + + private Token parsePuncuation() { + int start = _position-1; + while( isPunctuation(nextChar()) ); + return Token.newPunct(new String(_input,start,--_position-start)); + } + /** - * Gets the next token + * Gets the next Token */ public Token next() { - skipWhiteSpace(); - if (_cur == 0) + char ch = skipWhiteSpace(); + if (ch == Character.MAX_VALUE) return Token.EOF; - if (Character.isDigit(_cur)) + if (isFirstNumber(ch)) return parseNumber(); - if (isIdentifierLetter(_cur)) + if (isIdentifierLetter(ch)) return parseIdentifier(); - Token token = Token.newPunct(Character.toString(_cur)); - _cur = ' '; - return token; + if (isPunctuation(ch)) + return parsePuncuation(); + + throw error("Unknown token "+ch); + } + + private RuntimeException error(String msg) { + throw new RuntimeException(msg); } } + } diff --git a/chapter01/src/test/java/com/seaofnodes/simple/ParserTest.java b/chapter01/src/test/java/com/seaofnodes/simple/ParserTest.java index 9bc2fc1c..586c53de 100644 --- a/chapter01/src/test/java/com/seaofnodes/simple/ParserTest.java +++ b/chapter01/src/test/java/com/seaofnodes/simple/ParserTest.java @@ -1,8 +1,8 @@ package com.seaofnodes.simple; import com.seaofnodes.simple.node.*; -import org.junit.Assert; import org.junit.Test; +import static org.junit.Assert.*; public class ParserTest { @@ -10,23 +10,50 @@ public class ParserTest { public void testSimpleProgram() { Parser parser = new Parser(); StartNode startNode = parser.parse("return 1;"); - Assert.assertNotNull(startNode); - Assert.assertEquals(2, startNode.nOuts()); + assertNotNull(startNode); + assertEquals(2, startNode.nOuts()); for (int i = 0; i < startNode.nOuts(); i++) { switch( startNode.out(i) ) { case ReturnNode ret: - Assert.assertEquals(2, ret.nIns()); - Assert.assertEquals(startNode, ret.in(0)); - Assert.assertTrue(ret.in(1) instanceof ConstantNode); + assertEquals(2, ret.nIns()); + assertEquals(startNode, ret.in(0)); + assertTrue(ret.in(1) instanceof ConstantNode); break; case ConstantNode con: - Assert.assertEquals(1, con._value); - Assert.assertEquals(1, con.nIns()); - Assert.assertEquals(startNode, con.in(0)); + assertEquals(1, con._value); + assertEquals(1, con.nIns()); + assertEquals(startNode, con.in(0)); break; default: throw new IllegalStateException(); } } } + + @Test + public void testNegative() { + Parser parser = new Parser(); + StartNode start = parser.parse("return -123;"); + for( Node use : start._outputs ) + if( use instanceof ConstantNode con ) + assertEquals(-123,con._value); + } + + @Test + public void testBad1() { + try { new Parser().parse("ret"); } + catch( RuntimeException e ) { assertEquals("syntax error, expected return: ret",e.getMessage()); } + } + + @Test + public void testBad2() { + try { new Parser().parse("return 0123;"); } + catch( RuntimeException e ) { assertEquals("syntax error, expected nested expr or integer literal: 0123",e.getMessage()); } + } + + @Test + public void testBad3() { + try { new Parser().parse("return --12;"); } + catch( RuntimeException e ) { assertEquals("syntax error, expected ( got --: --",e.getMessage()); } + } } -- cgit v1.2.3