summaryrefslogtreecommitdiff
path: root/src/main/java/com/seaofnodes/simple/parser/Lexer.java
diff options
context:
space:
mode:
Diffstat (limited to 'src/main/java/com/seaofnodes/simple/parser/Lexer.java')
-rw-r--r--src/main/java/com/seaofnodes/simple/parser/Lexer.java384
1 files changed, 384 insertions, 0 deletions
diff --git a/src/main/java/com/seaofnodes/simple/parser/Lexer.java b/src/main/java/com/seaofnodes/simple/parser/Lexer.java
new file mode 100644
index 00000000..f2857c9f
--- /dev/null
+++ b/src/main/java/com/seaofnodes/simple/parser/Lexer.java
@@ -0,0 +1,384 @@
+package com.seaofnodes.simple.parser;
+
+import com.seaofnodes.simple.codegen.CodeGen;
+import com.seaofnodes.simple.codegen.CompUnit;
+import com.seaofnodes.simple.codegen.Serialize;
+import com.seaofnodes.simple.parser.Parser.ParseException;
+import com.seaofnodes.simple.type.*;
+import com.seaofnodes.simple.util.BAOS;
+import java.util.HashMap;
+
+// Lexer provides low level access to the raw file bytes, peeks and matches
+// short strings, parses numbers, skips comments and whitespace, tracks
+// line numbers, allows the parse position to be saved and restored, and
+// serves as a location indicator for errors.
+
+public class Lexer {
+
+ // Input buffer; an array of text bytes read from a file or a string
+ private final byte[] _input;
+ // Original source, retained across inlining.
+ public final CompUnit _cu;
+ // Tracks current position in input buffer
+ int _position = 0;
+
+ //
+ private int _line_number = 1;
+ // Start of current line
+ private int _line_start = 0;
+
+ /** True when this location precedes {@code that} in the same source. */
+ public boolean before(Lexer that) {
+ return that != null && (_cu!=null ? _cu==that._cu : _input==that._input) && _position < that._position;
+ }
+
+ public String position() { return _line_number+":"+(_position-_line_start+1); }
+ public int pos() { return _position; }
+
+ /**
+ * Record the source text for lexing
+ */
+ public Lexer(String source) {
+ this(source.getBytes());
+ }
+
+ /**
+ * Direct from disk file source
+ */
+ public Lexer(byte[] buf) {
+ _input = buf;
+ _cu = null;
+ }
+
+ Lexer(CompUnit cu) {
+ _input = cu._src.getBytes();
+ _cu = cu;
+ }
+
+ // Imported location: source text is only needed when reporting an error.
+ Lexer(CompUnit cu, int pos) {
+ _input = null;
+ _cu = cu;
+ _position = pos;
+ }
+
+ public static void gather(Lexer loc, HashMap<String,Integer> strs) {
+ if( loc==null || loc._cu==null ) return;
+ Serialize.gather(strs,loc._cu._cname);
+ String file = loc._cu.sourceFile();
+ if( file!=null ) Serialize.gather(strs,file);
+ }
+
+ public static void packed(Lexer loc, BAOS out, HashMap<String,Integer> strs) {
+ if( loc==null || loc._cu==null ) { out.packed4(0); return; }
+ out.packed4(loc._position+1);
+ out.packed4(strs.get(loc._cu._cname));
+ String file = loc._cu.sourceFile();
+ out.packed4(file==null ? 0 : strs.get(file));
+ }
+
+ public static Lexer unpack(BAOS in, String[] strs) {
+ int pos = in.packed4();
+ if( pos==0 ) return null;
+ String owner = strs[in.packed4()];
+ String file = strs[in.packed4()];
+ CompUnit cu = CodeGen.CODE._compunits.get(owner.replace('.','/'));
+ assert cu!=null : owner;
+ if( !file.isEmpty() ) cu._sourceFile = file;
+ return new Lexer(cu,pos-1);
+ }
+
+ /**
+ * Copy a lexer from a lexer
+ */
+ Lexer(Lexer l) {
+ _input = l._input;
+ _cu = l._cu;
+ _position = l._position;
+ _line_number = l._line_number;
+ _line_start = l._line_start;
+ }
+
+ // Very handy in the debugger, shows the unparsed program
+ @Override
+ public String toString() {
+ return _input==null ? _cu._cname+"@"+_position : new String(_input, _position, _input.length - _position);
+ }
+
+ // True if at EOF
+ boolean isEOF() {
+ return _position >= _input.length;
+ }
+
+ // Peek next character, or report EOF
+ char peek() {
+ return isEOF() ? Character.MAX_VALUE // Special value that causes parsing to terminate
+ : (char) _input[_position];
+ }
+ // Just crash if misused
+ public byte peek(int off) { return _input[_position+off]; }
+
+ private void inc() {
+ if( _position++ < _input.length && _input[_position-1] == '\n' ) {
+ _line_number++;
+ _line_start = _position;
+ }
+ }
+ // Does not honor LF, so caller must roll back position on a LF
+ private char nextChar() {
+ char ch = peek();
+ inc();
+ return ch;
+ }
+
+ // True if a white space
+ private boolean isWhiteSpace() {
+ return peek() <= ' '; // Includes all the use space, tab, newline, CR
+ }
+
+ /**
+ * Return the next non-white-space character
+ */
+ void skipWhiteSpace() {
+ while( true ) {
+ if( isWhiteSpace() ) inc();
+ // Skip // to end of line
+ else if( _position+2 < _input.length &&
+ _input[_position ] == '/' &&
+ _input[_position+1] == '/') {
+ inc(); inc();
+ while( !isEOF() && _input[_position] != '\n' )
+ inc();
+ } else if( _position+2 < _input.length &&
+ _input[_position ] == '/' &&
+ _input[_position+1] == '*') {
+ // Skip /*comment*/
+ while( !isEOF() && !(_input[_position-1] == '*' && _input[_position] == '/'))
+ inc();
+ inc();
+ } else break;
+ }
+ }
+
+ // Return true, if we find "syntax" after skipping white space; also
+ // then advance the cursor past syntax.
+ // Return false otherwise, and do not advance the cursor.
+ boolean match(String syntax) {
+ assert syntax.indexOf('\n')==-1; // No newlines in match
+ skipWhiteSpace();
+ int len = syntax.length();
+ if (_position + len > _input.length) return false;
+ for (int i = 0; i < len; i++)
+ if ((char) _input[_position + i] != syntax.charAt(i))
+ return false;
+ _position += len;
+ return true;
+ }
+
+ // Match must be exact and not followed by more ID characters.
+ // Prevents identifier "ifxy" from matching an "if" statement.
+ boolean matchx(String syntax) {
+ if( !match(syntax) ) return false;
+ if( !isIdLetter(peek()) ) return true;
+ _position -= syntax.length();
+ return false;
+ }
+ // Match this char, and the next char must be different.
+ // Handles '&&' vs '&'
+ boolean matchOp( char c0 ) {
+ skipWhiteSpace();
+ if( _position+1 >= _input.length || _input[_position]!=c0 || _input[_position+1]==c0 )
+ return false;
+ inc();
+ return true;
+ }
+ // Match these two characters in a row
+ boolean matchOpx(char c0, char c1) {
+ skipWhiteSpace();
+ if( _position+1 >= _input.length || _input[_position]!=c0 || _input[_position+1]==c1 )
+ return false;
+ inc();
+ return true;
+ }
+
+ boolean peek(char ch) {
+ skipWhiteSpace();
+ return peek()==ch;
+ }
+
+ boolean peekIsId() {
+ skipWhiteSpace();
+ return isIdStart(peek());
+ }
+
+ // Return an identifier or null
+ String matchId() {
+ return peekIsId() ? parseId() : null;
+ }
+
+ // Used for errors
+ String getAnyNextToken() {
+ if (isEOF()) return "EOF";
+ if (isIdStart(peek())) return parseId();
+ if (isNumber(peek())) return parseNumberString();
+ if (isPunctuation(peek())) return parsePunctuation();
+ return String.valueOf(peek());
+ }
+
+ boolean isNumber(char ch) {return Character.isDigit(ch);}
+
+ // Return a constant Type, either TypeInteger or TypeFloat
+ Type parseNumber() {
+ int old = _position;
+ int len = isLongOrDouble();
+ boolean f32 = peek()=='f';
+ if( f32 ) nextChar();
+ int n = Math.abs(len);
+ if( f32 )
+ return TypeFloat.constant(Float.parseFloat(new String(_input,old,n)));
+ if( len > 0 ) {
+ if( len > 1 && _input[old]=='0' )
+ throw Parser.error("Syntax error: integer values cannot start with '0'",this);
+ long i = Long.parseLong(new String(_input,old,len));
+ return TypeInteger.constant(i);
+ }
+ return TypeFloat.constant(Double.parseDouble(new String(_input,old,n)));
+ }
+ private String parseNumberString() {
+ int old = _position;
+ int len = Math.abs(isLongOrDouble());
+ if( peek()=='f' ) {
+ nextChar();
+ len++;
+ }
+ return new String(_input,old,len);
+ }
+
+ // Return +len that ends a long
+ // Return -len that ends a double
+ private int isLongOrDouble() {
+ int old = _position;
+ char c;
+ while( Character.isDigit(c = nextChar()) ) ;
+ if( !(c=='e' || c=='.') )
+ return --_position - old;
+ if( peek() == '-' ) nextChar();
+ while( Character.isDigit(c = nextChar()) || c=='e' || c=='.' ) ;
+ return -(--_position - old);
+ }
+
+ // First letter of an identifier
+ private boolean isIdStart(char ch) {
+ return Character.isAlphabetic(ch) || ch == '_';
+ }
+
+ // All characters of an identifier, e.g. "_x123"
+ private boolean isIdLetter(char ch) {
+ return Character.isLetterOrDigit(ch) || ch == '_';
+ }
+
+ private String parseId() {
+ int start = _position;
+ while (isIdLetter(nextChar())) ;
+ return new String(_input, start, --_position - start).intern();
+ }
+
+ //
+ private boolean isPunctuation(char ch) {
+ return "=;[]<>()+-/*&|^".indexOf(ch) != -1;
+ }
+
+ private String parsePunctuation() {
+ int start = _position;
+ return new String(_input, start, 1);
+ }
+
+ // Next oper= character, or 0.
+ // As a convenience, mark "++" as a char 1 and "--" as char -1 (65535)
+ // Distinct tags for the multi-character compound operators.
+ static final char SHL = 2, SAR = 3, SHR = 4;
+
+ public char matchOperAssign() {
+ skipWhiteSpace();
+ if( match("<<=" ) ) return SHL;
+ if( match(">>>=") ) return SHR;
+ if( match(">>=" ) ) return SAR;
+ if( _position+2 >= _input.length ) return 0;
+ char ch0 = (char)_input[_position];
+ if( "+-/*&|^".indexOf(ch0) == -1 ) return 0;
+ char ch1 = (char)_input[_position+1];
+ if( ch1 == '=' ) { _position += 2; return ch0; }
+ if( isIdLetter((char)_input[_position+2]) ) return 0;
+ if( ch0 == '+' && ch1 == '+' ) { _position += 2; return (char) 1; }
+ if( ch0 == '-' && ch1 == '-' ) { _position += 2; return (char)-1; }
+ return 0;
+ }
+
+ String parseString() {
+ if( !peek('"') ) return null;
+ inc();
+ StringBuilder sb = new StringBuilder();
+ while( !isEOF() ) {
+ char c = nextChar();
+ if( c=='"' ) return sb.toString();
+ if( c!='\\' ) {
+ sb.append(c);
+ continue;
+ }
+ if( isEOF() )
+ throw error("Unclosed string");
+ sb.append(escapedChar("string"));
+ }
+ throw error("Unclosed string");
+ }
+
+ private char escapedChar(String kind) {
+ char esc = nextChar();
+ return switch( esc ) {
+ case '0' -> '\u0000';
+ case 'n' -> '\n';
+ case 't' -> '\t';
+ case 'r' -> '\r';
+ case '\\' -> '\\';
+ case '"' -> '"';
+ case '\'' -> '\'';
+ default -> throw error("Unknown "+kind+" escape \\\\"+esc);
+ };
+ }
+
+ char parseChar() {
+ if( isEOF() ) throw error("Unclosed character");
+ char c = nextChar();
+ if( c=='\\' ) {
+ if( isEOF() ) throw error("Unclosed character");
+ c = escapedChar("character");
+ }
+ return c;
+ }
+
+ private ParseException error(String msg) { return Parser.error(msg,this); }
+
+ // Scan a deferred fragment, leaving its unmatched closing delimiter unread.
+ // Comments, quoted text, and escapes do not affect the nesting depth.
+ void skipTo(char open, char close, String what) {
+ int depth = 0;
+ while( true ) {
+ skipWhiteSpace();
+ if( isEOF() ) throw error("Unclosed "+what);
+ char ch = peek();
+ if( ch==close && depth==0 ) return;
+ nextChar();
+ if( ch==open ) depth++;
+ else if( ch==close ) depth--;
+ else if( ch=='"' || ch=='\'' ) {
+ char quote = ch;
+ while( !isEOF() ) {
+ ch = nextChar();
+ if( ch=='\\' ) nextChar();
+ else if( ch==quote ) break;
+ }
+ }
+ }
+ }
+
+}
generated by cgit v1.2.3 (git 2.25.1) at 2026-11-02 23:15:06 +0000