From a532a21b792f45359aab9e6259b2e9412f31b81a Mon Sep 17 00:00:00 2001 From: Antia Anderson Date: Tue, 25 Aug 2026 02:54:50 -0400 Subject: Start work on the lexer and added tests Signed-off-by: Antia Anderson --- src/moe/anita/mai/buffer/Buffer.java | 43 +++++ src/moe/anita/mai/buffer/MemBuf.java | 29 +++ src/moe/anita/mai/buffer/SrcBuf.java | 44 +++++ src/moe/anita/mai/lexer/Lexer.java | 332 +++++++++++++++++++++++++++++++++++ 4 files changed, 448 insertions(+) create mode 100644 src/moe/anita/mai/buffer/Buffer.java create mode 100644 src/moe/anita/mai/buffer/MemBuf.java create mode 100644 src/moe/anita/mai/buffer/SrcBuf.java create mode 100644 src/moe/anita/mai/lexer/Lexer.java (limited to 'src/moe') diff --git a/src/moe/anita/mai/buffer/Buffer.java b/src/moe/anita/mai/buffer/Buffer.java new file mode 100644 index 0000000..c235bbb --- /dev/null +++ b/src/moe/anita/mai/buffer/Buffer.java @@ -0,0 +1,43 @@ +package moe.anita.mai.buffer; + +public interface Buffer { + + /** + * Returns the number of characters in the buffer. + */ + int length(); + + /** + * Returns the character at the given offset. + * + * @param index character offset + * @return character at the offset + */ + char get(int index); + + /** + * Returns whether the given offset exists in the buffer. + * + * @param index character offset + * @return true when the offset is valid + */ + default boolean has(int index) { + return index >= 0 && index < length(); + } + + /** + * Returns whether the buffer is empty. + */ + default boolean isEmpty() { + return length() == 0; + } + + /** + * Returns a slice of the buffer in the range [start, end). + * + * @param start starting offset + * @param end ending offset + * @return characters in the requested range + */ + CharSequence slice(int start, int end); +} diff --git a/src/moe/anita/mai/buffer/MemBuf.java b/src/moe/anita/mai/buffer/MemBuf.java new file mode 100644 index 0000000..a2bc477 --- /dev/null +++ b/src/moe/anita/mai/buffer/MemBuf.java @@ -0,0 +1,29 @@ +package moe.anita.mai.buffer; + +import java.util.Objects; + +/** + * Buffer backed entirely by an in-memory string. + */ +public final class MemBuf implements Buffer { + + private final String data; + + public MemBuf(String data) { this.data = Objects.requireNonNull(data); + } + + @Override + public int length() { + return data.length(); + } + + @Override + public char get(int index) { + return data.charAt(index); + } + + @Override + public CharSequence slice(int start, int end) { + return data.subSequence(start, end); + } +} diff --git a/src/moe/anita/mai/buffer/SrcBuf.java b/src/moe/anita/mai/buffer/SrcBuf.java new file mode 100644 index 0000000..670bcbe --- /dev/null +++ b/src/moe/anita/mai/buffer/SrcBuf.java @@ -0,0 +1,44 @@ +package moe.anita.mai.buffer; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; + +/** + * A buffer backed by the contents of a source file. + */ +public final class SrcBuf implements Buffer { + + private final Path path; + private final String data; + + public SrcBuf(Path path) throws IOException { + this.path = Objects.requireNonNull(path); + this.data = Files.readString(path); + } + + /** + * Returns the path of the source file. + * + * @return source file path + */ + public Path path() { + return path; + } + + @Override + public int length() { + return data.length(); + } + + @Override + public char get(int index) { + return data.charAt(index); + } + + @Override + public CharSequence slice(int start, int end) { + return data.subSequence(start, end); + } +} diff --git a/src/moe/anita/mai/lexer/Lexer.java b/src/moe/anita/mai/lexer/Lexer.java new file mode 100644 index 0000000..56f4534 --- /dev/null +++ b/src/moe/anita/mai/lexer/Lexer.java @@ -0,0 +1,332 @@ +package moe.anita.mai.lexer; + +import moe.anita.mai.buffer.Buffer; +import moe.anita.mai.span.Pos; +import moe.anita.mai.span.Span; +import moe.anita.mai.token.Token; +import moe.anita.mai.token.TokenKind; + +public final class Lexer { + + private final Buffer buffer; + + private int offset; + private int line; + private int column; + + public Lexer(Buffer buffer) { + this.buffer = buffer; + + offset = 0; + line = 1; + column = 1; + } + + /** + * Returns the next token in the source. + * + * @return next token + */ + public Token next() { + skipWhitespace(); + + Pos start = pos(); + + if (eof()) { + return token(TokenKind.EOF, start); + } + + char c = current(); + + if (c == '\n') { + advance(); + return token(TokenKind.EOL, start); + } + + if (isDigit(c)) { + return number(start); + } + + if (isIdentStart(c)) { + return ident(start); + } + + advance(); + + return switch (c) { + case '(' -> token(TokenKind.LPAREN, start); + case ')' -> token(TokenKind.RPAREN, start); + + case '[' -> token(TokenKind.LBRACKET, start); + case ']' -> token(TokenKind.RBRACKET, start); + + case '{' -> token(TokenKind.LBRACE, start); + case '}' -> token(TokenKind.RBRACE, start); + + case ',' -> token(TokenKind.COMMA, start); + case '.' -> token(TokenKind.DOT, start); + case ';' -> token(TokenKind.SEMICOLON, start); + case '?' -> token(TokenKind.QUESTION, start); + case '@' -> token(TokenKind.AT, start); + + case '+' -> { + if (match('+')) { + yield token(TokenKind.INC, start); + } + + yield token(TokenKind.ADD, start); + } + + case '-' -> { + if (match('-')) { + yield token(TokenKind.DEC, start); + } + + if (match('>')) { + yield token(TokenKind.ARROW, start); + } + + yield token(TokenKind.SUB, start); + } + + case '*' -> token(TokenKind.MUL, start); + case '/' -> token(TokenKind.DIV, start); + case '%' -> token(TokenKind.MOD, start); + + case '=' -> { + if (match('=')) { + yield token(TokenKind.EQ, start); + } + + yield token(TokenKind.ASSIGN, start); + } + + case ':' -> { + if (match('=')) { + yield token(TokenKind.DECLARE, start); + } + + yield token(TokenKind.COLON, start); + } + + case '<' -> { + if (match('=')) { + yield token(TokenKind.LE, start); + } + + if (match('>')) { + yield token(TokenKind.NE, start); + } + + if (match('-')) { + yield token(TokenKind.BIND, start); + } + + yield token(TokenKind.LT, start); + } + + case '>' -> { + if (match('=')) { + yield token(TokenKind.GE, start); + } + + yield token(TokenKind.GT, start); + } + + case '&' -> { + if (match('&')) { + yield token(TokenKind.AND, start); + } + + yield token(TokenKind.BIT_AND, start); + } + + case '|' -> { + if (match('|')) { + yield token(TokenKind.OR, start); + } + + yield token(TokenKind.BIT_OR, start); + } + + case '^' -> token(TokenKind.BIT_XOR, start); + case '~' -> token(TokenKind.BIT_NOT, start); + case '!' -> token(TokenKind.NOT, start); + + default -> token(TokenKind.ERROR, start); + }; + } + + private Token number(Pos start) { + if (current() == '0' && has(1)) { + char prefix = peek(1); + + switch (prefix) { + case 'x', 'X' -> { + advance(); + advance(); + + while (!eof() && isHexDigitOrUnderscore(current())) { + advance(); + } + + return token(TokenKind.INT, start); + } + + case 'b', 'B' -> { + advance(); + advance(); + + while (!eof() && isBinaryDigitOrUnderscore(current())) { + advance(); + } + + return token(TokenKind.INT, start); + } + + case 'o', 'O' -> { + advance(); + advance(); + + while (!eof() && isOctalDigitOrUnderscore(current())) { + advance(); + } + + return token(TokenKind.INT, start); + } + } + } + + while (!eof() && isDigitOrUnderscore(current())) { + advance(); + } + + if ( + !eof() + && current() == '.' + && has(1) + && isDigit(peek(1)) + ) { + advance(); + + while (!eof() && isDigitOrUnderscore(current())) { + advance(); + } + + return token(TokenKind.FLOAT, start); + } + + return token(TokenKind.INT, start); + } + + private Token ident(Pos start) { + while (!eof() && isIdentContinue(current())) { + advance(); + } + + return token(TokenKind.IDENT, start); + } + + private void skipWhitespace() { + while (!eof()) { + char c = current(); + + if ( + c != ' ' + && c != '\t' + && c != '\r' + ) { + return; + } + + advance(); + } + } + + private boolean isDigit(char c) { + return c >= '0' && c <= '9'; + } + + private boolean isDigitOrUnderscore(char c) { + return isDigit(c) || c == '_'; + } + + private boolean isHexDigitOrUnderscore(char c) { + return isDigit(c) + || (c >= 'a' && c <= 'f') + || (c >= 'A' && c <= 'F') + || c == '_'; + } + + private boolean isBinaryDigitOrUnderscore(char c) { + return c == '0' + || c == '1' + || c == '_'; + } + + private boolean isOctalDigitOrUnderscore(char c) { + return (c >= '0' && c <= '7') + || c == '_'; + } + + private boolean isIdentStart(char c) { + return c == '_' + || Character.isUnicodeIdentifierStart(c); + } + + private boolean isIdentContinue(char c) { + return c == '_' + || Character.isUnicodeIdentifierPart(c); + } + + private boolean eof() { + return offset >= buffer.length(); + } + + private boolean has(int distance) { + return offset + distance < buffer.length(); + } + + private char current() { + return buffer.get(offset); + } + + private char peek(int distance) { + return buffer.get(offset + distance); + } + + private boolean match(char expected) { + if (eof() || current() != expected) { + return false; + } + + advance(); + return true; + } + + private char advance() { + char c = current(); + + offset++; + + if (c == '\n') { + line++; + column = 1; + } else { + column++; + } + + return c; + } + + private Pos pos() { + return Pos.of(offset, line, column); + } + + private Token token(TokenKind kind, Pos start) { + return new Token( + kind, + Span.of(start, pos()) + ); + } +} -- cgit v1.2.3