Start on the lexer

Signed-off-by: Antia Anderson <anita@ansan.io>

4 files changed, 268 insertions, 0 deletions

diff --git a/src/moe/anita/mai/span/Pos.java b/src/moe/anita/mai/span/Pos.java
new file mode 100644
index 0000000..c6d3043
--- /dev/null
+++ b/src/moe/anita/mai/span/Pos.java
@@ -0,0 +1,47 @@
+package moe.anita.mai.span;
+
+public final class Pos {
+
+	private final long value;
+
+	private static final long OFFSET_MASK = 0xFFFF_FFFFL;
+	private static final long U16_MASK = 0xFFFFL;
+
+	private Pos(long value) {
+		this.value = value;
+	}
+
+	public static Pos of(int offset, int line, int column) {
+		long value =
+			((long) offset << OFFSET_MASK)
+			| ((long) line & U16_MASK) << 16
+			| ((long) column & U16_MASK);
+
+		return new Pos(value);
+	}
+
+	public static Pos zero() {
+		return new Pos(0);
+	}
+
+	public int offset() {
+		return (int) (value >>> 32);
+	}
+
+	public int line() {
+		return (int) ((value >>> 16) & U16_MASK);
+	}
+
+	public int column() {
+		return (int) (value & U16_MASK);
+	}
+
+	public boolean isZero() {
+		return value == 0;
+	}
+
+	@Override
+	public String toString() {
+		return "%d:%d".formatted(line(), column());
+	}
+}
diff --git a/src/moe/anita/mai/span/Span.java b/src/moe/anita/mai/span/Span.java
new file mode 100644
index 0000000..fe27a91
--- /dev/null
+++ b/src/moe/anita/mai/span/Span.java
@@ -0,0 +1,45 @@
+package moe.anita.mai.span;
+
+public final class Span {
+
+	private final Pos start;
+	private final Pos end;
+
+	private Span(Pos start, Pos end) {
+		this.start = start;
+		this.end = end;
+	}
+
+	public static Span of(Pos start, Pos end) {
+		return new Span(start, end);
+	}
+
+	public static Span zero() {
+		return new Span(Pos.zero(), Pos.zero());
+	}
+
+	public Pos start() {
+		return start;
+	}
+
+	public Pos end() {
+		return end;
+	}
+
+	public int length() {
+		return end.offset() - start.offset();
+	}
+
+	public boolean isEmpty() {
+		return length() == 0;
+	}
+
+	public boolean isZero() {
+		return start.isZero() && end.isZero();
+	}
+
+	@Override
+	public String toString() {
+		return "%s-%s".formatted(start, end);
+	}
+}
diff --git a/src/moe/anita/mai/token/Token.java b/src/moe/anita/mai/token/Token.java
new file mode 100644
index 0000000..426507a
--- /dev/null
+++ b/src/moe/anita/mai/token/Token.java
@@ -0,0 +1,5 @@
+package moe.anita.mai.token;
+
+import moe.anita.mai.span.Span;
+
+public record Token(TokenKind kind, Span span) {}
diff --git a/src/moe/anita/mai/token/TokenKind.java b/src/moe/anita/mai/token/TokenKind.java
new file mode 100644
index 0000000..2beadb2
--- /dev/null
+++ b/src/moe/anita/mai/token/TokenKind.java
@@ -0,0 +1,171 @@
+package moe.anita.mai.token;
+
+public enum TokenKind {
+
+	// Special
+	EOF,
+	ERROR,
+	EOL,
+
+	// Trivia
+	WHITESPACE,
+	COMMENT,
+	DOC_COMMENT,
+
+	// Names
+	IDENT,
+
+	// Literals
+	INT,
+	FLOAT,
+	STRING,
+	RAW_STRING,
+	RUNE,
+
+	// Delimiters
+	LPAREN("("),
+	RPAREN(")"),
+	LBRACKET("["),
+	RBRACKET("]"),
+	LBRACE("{"),
+	RBRACE("}"),
+
+	COMMA(","),
+	DOT("."),
+	COLON(":"),
+	SEMICOLON(";"),
+
+	// Assignment / declaration
+	ASSIGN("="),
+	DECLARE(":="),
+
+	// Arithmetic
+	INC("++", Precedence.PREFIX, Flags.PREFIX),
+	DEC("--", Precedence.PREFIX, Flags.PREFIX),
+
+	ADD("+", Precedence.ADD, Flags.INFIX | Flags.PREFIX),
+	SUB("-", Precedence.ADD, Flags.INFIX | Flags.PREFIX),
+
+	MUL("*", Precedence.MUL, Flags.INFIX),
+	DIV("/", Precedence.MUL, Flags.INFIX),
+	MOD("%", Precedence.MUL, Flags.INFIX),
+
+	// Comparison
+	EQ("==", Precedence.COMPARE, Flags.INFIX),
+	NE("<>", Precedence.COMPARE, Flags.INFIX),
+
+	LT("<", Precedence.COMPARE, Flags.INFIX),
+	LE("<=", Precedence.COMPARE, Flags.INFIX),
+	GT(">", Precedence.COMPARE, Flags.INFIX),
+	GE(">=", Precedence.COMPARE, Flags.INFIX),
+
+	// Bitwise
+	BIT_AND("&", Precedence.BIT_AND, Flags.INFIX),
+	BIT_XOR("^", Precedence.BIT_XOR, Flags.INFIX),
+	BIT_OR("|", Precedence.BIT_OR, Flags.INFIX),
+
+	BIT_NOT("~", Precedence.PREFIX, Flags.PREFIX),
+
+	// Logical
+	AND("&&", Precedence.LOGICAL_AND, Flags.INFIX),
+	OR("||", Precedence.LOGICAL_OR, Flags.INFIX),
+
+	NOT("!", Precedence.PREFIX, Flags.PREFIX),
+
+	// Misc operators
+	ARROW("->"),
+	BIND("<-"),
+
+	RANGE(".."),
+
+	QUESTION("?"),
+	AT("@"),
+
+	/*
+	 * These are parser-created operators.
+	 *
+	 * The lexer emits two LT or GT tokens instead of directly
+	 * producing SHIFT_LEFT / SHIFT_RIGHT. This lets the parser
+	 * distinguish shifts from nested generic type syntax.
+	 */
+	SHIFT_LEFT("<<", Precedence.SHIFT, Flags.INFIX),
+	SHIFT_RIGHT(">>", Precedence.SHIFT, Flags.INFIX);
+
+	private final String spelling;
+	private final int precedence;
+	private final int flags;
+
+	TokenKind() {
+		this(null, Precedence.NONE, Flags.NONE);
+	}
+
+	TokenKind(String spelling) {
+		this(spelling, Precedence.NONE, Flags.NONE);
+	}
+
+	TokenKind(String spelling, int precedence, int flags) {
+		this.spelling = spelling;
+		this.precedence = precedence;
+		this.flags = flags;
+	}
+
+	public String spelling() {
+		return spelling;
+	}
+
+	public int precedence() {
+		return precedence;
+	}
+
+	public boolean hasSpelling() {
+		return spelling != null;
+	}
+
+	public boolean isInfix() {
+		return (flags & Flags.INFIX) != 0;
+	}
+
+	public boolean isPrefix() {
+		return (flags & Flags.PREFIX) != 0;
+	}
+
+
+
+	private static final class Flags {
+
+		private static final int NONE = 0;
+
+		private static final int INFIX = 1 << 0;
+		private static final int PREFIX = 1 << 1;
+
+		private Flags() {
+		}
+	}
+
+	private static final class Precedence {
+
+		private static final int NONE = 0;
+
+		/*
+		 * Larger values bind tighter.
+		 */
+		private static final int LOGICAL_OR = 10;
+		private static final int LOGICAL_AND = 20;
+
+		private static final int BIT_OR = 30;
+		private static final int BIT_XOR = 40;
+		private static final int BIT_AND = 50;
+
+		private static final int COMPARE = 60;
+
+		private static final int SHIFT = 70;
+
+		private static final int ADD = 80;
+		private static final int MUL = 90;
+
+		private static final int PREFIX = 100;
+
+		private Precedence() {
+		}
+	}
+}