Finish the lexerHEADtrunklexer

Signed-off-by: Antia Anderson <anita@ansan.io>

6 files changed, 537 insertions, 140 deletions

diff --git a/src/moe/anita/mai/buffer/Buffer.java b/src/moe/anita/mai/buffer/Buffer.java
index c235bbb..e119b40 100644
--- a/src/moe/anita/mai/buffer/Buffer.java
+++ b/src/moe/anita/mai/buffer/Buffer.java
@@ -1,43 +1,61 @@
 package moe.anita.mai.buffer;
 
+/**
+ * Provides random access to Unicode source text.
+ */
 public interface Buffer {
 
 	/**
-	 * Returns the number of characters in the buffer.
+	 * Returns the length of the underlying source representation.
+	 *
+	 * Offsets are measured in the backing representation's code units.
+	 *
+	 * @return source length
 	 */
 	int length();
 
 	/**
-	 * Returns the character at the given offset.
+	 * Returns the Unicode code point beginning at the given offset.
 	 *
-	 * @param index character offset
-	 * @return character at the offset
+	 * @param offset source offset
+	 * @return Unicode code point
 	 */
-	char get(int index);
+	int get(int offset);
+
+	/**
+	 * Returns the number of underlying code units occupied by the
+	 * code point beginning at the given offset.
+	 *
+	 * @param offset source offset
+	 * @return width of the code point
+	 */
+	int width(int offset);
 
 	/**
 	 * Returns whether the given offset exists in the buffer.
 	 *
-	 * @param index character offset
-	 * @return true when the offset is valid
+	 * @param offset source offset
+	 * @return true if the offset is valid
 	 */
-	default boolean has(int index) {
-		return index >= 0 && index < length();
+	default boolean has(int offset) {
+		return offset >= 0 && offset < length();
 	}
 
 	/**
-	 * Returns whether the buffer is empty.
+	 * Returns whether the buffer contains no source text.
+	 *
+	 * @return true if empty
 	 */
 	default boolean isEmpty() {
 		return length() == 0;
 	}
 
 	/**
-	 * Returns a slice of the buffer in the range [start, end).
+	 * Returns the source contents between two offsets.
 	 *
-	 * @param start starting offset
-	 * @param end ending offset
-	 * @return characters in the requested range
+	 * @param start starting offset, inclusive
+	 * @param end ending offset, exclusive
+	 * @return requested source text
 	 */
 	CharSequence slice(int start, int end);
 }
diff --git a/src/moe/anita/mai/buffer/MemBuf.java b/src/moe/anita/mai/buffer/MemBuf.java
index a2bc477..922e624 100644
--- a/src/moe/anita/mai/buffer/MemBuf.java
+++ b/src/moe/anita/mai/buffer/MemBuf.java
@@ -1,29 +1,72 @@
 package moe.anita.mai.buffer;
 
+import java.nio.charset.StandardCharsets;
 import java.util.Objects;
 
 /**
- * Buffer backed entirely by an in-memory string.
+ * An in-memory UTF-8 source buffer.
+ *
+ * Offsets and lengths are measured in UTF-8 bytes.
  */
 public final class MemBuf implements Buffer {
 
-	private final String data;
+	private final byte[] data;
 
-	public MemBuf(String data) { this.data = Objects.requireNonNull(data);
+	public MemBuf(String source) {
+		Objects.requireNonNull(source);
+		data = source.getBytes(StandardCharsets.UTF_8);
 	}
 
 	@Override
 	public int length() {
-		return data.length();
+		return data.length;
 	}
 
 	@Override
-	public char get(int index) {
-		return data.charAt(index);
+	public int get(int offset) {
+		int first = Byte.toUnsignedInt(data[offset]);
+
+		if ((first & 0x80) == 0) {
+			return first;
+		}
+
+		if ((first & 0xE0) == 0xC0) {
+			return ((first & 0x1F) << 6) | (Byte.toUnsignedInt(data[offset + 1]) & 0x3F);
+		}
+
+		if ((first & 0xF0) == 0xE0) {
+			return ((first & 0x0F) << 12)
+				| ((Byte.toUnsignedInt(data[offset + 1]) & 0x3F) << 6)
+				| (Byte.toUnsignedInt(data[offset + 2]) & 0x3F);
+		}
+
+		return ((first & 0x07) << 18)
+			|  ((Byte.toUnsignedInt(data[offset + 1]) & 0x3F) << 12)
+			| ((Byte.toUnsignedInt(data[offset + 2]) & 0x3F) << 6)
+			| (Byte.toUnsignedInt(data[offset + 3]) & 0x3F);
+	}
+
+	@Override
+	public int width(int offset) {
+		int first = Byte.toUnsignedInt(data[offset]);
+
+		if ((first & 0x80) == 0) {
+			return 1;
+		}
+
+		if ((first & 0xE0) == 0xC0) {
+			return 2;
+		}
+
+		if ((first & 0xF0) == 0xE0) {
+			return 3;
+		}
+
+		return 4;
 	}
 
 	@Override
 	public CharSequence slice(int start, int end) {
-		return data.subSequence(start, end);
+		return new String(data, start, end - start, StandardCharsets.UTF_8);
 	}
 }
diff --git a/src/moe/anita/mai/buffer/SrcBuf.java b/src/moe/anita/mai/buffer/SrcBuf.java
index 670bcbe..f288316 100644
--- a/src/moe/anita/mai/buffer/SrcBuf.java
+++ b/src/moe/anita/mai/buffer/SrcBuf.java
@@ -1,13 +1,11 @@
 package moe.anita.mai.buffer;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Objects;
 
-/**
- * A buffer backed by the contents of a source file.
- */
 public final class SrcBuf implements Buffer {
 
 	private final Path path;
@@ -15,13 +13,13 @@ public final class SrcBuf implements Buffer {
 
 	public SrcBuf(Path path) throws IOException {
 		this.path = Objects.requireNonNull(path);
-		this.data = Files.readString(path);
+		this.data = Files.readString(path, StandardCharsets.UTF_8);
 	}
 
 	/**
-	 * Returns the path of the source file.
+	 * Returns the source file path.
 	 *
-	 * @return source file path
+	 * @return source path
 	 */
 	public Path path() {
 		return path;
@@ -33,8 +31,13 @@ public final class SrcBuf implements Buffer {
 	}
 
 	@Override
-	public char get(int index) {
-		return data.charAt(index);
+	public int get(int offset) {
+		return data.codePointAt(offset);
+	}
+
+	@Override
+	public int width(int offset) {
+		return Character.charCount(data.codePointAt(offset));
 	}
 
 	@Override
diff --git a/src/moe/anita/mai/lexer/Lexer.java b/src/moe/anita/mai/lexer/Lexer.java
index 56f4534..85f0be1 100644
--- a/src/moe/anita/mai/lexer/Lexer.java
+++ b/src/moe/anita/mai/lexer/Lexer.java
@@ -6,6 +6,9 @@ import moe.anita.mai.span.Span;
 import moe.anita.mai.token.Token;
 import moe.anita.mai.token.TokenKind;
 
+/**
+ * Tokenizes mai source text.
+ */
 public final class Lexer {
 
 	private final Buffer buffer;
@@ -36,13 +39,17 @@ public final class Lexer {
 			return token(TokenKind.EOF, start);
 		}
 
-		char c = current();
+		int c = current();
 
 		if (c == '\n') {
 			advance();
 			return token(TokenKind.EOL, start);
 		}
 
+		if (c == 'r' && hasNext() && peek() == '"') {
+			return rawString(start);
+		}
+
 		if (isDigit(c)) {
 			return number(start);
 		}
@@ -51,20 +58,34 @@ public final class Lexer {
 			return ident(start);
 		}
 
+		if (c == '"') {
+			return string(start);
+		}
+
+		if (c == '\'') {
+			return rune(start);
+		}
+
 		advance();
 
 		return switch (c) {
 			case '(' -> token(TokenKind.LPAREN, start);
 			case ')' -> token(TokenKind.RPAREN, start);
-
 			case '[' -> token(TokenKind.LBRACKET, start);
 			case ']' -> token(TokenKind.RBRACKET, start);
-
 			case '{' -> token(TokenKind.LBRACE, start);
 			case '}' -> token(TokenKind.RBRACE, start);
 
 			case ',' -> token(TokenKind.COMMA, start);
-			case '.' -> token(TokenKind.DOT, start);
+
+			case '.' -> {
+				if (match('.')) {
+					yield token(TokenKind.RANGE, start);
+				}
+
+				yield token(TokenKind.DOT, start);
+			}
+
 			case ';' -> token(TokenKind.SEMICOLON, start);
 			case '?' -> token(TokenKind.QUESTION, start);
 			case '@' -> token(TokenKind.AT, start);
@@ -90,7 +111,23 @@ public final class Lexer {
 			}
 
 			case '*' -> token(TokenKind.MUL, start);
-			case '/' -> token(TokenKind.DIV, start);
+
+			case '/' -> {
+				if (match('/')) {
+					if (match('/')) {
+						yield docComment(start);
+					}
+
+					yield lineComment(start);
+				}
+
+				if (match('*')) {
+					yield blockComment(start);
+				}
+
+				yield token(TokenKind.DIV, start);
+			}
+
 			case '%' -> token(TokenKind.MOD, start);
 
 			case '=' -> {
@@ -158,59 +195,76 @@ public final class Lexer {
 	}
 
 	private Token number(Pos start) {
-		if (current() == '0' && has(1)) {
-			char prefix = peek(1);
+		if (current() == '0' && hasNext()) {
+			return switch (peek()) {
+				case 'x', 'X' -> basedNumber(start, 16);
+				case 'b', 'B' -> basedNumber(start, 2);
+				case 'o', 'O' -> basedNumber(start, 8);
+				default -> decimalNumber(start);
+			};
+		}
 
-			switch (prefix) {
-				case 'x', 'X' -> {
-					advance();
-					advance();
+		return decimalNumber(start);
+	}
 
-					while (!eof() && isHexDigitOrUnderscore(current())) {
-						advance();
-					}
+	private Token basedNumber(Pos start, int base) {
+		advance();
+		advance();
 
-					return token(TokenKind.INT, start);
-				}
+		if (eof() || !isDigitForBase(current(), base)) {
+			consumeNumberTail();
+			return token(TokenKind.ERROR, start);
+		}
 
-				case 'b', 'B' -> {
-					advance();
-					advance();
+		boolean underscore = false;
 
-					while (!eof() && isBinaryDigitOrUnderscore(current())) {
-						advance();
-					}
+		while (!eof()) {
+			int c = current();
 
-					return token(TokenKind.INT, start);
-				}
+			if (isDigitForBase(c, base)) {
+				underscore = false;
+				advance();
+				continue;
+			}
 
-				case 'o', 'O' -> {
-					advance();
-					advance();
+			if (c == '_') {
+				if (underscore) {
+					consumeNumberTail();
+					return token(TokenKind.ERROR, start);
+				}
 
-					while (!eof() && isOctalDigitOrUnderscore(current())) {
-						advance();
-					}
+				underscore = true;
+				advance();
+				continue;
+			}
 
-					return token(TokenKind.INT, start);
-				}
+			if (isIdentContinue(c) || isDigit(c)) {
+				consumeNumberTail();
+				return token(TokenKind.ERROR, start);
 			}
+
+			break;
 		}
 
-		while (!eof() && isDigitOrUnderscore(current())) {
-			advance();
+		if (underscore) {
+			return token(TokenKind.ERROR, start);
 		}
 
-		if (
-			!eof()
-			&& current() == '.'
-			&& has(1)
-			&& isDigit(peek(1))
-		) {
+		return token(TokenKind.INT, start);
+	}
+
+	private Token decimalNumber(Pos start) {
+		if (!consumeDecimalDigits()) {
+			consumeNumberTail();
+			return token(TokenKind.ERROR, start);
+		}
+
+		if (!eof() && current() == '.' && hasNext() && isDigit(peek())) {
 			advance();
 
-			while (!eof() && isDigitOrUnderscore(current())) {
-				advance();
+			if (!consumeDecimalDigits()) {
+				consumeNumberTail();
+				return token(TokenKind.ERROR, start);
 			}
 
 			return token(TokenKind.FLOAT, start);
@@ -219,23 +273,208 @@ public final class Lexer {
 		return token(TokenKind.INT, start);
 	}
 
+	private boolean consumeDecimalDigits() {
+		boolean underscore = false;
+
+		while (!eof()) {
+			int c = current();
+
+			if (isDigit(c)) {
+				underscore = false;
+				advance();
+				continue;
+			}
+
+			if (c == '_') {
+				if (underscore) {
+					return false;
+				}
+
+				underscore = true;
+				advance();
+				continue;
+			}
+
+			break;
+		}
+
+		return !underscore;
+	}
+
+	private void consumeNumberTail() {
+		while (!eof() && (isIdentContinue(current()) || isDigit(current()))) {
+			advance();
+		}
+	}
+
+	private boolean isDigitForBase(int c, int base) {
+		return switch (base) {
+			case 2 -> c == '0' || c == '1';
+			case 8 -> c >= '0' && c <= '7';
+			case 16 -> isDigit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
+			default -> false;
+		};
+	}
+
 	private Token ident(Pos start) {
 		while (!eof() && isIdentContinue(current())) {
 			advance();
 		}
 
-		return token(TokenKind.IDENT, start);
+		return token(keyword(start.offset(), offset), start);
 	}
 
-	private void skipWhitespace() {
+	private TokenKind keyword(int start, int end) {
+		String text = buffer.slice(start, end).toString();
+
+		return switch (text) {
+			case "pkg" -> TokenKind.PKG;
+			case "use" -> TokenKind.USE;
+			case "fn" -> TokenKind.FN;
+			case "ret" -> TokenKind.RET;
+			case "mut" -> TokenKind.MUT;
+			case "for" -> TokenKind.FOR;
+			case "if" -> TokenKind.IF;
+			case "else" -> TokenKind.ELSE;
+			case "match" -> TokenKind.MATCH;
+			case "with" -> TokenKind.WITH;
+			case "inlay" -> TokenKind.INLAY;
+			case "enum" -> TokenKind.ENUM;
+			case "alias" -> TokenKind.ALIAS;
+			case "link" -> TokenKind.LINK;
+			case "bind" -> TokenKind.BIND;
+			case "ex" -> TokenKind.EX;
+
+			case "i8" -> TokenKind.I8;
+			case "i16" -> TokenKind.I16;
+			case "i32" -> TokenKind.I32;
+			case "i64" -> TokenKind.I64;
+			case "isize" -> TokenKind.ISIZE;
+
+			case "u8" -> TokenKind.U8;
+			case "u16" -> TokenKind.U16;
+			case "u32" -> TokenKind.U32;
+			case "u64" -> TokenKind.U64;
+			case "usize" -> TokenKind.USIZE;
+
+			case "f32" -> TokenKind.F32;
+			case "f64" -> TokenKind.F64;
+
+			case "bool" -> TokenKind.BOOL;
+			case "rune" -> TokenKind.RUNE_TYPE;
+			case "void" -> TokenKind.VOID;
+			case "string" -> TokenKind.STRING_TYPE;
+
+			case "true" -> TokenKind.TRUE;
+			case "false" -> TokenKind.FALSE;
+			case "null" -> TokenKind.NULL;
+
+			default -> TokenKind.IDENT;
+		};
+	}
+
+	private Token string(Pos start) {
+		advance();
+
+		while (!eof()) {
+			int c = current();
+
+			if (c == '"') {
+				advance();
+				return token(TokenKind.STRING, start);
+			}
+
+			if (c == '\n') {
+				return token(TokenKind.ERROR, start);
+			}
+
+			if (c == '\\') {
+				if (!consumeEscape()) {
+					consumeStringTail();
+					return token(TokenKind.ERROR, start);
+				}
+
+				continue;
+			}
+
+			advance();
+		}
+
+		return token(TokenKind.ERROR, start);
+	}
+
+	private Token rawString(Pos start) {
+		advance();
+		advance();
+
 		while (!eof()) {
-			char c = current();
+			if (current() == '"') {
+				advance();
+				return token(TokenKind.RAW_STRING, start);
+			}
+
+			advance();
+		}
+
+		return token(TokenKind.ERROR, start);
+	}
+
+	private Token rune(Pos start) {
+		advance();
+
+		if (eof() || current() == '\n' || current() == '\'') {
+			if (!eof() && current() == '\'') {
+				advance();
+			}
+
+			return token(TokenKind.ERROR, start);
+		}
+
+		if (current() == '\\') {
+			if (!consumeEscape()) {
+				consumeRuneTail();
+				return token(TokenKind.ERROR, start);
+			}
+		} else {
+			advance();
+		}
+
+		if (eof() || current() != '\'') {
+			consumeRuneTail();
+			return token(TokenKind.ERROR, start);
+		}
+
+		advance();
+
+		return token(TokenKind.RUNE, start);
+	}
+
+	private boolean consumeEscape() {
+		if (eof() || current() != '\\') {
+			return false;
+		}
+
+		advance();
+
+		if (eof()) {
+			return false;
+		}
+
+		int c = current();
+
+		if (c != 'n' && c != 'r' && c != 't' && c != '0' && c != '\\' && c != '"' && c != '\'') {
+			return false;
+		}
+
+		advance();
 
-			if (
-				c != ' '
-				&& c != '\t'
-				&& c != '\r'
-			) {
+		return true;
+	}
+
+	private void consumeStringTail() {
+		while (!eof() && current() != '\n') {
+			if (current() == '"') {
+				advance();
 				return;
 			}
 
@@ -243,71 +482,111 @@ public final class Lexer {
 		}
 	}
 
-	private boolean isDigit(char c) {
-		return c >= '0' && c <= '9';
+	private void consumeRuneTail() {
+		while (!eof() && current() != '\n') {
+			if (current() == '\'') {
+				advance();
+				return;
+			}
+
+			advance();
+		}
 	}
 
-	private boolean isDigitOrUnderscore(char c) {
-		return isDigit(c) || c == '_';
+	private Token lineComment(Pos start) {
+		while (!eof() && current() != '\n') {
+			advance();
+		}
+
+		return token(TokenKind.COMMENT, start);
 	}
 
-	private boolean isHexDigitOrUnderscore(char c) {
-		return isDigit(c)
-			|| (c >= 'a' && c <= 'f')
-			|| (c >= 'A' && c <= 'F')
-			|| c == '_';
+	private Token docComment(Pos start) {
+		while (!eof() && current() != '\n') {
+			advance();
+		}
+
+		return token(TokenKind.DOC_COMMENT, start);
+	}
+
+	private Token blockComment(Pos start) {
+		while (!eof()) {
+			if (current() == '*' && hasNext() && peek() == '/') {
+				advance();
+				advance();
+
+				return token(TokenKind.COMMENT, start);
+			}
+
+			advance();
+		}
+
+		return token(TokenKind.ERROR, start);
 	}
 
-	private boolean isBinaryDigitOrUnderscore(char c) {
-		return c == '0'
-			|| c == '1'
-			|| c == '_';
+	private void skipWhitespace() {
+		while (!eof()) {
+			int c = current();
+
+			if (c != ' ' && c != '\t' && c != '\r') {
+				return;
+			}
+
+			advance();
+		}
 	}
 
-	private boolean isOctalDigitOrUnderscore(char c) {
-		return (c >= '0' && c <= '7')
-			|| c == '_';
+	private boolean isDigit(int c) {
+		return c >= '0' && c <= '9';
 	}
 
-	private boolean isIdentStart(char c) {
-		return c == '_'
-			|| Character.isUnicodeIdentifierStart(c);
+	private boolean isIdentStart(int c) {
+		return c == '_' || Character.isUnicodeIdentifierStart(c);
 	}
 
-	private boolean isIdentContinue(char c) {
-		return c == '_'
-			|| Character.isUnicodeIdentifierPart(c);
+	private boolean isIdentContinue(int c) {
+		return c == '_' || Character.isUnicodeIdentifierPart(c);
 	}
 
 	private boolean eof() {
 		return offset >= buffer.length();
 	}
 
-	private boolean has(int distance) {
-		return offset + distance < buffer.length();
+	private boolean hasNext() {
+		if (eof()) {
+			return false;
+		}
+
+		int next = offset + buffer.width(offset);
+
+		return next < buffer.length();
 	}
 
-	private char current() {
+	private int current() {
 		return buffer.get(offset);
 	}
 
-	private char peek(int distance) {
-		return buffer.get(offset + distance);
+	private int peek() {
+		int next = offset + buffer.width(offset);
+
+		return buffer.get(next);
 	}
 
-	private boolean match(char expected) {
+	private boolean match(int expected) {
 		if (eof() || current() != expected) {
 			return false;
 		}
 
 		advance();
+
 		return true;
 	}
 
-	private char advance() {
-		char c = current();
+	private int advance() {
+		int c = current();
+		int width = buffer.width(offset);
 
-		offset++;
+		offset += width;
 
 		if (c == '\n') {
 			line++;
@@ -324,9 +603,6 @@ public final class Lexer {
 	}
 
 	private Token token(TokenKind kind, Pos start) {
-		return new Token(
-			kind,
-			Span.of(start, pos())
-		);
+		return new Token(kind, Span.of(start, pos()));
 	}
 }
diff --git a/src/moe/anita/mai/span/Pos.java b/src/moe/anita/mai/span/Pos.java
index c6d3043..4b129e2 100644
--- a/src/moe/anita/mai/span/Pos.java
+++ b/src/moe/anita/mai/span/Pos.java
@@ -5,16 +5,15 @@ public final class Pos {
 	private final long value;
 
 	private static final long OFFSET_MASK = 0xFFFF_FFFFL;
-	private static final long U16_MASK = 0xFFFFL;
+	private static final long U16_MASK    = 0xFFFFL;
 
 	private Pos(long value) {
 		this.value = value;
 	}
 
 	public static Pos of(int offset, int line, int column) {
-		long value =
-			((long) offset << OFFSET_MASK)
-			| ((long) line & U16_MASK) << 16
+		long value = (((long) offset & OFFSET_MASK) << 32)
+			| (((long) line & U16_MASK) << 16)
 			| ((long) column & U16_MASK);
 
 		return new Pos(value);
@@ -25,7 +24,7 @@ public final class Pos {
 	}
 
 	public int offset() {
-		return (int) (value >>> 32);
+		return (int) ((value >>> 32) & OFFSET_MASK);
 	}
 
 	public int line() {
diff --git a/src/moe/anita/mai/token/TokenKind.java b/src/moe/anita/mai/token/TokenKind.java
index 2beadb2..7fd6439 100644
--- a/src/moe/anita/mai/token/TokenKind.java
+++ b/src/moe/anita/mai/token/TokenKind.java
@@ -1,5 +1,8 @@
 package moe.anita.mai.token;
 
+/**
+ * Identifies the kind of a lexical token.
+ */
 public enum TokenKind {
 
 	// Special
@@ -8,7 +11,6 @@ public enum TokenKind {
 	EOL,
 
 	// Trivia
-	WHITESPACE,
 	COMMENT,
 	DOC_COMMENT,
 
@@ -22,6 +24,48 @@ public enum TokenKind {
 	RAW_STRING,
 	RUNE,
 
+	TRUE,
+	FALSE,
+	NULL,
+
+	// Keywords
+	PKG,
+	USE,
+	FN,
+	RET,
+	MUT,
+	FOR,
+	IF,
+	ELSE,
+	MATCH,
+	WITH,
+	INLAY,
+	ENUM,
+	ALIAS,
+	LINK,
+	BIND,
+	EX,
+
+	I8,
+	I16,
+	I32,
+	I64,
+	ISIZE,
+
+	U8,
+	U16,
+	U32,
+	U64,
+	USIZE,
+
+	F32,
+	F64,
+
+	BOOL,
+	RUNE_TYPE,
+	STRING_TYPE,
+	VOID,
+
 	// Delimiters
 	LPAREN("("),
 	RPAREN(")"),
@@ -30,10 +74,14 @@ public enum TokenKind {
 	LBRACE("{"),
 	RBRACE("}"),
 
+	// Punctuation
 	COMMA(","),
 	DOT("."),
+	RANGE(".."),
 	COLON(":"),
 	SEMICOLON(";"),
+	QUESTION("?"),
+	AT("@"),
 
 	// Assignment / declaration
 	ASSIGN("="),
@@ -63,30 +111,22 @@ public enum TokenKind {
 	BIT_AND("&", Precedence.BIT_AND, Flags.INFIX),
 	BIT_XOR("^", Precedence.BIT_XOR, Flags.INFIX),
 	BIT_OR("|", Precedence.BIT_OR, Flags.INFIX),
-
 	BIT_NOT("~", Precedence.PREFIX, Flags.PREFIX),
 
 	// Logical
 	AND("&&", Precedence.LOGICAL_AND, Flags.INFIX),
 	OR("||", Precedence.LOGICAL_OR, Flags.INFIX),
-
 	NOT("!", Precedence.PREFIX, Flags.PREFIX),
 
-	// Misc operators
+	// Arrows
 	ARROW("->"),
-	BIND("<-"),
-
-	RANGE(".."),
-
-	QUESTION("?"),
-	AT("@"),
+	LEFT_ARROW("<-"),
 
 	/*
-	 * These are parser-created operators.
+	 * Parser-created operators.
 	 *
-	 * The lexer emits two LT or GT tokens instead of directly
-	 * producing SHIFT_LEFT / SHIFT_RIGHT. This lets the parser
-	 * distinguish shifts from nested generic type syntax.
+	 * The lexer emits separate LT/GT tokens. The parser combines
+	 * adjacent tokens into shifts where the grammar permits them.
 	 */
 	SHIFT_LEFT("<<", Precedence.SHIFT, Flags.INFIX),
 	SHIFT_RIGHT(">>", Precedence.SHIFT, Flags.INFIX);
@@ -109,32 +149,54 @@ public enum TokenKind {
 		this.flags = flags;
 	}
 
+	/**
+	 * Returns the fixed spelling of this token kind, if one exists.
+	 *
+	 * @return token spelling, or null
+	 */
 	public String spelling() {
 		return spelling;
 	}
 
+	/**
+	 * Returns this token's Pratt precedence.
+	 *
+	 * @return precedence
+	 */
 	public int precedence() {
 		return precedence;
 	}
 
+	/**
+	 * Returns whether this token has a fixed spelling.
+	 *
+	 * @return true if a spelling exists
+	 */
 	public boolean hasSpelling() {
 		return spelling != null;
 	}
 
+	/**
+	 * Returns whether this token may be used as an infix operator.
+	 *
+	 * @return true if infix
+	 */
 	public boolean isInfix() {
 		return (flags & Flags.INFIX) != 0;
 	}
 
+	/**
+	 * Returns whether this token may be used as a prefix operator.
+	 *
+	 * @return true if prefix
+	 */
 	public boolean isPrefix() {
 		return (flags & Flags.PREFIX) != 0;
 	}
 
-
-
 	private static final class Flags {
 
 		private static final int NONE = 0;
-
 		private static final int INFIX = 1 << 0;
 		private static final int PREFIX = 1 << 1;
 
@@ -146,9 +208,6 @@ public enum TokenKind {
 
 		private static final int NONE = 0;
 
-		/*
-		 * Larger values bind tighter.
-		 */
 		private static final int LOGICAL_OR = 10;
 		private static final int LOGICAL_AND = 20;
 
@@ -157,7 +216,6 @@ public enum TokenKind {
 		private static final int BIT_AND = 50;
 
 		private static final int COMPARE = 60;
-
 		private static final int SHIFT = 70;
 
 		private static final int ADD = 80;