Finish the lexerHEADtrunklexer

Signed-off-by: Antia Anderson <anita@ansan.io>

18 files changed, 1523 insertions, 211 deletions

diff --git a/src/moe/anita/mai/buffer/Buffer.java b/src/moe/anita/mai/buffer/Buffer.java
index c235bbb..e119b40 100644
--- a/src/moe/anita/mai/buffer/Buffer.java
+++ b/src/moe/anita/mai/buffer/Buffer.java
@@ -1,43 +1,61 @@
 package moe.anita.mai.buffer;
 
+/**
+ * Provides random access to Unicode source text.
+ */
 public interface Buffer {
 
 	/**
-	 * Returns the number of characters in the buffer.
+	 * Returns the length of the underlying source representation.
+	 *
+	 * Offsets are measured in the backing representation's code units.
+	 *
+	 * @return source length
 	 */
 	int length();
 
 	/**
-	 * Returns the character at the given offset.
+	 * Returns the Unicode code point beginning at the given offset.
 	 *
-	 * @param index character offset
-	 * @return character at the offset
+	 * @param offset source offset
+	 * @return Unicode code point
 	 */
-	char get(int index);
+	int get(int offset);
+
+	/**
+	 * Returns the number of underlying code units occupied by the
+	 * code point beginning at the given offset.
+	 *
+	 * @param offset source offset
+	 * @return width of the code point
+	 */
+	int width(int offset);
 
 	/**
 	 * Returns whether the given offset exists in the buffer.
 	 *
-	 * @param index character offset
-	 * @return true when the offset is valid
+	 * @param offset source offset
+	 * @return true if the offset is valid
 	 */
-	default boolean has(int index) {
-		return index >= 0 && index < length();
+	default boolean has(int offset) {
+		return offset >= 0 && offset < length();
 	}
 
 	/**
-	 * Returns whether the buffer is empty.
+	 * Returns whether the buffer contains no source text.
+	 *
+	 * @return true if empty
 	 */
 	default boolean isEmpty() {
 		return length() == 0;
 	}
 
 	/**
-	 * Returns a slice of the buffer in the range [start, end).
+	 * Returns the source contents between two offsets.
 	 *
-	 * @param start starting offset
-	 * @param end ending offset
-	 * @return characters in the requested range
+	 * @param start starting offset, inclusive
+	 * @param end ending offset, exclusive
+	 * @return requested source text
 	 */
 	CharSequence slice(int start, int end);
 }
diff --git a/src/moe/anita/mai/buffer/MemBuf.java b/src/moe/anita/mai/buffer/MemBuf.java
index a2bc477..922e624 100644
--- a/src/moe/anita/mai/buffer/MemBuf.java
+++ b/src/moe/anita/mai/buffer/MemBuf.java
@@ -1,29 +1,72 @@
 package moe.anita.mai.buffer;
 
+import java.nio.charset.StandardCharsets;
 import java.util.Objects;
 
 /**
- * Buffer backed entirely by an in-memory string.
+ * An in-memory UTF-8 source buffer.
+ *
+ * Offsets and lengths are measured in UTF-8 bytes.
  */
 public final class MemBuf implements Buffer {
 
-	private final String data;
+	private final byte[] data;
 
-	public MemBuf(String data) { this.data = Objects.requireNonNull(data);
+	public MemBuf(String source) {
+		Objects.requireNonNull(source);
+		data = source.getBytes(StandardCharsets.UTF_8);
 	}
 
 	@Override
 	public int length() {
-		return data.length();
+		return data.length;
 	}
 
 	@Override
-	public char get(int index) {
-		return data.charAt(index);
+	public int get(int offset) {
+		int first = Byte.toUnsignedInt(data[offset]);
+
+		if ((first & 0x80) == 0) {
+			return first;
+		}
+
+		if ((first & 0xE0) == 0xC0) {
+			return ((first & 0x1F) << 6) | (Byte.toUnsignedInt(data[offset + 1]) & 0x3F);
+		}
+
+		if ((first & 0xF0) == 0xE0) {
+			return ((first & 0x0F) << 12)
+				| ((Byte.toUnsignedInt(data[offset + 1]) & 0x3F) << 6)
+				| (Byte.toUnsignedInt(data[offset + 2]) & 0x3F);
+		}
+
+		return ((first & 0x07) << 18)
+			|  ((Byte.toUnsignedInt(data[offset + 1]) & 0x3F) << 12)
+			| ((Byte.toUnsignedInt(data[offset + 2]) & 0x3F) << 6)
+			| (Byte.toUnsignedInt(data[offset + 3]) & 0x3F);
+	}
+
+	@Override
+	public int width(int offset) {
+		int first = Byte.toUnsignedInt(data[offset]);
+
+		if ((first & 0x80) == 0) {
+			return 1;
+		}
+
+		if ((first & 0xE0) == 0xC0) {
+			return 2;
+		}
+
+		if ((first & 0xF0) == 0xE0) {
+			return 3;
+		}
+
+		return 4;
 	}
 
 	@Override
 	public CharSequence slice(int start, int end) {
-		return data.subSequence(start, end);
+		return new String(data, start, end - start, StandardCharsets.UTF_8);
 	}
 }
diff --git a/src/moe/anita/mai/buffer/SrcBuf.java b/src/moe/anita/mai/buffer/SrcBuf.java
index 670bcbe..f288316 100644
--- a/src/moe/anita/mai/buffer/SrcBuf.java
+++ b/src/moe/anita/mai/buffer/SrcBuf.java
@@ -1,13 +1,11 @@
 package moe.anita.mai.buffer;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Objects;
 
-/**
- * A buffer backed by the contents of a source file.
- */
 public final class SrcBuf implements Buffer {
 
 	private final Path path;
@@ -15,13 +13,13 @@ public final class SrcBuf implements Buffer {
 
 	public SrcBuf(Path path) throws IOException {
 		this.path = Objects.requireNonNull(path);
-		this.data = Files.readString(path);
+		this.data = Files.readString(path, StandardCharsets.UTF_8);
 	}
 
 	/**
-	 * Returns the path of the source file.
+	 * Returns the source file path.
 	 *
-	 * @return source file path
+	 * @return source path
 	 */
 	public Path path() {
 		return path;
@@ -33,8 +31,13 @@ public final class SrcBuf implements Buffer {
 	}
 
 	@Override
-	public char get(int index) {
-		return data.charAt(index);
+	public int get(int offset) {
+		return data.codePointAt(offset);
+	}
+
+	@Override
+	public int width(int offset) {
+		return Character.charCount(data.codePointAt(offset));
 	}
 
 	@Override
diff --git a/src/moe/anita/mai/lexer/Lexer.java b/src/moe/anita/mai/lexer/Lexer.java
index 56f4534..85f0be1 100644
--- a/src/moe/anita/mai/lexer/Lexer.java
+++ b/src/moe/anita/mai/lexer/Lexer.java
@@ -6,6 +6,9 @@ import moe.anita.mai.span.Span;
 import moe.anita.mai.token.Token;
 import moe.anita.mai.token.TokenKind;
 
+/**
+ * Tokenizes mai source text.
+ */
 public final class Lexer {
 
 	private final Buffer buffer;
@@ -36,13 +39,17 @@ public final class Lexer {
 			return token(TokenKind.EOF, start);
 		}
 
-		char c = current();
+		int c = current();
 
 		if (c == '\n') {
 			advance();
 			return token(TokenKind.EOL, start);
 		}
 
+		if (c == 'r' && hasNext() && peek() == '"') {
+			return rawString(start);
+		}
+
 		if (isDigit(c)) {
 			return number(start);
 		}
@@ -51,20 +58,34 @@ public final class Lexer {
 			return ident(start);
 		}
 
+		if (c == '"') {
+			return string(start);
+		}
+
+		if (c == '\'') {
+			return rune(start);
+		}
+
 		advance();
 
 		return switch (c) {
 			case '(' -> token(TokenKind.LPAREN, start);
 			case ')' -> token(TokenKind.RPAREN, start);
-
 			case '[' -> token(TokenKind.LBRACKET, start);
 			case ']' -> token(TokenKind.RBRACKET, start);
-
 			case '{' -> token(TokenKind.LBRACE, start);
 			case '}' -> token(TokenKind.RBRACE, start);
 
 			case ',' -> token(TokenKind.COMMA, start);
-			case '.' -> token(TokenKind.DOT, start);
+
+			case '.' -> {
+				if (match('.')) {
+					yield token(TokenKind.RANGE, start);
+				}
+
+				yield token(TokenKind.DOT, start);
+			}
+
 			case ';' -> token(TokenKind.SEMICOLON, start);
 			case '?' -> token(TokenKind.QUESTION, start);
 			case '@' -> token(TokenKind.AT, start);
@@ -90,7 +111,23 @@ public final class Lexer {
 			}
 
 			case '*' -> token(TokenKind.MUL, start);
-			case '/' -> token(TokenKind.DIV, start);
+
+			case '/' -> {
+				if (match('/')) {
+					if (match('/')) {
+						yield docComment(start);
+					}
+
+					yield lineComment(start);
+				}
+
+				if (match('*')) {
+					yield blockComment(start);
+				}
+
+				yield token(TokenKind.DIV, start);
+			}
+
 			case '%' -> token(TokenKind.MOD, start);
 
 			case '=' -> {
@@ -158,59 +195,76 @@ public final class Lexer {
 	}
 
 	private Token number(Pos start) {
-		if (current() == '0' && has(1)) {
-			char prefix = peek(1);
+		if (current() == '0' && hasNext()) {
+			return switch (peek()) {
+				case 'x', 'X' -> basedNumber(start, 16);
+				case 'b', 'B' -> basedNumber(start, 2);
+				case 'o', 'O' -> basedNumber(start, 8);
+				default -> decimalNumber(start);
+			};
+		}
 
-			switch (prefix) {
-				case 'x', 'X' -> {
-					advance();
-					advance();
+		return decimalNumber(start);
+	}
 
-					while (!eof() && isHexDigitOrUnderscore(current())) {
-						advance();
-					}
+	private Token basedNumber(Pos start, int base) {
+		advance();
+		advance();
 
-					return token(TokenKind.INT, start);
-				}
+		if (eof() || !isDigitForBase(current(), base)) {
+			consumeNumberTail();
+			return token(TokenKind.ERROR, start);
+		}
 
-				case 'b', 'B' -> {
-					advance();
-					advance();
+		boolean underscore = false;
 
-					while (!eof() && isBinaryDigitOrUnderscore(current())) {
-						advance();
-					}
+		while (!eof()) {
+			int c = current();
 
-					return token(TokenKind.INT, start);
-				}
+			if (isDigitForBase(c, base)) {
+				underscore = false;
+				advance();
+				continue;
+			}
 
-				case 'o', 'O' -> {
-					advance();
-					advance();
+			if (c == '_') {
+				if (underscore) {
+					consumeNumberTail();
+					return token(TokenKind.ERROR, start);
+				}
 
-					while (!eof() && isOctalDigitOrUnderscore(current())) {
-						advance();
-					}
+				underscore = true;
+				advance();
+				continue;
+			}
 
-					return token(TokenKind.INT, start);
-				}
+			if (isIdentContinue(c) || isDigit(c)) {
+				consumeNumberTail();
+				return token(TokenKind.ERROR, start);
 			}
+
+			break;
 		}
 
-		while (!eof() && isDigitOrUnderscore(current())) {
-			advance();
+		if (underscore) {
+			return token(TokenKind.ERROR, start);
 		}
 
-		if (
-			!eof()
-			&& current() == '.'
-			&& has(1)
-			&& isDigit(peek(1))
-		) {
+		return token(TokenKind.INT, start);
+	}
+
+	private Token decimalNumber(Pos start) {
+		if (!consumeDecimalDigits()) {
+			consumeNumberTail();
+			return token(TokenKind.ERROR, start);
+		}
+
+		if (!eof() && current() == '.' && hasNext() && isDigit(peek())) {
 			advance();
 
-			while (!eof() && isDigitOrUnderscore(current())) {
-				advance();
+			if (!consumeDecimalDigits()) {
+				consumeNumberTail();
+				return token(TokenKind.ERROR, start);
 			}
 
 			return token(TokenKind.FLOAT, start);
@@ -219,23 +273,208 @@ public final class Lexer {
 		return token(TokenKind.INT, start);
 	}
 
+	private boolean consumeDecimalDigits() {
+		boolean underscore = false;
+
+		while (!eof()) {
+			int c = current();
+
+			if (isDigit(c)) {
+				underscore = false;
+				advance();
+				continue;
+			}
+
+			if (c == '_') {
+				if (underscore) {
+					return false;
+				}
+
+				underscore = true;
+				advance();
+				continue;
+			}
+
+			break;
+		}
+
+		return !underscore;
+	}
+
+	private void consumeNumberTail() {
+		while (!eof() && (isIdentContinue(current()) || isDigit(current()))) {
+			advance();
+		}
+	}
+
+	private boolean isDigitForBase(int c, int base) {
+		return switch (base) {
+			case 2 -> c == '0' || c == '1';
+			case 8 -> c >= '0' && c <= '7';
+			case 16 -> isDigit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
+			default -> false;
+		};
+	}
+
 	private Token ident(Pos start) {
 		while (!eof() && isIdentContinue(current())) {
 			advance();
 		}
 
-		return token(TokenKind.IDENT, start);
+		return token(keyword(start.offset(), offset), start);
 	}
 
-	private void skipWhitespace() {
+	private TokenKind keyword(int start, int end) {
+		String text = buffer.slice(start, end).toString();
+
+		return switch (text) {
+			case "pkg" -> TokenKind.PKG;
+			case "use" -> TokenKind.USE;
+			case "fn" -> TokenKind.FN;
+			case "ret" -> TokenKind.RET;
+			case "mut" -> TokenKind.MUT;
+			case "for" -> TokenKind.FOR;
+			case "if" -> TokenKind.IF;
+			case "else" -> TokenKind.ELSE;
+			case "match" -> TokenKind.MATCH;
+			case "with" -> TokenKind.WITH;
+			case "inlay" -> TokenKind.INLAY;
+			case "enum" -> TokenKind.ENUM;
+			case "alias" -> TokenKind.ALIAS;
+			case "link" -> TokenKind.LINK;
+			case "bind" -> TokenKind.BIND;
+			case "ex" -> TokenKind.EX;
+
+			case "i8" -> TokenKind.I8;
+			case "i16" -> TokenKind.I16;
+			case "i32" -> TokenKind.I32;
+			case "i64" -> TokenKind.I64;
+			case "isize" -> TokenKind.ISIZE;
+
+			case "u8" -> TokenKind.U8;
+			case "u16" -> TokenKind.U16;
+			case "u32" -> TokenKind.U32;
+			case "u64" -> TokenKind.U64;
+			case "usize" -> TokenKind.USIZE;
+
+			case "f32" -> TokenKind.F32;
+			case "f64" -> TokenKind.F64;
+
+			case "bool" -> TokenKind.BOOL;
+			case "rune" -> TokenKind.RUNE_TYPE;
+			case "void" -> TokenKind.VOID;
+			case "string" -> TokenKind.STRING_TYPE;
+
+			case "true" -> TokenKind.TRUE;
+			case "false" -> TokenKind.FALSE;
+			case "null" -> TokenKind.NULL;
+
+			default -> TokenKind.IDENT;
+		};
+	}
+
+	private Token string(Pos start) {
+		advance();
+
+		while (!eof()) {
+			int c = current();
+
+			if (c == '"') {
+				advance();
+				return token(TokenKind.STRING, start);
+			}
+
+			if (c == '\n') {
+				return token(TokenKind.ERROR, start);
+			}
+
+			if (c == '\\') {
+				if (!consumeEscape()) {
+					consumeStringTail();
+					return token(TokenKind.ERROR, start);
+				}
+
+				continue;
+			}
+
+			advance();
+		}
+
+		return token(TokenKind.ERROR, start);
+	}
+
+	private Token rawString(Pos start) {
+		advance();
+		advance();
+
 		while (!eof()) {
-			char c = current();
+			if (current() == '"') {
+				advance();
+				return token(TokenKind.RAW_STRING, start);
+			}
+
+			advance();
+		}
+
+		return token(TokenKind.ERROR, start);
+	}
+
+	private Token rune(Pos start) {
+		advance();
+
+		if (eof() || current() == '\n' || current() == '\'') {
+			if (!eof() && current() == '\'') {
+				advance();
+			}
+
+			return token(TokenKind.ERROR, start);
+		}
+
+		if (current() == '\\') {
+			if (!consumeEscape()) {
+				consumeRuneTail();
+				return token(TokenKind.ERROR, start);
+			}
+		} else {
+			advance();
+		}
+
+		if (eof() || current() != '\'') {
+			consumeRuneTail();
+			return token(TokenKind.ERROR, start);
+		}
+
+		advance();
+
+		return token(TokenKind.RUNE, start);
+	}
+
+	private boolean consumeEscape() {
+		if (eof() || current() != '\\') {
+			return false;
+		}
+
+		advance();
+
+		if (eof()) {
+			return false;
+		}
+
+		int c = current();
+
+		if (c != 'n' && c != 'r' && c != 't' && c != '0' && c != '\\' && c != '"' && c != '\'') {
+			return false;
+		}
+
+		advance();
 
-			if (
-				c != ' '
-				&& c != '\t'
-				&& c != '\r'
-			) {
+		return true;
+	}
+
+	private void consumeStringTail() {
+		while (!eof() && current() != '\n') {
+			if (current() == '"') {
+				advance();
 				return;
 			}
 
@@ -243,71 +482,111 @@ public final class Lexer {
 		}
 	}
 
-	private boolean isDigit(char c) {
-		return c >= '0' && c <= '9';
+	private void consumeRuneTail() {
+		while (!eof() && current() != '\n') {
+			if (current() == '\'') {
+				advance();
+				return;
+			}
+
+			advance();
+		}
 	}
 
-	private boolean isDigitOrUnderscore(char c) {
-		return isDigit(c) || c == '_';
+	private Token lineComment(Pos start) {
+		while (!eof() && current() != '\n') {
+			advance();
+		}
+
+		return token(TokenKind.COMMENT, start);
 	}
 
-	private boolean isHexDigitOrUnderscore(char c) {
-		return isDigit(c)
-			|| (c >= 'a' && c <= 'f')
-			|| (c >= 'A' && c <= 'F')
-			|| c == '_';
+	private Token docComment(Pos start) {
+		while (!eof() && current() != '\n') {
+			advance();
+		}
+
+		return token(TokenKind.DOC_COMMENT, start);
+	}
+
+	private Token blockComment(Pos start) {
+		while (!eof()) {
+			if (current() == '*' && hasNext() && peek() == '/') {
+				advance();
+				advance();
+
+				return token(TokenKind.COMMENT, start);
+			}
+
+			advance();
+		}
+
+		return token(TokenKind.ERROR, start);
 	}
 
-	private boolean isBinaryDigitOrUnderscore(char c) {
-		return c == '0'
-			|| c == '1'
-			|| c == '_';
+	private void skipWhitespace() {
+		while (!eof()) {
+			int c = current();
+
+			if (c != ' ' && c != '\t' && c != '\r') {
+				return;
+			}
+
+			advance();
+		}
 	}
 
-	private boolean isOctalDigitOrUnderscore(char c) {
-		return (c >= '0' && c <= '7')
-			|| c == '_';
+	private boolean isDigit(int c) {
+		return c >= '0' && c <= '9';
 	}
 
-	private boolean isIdentStart(char c) {
-		return c == '_'
-			|| Character.isUnicodeIdentifierStart(c);
+	private boolean isIdentStart(int c) {
+		return c == '_' || Character.isUnicodeIdentifierStart(c);
 	}
 
-	private boolean isIdentContinue(char c) {
-		return c == '_'
-			|| Character.isUnicodeIdentifierPart(c);
+	private boolean isIdentContinue(int c) {
+		return c == '_' || Character.isUnicodeIdentifierPart(c);
 	}
 
 	private boolean eof() {
 		return offset >= buffer.length();
 	}
 
-	private boolean has(int distance) {
-		return offset + distance < buffer.length();
+	private boolean hasNext() {
+		if (eof()) {
+			return false;
+		}
+
+		int next = offset + buffer.width(offset);
+
+		return next < buffer.length();
 	}
 
-	private char current() {
+	private int current() {
 		return buffer.get(offset);
 	}
 
-	private char peek(int distance) {
-		return buffer.get(offset + distance);
+	private int peek() {
+		int next = offset + buffer.width(offset);
+
+		return buffer.get(next);
 	}
 
-	private boolean match(char expected) {
+	private boolean match(int expected) {
 		if (eof() || current() != expected) {
 			return false;
 		}
 
 		advance();
+
 		return true;
 	}
 
-	private char advance() {
-		char c = current();
+	private int advance() {
+		int c = current();
+		int width = buffer.width(offset);
 
-		offset++;
+		offset += width;
 
 		if (c == '\n') {
 			line++;
@@ -324,9 +603,6 @@ public final class Lexer {
 	}
 
 	private Token token(TokenKind kind, Pos start) {
-		return new Token(
-			kind,
-			Span.of(start, pos())
-		);
+		return new Token(kind, Span.of(start, pos()));
 	}
 }
diff --git a/src/moe/anita/mai/span/Pos.java b/src/moe/anita/mai/span/Pos.java
index c6d3043..4b129e2 100644
--- a/src/moe/anita/mai/span/Pos.java
+++ b/src/moe/anita/mai/span/Pos.java
@@ -5,16 +5,15 @@ public final class Pos {
 	private final long value;
 
 	private static final long OFFSET_MASK = 0xFFFF_FFFFL;
-	private static final long U16_MASK = 0xFFFFL;
+	private static final long U16_MASK    = 0xFFFFL;
 
 	private Pos(long value) {
 		this.value = value;
 	}
 
 	public static Pos of(int offset, int line, int column) {
-		long value =
-			((long) offset << OFFSET_MASK)
-			| ((long) line & U16_MASK) << 16
+		long value = (((long) offset & OFFSET_MASK) << 32)
+			| (((long) line & U16_MASK) << 16)
 			| ((long) column & U16_MASK);
 
 		return new Pos(value);
@@ -25,7 +24,7 @@ public final class Pos {
 	}
 
 	public int offset() {
-		return (int) (value >>> 32);
+		return (int) ((value >>> 32) & OFFSET_MASK);
 	}
 
 	public int line() {
diff --git a/src/moe/anita/mai/token/TokenKind.java b/src/moe/anita/mai/token/TokenKind.java
index 2beadb2..7fd6439 100644
--- a/src/moe/anita/mai/token/TokenKind.java
+++ b/src/moe/anita/mai/token/TokenKind.java
@@ -1,5 +1,8 @@
 package moe.anita.mai.token;
 
+/**
+ * Identifies the kind of a lexical token.
+ */
 public enum TokenKind {
 
 	// Special
@@ -8,7 +11,6 @@ public enum TokenKind {
 	EOL,
 
 	// Trivia
-	WHITESPACE,
 	COMMENT,
 	DOC_COMMENT,
 
@@ -22,6 +24,48 @@ public enum TokenKind {
 	RAW_STRING,
 	RUNE,
 
+	TRUE,
+	FALSE,
+	NULL,
+
+	// Keywords
+	PKG,
+	USE,
+	FN,
+	RET,
+	MUT,
+	FOR,
+	IF,
+	ELSE,
+	MATCH,
+	WITH,
+	INLAY,
+	ENUM,
+	ALIAS,
+	LINK,
+	BIND,
+	EX,
+
+	I8,
+	I16,
+	I32,
+	I64,
+	ISIZE,
+
+	U8,
+	U16,
+	U32,
+	U64,
+	USIZE,
+
+	F32,
+	F64,
+
+	BOOL,
+	RUNE_TYPE,
+	STRING_TYPE,
+	VOID,
+
 	// Delimiters
 	LPAREN("("),
 	RPAREN(")"),
@@ -30,10 +74,14 @@ public enum TokenKind {
 	LBRACE("{"),
 	RBRACE("}"),
 
+	// Punctuation
 	COMMA(","),
 	DOT("."),
+	RANGE(".."),
 	COLON(":"),
 	SEMICOLON(";"),
+	QUESTION("?"),
+	AT("@"),
 
 	// Assignment / declaration
 	ASSIGN("="),
@@ -63,30 +111,22 @@ public enum TokenKind {
 	BIT_AND("&", Precedence.BIT_AND, Flags.INFIX),
 	BIT_XOR("^", Precedence.BIT_XOR, Flags.INFIX),
 	BIT_OR("|", Precedence.BIT_OR, Flags.INFIX),
-
 	BIT_NOT("~", Precedence.PREFIX, Flags.PREFIX),
 
 	// Logical
 	AND("&&", Precedence.LOGICAL_AND, Flags.INFIX),
 	OR("||", Precedence.LOGICAL_OR, Flags.INFIX),
-
 	NOT("!", Precedence.PREFIX, Flags.PREFIX),
 
-	// Misc operators
+	// Arrows
 	ARROW("->"),
-	BIND("<-"),
-
-	RANGE(".."),
-
-	QUESTION("?"),
-	AT("@"),
+	LEFT_ARROW("<-"),
 
 	/*
-	 * These are parser-created operators.
+	 * Parser-created operators.
 	 *
-	 * The lexer emits two LT or GT tokens instead of directly
-	 * producing SHIFT_LEFT / SHIFT_RIGHT. This lets the parser
-	 * distinguish shifts from nested generic type syntax.
+	 * The lexer emits separate LT/GT tokens. The parser combines
+	 * adjacent tokens into shifts where the grammar permits them.
 	 */
 	SHIFT_LEFT("<<", Precedence.SHIFT, Flags.INFIX),
 	SHIFT_RIGHT(">>", Precedence.SHIFT, Flags.INFIX);
@@ -109,32 +149,54 @@ public enum TokenKind {
 		this.flags = flags;
 	}
 
+	/**
+	 * Returns the fixed spelling of this token kind, if one exists.
+	 *
+	 * @return token spelling, or null
+	 */
 	public String spelling() {
 		return spelling;
 	}
 
+	/**
+	 * Returns this token's Pratt precedence.
+	 *
+	 * @return precedence
+	 */
 	public int precedence() {
 		return precedence;
 	}
 
+	/**
+	 * Returns whether this token has a fixed spelling.
+	 *
+	 * @return true if a spelling exists
+	 */
 	public boolean hasSpelling() {
 		return spelling != null;
 	}
 
+	/**
+	 * Returns whether this token may be used as an infix operator.
+	 *
+	 * @return true if infix
+	 */
 	public boolean isInfix() {
 		return (flags & Flags.INFIX) != 0;
 	}
 
+	/**
+	 * Returns whether this token may be used as a prefix operator.
+	 *
+	 * @return true if prefix
+	 */
 	public boolean isPrefix() {
 		return (flags & Flags.PREFIX) != 0;
 	}
 
-
-
 	private static final class Flags {
 
 		private static final int NONE = 0;
-
 		private static final int INFIX = 1 << 0;
 		private static final int PREFIX = 1 << 1;
 
@@ -146,9 +208,6 @@ public enum TokenKind {
 
 		private static final int NONE = 0;
 
-		/*
-		 * Larger values bind tighter.
-		 */
 		private static final int LOGICAL_OR = 10;
 		private static final int LOGICAL_AND = 20;
 
@@ -157,7 +216,6 @@ public enum TokenKind {
 		private static final int BIT_AND = 50;
 
 		private static final int COMPARE = 60;
-
 		private static final int SHIFT = 70;
 
 		private static final int ADD = 80;
diff --git a/test/moe/anita/mai/buffer/MemBufTest.java b/test/moe/anita/mai/buffer/MemBufTest.java
index 450ac31..28dc5c9 100644
--- a/test/moe/anita/mai/buffer/MemBufTest.java
+++ b/test/moe/anita/mai/buffer/MemBufTest.java
@@ -2,8 +2,7 @@ package moe.anita.mai.buffer;
 
 import moe.anita.mai.test.Case;
 import moe.anita.mai.test.Runner;
-
-@Runner(name = "MemBuf")
+@Runner(name = "MemBuffer")
 public final class MemBufTest {
 
 	@Case(name = "length")
@@ -13,13 +12,57 @@ public final class MemBufTest {
 		assert buffer.length() == 3;
 	}
 
-	@Case(name = "read")
-	static void read() {
+	@Case(name = "empty")
+	static void empty() {
+		assert new MemBuf("").isEmpty();
+		assert !new MemBuf("mai").isEmpty();
+	}
+
+	@Case(name = "ASCII code points")
+	static void ascii() {
 		var buffer = new MemBuf("mai");
 
 		assert buffer.get(0) == 'm';
 		assert buffer.get(1) == 'a';
 		assert buffer.get(2) == 'i';
+
+		assert buffer.width(0) == 1;
+		assert buffer.width(1) == 1;
+		assert buffer.width(2) == 1;
+	}
+
+	@Case(name = "Unicode code point")
+	static void unicode() {
+		var buffer = new MemBuf("猫");
+
+		assert buffer.length() == 3;
+		assert buffer.get(0) == '猫';
+		assert buffer.width(0) == 3;
+	}
+
+	@Case(name = "supplementary Unicode code point")
+	static void supplementaryUnicode() {
+		var buffer = new MemBuf("😀");
+
+		assert buffer.length() == 4;
+		assert buffer.get(0) == 0x1F600;
+		assert buffer.width(0) == 4;
+	}
+
+	@Case(name = "mixed Unicode")
+	static void mixedUnicode() {
+		var buffer = new MemBuf("a😀猫");
+
+		assert buffer.length() == 8;
+
+		assert buffer.get(0) == 'a';
+		assert buffer.width(0) == 1;
+
+		assert buffer.get(1) == 0x1F600;
+		assert buffer.width(1) == 4;
+
+		assert buffer.get(5) == '猫';
+		assert buffer.width(5) == 3;
 	}
 
 	@Case(name = "bounds")
@@ -28,22 +71,17 @@ public final class MemBufTest {
 
 		assert !buffer.has(-1);
 		assert buffer.has(0);
+		assert buffer.has(1);
 		assert buffer.has(2);
 		assert !buffer.has(3);
 	}
 
-	@Case(name = "empty")
-	static void empty() {
-		assert new MemBuf("").isEmpty();
-		assert !new MemBuf("mai").isEmpty();
-	}
-
 	@Case(name = "slice")
 	static void slice() {
-		var buffer = new MemBuf("hello mai");
+		var buffer = new MemBuf("hello 😀");
 
 		assert buffer.slice(0, 5).toString().equals("hello");
-		assert buffer.slice(6, 9).toString().equals("mai");
+		assert buffer.slice(6, 10).toString().equals("😀");
 	}
 
 	private MemBufTest() {
diff --git a/test/moe/anita/mai/buffer/SrcBufTest.java b/test/moe/anita/mai/buffer/SrcBufTest.java
index ced5d1c..9d1f7c1 100644
--- a/test/moe/anita/mai/buffer/SrcBufTest.java
+++ b/test/moe/anita/mai/buffer/SrcBufTest.java
@@ -4,7 +4,6 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
-import moe.anita.mai.buffer.SrcBuf;
 
 import moe.anita.mai.test.Case;
 import moe.anita.mai.test.Runner;
@@ -54,7 +53,7 @@ public final class SrcBufTest {
 		}
 	}
 
-	@Case(name = "retains source path")
+	@Case(name = "keeps source path")
 	static void path() throws IOException {
 		Path path = Files.createTempFile("mai-src-", ".mai");
 
diff --git a/test/moe/anita/mai/lexer/LexerCommentTest.java b/test/moe/anita/mai/lexer/LexerCommentTest.java
new file mode 100644
index 0000000..11b47ad
--- /dev/null
+++ b/test/moe/anita/mai/lexer/LexerCommentTest.java
@@ -0,0 +1,104 @@
+package moe.anita.mai.lexer;
+
+import moe.anita.mai.buffer.MemBuf;
+import moe.anita.mai.test.Case;
+import moe.anita.mai.test.Runner;
+import moe.anita.mai.token.TokenKind;
+
+@Runner(name = "Lexer Comments")
+public final class LexerCommentTest {
+
+	@Case(name = "line comment")
+	static void lineComment() {
+		var lexer = lexer("// hello");
+
+		assert lexer.next().kind() == TokenKind.COMMENT;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "line comment stops before newline")
+	static void lineCommentNewline() {
+		var lexer = lexer("// hello\nfoo");
+
+		assert lexer.next().kind() == TokenKind.COMMENT;
+		assert lexer.next().kind() == TokenKind.EOL;
+		assert lexer.next().kind() == TokenKind.IDENT;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "documentation comment")
+	static void docComment() {
+		var lexer = lexer("/// hello");
+
+		assert lexer.next().kind() == TokenKind.DOC_COMMENT;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "documentation comment stops before newline")
+	static void docCommentNewline() {
+		var lexer = lexer("/// hello\nfoo");
+
+		assert lexer.next().kind() == TokenKind.DOC_COMMENT;
+		assert lexer.next().kind() == TokenKind.EOL;
+		assert lexer.next().kind() == TokenKind.IDENT;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "block comment")
+	static void blockComment() {
+		var lexer = lexer("/* hello */");
+
+		assert lexer.next().kind() == TokenKind.COMMENT;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "multiline block comment")
+	static void multilineBlockComment() {
+		var lexer = lexer("/* hello\nworld */foo");
+
+		assert lexer.next().kind() == TokenKind.COMMENT;
+		assert lexer.next().kind() == TokenKind.IDENT;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "empty block comment")
+	static void emptyBlockComment() {
+		var lexer = lexer("/**/");
+
+		assert lexer.next().kind() == TokenKind.COMMENT;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "unterminated block comment")
+	static void unterminatedBlockComment() {
+		var lexer = lexer("/* hello");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "division operator")
+	static void division() {
+		var lexer = lexer("/");
+
+		assert lexer.next().kind() == TokenKind.DIV;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "division expression")
+	static void divisionExpression() {
+		var lexer = lexer("a / b");
+
+		assert lexer.next().kind() == TokenKind.IDENT;
+		assert lexer.next().kind() == TokenKind.DIV;
+		assert lexer.next().kind() == TokenKind.IDENT;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	private static Lexer lexer(String source) {
+		return new Lexer(new MemBuf(source));
+	}
+
+	private LexerCommentTest() {
+	}
+}
diff --git a/test/moe/anita/mai/lexer/LexerEscapeTest.java b/test/moe/anita/mai/lexer/LexerEscapeTest.java
new file mode 100644
index 0000000..fe10990
--- /dev/null
+++ b/test/moe/anita/mai/lexer/LexerEscapeTest.java
@@ -0,0 +1,100 @@
+package moe.anita.mai.lexer;
+
+import moe.anita.mai.buffer.MemBuf;
+import moe.anita.mai.test.Case;
+import moe.anita.mai.test.Runner;
+import moe.anita.mai.token.TokenKind;
+
+@Runner(name = "Lexer Escapes")
+public final class LexerEscapeTest {
+
+	@Case(name = "newline escape")
+	static void newline() {
+		assertString("\"hello\\nworld\"");
+	}
+
+	@Case(name = "carriage return escape")
+	static void carriageReturn() {
+		assertString("\"hello\\rworld\"");
+	}
+
+	@Case(name = "tab escape")
+	static void tab() {
+		assertString("\"hello\\tworld\"");
+	}
+
+	@Case(name = "null escape")
+	static void nullEscape() {
+		assertString("\"hello\\0world\"");
+	}
+
+	@Case(name = "backslash escape")
+	static void backslash() {
+		assertString("\"hello\\\\world\"");
+	}
+
+	@Case(name = "quote escape")
+	static void quote() {
+		assertString("\"hello \\\"mai\\\"\"");
+	}
+
+	@Case(name = "escaped newline rune")
+	static void newlineRune() {
+		assertRune("'\\n'");
+	}
+
+	@Case(name = "escaped quote rune")
+	static void quoteRune() {
+		assertRune("'\\''");
+	}
+
+	@Case(name = "escaped backslash rune")
+	static void backslashRune() {
+		assertRune("'\\\\'");
+	}
+
+	@Case(name = "invalid string escape")
+	static void invalidStringEscape() {
+		var lexer = lexer("\"hello\\qworld\"");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "invalid rune escape")
+	static void invalidRuneEscape() {
+		var lexer = lexer("'\\q'");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "unterminated escape")
+	static void unterminatedEscape() {
+		var lexer = lexer("\"hello\\");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	private static void assertString(String source) {
+		var lexer = lexer(source);
+
+		assert lexer.next().kind() == TokenKind.STRING;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	private static void assertRune(String source) {
+		var lexer = lexer(source);
+
+		assert lexer.next().kind() == TokenKind.RUNE;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	private static Lexer lexer(String source) {
+		return new Lexer(new MemBuf(source));
+	}
+
+	private LexerEscapeTest() {
+	}
+}
diff --git a/test/moe/anita/mai/lexer/LexerInvalidNumbersTest.java b/test/moe/anita/mai/lexer/LexerInvalidNumbersTest.java
new file mode 100644
index 0000000..07c602a
--- /dev/null
+++ b/test/moe/anita/mai/lexer/LexerInvalidNumbersTest.java
@@ -0,0 +1,97 @@
+package moe.anita.mai.lexer;
+
+import moe.anita.mai.buffer.MemBuf;
+import moe.anita.mai.test.Case;
+import moe.anita.mai.test.Runner;
+import moe.anita.mai.token.TokenKind;
+
+@Runner(name = "Lexer Invalid Numbers")
+public final class LexerInvalidNumbersTest {
+
+	@Case(name = "hex prefix requires digits")
+	static void emptyHex() {
+		var lexer = lexer("0x");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "binary prefix requires digits")
+	static void emptyBinary() {
+		var lexer = lexer("0b");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "octal prefix requires digits")
+	static void emptyOctal() {
+		var lexer = lexer("0o");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "underscore cannot follow prefix")
+	static void underscoreAfterPrefix() {
+		var lexer = lexer("0x_DEAD");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "integer cannot end with underscore")
+	static void trailingUnderscore() {
+		var lexer = lexer("123_");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "integer cannot contain consecutive underscores")
+	static void consecutiveUnderscores() {
+		var lexer = lexer("123__456");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "binary integer rejects invalid digit")
+	static void invalidBinaryDigit() {
+		var lexer = lexer("0b102");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "octal integer rejects invalid digit")
+	static void invalidOctalDigit() {
+		var lexer = lexer("0o89");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "hex integer rejects invalid digit")
+	static void invalidHexDigit() {
+		var lexer = lexer("0x12G");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "float fraction cannot end with underscore")
+	static void floatTrailingUnderscore() {
+		var lexer = lexer("12.34_");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	private static Lexer lexer(String source) {
+		return new Lexer(new MemBuf(source));
+	}
+
+	private LexerInvalidNumbersTest() {
+	}
+}
diff --git a/test/moe/anita/mai/lexer/LexerKeywordTest.java b/test/moe/anita/mai/lexer/LexerKeywordTest.java
new file mode 100644
index 0000000..78b7a2c
--- /dev/null
+++ b/test/moe/anita/mai/lexer/LexerKeywordTest.java
@@ -0,0 +1,144 @@
+package moe.anita.mai.lexer;
+
+import moe.anita.mai.buffer.MemBuf;
+import moe.anita.mai.test.Case;
+import moe.anita.mai.test.Runner;
+import moe.anita.mai.token.TokenKind;
+
+@Runner(name = "Lexer Keywords")
+public final class LexerKeywordTest {
+
+	@Case(name = "declaration keywords")
+	static void declarations() {
+		var lexer = lexer("pkg use fn mut");
+
+		assertNext(lexer, TokenKind.PKG);
+		assertNext(lexer, TokenKind.USE);
+		assertNext(lexer, TokenKind.FN);
+		assertNext(lexer, TokenKind.MUT);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "control flow keywords")
+	static void controlFlow() {
+		var lexer = lexer("ret for if else match");
+
+		assertNext(lexer, TokenKind.RET);
+		assertNext(lexer, TokenKind.FOR);
+		assertNext(lexer, TokenKind.IF);
+		assertNext(lexer, TokenKind.ELSE);
+		assertNext(lexer, TokenKind.MATCH);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "type declaration keywords")
+	static void typeDeclarations() {
+		var lexer = lexer("with inlay enum alias");
+
+		assertNext(lexer, TokenKind.WITH);
+		assertNext(lexer, TokenKind.INLAY);
+		assertNext(lexer, TokenKind.ENUM);
+		assertNext(lexer, TokenKind.ALIAS);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "linkage keywords")
+	static void linkage() {
+		var lexer = lexer("link bind ex");
+
+		assertNext(lexer, TokenKind.LINK);
+		assertNext(lexer, TokenKind.BIND);
+		assertNext(lexer, TokenKind.EX);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "literal keywords")
+	static void literals() {
+		var lexer = lexer("true false null");
+
+		assertNext(lexer, TokenKind.TRUE);
+		assertNext(lexer, TokenKind.FALSE);
+		assertNext(lexer, TokenKind.NULL);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "signed integer primitives")
+	static void signedIntegers() {
+		var lexer = lexer("i8 i16 i32 i64 isize");
+
+		assertNext(lexer, TokenKind.I8);
+		assertNext(lexer, TokenKind.I16);
+		assertNext(lexer, TokenKind.I32);
+		assertNext(lexer, TokenKind.I64);
+		assertNext(lexer, TokenKind.ISIZE);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "unsigned integer primitives")
+	static void unsignedIntegers() {
+		var lexer = lexer("u8 u16 u32 u64 usize");
+
+		assertNext(lexer, TokenKind.U8);
+		assertNext(lexer, TokenKind.U16);
+		assertNext(lexer, TokenKind.U32);
+		assertNext(lexer, TokenKind.U64);
+		assertNext(lexer, TokenKind.USIZE);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "floating point primitives")
+	static void floatingPoint() {
+		var lexer = lexer("f32 f64");
+
+		assertNext(lexer, TokenKind.F32);
+		assertNext(lexer, TokenKind.F64);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "other primitives")
+	static void otherPrimitives() {
+		var lexer = lexer("bool rune string void");
+
+		assertNext(lexer, TokenKind.BOOL);
+		assertNext(lexer, TokenKind.RUNE_TYPE);
+		assertNext(lexer, TokenKind.STRING_TYPE);
+		assertNext(lexer, TokenKind.VOID);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "keyword prefixes remain identifiers")
+	static void keywordPrefixes() {
+		var lexer = lexer("function mutable matching stringBuilder usizeOf");
+
+		assertNext(lexer, TokenKind.IDENT);
+		assertNext(lexer, TokenKind.IDENT);
+		assertNext(lexer, TokenKind.IDENT);
+		assertNext(lexer, TokenKind.IDENT);
+		assertNext(lexer, TokenKind.IDENT);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "keywords are case sensitive")
+	static void caseSensitive() {
+		var lexer = lexer("Fn STRING Bool I32");
+
+		assertNext(lexer, TokenKind.IDENT);
+		assertNext(lexer, TokenKind.IDENT);
+		assertNext(lexer, TokenKind.IDENT);
+		assertNext(lexer, TokenKind.IDENT);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	private static void assertNext(Lexer lexer, TokenKind expected) {
+		var token = lexer.next();
+
+		assert token.kind() == expected : "expected " + expected + ", got " + token.kind();
+	}
+
+	private static Lexer lexer(String source) {
+		return new Lexer(new MemBuf(source));
+	}
+
+	private LexerKeywordTest() {
+	}
+}
diff --git a/test/moe/anita/mai/lexer/LexerNumberTest.java b/test/moe/anita/mai/lexer/LexerNumberTest.java
index 1f4ab1f..50147cb 100644
--- a/test/moe/anita/mai/lexer/LexerNumberTest.java
+++ b/test/moe/anita/mai/lexer/LexerNumberTest.java
@@ -69,8 +69,7 @@ public final class LexerNumberTest {
 		var lexer = lexer("10..20");
 
 		assert lexer.next().kind() == TokenKind.INT;
-		assert lexer.next().kind() == TokenKind.DOT;
-		assert lexer.next().kind() == TokenKind.DOT;
+		assert lexer.next().kind() == TokenKind.RANGE;
 		assert lexer.next().kind() == TokenKind.INT;
 		assert lexer.next().kind() == TokenKind.EOF;
 	}
diff --git a/test/moe/anita/mai/lexer/LexerPunctuationTest.java b/test/moe/anita/mai/lexer/LexerPunctuationTest.java
new file mode 100644
index 0000000..d9a6439
--- /dev/null
+++ b/test/moe/anita/mai/lexer/LexerPunctuationTest.java
@@ -0,0 +1,127 @@
+package moe.anita.mai.lexer;
+
+import moe.anita.mai.buffer.MemBuf;
+import moe.anita.mai.test.Case;
+import moe.anita.mai.test.Runner;
+import moe.anita.mai.token.TokenKind;
+
+@Runner(name = "Lexer Punctuation")
+public final class LexerPunctuationTest {
+
+	@Case(name = "delimiters")
+	static void delimiters() {
+		var lexer = lexer("( ) [ ] { }");
+
+		assertNext(lexer, TokenKind.LPAREN);
+		assertNext(lexer, TokenKind.RPAREN);
+		assertNext(lexer, TokenKind.LBRACKET);
+		assertNext(lexer, TokenKind.RBRACKET);
+		assertNext(lexer, TokenKind.LBRACE);
+		assertNext(lexer, TokenKind.RBRACE);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "punctuation")
+	static void punctuation() {
+		var lexer = lexer(", . .. : ; ? @");
+
+		assertNext(lexer, TokenKind.COMMA);
+		assertNext(lexer, TokenKind.DOT);
+		assertNext(lexer, TokenKind.RANGE);
+		assertNext(lexer, TokenKind.COLON);
+		assertNext(lexer, TokenKind.SEMICOLON);
+		assertNext(lexer, TokenKind.QUESTION);
+		assertNext(lexer, TokenKind.AT);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "assignment and declaration")
+	static void assignment() {
+		var lexer = lexer("= :=");
+
+		assertNext(lexer, TokenKind.ASSIGN);
+		assertNext(lexer, TokenKind.DECLARE);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "arithmetic operators")
+	static void arithmetic() {
+		var lexer = lexer("+ - * / % ++ --");
+
+		assertNext(lexer, TokenKind.ADD);
+		assertNext(lexer, TokenKind.SUB);
+		assertNext(lexer, TokenKind.MUL);
+		assertNext(lexer, TokenKind.DIV);
+		assertNext(lexer, TokenKind.MOD);
+		assertNext(lexer, TokenKind.INC);
+		assertNext(lexer, TokenKind.DEC);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "comparison operators")
+	static void comparison() {
+		var lexer = lexer("== <> < <= > >=");
+
+		assertNext(lexer, TokenKind.EQ);
+		assertNext(lexer, TokenKind.NE);
+		assertNext(lexer, TokenKind.LT);
+		assertNext(lexer, TokenKind.LE);
+		assertNext(lexer, TokenKind.GT);
+		assertNext(lexer, TokenKind.GE);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "bitwise operators")
+	static void bitwise() {
+		var lexer = lexer("& | ^ ~");
+
+		assertNext(lexer, TokenKind.BIT_AND);
+		assertNext(lexer, TokenKind.BIT_OR);
+		assertNext(lexer, TokenKind.BIT_XOR);
+		assertNext(lexer, TokenKind.BIT_NOT);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "logical operators")
+	static void logical() {
+		var lexer = lexer("&& || !");
+
+		assertNext(lexer, TokenKind.AND);
+		assertNext(lexer, TokenKind.OR);
+		assertNext(lexer, TokenKind.NOT);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "arrows")
+	static void arrows() {
+		var lexer = lexer("-> <-");
+
+		assertNext(lexer, TokenKind.ARROW);
+		assertNext(lexer, TokenKind.BIND);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "shifts remain parser constructed")
+	static void shifts() {
+		var lexer = lexer("<< >>");
+
+		assertNext(lexer, TokenKind.LT);
+		assertNext(lexer, TokenKind.LT);
+		assertNext(lexer, TokenKind.GT);
+		assertNext(lexer, TokenKind.GT);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	private static void assertNext(Lexer lexer, TokenKind expected) {
+		var token = lexer.next();
+
+		assert token.kind() == expected : "expected " + expected + ", got " + token.kind();
+	}
+
+	private static Lexer lexer(String source) {
+		return new Lexer(new MemBuf(source));
+	}
+
+	private LexerPunctuationTest() {
+	}
+}
diff --git a/test/moe/anita/mai/lexer/LexerRawStringTest.java b/test/moe/anita/mai/lexer/LexerRawStringTest.java
new file mode 100644
index 0000000..7503679
--- /dev/null
+++ b/test/moe/anita/mai/lexer/LexerRawStringTest.java
@@ -0,0 +1,85 @@
+package moe.anita.mai.lexer;
+
+import moe.anita.mai.buffer.MemBuf;
+import moe.anita.mai.test.Case;
+import moe.anita.mai.test.Runner;
+import moe.anita.mai.token.TokenKind;
+
+@Runner(name = "Lexer Raw Strings")
+public final class LexerRawStringTest {
+
+	@Case(name = "raw string")
+	static void rawString() {
+		var lexer = lexer("r\"hello mai\"");
+
+		assert lexer.next().kind() == TokenKind.RAW_STRING;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "empty raw string")
+	static void emptyRawString() {
+		var lexer = lexer("r\"\"");
+
+		assert lexer.next().kind() == TokenKind.RAW_STRING;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "raw string preserves escapes")
+	static void escapes() {
+		var lexer = lexer("r\"hello\\nworld\\t!\"");
+
+		assert lexer.next().kind() == TokenKind.RAW_STRING;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "raw string can contain single quotes")
+	static void singleQuotes() {
+		var lexer = lexer("r\"'hello mai'\"");
+
+		assert lexer.next().kind() == TokenKind.RAW_STRING;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "raw string can span lines")
+	static void multiline() {
+		var lexer = lexer("r\"hello\nmai\"");
+
+		var token = lexer.next();
+
+		assert token.kind() == TokenKind.RAW_STRING;
+		assert token.span().start().line() == 1;
+		assert token.span().end().line() == 2;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "raw string supports Unicode")
+	static void unicode() {
+		var lexer = lexer("r\"猫 😀 λ\"");
+
+		assert lexer.next().kind() == TokenKind.RAW_STRING;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "unterminated raw string")
+	static void unterminated() {
+		var lexer = lexer("r\"hello mai");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "r identifier is not a raw string")
+	static void identifier() {
+		var lexer = lexer("raw");
+
+		assert lexer.next().kind() == TokenKind.IDENT;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	private static Lexer lexer(String source) {
+		return new Lexer(new MemBuf(source));
+	}
+
+	private LexerRawStringTest() {
+	}
+}
diff --git a/test/moe/anita/mai/lexer/LexerSpanTest.java b/test/moe/anita/mai/lexer/LexerSpanTest.java
new file mode 100644
index 0000000..65bb552
--- /dev/null
+++ b/test/moe/anita/mai/lexer/LexerSpanTest.java
@@ -0,0 +1,158 @@
+package moe.anita.mai.lexer;
+
+import moe.anita.mai.buffer.MemBuf;
+import moe.anita.mai.span.Pos;
+import moe.anita.mai.test.Case;
+import moe.anita.mai.test.Runner;
+import moe.anita.mai.token.TokenKind;
+
+@Runner(name = "Lexer Spans")
+public final class LexerSpanTest {
+
+	@Case(name = "ASCII identifier span")
+	static void asciiIdentifier() {
+		var lexer = lexer("hello");
+		var token = lexer.next();
+
+		assert token.kind() == TokenKind.IDENT;
+
+		assert token.span().start().offset() == 0;
+		assert token.span().start().line() == 1;
+		assert token.span().start().column() == 1;
+
+		assert token.span().end().offset() == 5;
+		assert token.span().end().line() == 1;
+		assert token.span().end().column() == 6;
+	}
+
+	@Case(name = "token after whitespace")
+	static void whitespace() {
+		var lexer = lexer("   foo");
+		var token = lexer.next();
+
+		assert token.kind() == TokenKind.IDENT;
+
+		assert token.span().start().offset() == 3;
+		assert token.span().start().line() == 1;
+		assert token.span().start().column() == 4;
+
+		assert token.span().end().offset() == 6;
+		assert token.span().end().line() == 1;
+		assert token.span().end().column() == 7;
+	}
+
+	@Case(name = "newline span")
+	static void newline() {
+		var lexer = lexer("\n");
+		var token = lexer.next();
+
+		assert token.kind() == TokenKind.EOL;
+
+		assert token.span().start().offset() == 0;
+		assert token.span().start().line() == 1;
+		assert token.span().start().column() == 1;
+
+		assert token.span().end().offset() == 1;
+		assert token.span().end().line() == 2;
+		assert token.span().end().column() == 1;
+	}
+
+	@Case(name = "token after newline")
+	static void afterNewline() {
+		var lexer = lexer("\nfoo");
+
+		assert lexer.next().kind() == TokenKind.EOL;
+
+		var token = lexer.next();
+
+		assert token.kind() == TokenKind.IDENT;
+
+		assert token.span().start().offset() == 1;
+		assert token.span().start().line() == 2;
+		assert token.span().start().column() == 1;
+
+		assert token.span().end().offset() == 4;
+		assert token.span().end().line() == 2;
+		assert token.span().end().column() == 4;
+	}
+
+	@Case(name = "BMP Unicode span")
+	static void bmpUnicode() {
+		var lexer = lexer("猫");
+		var token = lexer.next();
+
+		assert token.kind() == TokenKind.IDENT;
+
+		assert token.span().start().offset() == 0;
+		assert token.span().start().column() == 1;
+
+		assert token.span().end().offset() == 3;
+		assert token.span().end().column() == 2;
+	}
+
+	@Case(name = "supplementary Unicode rune span")
+	static void supplementaryRune() {
+		var lexer = lexer("'😀'");
+		var token = lexer.next();
+
+		assert token.kind() == TokenKind.RUNE;
+
+		assert token.span().start().offset() == 0;
+		assert token.span().start().line() == 1;
+		assert token.span().start().column() == 1;
+
+		assert token.span().end().offset() == 6;
+		assert token.span().end().line() == 1;
+		assert token.span().end().column() == 4;
+	}
+
+	@Case(name = "multiline raw string span")
+	static void multilineRawString() {
+		var lexer = lexer("r\"hello\nworld\"");
+		var token = lexer.next();
+
+		assert token.kind() == TokenKind.RAW_STRING;
+
+		assert token.span().start().offset() == 0;
+		assert token.span().start().line() == 1;
+		assert token.span().start().column() == 1;
+
+		assert token.span().end().offset() == 14;
+		assert token.span().end().line() == 2;
+		assert token.span().end().column() == 7;
+	}
+
+	@Case(name = "multibyte text before token")
+	static void multibyteBeforeToken() {
+		var lexer = lexer("猫 foo");
+
+		var first = lexer.next();
+		var second = lexer.next();
+
+		assert first.kind() == TokenKind.IDENT;
+		assert second.kind() == TokenKind.IDENT;
+
+		assert second.span().start().offset() == 4;
+		assert second.span().start().line() == 1;
+		assert second.span().start().column() == 3;
+
+		assert second.span().end().offset() == 7;
+		assert second.span().end().column() == 6;
+	}
+
+	@Case(name = "position packing")
+	static void positionPacking() {
+		var pos = Pos.of(5, 2, 7);
+
+		assert pos.offset() == 5 : "offset = " + pos.offset();
+		assert pos.line() == 2 : "line = " + pos.line();
+		assert pos.column() == 7 : "column = " + pos.column();
+	}
+
+	private static Lexer lexer(String source) {
+		return new Lexer(new MemBuf(source));
+	}
+
+	private LexerSpanTest() {
+	}
+}
diff --git a/test/moe/anita/mai/lexer/LexerStringTest.java b/test/moe/anita/mai/lexer/LexerStringTest.java
new file mode 100644
index 0000000..4707264
--- /dev/null
+++ b/test/moe/anita/mai/lexer/LexerStringTest.java
@@ -0,0 +1,102 @@
+package moe.anita.mai.lexer;
+
+import moe.anita.mai.buffer.MemBuf;
+import moe.anita.mai.test.Case;
+import moe.anita.mai.test.Runner;
+import moe.anita.mai.token.TokenKind;
+
+@Runner(name = "Lexer Strings")
+public final class LexerStringTest {
+
+	@Case(name = "string literal")
+	static void string() {
+		var lexer = lexer("\"hello\"");
+
+		assert lexer.next().kind() == TokenKind.STRING;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "empty string")
+	static void emptyString() {
+		var lexer = lexer("\"\"");
+
+		assert lexer.next().kind() == TokenKind.STRING;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "escaped quote")
+	static void escapedQuote() {
+		var lexer = lexer("\"hello \\\"mai\\\"\"");
+
+		assert lexer.next().kind() == TokenKind.STRING;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "escaped backslash")
+	static void escapedBackslash() {
+		var lexer = lexer("\"foo\\\\bar\"");
+
+		assert lexer.next().kind() == TokenKind.STRING;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "unterminated string")
+	static void unterminatedString() {
+		var lexer = lexer("\"hello");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "string cannot cross newline")
+	static void stringNewline() {
+		var lexer = lexer("\"hello\nworld\"");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+	}
+
+	@Case(name = "rune literal")
+	static void rune() {
+		var lexer = lexer("'a'");
+
+		assert lexer.next().kind() == TokenKind.RUNE;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "escaped rune")
+	static void escapedRune() {
+		var lexer = lexer("'\\n'");
+
+		assert lexer.next().kind() == TokenKind.RUNE;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "quote rune")
+	static void quoteRune() {
+		var lexer = lexer("'\\''");
+
+		assert lexer.next().kind() == TokenKind.RUNE;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "empty rune is invalid")
+	static void emptyRune() {
+		var lexer = lexer("''");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+	}
+
+	@Case(name = "multiple characters are invalid rune")
+	static void multipleRune() {
+		var lexer = lexer("'ab'");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+	}
+
+	private static Lexer lexer(String source) {
+		return new Lexer(new MemBuf(source));
+	}
+
+	private LexerStringTest() {
+	}
+}
diff --git a/test/moe/anita/mai/test/TestRunner.java b/test/moe/anita/mai/test/TestRunner.java
index 7c74bf3..8103cad 100644
--- a/test/moe/anita/mai/test/TestRunner.java
+++ b/test/moe/anita/mai/test/TestRunner.java
@@ -42,15 +42,11 @@ public final class TestRunner {
 	}
 
 	private static boolean isTopLevelClass(Path path) {
-		return !path.getFileName()
-			.toString()
-			.contains("$");
+		return !path.getFileName().toString().contains("$");
 	}
 
 	private static String className(Path root, Path classFile) {
-		String relative = root
-			.relativize(classFile)
-			.toString();
+		String relative = root.relativize(classFile).toString();
 
 		String name = relative.substring(0, relative.length() - ".class".length()
 		);
@@ -62,8 +58,7 @@ public final class TestRunner {
 		try {
 			Class<?> testClass = Class.forName(className);
 
-			Runner runner =
-				testClass.getAnnotation(Runner.class);
+			Runner runner = testClass.getAnnotation(Runner.class);
 
 			if (runner == null) {
 				return;
@@ -72,21 +67,13 @@ public final class TestRunner {
 			runClass(testClass, runner, result);
 
 		} catch (ClassNotFoundException e) {
-			System.err.printf(
-				"[FAIL] unable to load %s: %s%n",
-				className,
-				e
-			);
+			System.err.printf("[FAIL] unable to load %s: %s%n", className, e);
 
 			result.failed++;
 		}
 	}
 
-	private static void runClass(
-		Class<?> testClass,
-		Runner runner,
-		Result result
-	) {
+	private static void runClass(Class<?> testClass, Runner runner, Result result) {
 		String name = runner.name();
 
 		if (name.isEmpty()) {
@@ -115,21 +102,14 @@ public final class TestRunner {
 		}
 
 		if (!testCase.enabled()) {
-			System.out.printf(
-				"  [SKIP] %s%n",
-				name
-			);
-
+			System.out.printf("  [SKIP] %s%n", name);
 			result.skipped++;
 			return;
 		}
 
 		if (!validCase(method)) {
-			System.err.printf(
-				"  [FAIL] %s: invalid test case%n",
-				name
-			);
-
+			System.out.printf("  [FAIL] %s:%n", name);
+			System.out.println("     invalid test case");
 			result.failed++;
 			return;
 		}
@@ -144,43 +124,30 @@ public final class TestRunner {
 			method.setAccessible(true);
 			method.invoke(instance);
 
-			System.out.printf(
-				"  [PASS] %s%n",
-				name
-			);
-
+			System.out.printf("  [PASS] %s%n", name);
 			result.passed++;
-
 		} catch (InvocationTargetException e) {
 			Throwable cause = e.getCause();
 
-			System.err.printf(
-				"  [FAIL] %s: %s%n",
-				name,
-				cause
-			);
+			System.out.printf("  [FAIL] %s:%n", name);
+			System.out.printf("     %s%n", cause);
 
 			result.failed++;
-
 		} catch (ReflectiveOperationException e) {
-			System.err.printf(
-				"  [FAIL] %s: %s%n",
-				name,
-				e
-			);
+			System.out.printf("  [FAIL] %s:%n", name);
+			System.out.printf("     %s%n", e);
 
 			result.failed++;
 		}
 	}
 
+
 	private static boolean validCase(Method method) {
-		return method.getParameterCount() == 0
-			&& method.getReturnType() == void.class;
+		return method.getParameterCount() == 0 && method.getReturnType() == void.class;
 	}
 
 	private static Object createInstance(Class<?> testClass) throws ReflectiveOperationException {
-		var constructor =
-			testClass.getDeclaredConstructor();
+		var constructor = testClass.getDeclaredConstructor();
 
 		constructor.setAccessible(true);
 
@@ -190,12 +157,7 @@ public final class TestRunner {
 	private static void printSummary(Result result) {
 		System.out.println();
 
-		System.out.printf(
-			"Tests: %d passed, %d failed, %d skipped%n",
-			result.passed,
-			result.failed,
-			result.skipped
-		);
+		System.out.printf("Tests: %d passed, %d failed, %d skipped%n", result.passed, result.failed, result.skipped);
 	}
 
 	private static final class Result {