Finish the lexerHEADtrunklexer

Signed-off-by: Antia Anderson <anita@ansan.io>

12 files changed, 986 insertions, 71 deletions

diff --git a/test/moe/anita/mai/buffer/MemBufTest.java b/test/moe/anita/mai/buffer/MemBufTest.java
index 450ac31..28dc5c9 100644
--- a/test/moe/anita/mai/buffer/MemBufTest.java
+++ b/test/moe/anita/mai/buffer/MemBufTest.java
@@ -2,8 +2,7 @@ package moe.anita.mai.buffer;
 
 import moe.anita.mai.test.Case;
 import moe.anita.mai.test.Runner;
-
-@Runner(name = "MemBuf")
+@Runner(name = "MemBuffer")
 public final class MemBufTest {
 
 	@Case(name = "length")
@@ -13,13 +12,57 @@ public final class MemBufTest {
 		assert buffer.length() == 3;
 	}
 
-	@Case(name = "read")
-	static void read() {
+	@Case(name = "empty")
+	static void empty() {
+		assert new MemBuf("").isEmpty();
+		assert !new MemBuf("mai").isEmpty();
+	}
+
+	@Case(name = "ASCII code points")
+	static void ascii() {
 		var buffer = new MemBuf("mai");
 
 		assert buffer.get(0) == 'm';
 		assert buffer.get(1) == 'a';
 		assert buffer.get(2) == 'i';
+
+		assert buffer.width(0) == 1;
+		assert buffer.width(1) == 1;
+		assert buffer.width(2) == 1;
+	}
+
+	@Case(name = "Unicode code point")
+	static void unicode() {
+		var buffer = new MemBuf("猫");
+
+		assert buffer.length() == 3;
+		assert buffer.get(0) == '猫';
+		assert buffer.width(0) == 3;
+	}
+
+	@Case(name = "supplementary Unicode code point")
+	static void supplementaryUnicode() {
+		var buffer = new MemBuf("😀");
+
+		assert buffer.length() == 4;
+		assert buffer.get(0) == 0x1F600;
+		assert buffer.width(0) == 4;
+	}
+
+	@Case(name = "mixed Unicode")
+	static void mixedUnicode() {
+		var buffer = new MemBuf("a😀猫");
+
+		assert buffer.length() == 8;
+
+		assert buffer.get(0) == 'a';
+		assert buffer.width(0) == 1;
+
+		assert buffer.get(1) == 0x1F600;
+		assert buffer.width(1) == 4;
+
+		assert buffer.get(5) == '猫';
+		assert buffer.width(5) == 3;
 	}
 
 	@Case(name = "bounds")
@@ -28,22 +71,17 @@ public final class MemBufTest {
 
 		assert !buffer.has(-1);
 		assert buffer.has(0);
+		assert buffer.has(1);
 		assert buffer.has(2);
 		assert !buffer.has(3);
 	}
 
-	@Case(name = "empty")
-	static void empty() {
-		assert new MemBuf("").isEmpty();
-		assert !new MemBuf("mai").isEmpty();
-	}
-
 	@Case(name = "slice")
 	static void slice() {
-		var buffer = new MemBuf("hello mai");
+		var buffer = new MemBuf("hello 😀");
 
 		assert buffer.slice(0, 5).toString().equals("hello");
-		assert buffer.slice(6, 9).toString().equals("mai");
+		assert buffer.slice(6, 10).toString().equals("😀");
 	}
 
 	private MemBufTest() {
diff --git a/test/moe/anita/mai/buffer/SrcBufTest.java b/test/moe/anita/mai/buffer/SrcBufTest.java
index ced5d1c..9d1f7c1 100644
--- a/test/moe/anita/mai/buffer/SrcBufTest.java
+++ b/test/moe/anita/mai/buffer/SrcBufTest.java
@@ -4,7 +4,6 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
-import moe.anita.mai.buffer.SrcBuf;
 
 import moe.anita.mai.test.Case;
 import moe.anita.mai.test.Runner;
@@ -54,7 +53,7 @@ public final class SrcBufTest {
 		}
 	}
 
-	@Case(name = "retains source path")
+	@Case(name = "keeps source path")
 	static void path() throws IOException {
 		Path path = Files.createTempFile("mai-src-", ".mai");
 
diff --git a/test/moe/anita/mai/lexer/LexerCommentTest.java b/test/moe/anita/mai/lexer/LexerCommentTest.java
new file mode 100644
index 0000000..11b47ad
--- /dev/null
+++ b/test/moe/anita/mai/lexer/LexerCommentTest.java
@@ -0,0 +1,104 @@
+package moe.anita.mai.lexer;
+
+import moe.anita.mai.buffer.MemBuf;
+import moe.anita.mai.test.Case;
+import moe.anita.mai.test.Runner;
+import moe.anita.mai.token.TokenKind;
+
+@Runner(name = "Lexer Comments")
+public final class LexerCommentTest {
+
+	@Case(name = "line comment")
+	static void lineComment() {
+		var lexer = lexer("// hello");
+
+		assert lexer.next().kind() == TokenKind.COMMENT;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "line comment stops before newline")
+	static void lineCommentNewline() {
+		var lexer = lexer("// hello\nfoo");
+
+		assert lexer.next().kind() == TokenKind.COMMENT;
+		assert lexer.next().kind() == TokenKind.EOL;
+		assert lexer.next().kind() == TokenKind.IDENT;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "documentation comment")
+	static void docComment() {
+		var lexer = lexer("/// hello");
+
+		assert lexer.next().kind() == TokenKind.DOC_COMMENT;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "documentation comment stops before newline")
+	static void docCommentNewline() {
+		var lexer = lexer("/// hello\nfoo");
+
+		assert lexer.next().kind() == TokenKind.DOC_COMMENT;
+		assert lexer.next().kind() == TokenKind.EOL;
+		assert lexer.next().kind() == TokenKind.IDENT;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "block comment")
+	static void blockComment() {
+		var lexer = lexer("/* hello */");
+
+		assert lexer.next().kind() == TokenKind.COMMENT;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "multiline block comment")
+	static void multilineBlockComment() {
+		var lexer = lexer("/* hello\nworld */foo");
+
+		assert lexer.next().kind() == TokenKind.COMMENT;
+		assert lexer.next().kind() == TokenKind.IDENT;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "empty block comment")
+	static void emptyBlockComment() {
+		var lexer = lexer("/**/");
+
+		assert lexer.next().kind() == TokenKind.COMMENT;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "unterminated block comment")
+	static void unterminatedBlockComment() {
+		var lexer = lexer("/* hello");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "division operator")
+	static void division() {
+		var lexer = lexer("/");
+
+		assert lexer.next().kind() == TokenKind.DIV;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "division expression")
+	static void divisionExpression() {
+		var lexer = lexer("a / b");
+
+		assert lexer.next().kind() == TokenKind.IDENT;
+		assert lexer.next().kind() == TokenKind.DIV;
+		assert lexer.next().kind() == TokenKind.IDENT;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	private static Lexer lexer(String source) {
+		return new Lexer(new MemBuf(source));
+	}
+
+	private LexerCommentTest() {
+	}
+}
diff --git a/test/moe/anita/mai/lexer/LexerEscapeTest.java b/test/moe/anita/mai/lexer/LexerEscapeTest.java
new file mode 100644
index 0000000..fe10990
--- /dev/null
+++ b/test/moe/anita/mai/lexer/LexerEscapeTest.java
@@ -0,0 +1,100 @@
+package moe.anita.mai.lexer;
+
+import moe.anita.mai.buffer.MemBuf;
+import moe.anita.mai.test.Case;
+import moe.anita.mai.test.Runner;
+import moe.anita.mai.token.TokenKind;
+
+@Runner(name = "Lexer Escapes")
+public final class LexerEscapeTest {
+
+	@Case(name = "newline escape")
+	static void newline() {
+		assertString("\"hello\\nworld\"");
+	}
+
+	@Case(name = "carriage return escape")
+	static void carriageReturn() {
+		assertString("\"hello\\rworld\"");
+	}
+
+	@Case(name = "tab escape")
+	static void tab() {
+		assertString("\"hello\\tworld\"");
+	}
+
+	@Case(name = "null escape")
+	static void nullEscape() {
+		assertString("\"hello\\0world\"");
+	}
+
+	@Case(name = "backslash escape")
+	static void backslash() {
+		assertString("\"hello\\\\world\"");
+	}
+
+	@Case(name = "quote escape")
+	static void quote() {
+		assertString("\"hello \\\"mai\\\"\"");
+	}
+
+	@Case(name = "escaped newline rune")
+	static void newlineRune() {
+		assertRune("'\\n'");
+	}
+
+	@Case(name = "escaped quote rune")
+	static void quoteRune() {
+		assertRune("'\\''");
+	}
+
+	@Case(name = "escaped backslash rune")
+	static void backslashRune() {
+		assertRune("'\\\\'");
+	}
+
+	@Case(name = "invalid string escape")
+	static void invalidStringEscape() {
+		var lexer = lexer("\"hello\\qworld\"");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "invalid rune escape")
+	static void invalidRuneEscape() {
+		var lexer = lexer("'\\q'");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "unterminated escape")
+	static void unterminatedEscape() {
+		var lexer = lexer("\"hello\\");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	private static void assertString(String source) {
+		var lexer = lexer(source);
+
+		assert lexer.next().kind() == TokenKind.STRING;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	private static void assertRune(String source) {
+		var lexer = lexer(source);
+
+		assert lexer.next().kind() == TokenKind.RUNE;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	private static Lexer lexer(String source) {
+		return new Lexer(new MemBuf(source));
+	}
+
+	private LexerEscapeTest() {
+	}
+}
diff --git a/test/moe/anita/mai/lexer/LexerInvalidNumbersTest.java b/test/moe/anita/mai/lexer/LexerInvalidNumbersTest.java
new file mode 100644
index 0000000..07c602a
--- /dev/null
+++ b/test/moe/anita/mai/lexer/LexerInvalidNumbersTest.java
@@ -0,0 +1,97 @@
+package moe.anita.mai.lexer;
+
+import moe.anita.mai.buffer.MemBuf;
+import moe.anita.mai.test.Case;
+import moe.anita.mai.test.Runner;
+import moe.anita.mai.token.TokenKind;
+
+@Runner(name = "Lexer Invalid Numbers")
+public final class LexerInvalidNumbersTest {
+
+	@Case(name = "hex prefix requires digits")
+	static void emptyHex() {
+		var lexer = lexer("0x");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "binary prefix requires digits")
+	static void emptyBinary() {
+		var lexer = lexer("0b");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "octal prefix requires digits")
+	static void emptyOctal() {
+		var lexer = lexer("0o");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "underscore cannot follow prefix")
+	static void underscoreAfterPrefix() {
+		var lexer = lexer("0x_DEAD");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "integer cannot end with underscore")
+	static void trailingUnderscore() {
+		var lexer = lexer("123_");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "integer cannot contain consecutive underscores")
+	static void consecutiveUnderscores() {
+		var lexer = lexer("123__456");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "binary integer rejects invalid digit")
+	static void invalidBinaryDigit() {
+		var lexer = lexer("0b102");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "octal integer rejects invalid digit")
+	static void invalidOctalDigit() {
+		var lexer = lexer("0o89");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "hex integer rejects invalid digit")
+	static void invalidHexDigit() {
+		var lexer = lexer("0x12G");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "float fraction cannot end with underscore")
+	static void floatTrailingUnderscore() {
+		var lexer = lexer("12.34_");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	private static Lexer lexer(String source) {
+		return new Lexer(new MemBuf(source));
+	}
+
+	private LexerInvalidNumbersTest() {
+	}
+}
diff --git a/test/moe/anita/mai/lexer/LexerKeywordTest.java b/test/moe/anita/mai/lexer/LexerKeywordTest.java
new file mode 100644
index 0000000..78b7a2c
--- /dev/null
+++ b/test/moe/anita/mai/lexer/LexerKeywordTest.java
@@ -0,0 +1,144 @@
+package moe.anita.mai.lexer;
+
+import moe.anita.mai.buffer.MemBuf;
+import moe.anita.mai.test.Case;
+import moe.anita.mai.test.Runner;
+import moe.anita.mai.token.TokenKind;
+
+@Runner(name = "Lexer Keywords")
+public final class LexerKeywordTest {
+
+	@Case(name = "declaration keywords")
+	static void declarations() {
+		var lexer = lexer("pkg use fn mut");
+
+		assertNext(lexer, TokenKind.PKG);
+		assertNext(lexer, TokenKind.USE);
+		assertNext(lexer, TokenKind.FN);
+		assertNext(lexer, TokenKind.MUT);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "control flow keywords")
+	static void controlFlow() {
+		var lexer = lexer("ret for if else match");
+
+		assertNext(lexer, TokenKind.RET);
+		assertNext(lexer, TokenKind.FOR);
+		assertNext(lexer, TokenKind.IF);
+		assertNext(lexer, TokenKind.ELSE);
+		assertNext(lexer, TokenKind.MATCH);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "type declaration keywords")
+	static void typeDeclarations() {
+		var lexer = lexer("with inlay enum alias");
+
+		assertNext(lexer, TokenKind.WITH);
+		assertNext(lexer, TokenKind.INLAY);
+		assertNext(lexer, TokenKind.ENUM);
+		assertNext(lexer, TokenKind.ALIAS);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "linkage keywords")
+	static void linkage() {
+		var lexer = lexer("link bind ex");
+
+		assertNext(lexer, TokenKind.LINK);
+		assertNext(lexer, TokenKind.BIND);
+		assertNext(lexer, TokenKind.EX);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "literal keywords")
+	static void literals() {
+		var lexer = lexer("true false null");
+
+		assertNext(lexer, TokenKind.TRUE);
+		assertNext(lexer, TokenKind.FALSE);
+		assertNext(lexer, TokenKind.NULL);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "signed integer primitives")
+	static void signedIntegers() {
+		var lexer = lexer("i8 i16 i32 i64 isize");
+
+		assertNext(lexer, TokenKind.I8);
+		assertNext(lexer, TokenKind.I16);
+		assertNext(lexer, TokenKind.I32);
+		assertNext(lexer, TokenKind.I64);
+		assertNext(lexer, TokenKind.ISIZE);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "unsigned integer primitives")
+	static void unsignedIntegers() {
+		var lexer = lexer("u8 u16 u32 u64 usize");
+
+		assertNext(lexer, TokenKind.U8);
+		assertNext(lexer, TokenKind.U16);
+		assertNext(lexer, TokenKind.U32);
+		assertNext(lexer, TokenKind.U64);
+		assertNext(lexer, TokenKind.USIZE);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "floating point primitives")
+	static void floatingPoint() {
+		var lexer = lexer("f32 f64");
+
+		assertNext(lexer, TokenKind.F32);
+		assertNext(lexer, TokenKind.F64);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "other primitives")
+	static void otherPrimitives() {
+		var lexer = lexer("bool rune string void");
+
+		assertNext(lexer, TokenKind.BOOL);
+		assertNext(lexer, TokenKind.RUNE_TYPE);
+		assertNext(lexer, TokenKind.STRING_TYPE);
+		assertNext(lexer, TokenKind.VOID);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "keyword prefixes remain identifiers")
+	static void keywordPrefixes() {
+		var lexer = lexer("function mutable matching stringBuilder usizeOf");
+
+		assertNext(lexer, TokenKind.IDENT);
+		assertNext(lexer, TokenKind.IDENT);
+		assertNext(lexer, TokenKind.IDENT);
+		assertNext(lexer, TokenKind.IDENT);
+		assertNext(lexer, TokenKind.IDENT);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "keywords are case sensitive")
+	static void caseSensitive() {
+		var lexer = lexer("Fn STRING Bool I32");
+
+		assertNext(lexer, TokenKind.IDENT);
+		assertNext(lexer, TokenKind.IDENT);
+		assertNext(lexer, TokenKind.IDENT);
+		assertNext(lexer, TokenKind.IDENT);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	private static void assertNext(Lexer lexer, TokenKind expected) {
+		var token = lexer.next();
+
+		assert token.kind() == expected : "expected " + expected + ", got " + token.kind();
+	}
+
+	private static Lexer lexer(String source) {
+		return new Lexer(new MemBuf(source));
+	}
+
+	private LexerKeywordTest() {
+	}
+}
diff --git a/test/moe/anita/mai/lexer/LexerNumberTest.java b/test/moe/anita/mai/lexer/LexerNumberTest.java
index 1f4ab1f..50147cb 100644
--- a/test/moe/anita/mai/lexer/LexerNumberTest.java
+++ b/test/moe/anita/mai/lexer/LexerNumberTest.java
@@ -69,8 +69,7 @@ public final class LexerNumberTest {
 		var lexer = lexer("10..20");
 
 		assert lexer.next().kind() == TokenKind.INT;
-		assert lexer.next().kind() == TokenKind.DOT;
-		assert lexer.next().kind() == TokenKind.DOT;
+		assert lexer.next().kind() == TokenKind.RANGE;
 		assert lexer.next().kind() == TokenKind.INT;
 		assert lexer.next().kind() == TokenKind.EOF;
 	}
diff --git a/test/moe/anita/mai/lexer/LexerPunctuationTest.java b/test/moe/anita/mai/lexer/LexerPunctuationTest.java
new file mode 100644
index 0000000..d9a6439
--- /dev/null
+++ b/test/moe/anita/mai/lexer/LexerPunctuationTest.java
@@ -0,0 +1,127 @@
+package moe.anita.mai.lexer;
+
+import moe.anita.mai.buffer.MemBuf;
+import moe.anita.mai.test.Case;
+import moe.anita.mai.test.Runner;
+import moe.anita.mai.token.TokenKind;
+
+@Runner(name = "Lexer Punctuation")
+public final class LexerPunctuationTest {
+
+	@Case(name = "delimiters")
+	static void delimiters() {
+		var lexer = lexer("( ) [ ] { }");
+
+		assertNext(lexer, TokenKind.LPAREN);
+		assertNext(lexer, TokenKind.RPAREN);
+		assertNext(lexer, TokenKind.LBRACKET);
+		assertNext(lexer, TokenKind.RBRACKET);
+		assertNext(lexer, TokenKind.LBRACE);
+		assertNext(lexer, TokenKind.RBRACE);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "punctuation")
+	static void punctuation() {
+		var lexer = lexer(", . .. : ; ? @");
+
+		assertNext(lexer, TokenKind.COMMA);
+		assertNext(lexer, TokenKind.DOT);
+		assertNext(lexer, TokenKind.RANGE);
+		assertNext(lexer, TokenKind.COLON);
+		assertNext(lexer, TokenKind.SEMICOLON);
+		assertNext(lexer, TokenKind.QUESTION);
+		assertNext(lexer, TokenKind.AT);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "assignment and declaration")
+	static void assignment() {
+		var lexer = lexer("= :=");
+
+		assertNext(lexer, TokenKind.ASSIGN);
+		assertNext(lexer, TokenKind.DECLARE);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "arithmetic operators")
+	static void arithmetic() {
+		var lexer = lexer("+ - * / % ++ --");
+
+		assertNext(lexer, TokenKind.ADD);
+		assertNext(lexer, TokenKind.SUB);
+		assertNext(lexer, TokenKind.MUL);
+		assertNext(lexer, TokenKind.DIV);
+		assertNext(lexer, TokenKind.MOD);
+		assertNext(lexer, TokenKind.INC);
+		assertNext(lexer, TokenKind.DEC);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "comparison operators")
+	static void comparison() {
+		var lexer = lexer("== <> < <= > >=");
+
+		assertNext(lexer, TokenKind.EQ);
+		assertNext(lexer, TokenKind.NE);
+		assertNext(lexer, TokenKind.LT);
+		assertNext(lexer, TokenKind.LE);
+		assertNext(lexer, TokenKind.GT);
+		assertNext(lexer, TokenKind.GE);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "bitwise operators")
+	static void bitwise() {
+		var lexer = lexer("& | ^ ~");
+
+		assertNext(lexer, TokenKind.BIT_AND);
+		assertNext(lexer, TokenKind.BIT_OR);
+		assertNext(lexer, TokenKind.BIT_XOR);
+		assertNext(lexer, TokenKind.BIT_NOT);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "logical operators")
+	static void logical() {
+		var lexer = lexer("&& || !");
+
+		assertNext(lexer, TokenKind.AND);
+		assertNext(lexer, TokenKind.OR);
+		assertNext(lexer, TokenKind.NOT);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "arrows")
+	static void arrows() {
+		var lexer = lexer("-> <-");
+
+		assertNext(lexer, TokenKind.ARROW);
+		assertNext(lexer, TokenKind.BIND);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	@Case(name = "shifts remain parser constructed")
+	static void shifts() {
+		var lexer = lexer("<< >>");
+
+		assertNext(lexer, TokenKind.LT);
+		assertNext(lexer, TokenKind.LT);
+		assertNext(lexer, TokenKind.GT);
+		assertNext(lexer, TokenKind.GT);
+		assertNext(lexer, TokenKind.EOF);
+	}
+
+	private static void assertNext(Lexer lexer, TokenKind expected) {
+		var token = lexer.next();
+
+		assert token.kind() == expected : "expected " + expected + ", got " + token.kind();
+	}
+
+	private static Lexer lexer(String source) {
+		return new Lexer(new MemBuf(source));
+	}
+
+	private LexerPunctuationTest() {
+	}
+}
diff --git a/test/moe/anita/mai/lexer/LexerRawStringTest.java b/test/moe/anita/mai/lexer/LexerRawStringTest.java
new file mode 100644
index 0000000..7503679
--- /dev/null
+++ b/test/moe/anita/mai/lexer/LexerRawStringTest.java
@@ -0,0 +1,85 @@
+package moe.anita.mai.lexer;
+
+import moe.anita.mai.buffer.MemBuf;
+import moe.anita.mai.test.Case;
+import moe.anita.mai.test.Runner;
+import moe.anita.mai.token.TokenKind;
+
+@Runner(name = "Lexer Raw Strings")
+public final class LexerRawStringTest {
+
+	@Case(name = "raw string")
+	static void rawString() {
+		var lexer = lexer("r\"hello mai\"");
+
+		assert lexer.next().kind() == TokenKind.RAW_STRING;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "empty raw string")
+	static void emptyRawString() {
+		var lexer = lexer("r\"\"");
+
+		assert lexer.next().kind() == TokenKind.RAW_STRING;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "raw string preserves escapes")
+	static void escapes() {
+		var lexer = lexer("r\"hello\\nworld\\t!\"");
+
+		assert lexer.next().kind() == TokenKind.RAW_STRING;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "raw string can contain single quotes")
+	static void singleQuotes() {
+		var lexer = lexer("r\"'hello mai'\"");
+
+		assert lexer.next().kind() == TokenKind.RAW_STRING;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "raw string can span lines")
+	static void multiline() {
+		var lexer = lexer("r\"hello\nmai\"");
+
+		var token = lexer.next();
+
+		assert token.kind() == TokenKind.RAW_STRING;
+		assert token.span().start().line() == 1;
+		assert token.span().end().line() == 2;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "raw string supports Unicode")
+	static void unicode() {
+		var lexer = lexer("r\"猫 😀 λ\"");
+
+		assert lexer.next().kind() == TokenKind.RAW_STRING;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "unterminated raw string")
+	static void unterminated() {
+		var lexer = lexer("r\"hello mai");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "r identifier is not a raw string")
+	static void identifier() {
+		var lexer = lexer("raw");
+
+		assert lexer.next().kind() == TokenKind.IDENT;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	private static Lexer lexer(String source) {
+		return new Lexer(new MemBuf(source));
+	}
+
+	private LexerRawStringTest() {
+	}
+}
diff --git a/test/moe/anita/mai/lexer/LexerSpanTest.java b/test/moe/anita/mai/lexer/LexerSpanTest.java
new file mode 100644
index 0000000..65bb552
--- /dev/null
+++ b/test/moe/anita/mai/lexer/LexerSpanTest.java
@@ -0,0 +1,158 @@
+package moe.anita.mai.lexer;
+
+import moe.anita.mai.buffer.MemBuf;
+import moe.anita.mai.span.Pos;
+import moe.anita.mai.test.Case;
+import moe.anita.mai.test.Runner;
+import moe.anita.mai.token.TokenKind;
+
+@Runner(name = "Lexer Spans")
+public final class LexerSpanTest {
+
+	@Case(name = "ASCII identifier span")
+	static void asciiIdentifier() {
+		var lexer = lexer("hello");
+		var token = lexer.next();
+
+		assert token.kind() == TokenKind.IDENT;
+
+		assert token.span().start().offset() == 0;
+		assert token.span().start().line() == 1;
+		assert token.span().start().column() == 1;
+
+		assert token.span().end().offset() == 5;
+		assert token.span().end().line() == 1;
+		assert token.span().end().column() == 6;
+	}
+
+	@Case(name = "token after whitespace")
+	static void whitespace() {
+		var lexer = lexer("   foo");
+		var token = lexer.next();
+
+		assert token.kind() == TokenKind.IDENT;
+
+		assert token.span().start().offset() == 3;
+		assert token.span().start().line() == 1;
+		assert token.span().start().column() == 4;
+
+		assert token.span().end().offset() == 6;
+		assert token.span().end().line() == 1;
+		assert token.span().end().column() == 7;
+	}
+
+	@Case(name = "newline span")
+	static void newline() {
+		var lexer = lexer("\n");
+		var token = lexer.next();
+
+		assert token.kind() == TokenKind.EOL;
+
+		assert token.span().start().offset() == 0;
+		assert token.span().start().line() == 1;
+		assert token.span().start().column() == 1;
+
+		assert token.span().end().offset() == 1;
+		assert token.span().end().line() == 2;
+		assert token.span().end().column() == 1;
+	}
+
+	@Case(name = "token after newline")
+	static void afterNewline() {
+		var lexer = lexer("\nfoo");
+
+		assert lexer.next().kind() == TokenKind.EOL;
+
+		var token = lexer.next();
+
+		assert token.kind() == TokenKind.IDENT;
+
+		assert token.span().start().offset() == 1;
+		assert token.span().start().line() == 2;
+		assert token.span().start().column() == 1;
+
+		assert token.span().end().offset() == 4;
+		assert token.span().end().line() == 2;
+		assert token.span().end().column() == 4;
+	}
+
+	@Case(name = "BMP Unicode span")
+	static void bmpUnicode() {
+		var lexer = lexer("猫");
+		var token = lexer.next();
+
+		assert token.kind() == TokenKind.IDENT;
+
+		assert token.span().start().offset() == 0;
+		assert token.span().start().column() == 1;
+
+		assert token.span().end().offset() == 3;
+		assert token.span().end().column() == 2;
+	}
+
+	@Case(name = "supplementary Unicode rune span")
+	static void supplementaryRune() {
+		var lexer = lexer("'😀'");
+		var token = lexer.next();
+
+		assert token.kind() == TokenKind.RUNE;
+
+		assert token.span().start().offset() == 0;
+		assert token.span().start().line() == 1;
+		assert token.span().start().column() == 1;
+
+		assert token.span().end().offset() == 6;
+		assert token.span().end().line() == 1;
+		assert token.span().end().column() == 4;
+	}
+
+	@Case(name = "multiline raw string span")
+	static void multilineRawString() {
+		var lexer = lexer("r\"hello\nworld\"");
+		var token = lexer.next();
+
+		assert token.kind() == TokenKind.RAW_STRING;
+
+		assert token.span().start().offset() == 0;
+		assert token.span().start().line() == 1;
+		assert token.span().start().column() == 1;
+
+		assert token.span().end().offset() == 14;
+		assert token.span().end().line() == 2;
+		assert token.span().end().column() == 7;
+	}
+
+	@Case(name = "multibyte text before token")
+	static void multibyteBeforeToken() {
+		var lexer = lexer("猫 foo");
+
+		var first = lexer.next();
+		var second = lexer.next();
+
+		assert first.kind() == TokenKind.IDENT;
+		assert second.kind() == TokenKind.IDENT;
+
+		assert second.span().start().offset() == 4;
+		assert second.span().start().line() == 1;
+		assert second.span().start().column() == 3;
+
+		assert second.span().end().offset() == 7;
+		assert second.span().end().column() == 6;
+	}
+
+	@Case(name = "position packing")
+	static void positionPacking() {
+		var pos = Pos.of(5, 2, 7);
+
+		assert pos.offset() == 5 : "offset = " + pos.offset();
+		assert pos.line() == 2 : "line = " + pos.line();
+		assert pos.column() == 7 : "column = " + pos.column();
+	}
+
+	private static Lexer lexer(String source) {
+		return new Lexer(new MemBuf(source));
+	}
+
+	private LexerSpanTest() {
+	}
+}
diff --git a/test/moe/anita/mai/lexer/LexerStringTest.java b/test/moe/anita/mai/lexer/LexerStringTest.java
new file mode 100644
index 0000000..4707264
--- /dev/null
+++ b/test/moe/anita/mai/lexer/LexerStringTest.java
@@ -0,0 +1,102 @@
+package moe.anita.mai.lexer;
+
+import moe.anita.mai.buffer.MemBuf;
+import moe.anita.mai.test.Case;
+import moe.anita.mai.test.Runner;
+import moe.anita.mai.token.TokenKind;
+
+@Runner(name = "Lexer Strings")
+public final class LexerStringTest {
+
+	@Case(name = "string literal")
+	static void string() {
+		var lexer = lexer("\"hello\"");
+
+		assert lexer.next().kind() == TokenKind.STRING;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "empty string")
+	static void emptyString() {
+		var lexer = lexer("\"\"");
+
+		assert lexer.next().kind() == TokenKind.STRING;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "escaped quote")
+	static void escapedQuote() {
+		var lexer = lexer("\"hello \\\"mai\\\"\"");
+
+		assert lexer.next().kind() == TokenKind.STRING;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "escaped backslash")
+	static void escapedBackslash() {
+		var lexer = lexer("\"foo\\\\bar\"");
+
+		assert lexer.next().kind() == TokenKind.STRING;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "unterminated string")
+	static void unterminatedString() {
+		var lexer = lexer("\"hello");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "string cannot cross newline")
+	static void stringNewline() {
+		var lexer = lexer("\"hello\nworld\"");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+	}
+
+	@Case(name = "rune literal")
+	static void rune() {
+		var lexer = lexer("'a'");
+
+		assert lexer.next().kind() == TokenKind.RUNE;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "escaped rune")
+	static void escapedRune() {
+		var lexer = lexer("'\\n'");
+
+		assert lexer.next().kind() == TokenKind.RUNE;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "quote rune")
+	static void quoteRune() {
+		var lexer = lexer("'\\''");
+
+		assert lexer.next().kind() == TokenKind.RUNE;
+		assert lexer.next().kind() == TokenKind.EOF;
+	}
+
+	@Case(name = "empty rune is invalid")
+	static void emptyRune() {
+		var lexer = lexer("''");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+	}
+
+	@Case(name = "multiple characters are invalid rune")
+	static void multipleRune() {
+		var lexer = lexer("'ab'");
+
+		assert lexer.next().kind() == TokenKind.ERROR;
+	}
+
+	private static Lexer lexer(String source) {
+		return new Lexer(new MemBuf(source));
+	}
+
+	private LexerStringTest() {
+	}
+}
diff --git a/test/moe/anita/mai/test/TestRunner.java b/test/moe/anita/mai/test/TestRunner.java
index 7c74bf3..8103cad 100644
--- a/test/moe/anita/mai/test/TestRunner.java
+++ b/test/moe/anita/mai/test/TestRunner.java
@@ -42,15 +42,11 @@ public final class TestRunner {
 	}
 
 	private static boolean isTopLevelClass(Path path) {
-		return !path.getFileName()
-			.toString()
-			.contains("$");
+		return !path.getFileName().toString().contains("$");
 	}
 
 	private static String className(Path root, Path classFile) {
-		String relative = root
-			.relativize(classFile)
-			.toString();
+		String relative = root.relativize(classFile).toString();
 
 		String name = relative.substring(0, relative.length() - ".class".length()
 		);
@@ -62,8 +58,7 @@ public final class TestRunner {
 		try {
 			Class<?> testClass = Class.forName(className);
 
-			Runner runner =
-				testClass.getAnnotation(Runner.class);
+			Runner runner = testClass.getAnnotation(Runner.class);
 
 			if (runner == null) {
 				return;
@@ -72,21 +67,13 @@ public final class TestRunner {
 			runClass(testClass, runner, result);
 
 		} catch (ClassNotFoundException e) {
-			System.err.printf(
-				"[FAIL] unable to load %s: %s%n",
-				className,
-				e
-			);
+			System.err.printf("[FAIL] unable to load %s: %s%n", className, e);
 
 			result.failed++;
 		}
 	}
 
-	private static void runClass(
-		Class<?> testClass,
-		Runner runner,
-		Result result
-	) {
+	private static void runClass(Class<?> testClass, Runner runner, Result result) {
 		String name = runner.name();
 
 		if (name.isEmpty()) {
@@ -115,21 +102,14 @@ public final class TestRunner {
 		}
 
 		if (!testCase.enabled()) {
-			System.out.printf(
-				"  [SKIP] %s%n",
-				name
-			);
-
+			System.out.printf("  [SKIP] %s%n", name);
 			result.skipped++;
 			return;
 		}
 
 		if (!validCase(method)) {
-			System.err.printf(
-				"  [FAIL] %s: invalid test case%n",
-				name
-			);
-
+			System.out.printf("  [FAIL] %s:%n", name);
+			System.out.println("     invalid test case");
 			result.failed++;
 			return;
 		}
@@ -144,43 +124,30 @@ public final class TestRunner {
 			method.setAccessible(true);
 			method.invoke(instance);
 
-			System.out.printf(
-				"  [PASS] %s%n",
-				name
-			);
-
+			System.out.printf("  [PASS] %s%n", name);
 			result.passed++;
-
 		} catch (InvocationTargetException e) {
 			Throwable cause = e.getCause();
 
-			System.err.printf(
-				"  [FAIL] %s: %s%n",
-				name,
-				cause
-			);
+			System.out.printf("  [FAIL] %s:%n", name);
+			System.out.printf("     %s%n", cause);
 
 			result.failed++;
-
 		} catch (ReflectiveOperationException e) {
-			System.err.printf(
-				"  [FAIL] %s: %s%n",
-				name,
-				e
-			);
+			System.out.printf("  [FAIL] %s:%n", name);
+			System.out.printf("     %s%n", e);
 
 			result.failed++;
 		}
 	}
 
+
 	private static boolean validCase(Method method) {
-		return method.getParameterCount() == 0
-			&& method.getReturnType() == void.class;
+		return method.getParameterCount() == 0 && method.getReturnType() == void.class;
 	}
 
 	private static Object createInstance(Class<?> testClass) throws ReflectiveOperationException {
-		var constructor =
-			testClass.getDeclaredConstructor();
+		var constructor = testClass.getDeclaredConstructor();
 
 		constructor.setAccessible(true);
 
@@ -190,12 +157,7 @@ public final class TestRunner {
 	private static void printSummary(Result result) {
 		System.out.println();
 
-		System.out.printf(
-			"Tests: %d passed, %d failed, %d skipped%n",
-			result.passed,
-			result.failed,
-			result.skipped
-		);
+		System.out.printf("Tests: %d passed, %d failed, %d skipped%n", result.passed, result.failed, result.skipped);
 	}
 
 	private static final class Result {