1 files changed, 372 insertions, 0 deletions

diff --git a/compiler/src/lexer/tests.rs b/compiler/src/lexer/tests.rs
new file mode 100644
index 0000000..65dd2f2
--- /dev/null
+++ b/compiler/src/lexer/tests.rs
@@ -0,0 +1,372 @@
+use crate::span::Pos;
+
+use super::Token::*;
+use super::*;
+
+fn tokenize<'a>(input: &'a str) -> Vec<Token<'a>> {
+    Lexer::new(input).map(|s| s.into_inner().unwrap()).collect()
+}
+
+#[test]
+fn test_spaces() {
+    let cases = vec![
+        ("", vec![]),
+        ("        ", vec![]),
+        ("\n", vec![]),
+        ("\t\n  \r\n", vec![]),
+    ];
+    for (code, tokens) in cases {
+        assert_eq!(tokenize(code), tokens);
+    }
+}
+
+#[test]
+fn test_parens() {
+    let cases = vec![
+        ("()", vec![LeftPar, RightPar]),
+        ("( )", vec![LeftPar, RightPar]),
+        ("(())", vec![LeftPar, LeftPar, RightPar, RightPar]),
+        (
+            "((()))",
+            vec![LeftPar, LeftPar, LeftPar, RightPar, RightPar, RightPar],
+        ),
+        (")(", vec![RightPar, LeftPar]),
+    ];
+    for (code, tokens) in cases {
+        assert_eq!(tokenize(code), tokens);
+    }
+}
+
+#[test]
+fn test_quote() {
+    let cases = vec![
+        ("'", vec![Quote]),
+        ("'a", vec![Quote, Symbol("a")]),
+        ("''a", vec![Quote, Quote, Symbol("a")]),
+        ("'()", vec![Quote, LeftPar, RightPar]),
+        (
+            "'(1 2)",
+            vec![Quote, LeftPar, Number("1"), Number("2"), RightPar],
+        ),
+        ("(' )", vec![LeftPar, Quote, RightPar]),
+    ];
+    for (code, tokens) in cases {
+        assert_eq!(tokenize(code), tokens);
+    }
+}
+
+#[test]
+fn test_numbers() {
+    let cases = vec![
+        ("0", vec![Number("0")]),
+        ("42", vec![Number("42")]),
+        ("3.14", vec![Number("3.14")]),
+        ("-7", vec![Number("-7")]),
+        ("+5", vec![Number("+5")]),
+        ("-0.5", vec![Number("-0.5")]),
+        ("1e10", vec![Number("1e10")]),
+        ("1.5e-3", vec![Number("1.5e-3")]),
+        (".5", vec![Number(".5")]),
+    ];
+    for (code, tokens) in cases {
+        assert_eq!(tokenize(code), tokens);
+    }
+}
+
+#[test]
+fn test_strings() {
+    let cases = vec![
+        (r#""""#, vec![String("")]),
+        (r#""hello""#, vec![String("hello")]),
+        (r#""hello world""#, vec![String("hello world")]),
+        (r#""(not a list)""#, vec![String("(not a list)")]),
+        (r#""'not a quote""#, vec![String("'not a quote")]),
+        (r#""; not a comment""#, vec![String("; not a comment")]),
+        (r#""  spaces  ""#, vec![String("  spaces  ")]),
+    ];
+    for (code, tokens) in cases {
+        assert_eq!(tokenize(code), tokens);
+    }
+}
+
+#[test]
+fn test_string_escapes() {
+    let cases = vec![
+        (r#""line\nbreak""#, vec![String(r"line\nbreak")]),
+        (r#""with \"quotes\"""#, vec![String(r#"with \"quotes\""#)]),
+        (r#""\\""#, vec![String(r"\\")]),
+        ("\"single\\\nline\"", vec![String("single\\\nline")]),
+    ];
+    for (code, tokens) in cases {
+        assert_eq!(tokenize(code), tokens);
+    }
+}
+
+#[test]
+fn test_symbols() {
+    let cases = vec![
+        ("foo", vec![Symbol("foo")]),
+        ("foo-bar", vec![Symbol("foo-bar")]),
+        ("foo!", vec![Symbol("foo!")]),
+        ("empty?", vec![Symbol("empty?")]),
+        ("set!", vec![Symbol("set!")]),
+        ("->", vec![Symbol("->")]),
+        ("+", vec![Symbol("+")]),
+        ("-", vec![Symbol("-")]),
+        ("*", vec![Symbol("*")]),
+        ("/", vec![Symbol("/")]),
+        ("=", vec![Symbol("=")]),
+        ("<=", vec![Symbol("<=")]),
+        (">=", vec![Symbol(">=")]),
+        ("a1b2", vec![Symbol("a1b2")]),
+        ("x", vec![Symbol("x")]),
+    ];
+    for (code, tokens) in cases {
+        assert_eq!(tokenize(code), tokens);
+    }
+}
+
+#[test]
+fn test_ambiguous() {
+    let cases = vec![
+        ("-x", vec![Symbol("-x")]),
+        ("+foo", vec![Symbol("+foo")]),
+        ("...", vec![Symbol("...")]),
+        (".foo", vec![Symbol(".foo")]),
+    ];
+    for (code, tokens) in cases {
+        assert_eq!(tokenize(code), tokens);
+    }
+}
+
+#[test]
+fn test_no_separators() {
+    let cases = vec![
+        ("(foo)", vec![LeftPar, Symbol("foo"), RightPar]),
+        ("(1)", vec![LeftPar, Number("1"), RightPar]),
+        ("(a)b", vec![LeftPar, Symbol("a"), RightPar, Symbol("b")]),
+        ("'(a)", vec![Quote, LeftPar, Symbol("a"), RightPar]),
+        (r#"("s")"#, vec![LeftPar, String("s"), RightPar]),
+    ];
+    for (code, tokens) in cases {
+        assert_eq!(tokenize(code), tokens);
+    }
+}
+
+#[test]
+fn test_whitespace_separators() {
+    let cases = vec![
+        (
+            "(\n  foo\n  bar\n)",
+            vec![LeftPar, Symbol("foo"), Symbol("bar"), RightPar],
+        ),
+        (
+            "(\tfoo\tbar\t)",
+            vec![LeftPar, Symbol("foo"), Symbol("bar"), RightPar],
+        ),
+    ];
+    for (code, tokens) in cases {
+        assert_eq!(tokenize(code), tokens);
+    }
+}
+
+#[test]
+fn test_expressions() {
+    let cases = vec![
+        (
+            "(define x 42)",
+            vec![
+                LeftPar,
+                Symbol("define"),
+                Symbol("x"),
+                Number("42"),
+                RightPar,
+            ],
+        ),
+        (
+            "(+ 1 2)",
+            vec![LeftPar, Symbol("+"), Number("1"), Number("2"), RightPar],
+        ),
+        (
+            "(if (= x 0) 'zero 'nonzero)",
+            vec![
+                LeftPar,
+                Symbol("if"),
+                LeftPar,
+                Symbol("="),
+                Symbol("x"),
+                Number("0"),
+                RightPar,
+                Quote,
+                Symbol("zero"),
+                Quote,
+                Symbol("nonzero"),
+                RightPar,
+            ],
+        ),
+        (
+            r#"(print "hello, world")"#,
+            vec![LeftPar, Symbol("print"), String("hello, world"), RightPar],
+        ),
+        (
+            "(lambda (x) (* x x))",
+            vec![
+                LeftPar,
+                Symbol("lambda"),
+                LeftPar,
+                Symbol("x"),
+                RightPar,
+                LeftPar,
+                Symbol("*"),
+                Symbol("x"),
+                Symbol("x"),
+                RightPar,
+                RightPar,
+            ],
+        ),
+        (
+            "'(1 2 3)",
+            vec![
+                Quote,
+                LeftPar,
+                Number("1"),
+                Number("2"),
+                Number("3"),
+                RightPar,
+            ],
+        ),
+    ];
+    for (code, tokens) in cases {
+        assert_eq!(tokenize(code), tokens);
+    }
+}
+
+#[test]
+fn test_comments() {
+    let cases = vec![
+        (";", vec![]),
+        (";\n", vec![]),
+        ("; comment", vec![]),
+        ("; comment\n", vec![]),
+        ("; comment\n42", vec![Number("42")]),
+        ("42 ; comment", vec![Number("42")]),
+        ("42; comment", vec![Number("42")]),
+        (
+            "(+ 1 2) ; calc\n(- 3 4)",
+            vec![
+                LeftPar,
+                Symbol("+"),
+                Number("1"),
+                Number("2"),
+                RightPar,
+                LeftPar,
+                Symbol("-"),
+                Number("3"),
+                Number("4"),
+                RightPar,
+            ],
+        ),
+    ];
+    for (code, tokens) in cases {
+        assert_eq!(tokenize(code), tokens);
+    }
+}
+
+fn first_error(input: &str) -> Error {
+    Lexer::new(input)
+        .find_map(|s| s.into_inner().err())
+        .expect("error expected")
+}
+
+#[test]
+fn test_unclosed_string_at_eof() {
+    assert_eq!(first_error(r#""abc"#), Error::UnclosedString);
+    assert_eq!(first_error(r#"""#), Error::UnclosedString);
+}
+
+#[test]
+fn test_unclosed_string_with_trailing_escape() {
+    assert_eq!(first_error("\"abc\\"), Error::UnclosedString);
+}
+
+#[test]
+fn test_unclosed_string_with_newline() {
+    assert_eq!(first_error("\"abc\ndef\""), Error::UnclosedString);
+}
+
+#[test]
+fn test_lexer_stops_after_string_error() {
+    let mut lex = Lexer::new(r#""abc"#);
+    assert!(lex.next().unwrap().into_inner().is_err());
+    assert!(lex.next().is_none());
+}
+
+fn spans(input: &str) -> Vec<(Pos, Pos)> {
+    Lexer::new(input).map(|s| (s.start(), s.end())).collect()
+}
+
+#[test]
+fn test_span_single_char() {
+    let s = spans("(");
+    assert_eq!(s, vec![(Pos::new(1, 0, 0), Pos::new(1, 1, 1))]);
+}
+
+#[test]
+fn test_span_after_leading_whitespace() {
+    let s = spans("   (");
+    assert_eq!(s, vec![(Pos::new(1, 3, 3), Pos::new(1, 4, 4))]);
+}
+
+#[test]
+fn test_span_after_newline() {
+    let s = spans("\n(");
+    assert_eq!(s, vec![(Pos::new(2, 0, 1), Pos::new(2, 1, 2))]);
+}
+
+#[test]
+fn test_span_multi_char_() {
+    let s = spans("foo");
+    assert_eq!(s, vec![(Pos::new(1, 0, 0), Pos::new(1, 3, 3))]);
+}
+
+#[test]
+fn test_span_string() {
+    let s = spans(r#""hi""#);
+    assert_eq!(s, vec![(Pos::new(1, 0, 0), Pos::new(1, 4, 4))]);
+}
+
+#[test]
+fn test_span_sequence() {
+    // (foo 42)
+    // 012345678
+    let s = spans("(foo 42)");
+    assert_eq!(
+        s,
+        vec![
+            (Pos::new(1, 0, 0), Pos::new(1, 1, 1)), // (
+            (Pos::new(1, 1, 1), Pos::new(1, 4, 4)), // foo
+            (Pos::new(1, 5, 5), Pos::new(1, 7, 7)), // 42
+            (Pos::new(1, 7, 7), Pos::new(1, 8, 8)), // )
+        ],
+    );
+}
+
+#[test]
+fn test_span_lines() {
+    let s = spans("foo\nbar");
+    assert_eq!(
+        s,
+        vec![
+            (Pos::new(1, 0, 0), Pos::new(1, 3, 3)),
+            (Pos::new(2, 0, 4), Pos::new(2, 3, 7)),
+        ],
+    );
+}
+
+#[test]
+fn test_span_after_comment() {
+    // ; cm\nfoo
+    // 01234 5678
+    let s = spans("; cm\nfoo");
+    assert_eq!(s, vec![(Pos::new(2, 0, 5), Pos::new(2, 3, 8))]);
+}