use crate::{ lex::{Lexer, Token, Token::*}, span::Pos, }; fn tokenize<'a>(input: &'a str) -> Vec> { Lexer::new(input).map(|s| s.inner).collect() } #[test] fn test_spaces() { let cases = vec![ ("", vec![]), (" ", vec![]), ("\n", vec![]), ("\t\n \r\n", vec![]), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_parens() { let cases = vec![ ("()", vec![LeftPar, RightPar]), ("( )", vec![LeftPar, RightPar]), ("(())", vec![LeftPar, LeftPar, RightPar, RightPar]), ( "((()))", vec![LeftPar, LeftPar, LeftPar, RightPar, RightPar, RightPar], ), (")(", vec![RightPar, LeftPar]), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_numbers() { let cases = vec![ ("0", vec![Number("0")]), ("42", vec![Number("42")]), ("-7", vec![Number("-7")]), ("+5", vec![Number("+5")]), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_strings() { let cases = vec![ (r#""""#, vec![String("")]), (r#""hello""#, vec![String("hello")]), (r#""hello world""#, vec![String("hello world")]), (r#""(not a list)""#, vec![String("(not a list)")]), (r#""; not a comment""#, vec![String("; not a comment")]), (r#"" spaces ""#, vec![String(" spaces ")]), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_string_escapes() { let cases = vec![ (r#""line\nbreak""#, vec![String(r"line\nbreak")]), (r#""\\""#, vec![String(r"\\")]), ("\"single\\\nline\"", vec![String("single\\\nline")]), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_unclosed_strings() { let cases = vec![ (r#""abc"#, vec![UnclosedString("abc")]), (r#""abc\""#, vec![UnclosedString(r#"abc\""#)]), ("\"abc\n", vec![UnclosedString("abc")]), ("\"abc\\\ndef", vec![UnclosedString("abc\\\ndef")]), ("\"abc\n\"def\"", vec![UnclosedString("abc"), String("def")]), (r#"""#, vec![UnclosedString("")]), ("\"\n\"", vec![UnclosedString(""), UnclosedString("")]), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_symbols() { let cases = vec![ ("foo", vec![Symbol("foo")]), ("foo-bar", vec![Symbol("foo-bar")]), ("foo!", vec![Symbol("foo!")]), ("empty?", vec![Symbol("empty?")]), ("set!", vec![Symbol("set!")]), ("->", vec![Symbol("->")]), ("+", vec![Symbol("+")]), ("-", vec![Symbol("-")]), ("*", vec![Symbol("*")]), ("/", vec![Symbol("/")]), ("=", vec![Symbol("=")]), ("<=", vec![Symbol("<=")]), (">=", vec![Symbol(">=")]), ("a1b2", vec![Symbol("a1b2")]), ("x", vec![Symbol("x")]), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_ambiguous() { let cases = vec![ ("-x", vec![Symbol("-x")]), ("+foo", vec![Symbol("+foo")]), ("...", vec![Symbol("...")]), (".foo", vec![Symbol(".foo")]), ("-.", vec![Symbol("-.")]), ("+.", vec![Symbol("+.")]), (".", vec![Symbol(".")]), ("+.a", vec![Symbol("+.a")]), ("-.a", vec![Symbol("-.a")]), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_no_separators() { let cases = vec![ ("(foo)", vec![LeftPar, Symbol("foo"), RightPar]), ("(1)", vec![LeftPar, Number("1"), RightPar]), ("(a)b", vec![LeftPar, Symbol("a"), RightPar, Symbol("b")]), (r#"("s")"#, vec![LeftPar, String("s"), RightPar]), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_whitespace_separators() { let cases = vec![ ( "(\n foo\n bar\n)", vec![LeftPar, Symbol("foo"), Symbol("bar"), RightPar], ), ( "(\tfoo\tbar\t)", vec![LeftPar, Symbol("foo"), Symbol("bar"), RightPar], ), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_expressions() { let cases = vec![ ( "(define x 42)", vec![ LeftPar, Symbol("define"), Symbol("x"), Number("42"), RightPar, ], ), ( "(+ 1 2)", vec![LeftPar, Symbol("+"), Number("1"), Number("2"), RightPar], ), ( r#"(print "hello, world")"#, vec![LeftPar, Symbol("print"), String("hello, world"), RightPar], ), ( "(lambda (x) (* x x))", vec![ LeftPar, Symbol("lambda"), LeftPar, Symbol("x"), RightPar, LeftPar, Symbol("*"), Symbol("x"), Symbol("x"), RightPar, RightPar, ], ), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_comments() { let cases = vec![ (";", vec![]), (";\n", vec![]), ("; comment", vec![]), ("; comment\n", vec![]), ("; comment\n42", vec![Number("42")]), ("42 ; comment", vec![Number("42")]), ("42; comment", vec![Number("42")]), ( "(+ 1 2) ; calc\n(- 3 4)", vec![ LeftPar, Symbol("+"), Number("1"), Number("2"), RightPar, LeftPar, Symbol("-"), Number("3"), Number("4"), RightPar, ], ), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } fn spans(input: &str) -> Vec<(Pos, Pos)> { Lexer::new(input) .map(|s| (s.span.start, s.span.end)) .collect() } #[test] fn test_span_single_char() { let s = spans("("); assert_eq!(s, vec![(Pos::new(1, 0, 0), Pos::new(1, 1, 1))]); } #[test] fn test_span_after_leading_whitespace() { let s = spans(" ("); assert_eq!(s, vec![(Pos::new(1, 3, 3), Pos::new(1, 4, 4))]); } #[test] fn test_span_after_newline() { let s = spans("\n("); assert_eq!(s, vec![(Pos::new(2, 0, 1), Pos::new(2, 1, 2))]); } #[test] fn test_span_multi_char() { let s = spans("foo"); assert_eq!(s, vec![(Pos::new(1, 0, 0), Pos::new(1, 3, 3))]); } #[test] fn test_span_string() { let s = spans(r#""hi""#); assert_eq!(s, vec![(Pos::new(1, 0, 0), Pos::new(1, 4, 4))]); } #[test] fn test_span_sequence() { // (foo 42) // 012345678 let s = spans("(foo 42)"); assert_eq!( s, vec![ (Pos::new(1, 0, 0), Pos::new(1, 1, 1)), // ( (Pos::new(1, 1, 1), Pos::new(1, 4, 4)), // foo (Pos::new(1, 5, 5), Pos::new(1, 7, 7)), // 42 (Pos::new(1, 7, 7), Pos::new(1, 8, 8)), // ) ], ); } #[test] fn test_span_lines() { let s = spans("foo\nbar"); assert_eq!( s, vec![ (Pos::new(1, 0, 0), Pos::new(1, 3, 3)), (Pos::new(2, 0, 4), Pos::new(2, 3, 7)), ], ); } #[test] fn test_span_after_comment() { // ; cm\nfoo // 01234 5678 let s = spans("; cm\nfoo"); assert_eq!(s, vec![(Pos::new(2, 0, 5), Pos::new(2, 3, 8))]); }