use crate::span::Pos; use super::Token::*; use super::*; fn tokenize<'a>(input: &'a str) -> Vec> { Lexer::new(input).map(|s| s.into_inner().unwrap()).collect() } #[test] fn test_spaces() { let cases = vec![ ("", vec![]), (" ", vec![]), ("\n", vec![]), ("\t\n \r\n", vec![]), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_parens() { let cases = vec![ ("()", vec![LeftPar, RightPar]), ("( )", vec![LeftPar, RightPar]), ("(())", vec![LeftPar, LeftPar, RightPar, RightPar]), ( "((()))", vec![LeftPar, LeftPar, LeftPar, RightPar, RightPar, RightPar], ), (")(", vec![RightPar, LeftPar]), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_quote() { let cases = vec![ ("'", vec![Quote]), ("'a", vec![Quote, Symbol("a")]), ("''a", vec![Quote, Quote, Symbol("a")]), ("'()", vec![Quote, LeftPar, RightPar]), ( "'(1 2)", vec![Quote, LeftPar, Number("1"), Number("2"), RightPar], ), ("(' )", vec![LeftPar, Quote, RightPar]), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_numbers() { let cases = vec![ ("0", vec![Number("0")]), ("42", vec![Number("42")]), ("3.14", vec![Number("3.14")]), ("-7", vec![Number("-7")]), ("+5", vec![Number("+5")]), ("-0.5", vec![Number("-0.5")]), ("1e10", vec![Number("1e10")]), ("1.5e-3", vec![Number("1.5e-3")]), (".5", vec![Number(".5")]), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_strings() { let cases = vec![ (r#""""#, vec![String("")]), (r#""hello""#, vec![String("hello")]), (r#""hello world""#, vec![String("hello world")]), (r#""(not a list)""#, vec![String("(not a list)")]), (r#""'not a quote""#, vec![String("'not a quote")]), (r#""; not a comment""#, vec![String("; not a comment")]), (r#"" spaces ""#, vec![String(" spaces ")]), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_string_escapes() { let cases = vec![ (r#""line\nbreak""#, vec![String(r"line\nbreak")]), (r#""with \"quotes\"""#, vec![String(r#"with \"quotes\""#)]), (r#""\\""#, vec![String(r"\\")]), ("\"single\\\nline\"", vec![String("single\\\nline")]), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_symbols() { let cases = vec![ ("foo", vec![Symbol("foo")]), ("foo-bar", vec![Symbol("foo-bar")]), ("foo!", vec![Symbol("foo!")]), ("empty?", vec![Symbol("empty?")]), ("set!", vec![Symbol("set!")]), ("->", vec![Symbol("->")]), ("+", vec![Symbol("+")]), ("-", vec![Symbol("-")]), ("*", vec![Symbol("*")]), ("/", vec![Symbol("/")]), ("=", vec![Symbol("=")]), ("<=", vec![Symbol("<=")]), (">=", vec![Symbol(">=")]), ("a1b2", vec![Symbol("a1b2")]), ("x", vec![Symbol("x")]), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_ambiguous() { let cases = vec![ ("-x", vec![Symbol("-x")]), ("+foo", vec![Symbol("+foo")]), ("...", vec![Symbol("...")]), (".foo", vec![Symbol(".foo")]), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_no_separators() { let cases = vec![ ("(foo)", vec![LeftPar, Symbol("foo"), RightPar]), ("(1)", vec![LeftPar, Number("1"), RightPar]), ("(a)b", vec![LeftPar, Symbol("a"), RightPar, Symbol("b")]), ("'(a)", vec![Quote, LeftPar, Symbol("a"), RightPar]), (r#"("s")"#, vec![LeftPar, String("s"), RightPar]), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_whitespace_separators() { let cases = vec![ ( "(\n foo\n bar\n)", vec![LeftPar, Symbol("foo"), Symbol("bar"), RightPar], ), ( "(\tfoo\tbar\t)", vec![LeftPar, Symbol("foo"), Symbol("bar"), RightPar], ), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_expressions() { let cases = vec![ ( "(define x 42)", vec![ LeftPar, Symbol("define"), Symbol("x"), Number("42"), RightPar, ], ), ( "(+ 1 2)", vec![LeftPar, Symbol("+"), Number("1"), Number("2"), RightPar], ), ( "(if (= x 0) 'zero 'nonzero)", vec![ LeftPar, Symbol("if"), LeftPar, Symbol("="), Symbol("x"), Number("0"), RightPar, Quote, Symbol("zero"), Quote, Symbol("nonzero"), RightPar, ], ), ( r#"(print "hello, world")"#, vec![LeftPar, Symbol("print"), String("hello, world"), RightPar], ), ( "(lambda (x) (* x x))", vec![ LeftPar, Symbol("lambda"), LeftPar, Symbol("x"), RightPar, LeftPar, Symbol("*"), Symbol("x"), Symbol("x"), RightPar, RightPar, ], ), ( "'(1 2 3)", vec![ Quote, LeftPar, Number("1"), Number("2"), Number("3"), RightPar, ], ), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } #[test] fn test_comments() { let cases = vec![ (";", vec![]), (";\n", vec![]), ("; comment", vec![]), ("; comment\n", vec![]), ("; comment\n42", vec![Number("42")]), ("42 ; comment", vec![Number("42")]), ("42; comment", vec![Number("42")]), ( "(+ 1 2) ; calc\n(- 3 4)", vec![ LeftPar, Symbol("+"), Number("1"), Number("2"), RightPar, LeftPar, Symbol("-"), Number("3"), Number("4"), RightPar, ], ), ]; for (code, tokens) in cases { assert_eq!(tokenize(code), tokens); } } fn first_error(input: &str) -> Error { Lexer::new(input) .find_map(|s| s.into_inner().err()) .expect("error expected") } #[test] fn test_unclosed_string_at_eof() { assert_eq!(first_error(r#""abc"#), Error::UnclosedString); assert_eq!(first_error(r#"""#), Error::UnclosedString); } #[test] fn test_unclosed_string_with_trailing_escape() { assert_eq!(first_error("\"abc\\"), Error::UnclosedString); } #[test] fn test_unclosed_string_with_newline() { assert_eq!(first_error("\"abc\ndef\""), Error::UnclosedString); } #[test] fn test_lexer_stops_after_string_error() { let mut lex = Lexer::new(r#""abc"#); assert!(lex.next().unwrap().into_inner().is_err()); assert!(lex.next().is_none()); } fn spans(input: &str) -> Vec<(Pos, Pos)> { Lexer::new(input).map(|s| (s.start(), s.end())).collect() } #[test] fn test_span_single_char() { let s = spans("("); assert_eq!(s, vec![(Pos::new(1, 0, 0), Pos::new(1, 1, 1))]); } #[test] fn test_span_after_leading_whitespace() { let s = spans(" ("); assert_eq!(s, vec![(Pos::new(1, 3, 3), Pos::new(1, 4, 4))]); } #[test] fn test_span_after_newline() { let s = spans("\n("); assert_eq!(s, vec![(Pos::new(2, 0, 1), Pos::new(2, 1, 2))]); } #[test] fn test_span_multi_char_() { let s = spans("foo"); assert_eq!(s, vec![(Pos::new(1, 0, 0), Pos::new(1, 3, 3))]); } #[test] fn test_span_string() { let s = spans(r#""hi""#); assert_eq!(s, vec![(Pos::new(1, 0, 0), Pos::new(1, 4, 4))]); } #[test] fn test_span_sequence() { // (foo 42) // 012345678 let s = spans("(foo 42)"); assert_eq!( s, vec![ (Pos::new(1, 0, 0), Pos::new(1, 1, 1)), // ( (Pos::new(1, 1, 1), Pos::new(1, 4, 4)), // foo (Pos::new(1, 5, 5), Pos::new(1, 7, 7)), // 42 (Pos::new(1, 7, 7), Pos::new(1, 8, 8)), // ) ], ); } #[test] fn test_span_lines() { let s = spans("foo\nbar"); assert_eq!( s, vec![ (Pos::new(1, 0, 0), Pos::new(1, 3, 3)), (Pos::new(2, 0, 4), Pos::new(2, 3, 7)), ], ); } #[test] fn test_span_after_comment() { // ; cm\nfoo // 01234 5678 let s = spans("; cm\nfoo"); assert_eq!(s, vec![(Pos::new(2, 0, 5), Pos::new(2, 3, 8))]); }