From 58b937521f3e459089c0d475551bf9a49f930657 Mon Sep 17 00:00:00 2001 From: Tolmachev Igor Date: Fri, 8 May 2026 15:25:55 +0300 Subject: Fold unclosed string error into Token variant UnclosedString was the only error variant, making lexer::Error redundant. Also removes Result from the Iterator impl. --- compiler/src/lexer/error.rs | 18 ----------------- compiler/src/lexer/mod.rs | 31 ++++++++++++++++------------ compiler/src/lexer/tests.rs | 49 +++++++++++++++++---------------------------- 3 files changed, 36 insertions(+), 62 deletions(-) delete mode 100644 compiler/src/lexer/error.rs (limited to 'compiler') diff --git a/compiler/src/lexer/error.rs b/compiler/src/lexer/error.rs deleted file mode 100644 index f251167..0000000 --- a/compiler/src/lexer/error.rs +++ /dev/null @@ -1,18 +0,0 @@ -use std::{error, fmt, result}; - -pub type Result = result::Result; - -#[derive(Debug, PartialEq, Eq)] -pub enum Error { - UnclosedString, -} - -impl fmt::Display for Error { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - Error::UnclosedString => write!(f, "unclosed string literal"), - } - } -} - -impl error::Error for Error {} diff --git a/compiler/src/lexer/mod.rs b/compiler/src/lexer/mod.rs index 2ef4922..ff7d51d 100644 --- a/compiler/src/lexer/mod.rs +++ b/compiler/src/lexer/mod.rs @@ -1,7 +1,4 @@ -mod error; - use crate::span::{Pos, Span}; -pub use error::{Error, Result}; #[cfg(test)] mod tests; @@ -17,6 +14,7 @@ pub enum Token<'a> { Quote, Number(&'a str), String(&'a str), + UnclosedString(&'a str), Symbol(&'a str), } @@ -82,7 +80,7 @@ impl<'a> Lexer<'a> { self.next_while(|ch| !is_terminator(ch)) } - fn next_string(&mut self) -> Result<&'a str> { + fn next_string(&mut self) -> Result<&'a str, &'a str> { debug_assert_eq!(self.peek(), Some('"')); self.consume(); @@ -95,7 +93,11 @@ impl<'a> Lexer<'a> { self.consume(); return Ok(string); } - '\n' => return Err(Error::UnclosedString), + '\n' => { + let string = &self.input[start..self.cursor]; + self.consume(); + return Err(string); + } '\\' => { self.consume(); self.consume(); @@ -106,12 +108,12 @@ impl<'a> Lexer<'a> { } } - Err(Error::UnclosedString) + Err(&self.input[start..self.cursor]) } } impl<'a> Iterator for Lexer<'a> { - type Item = Span>>; + type Item = Span>; fn next(&mut self) -> Option { loop { @@ -131,15 +133,15 @@ impl<'a> Iterator for Lexer<'a> { let token = match self.peek()? { '(' => { self.consume(); - Ok(Token::LeftPar) + Token::LeftPar } ')' => { self.consume(); - Ok(Token::RightPar) + Token::RightPar } '\'' => { self.consume(); - Ok(Token::Quote) + Token::Quote } // Number @@ -147,14 +149,17 @@ impl<'a> Iterator for Lexer<'a> { || matches!(ch, '+' | '-' | '.') && self.peek_nth(1).is_some_and(|ch| ch.is_ascii_digit()) => { - Ok(Token::Number(self.next_atom())) + Token::Number(self.next_atom()) } // String - '"' => self.next_string().map(Token::String), + '"' => match self.next_string() { + Ok(string) => Token::String(string), + Err(string) => Token::UnclosedString(string), + }, // Symbol - _ => Ok(Token::Symbol(self.next_atom())), + _ => Token::Symbol(self.next_atom()), }; let end = Pos::new(self.line, self.column, self.cursor); diff --git a/compiler/src/lexer/tests.rs b/compiler/src/lexer/tests.rs index 65dd2f2..30be85a 100644 --- a/compiler/src/lexer/tests.rs +++ b/compiler/src/lexer/tests.rs @@ -4,7 +4,7 @@ use super::Token::*; use super::*; fn tokenize<'a>(input: &'a str) -> Vec> { - Lexer::new(input).map(|s| s.into_inner().unwrap()).collect() + Lexer::new(input).map(|s| s.into_inner()).collect() } #[test] @@ -102,6 +102,22 @@ fn test_string_escapes() { } } +#[test] +fn test_unclosed_strings() { + let cases = vec![ + (r#""abc"#, vec![UnclosedString("abc")]), + (r#""abc\""#, vec![UnclosedString(r#"abc\""#)]), + ("\"abc\n", vec![UnclosedString("abc")]), + ("\"abc\\\ndef", vec![UnclosedString("abc\\\ndef")]), + ("\"abc\n\"def\"", vec![UnclosedString("abc"), String("def")]), + (r#"""#, vec![UnclosedString("")]), + ("\"\n\"", vec![UnclosedString(""), UnclosedString("")]), + ]; + for (code, tokens) in cases { + assert_eq!(tokenize(code), tokens); + } +} + #[test] fn test_symbols() { let cases = vec![ @@ -272,35 +288,6 @@ fn test_comments() { } } -fn first_error(input: &str) -> Error { - Lexer::new(input) - .find_map(|s| s.into_inner().err()) - .expect("error expected") -} - -#[test] -fn test_unclosed_string_at_eof() { - assert_eq!(first_error(r#""abc"#), Error::UnclosedString); - assert_eq!(first_error(r#"""#), Error::UnclosedString); -} - -#[test] -fn test_unclosed_string_with_trailing_escape() { - assert_eq!(first_error("\"abc\\"), Error::UnclosedString); -} - -#[test] -fn test_unclosed_string_with_newline() { - assert_eq!(first_error("\"abc\ndef\""), Error::UnclosedString); -} - -#[test] -fn test_lexer_stops_after_string_error() { - let mut lex = Lexer::new(r#""abc"#); - assert!(lex.next().unwrap().into_inner().is_err()); - assert!(lex.next().is_none()); -} - fn spans(input: &str) -> Vec<(Pos, Pos)> { Lexer::new(input).map(|s| (s.start(), s.end())).collect() } @@ -324,7 +311,7 @@ fn test_span_after_newline() { } #[test] -fn test_span_multi_char_() { +fn test_span_multi_char() { let s = spans("foo"); assert_eq!(s, vec![(Pos::new(1, 0, 0), Pos::new(1, 3, 3))]); } -- cgit v1.3