diff options
Diffstat (limited to 'compiler/src/lexer/mod.rs')
| -rw-r--r-- | compiler/src/lexer/mod.rs | 172 |
1 files changed, 0 insertions, 172 deletions
diff --git a/compiler/src/lexer/mod.rs b/compiler/src/lexer/mod.rs deleted file mode 100644 index f3c8b76..0000000 --- a/compiler/src/lexer/mod.rs +++ /dev/null | |||
| @@ -1,172 +0,0 @@ | |||
| 1 | use crate::span::{Pos, Span, Spanned}; | ||
| 2 | |||
| 3 | #[cfg(test)] | ||
| 4 | mod tests; | ||
| 5 | |||
| 6 | fn is_terminator(ch: char) -> bool { | ||
| 7 | ch.is_whitespace() || matches!(ch, '(' | ')' | '\'' | '"' | ';') | ||
| 8 | } | ||
| 9 | |||
| 10 | #[derive(Clone, Copy, Debug, PartialEq, Eq)] | ||
| 11 | pub enum Token<'a> { | ||
| 12 | LeftPar, | ||
| 13 | RightPar, | ||
| 14 | Quote, | ||
| 15 | Number(&'a str), | ||
| 16 | String(&'a str), | ||
| 17 | UnclosedString(&'a str), | ||
| 18 | Symbol(&'a str), | ||
| 19 | } | ||
| 20 | |||
| 21 | pub struct Lexer<'a> { | ||
| 22 | input: &'a str, | ||
| 23 | cursor: usize, | ||
| 24 | |||
| 25 | line: usize, | ||
| 26 | column: usize, | ||
| 27 | } | ||
| 28 | |||
| 29 | impl<'a> Lexer<'a> { | ||
| 30 | pub fn new(input: &'a str) -> Self { | ||
| 31 | Self { | ||
| 32 | input, | ||
| 33 | cursor: 0, | ||
| 34 | |||
| 35 | line: 1, | ||
| 36 | column: 0, | ||
| 37 | } | ||
| 38 | } | ||
| 39 | |||
| 40 | fn rest(&self) -> &str { | ||
| 41 | &self.input[self.cursor..] | ||
| 42 | } | ||
| 43 | |||
| 44 | fn peek(&self) -> Option<char> { | ||
| 45 | self.rest().chars().next() | ||
| 46 | } | ||
| 47 | |||
| 48 | fn peek_nth(&self, n: usize) -> Option<char> { | ||
| 49 | self.rest().chars().nth(n) | ||
| 50 | } | ||
| 51 | |||
| 52 | fn consume(&mut self) -> Option<char> { | ||
| 53 | let ch = self.peek()?; | ||
| 54 | |||
| 55 | self.cursor += ch.len_utf8(); | ||
| 56 | if ch == '\n' { | ||
| 57 | self.line += 1; | ||
| 58 | self.column = 0; | ||
| 59 | } else { | ||
| 60 | self.column += 1; | ||
| 61 | } | ||
| 62 | |||
| 63 | Some(ch) | ||
| 64 | } | ||
| 65 | |||
| 66 | fn next_while(&mut self, mut predicate: impl FnMut(char) -> bool) -> &'a str { | ||
| 67 | let start = self.cursor; | ||
| 68 | |||
| 69 | while let Some(ch) = self.peek() { | ||
| 70 | if !predicate(ch) { | ||
| 71 | break; | ||
| 72 | } | ||
| 73 | self.consume(); | ||
| 74 | } | ||
| 75 | |||
| 76 | &self.input[start..self.cursor] | ||
| 77 | } | ||
| 78 | |||
| 79 | fn next_atom(&mut self) -> &'a str { | ||
| 80 | self.next_while(|ch| !is_terminator(ch)) | ||
| 81 | } | ||
| 82 | |||
| 83 | fn next_string(&mut self) -> Result<&'a str, &'a str> { | ||
| 84 | debug_assert_eq!(self.peek(), Some('"')); | ||
| 85 | self.consume(); | ||
| 86 | |||
| 87 | let start = self.cursor; | ||
| 88 | |||
| 89 | while let Some(ch) = self.peek() { | ||
| 90 | match ch { | ||
| 91 | '"' => { | ||
| 92 | let string = &self.input[start..self.cursor]; | ||
| 93 | self.consume(); | ||
| 94 | return Ok(string); | ||
| 95 | } | ||
| 96 | '\n' => { | ||
| 97 | let string = &self.input[start..self.cursor]; | ||
| 98 | self.consume(); | ||
| 99 | return Err(string); | ||
| 100 | } | ||
| 101 | '\\' => { | ||
| 102 | self.consume(); | ||
| 103 | self.consume(); | ||
| 104 | } | ||
| 105 | _ => { | ||
| 106 | self.consume(); | ||
| 107 | } | ||
| 108 | } | ||
| 109 | } | ||
| 110 | |||
| 111 | Err(&self.input[start..self.cursor]) | ||
| 112 | } | ||
| 113 | } | ||
| 114 | |||
| 115 | impl<'a> Iterator for Lexer<'a> { | ||
| 116 | type Item = Spanned<Token<'a>>; | ||
| 117 | |||
| 118 | fn next(&mut self) -> Option<Self::Item> { | ||
| 119 | loop { | ||
| 120 | match self.peek()? { | ||
| 121 | ch if ch.is_whitespace() => { | ||
| 122 | self.next_while(char::is_whitespace); | ||
| 123 | } | ||
| 124 | ';' => { | ||
| 125 | self.next_while(|ch| ch != '\n'); | ||
| 126 | } | ||
| 127 | _ => break, | ||
| 128 | } | ||
| 129 | } | ||
| 130 | |||
| 131 | let start = Pos::new(self.line, self.column, self.cursor); | ||
| 132 | |||
| 133 | let token = match self.peek()? { | ||
| 134 | '(' => { | ||
| 135 | self.consume(); | ||
| 136 | Token::LeftPar | ||
| 137 | } | ||
| 138 | ')' => { | ||
| 139 | self.consume(); | ||
| 140 | Token::RightPar | ||
| 141 | } | ||
| 142 | '\'' => { | ||
| 143 | self.consume(); | ||
| 144 | Token::Quote | ||
| 145 | } | ||
| 146 | |||
| 147 | // Number | ||
| 148 | ch if ch.is_ascii_digit() | ||
| 149 | || ch == '.' && self.peek_nth(1).is_some_and(|ch| ch.is_ascii_digit()) | ||
| 150 | || matches!(ch, '+' | '-') | ||
| 151 | && self.peek_nth(1).is_some_and(|ch| ch.is_ascii_digit()) | ||
| 152 | || matches!(ch, '+' | '-') | ||
| 153 | && self.peek_nth(1).is_some_and(|ch| ch == '.') | ||
| 154 | && self.peek_nth(2).is_some_and(|ch| ch.is_ascii_digit()) => | ||
| 155 | { | ||
| 156 | Token::Number(self.next_atom()) | ||
| 157 | } | ||
| 158 | |||
| 159 | // String | ||
| 160 | '"' => match self.next_string() { | ||
| 161 | Ok(string) => Token::String(string), | ||
| 162 | Err(string) => Token::UnclosedString(string), | ||
| 163 | }, | ||
| 164 | |||
| 165 | // Symbol | ||
| 166 | _ => Token::Symbol(self.next_atom()), | ||
| 167 | }; | ||
| 168 | |||
| 169 | let end = Pos::new(self.line, self.column, self.cursor); | ||
| 170 | Some(Spanned::new(token, Span::new(start, end))) | ||
| 171 | } | ||
| 172 | } | ||
