aboutsummaryrefslogtreecommitdiff
path: root/compiler/src/lexer/mod.rs
diff options
context:
space:
mode:
authorTolmachev Igor <me@igorek.dev>2026-05-09 20:47:04 +0300
committerTolmachev Igor <me@igorek.dev>2026-05-09 20:47:04 +0300
commit160b64427d79290a59ac48c9babca064232d8dfd (patch)
tree0c2cc79f0a266761866ff325abdd4f2f0c7e7301 /compiler/src/lexer/mod.rs
parent6be28381d6081dfb3a1dc9d1ec15062b67ba1ef9 (diff)
downloadcrisp-160b64427d79290a59ac48c9babca064232d8dfd.tar.gz
crisp-160b64427d79290a59ac48c9babca064232d8dfd.zip
Make project structure more consistentdev
Diffstat (limited to 'compiler/src/lexer/mod.rs')
-rw-r--r--compiler/src/lexer/mod.rs172
1 files changed, 0 insertions, 172 deletions
diff --git a/compiler/src/lexer/mod.rs b/compiler/src/lexer/mod.rs
deleted file mode 100644
index f3c8b76..0000000
--- a/compiler/src/lexer/mod.rs
+++ /dev/null
@@ -1,172 +0,0 @@
1use crate::span::{Pos, Span, Spanned};
2
3#[cfg(test)]
4mod tests;
5
6fn is_terminator(ch: char) -> bool {
7 ch.is_whitespace() || matches!(ch, '(' | ')' | '\'' | '"' | ';')
8}
9
10#[derive(Clone, Copy, Debug, PartialEq, Eq)]
11pub enum Token<'a> {
12 LeftPar,
13 RightPar,
14 Quote,
15 Number(&'a str),
16 String(&'a str),
17 UnclosedString(&'a str),
18 Symbol(&'a str),
19}
20
21pub struct Lexer<'a> {
22 input: &'a str,
23 cursor: usize,
24
25 line: usize,
26 column: usize,
27}
28
29impl<'a> Lexer<'a> {
30 pub fn new(input: &'a str) -> Self {
31 Self {
32 input,
33 cursor: 0,
34
35 line: 1,
36 column: 0,
37 }
38 }
39
40 fn rest(&self) -> &str {
41 &self.input[self.cursor..]
42 }
43
44 fn peek(&self) -> Option<char> {
45 self.rest().chars().next()
46 }
47
48 fn peek_nth(&self, n: usize) -> Option<char> {
49 self.rest().chars().nth(n)
50 }
51
52 fn consume(&mut self) -> Option<char> {
53 let ch = self.peek()?;
54
55 self.cursor += ch.len_utf8();
56 if ch == '\n' {
57 self.line += 1;
58 self.column = 0;
59 } else {
60 self.column += 1;
61 }
62
63 Some(ch)
64 }
65
66 fn next_while(&mut self, mut predicate: impl FnMut(char) -> bool) -> &'a str {
67 let start = self.cursor;
68
69 while let Some(ch) = self.peek() {
70 if !predicate(ch) {
71 break;
72 }
73 self.consume();
74 }
75
76 &self.input[start..self.cursor]
77 }
78
79 fn next_atom(&mut self) -> &'a str {
80 self.next_while(|ch| !is_terminator(ch))
81 }
82
83 fn next_string(&mut self) -> Result<&'a str, &'a str> {
84 debug_assert_eq!(self.peek(), Some('"'));
85 self.consume();
86
87 let start = self.cursor;
88
89 while let Some(ch) = self.peek() {
90 match ch {
91 '"' => {
92 let string = &self.input[start..self.cursor];
93 self.consume();
94 return Ok(string);
95 }
96 '\n' => {
97 let string = &self.input[start..self.cursor];
98 self.consume();
99 return Err(string);
100 }
101 '\\' => {
102 self.consume();
103 self.consume();
104 }
105 _ => {
106 self.consume();
107 }
108 }
109 }
110
111 Err(&self.input[start..self.cursor])
112 }
113}
114
115impl<'a> Iterator for Lexer<'a> {
116 type Item = Spanned<Token<'a>>;
117
118 fn next(&mut self) -> Option<Self::Item> {
119 loop {
120 match self.peek()? {
121 ch if ch.is_whitespace() => {
122 self.next_while(char::is_whitespace);
123 }
124 ';' => {
125 self.next_while(|ch| ch != '\n');
126 }
127 _ => break,
128 }
129 }
130
131 let start = Pos::new(self.line, self.column, self.cursor);
132
133 let token = match self.peek()? {
134 '(' => {
135 self.consume();
136 Token::LeftPar
137 }
138 ')' => {
139 self.consume();
140 Token::RightPar
141 }
142 '\'' => {
143 self.consume();
144 Token::Quote
145 }
146
147 // Number
148 ch if ch.is_ascii_digit()
149 || ch == '.' && self.peek_nth(1).is_some_and(|ch| ch.is_ascii_digit())
150 || matches!(ch, '+' | '-')
151 && self.peek_nth(1).is_some_and(|ch| ch.is_ascii_digit())
152 || matches!(ch, '+' | '-')
153 && self.peek_nth(1).is_some_and(|ch| ch == '.')
154 && self.peek_nth(2).is_some_and(|ch| ch.is_ascii_digit()) =>
155 {
156 Token::Number(self.next_atom())
157 }
158
159 // String
160 '"' => match self.next_string() {
161 Ok(string) => Token::String(string),
162 Err(string) => Token::UnclosedString(string),
163 },
164
165 // Symbol
166 _ => Token::Symbol(self.next_atom()),
167 };
168
169 let end = Pos::new(self.line, self.column, self.cursor);
170 Some(Spanned::new(token, Span::new(start, end)))
171 }
172}