cranelift_isle/
lexer.rs

1//! Lexer for the ISLE language.
2
3use std::borrow::Cow;
4
5use crate::error::{Error, Span};
6use crate::files::Files;
7
8type Result<T> = std::result::Result<T, Error>;
9
10/// The lexer.
11///
12/// Breaks source text up into a sequence of tokens (with source positions).
13#[derive(Clone, Debug)]
14pub struct Lexer<'src> {
15    src: &'src str,
16    pos: Pos,
17    lookahead: Option<(Pos, Token)>,
18}
19
20/// A source position.
21#[derive(Clone, Copy, Debug, PartialEq, Eq, Default, Hash, PartialOrd, Ord)]
22pub struct Pos {
23    /// This source position's file.
24    ///
25    /// Indexes into `Lexer::filenames` early in the compiler pipeline, and
26    /// later into `TypeEnv::filenames` once we get into semantic analysis.
27    pub file: usize,
28    /// This source position's byte offset in the file.
29    pub offset: usize,
30}
31
32impl Pos {
33    /// Create a new `Pos`.
34    pub fn new(file: usize, offset: usize) -> Self {
35        Self { file, offset }
36    }
37
38    /// Report whether the position is unknown.
39    pub fn is_unknown(&self) -> bool {
40        *self == Self::default()
41    }
42
43    /// Print this source position as `file.isle line 12`.
44    pub fn pretty_print_line(&self, files: &Files) -> String {
45        format!(
46            "{} line {}",
47            files.file_name(self.file).unwrap(),
48            files.file_line_map(self.file).unwrap().line(self.offset)
49        )
50    }
51}
52
53/// A token of ISLE source.
54#[derive(Clone, Debug, PartialEq, Eq)]
55pub enum Token {
56    /// Left paren.
57    LParen,
58    /// Right paren.
59    RParen,
60    /// A symbol, e.g. `Foo`.
61    Symbol(String),
62    /// An integer.
63    Int(i128),
64    /// `@`
65    At,
66}
67
68impl<'src> Lexer<'src> {
69    /// Create a new lexer for the given source contents
70    pub fn new(file: usize, src: &'src str) -> Result<Lexer<'src>> {
71        let mut l = Lexer {
72            src,
73            pos: Pos::new(file, 0),
74            lookahead: None,
75        };
76        l.reload()?;
77        Ok(l)
78    }
79
80    /// Get the lexer's current source position.
81    pub fn pos(&self) -> Pos {
82        self.pos
83    }
84
85    fn advance_pos(&mut self) {
86        self.advance_by(1)
87    }
88
89    fn advance_by(&mut self, n: usize) {
90        self.pos.offset += n;
91    }
92
93    fn error(&self, pos: Pos, msg: impl Into<String>) -> Error {
94        Error::ParseError {
95            msg: msg.into(),
96            span: Span::new_single(pos),
97        }
98    }
99
100    fn next_token(&mut self) -> Result<Option<(Pos, Token)>> {
101        fn is_sym_first_char(c: u8) -> bool {
102            match c {
103                b'-' | b'0'..=b'9' | b'(' | b')' | b';' => false,
104                c if c.is_ascii_whitespace() => false,
105                _ => true,
106            }
107        }
108        fn is_sym_other_char(c: u8) -> bool {
109            match c {
110                b'(' | b')' | b';' | b'@' => false,
111                c if c.is_ascii_whitespace() => false,
112                _ => true,
113            }
114        }
115
116        // Skip any whitespace and any comments.
117        while let Some(c) = self.peek_byte() {
118            match c {
119                b' ' | b'\t' | b'\n' | b'\r' => self.advance_pos(),
120                b';' => {
121                    while let Some(c) = self.peek_byte() {
122                        match c {
123                            b'\n' | b'\r' => break,
124                            _ => self.advance_pos(),
125                        }
126                    }
127                }
128                b'(' if self.lookahead_byte(1) == Some(b';') => {
129                    let pos = self.pos();
130                    self.advance_by(2);
131                    let mut depth = 1usize;
132                    loop {
133                        match self.peek_byte() {
134                            None => return Err(self.error(pos, "unterminated block comment")),
135                            Some(b'(') if self.lookahead_byte(1) == Some(b';') => {
136                                self.advance_by(2);
137                                depth += 1;
138                            }
139                            Some(b';') if self.lookahead_byte(1) == Some(b')') => {
140                                self.advance_by(2);
141                                depth -= 1;
142                                if depth == 0 {
143                                    break;
144                                }
145                            }
146                            Some(_) => self.advance_pos(),
147                        }
148                    }
149                }
150                _ => break,
151            }
152        }
153
154        let Some(c) = self.peek_byte() else {
155            return Ok(None);
156        };
157        let char_pos = self.pos();
158        match c {
159            b'(' => {
160                self.advance_pos();
161                Ok(Some((char_pos, Token::LParen)))
162            }
163            b')' => {
164                self.advance_pos();
165                Ok(Some((char_pos, Token::RParen)))
166            }
167            b'@' => {
168                self.advance_pos();
169                Ok(Some((char_pos, Token::At)))
170            }
171            c if is_sym_first_char(c) => {
172                let start = self.pos.offset;
173                let start_pos = self.pos();
174                while let Some(c) = self.peek_byte() {
175                    match c {
176                        c if is_sym_other_char(c) => self.advance_pos(),
177                        _ => break,
178                    }
179                }
180                let end = self.pos.offset;
181                let s = &self.src[start..end];
182                debug_assert!(!s.is_empty());
183                Ok(Some((start_pos, Token::Symbol(s.to_string()))))
184            }
185            c @ (b'0'..=b'9' | b'-') => {
186                let start_pos = self.pos();
187                let mut neg = false;
188                if c == b'-' {
189                    self.advance_pos();
190                    neg = true;
191                }
192
193                let mut radix = 10;
194
195                // Check for prefixed literals.
196                match (
197                    self.src.as_bytes().get(self.pos.offset),
198                    self.src.as_bytes().get(self.pos.offset + 1),
199                ) {
200                    (Some(b'0'), Some(b'x' | b'X')) => {
201                        self.advance_by(2);
202                        radix = 16;
203                    }
204                    (Some(b'0'), Some(b'o' | b'O')) => {
205                        self.advance_by(2);
206                        radix = 8;
207                    }
208                    (Some(b'0'), Some(b'b' | b'B')) => {
209                        self.advance_by(2);
210                        radix = 2;
211                    }
212                    _ => {}
213                }
214
215                // Find the range in the buffer for this integer literal. We'll
216                // pass this range to `i64::from_str_radix` to do the actual
217                // string-to-integer conversion.
218                let start = self.pos.offset;
219                while let Some(c) = self.peek_byte() {
220                    match c {
221                        b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F' | b'_' => self.advance_pos(),
222                        _ => break,
223                    }
224                }
225                let end = self.pos.offset;
226                let s = &self.src[start..end];
227                let s = if s.contains('_') {
228                    Cow::Owned(s.replace('_', ""))
229                } else {
230                    Cow::Borrowed(s)
231                };
232
233                // Support either signed range (-2^127..2^127) or
234                // unsigned range (0..2^128).
235                let num = match u128::from_str_radix(&s, radix) {
236                    Ok(num) => num,
237                    Err(err) => return Err(self.error(start_pos, err.to_string())),
238                };
239
240                let num = match (neg, num) {
241                    (true, 0x80000000000000000000000000000000) => {
242                        return Err(self.error(start_pos, "integer literal cannot fit in i128"));
243                    }
244                    (true, _) => -(num as i128),
245                    (false, _) => num as i128,
246                };
247                let tok = Token::Int(num);
248
249                Ok(Some((start_pos, tok)))
250            }
251            c => Err(self.error(self.pos, format!("Unexpected character '{c}'"))),
252        }
253    }
254
255    /// Get the next token from this lexer's token stream, if any.
256    pub fn next(&mut self) -> Result<Option<(Pos, Token)>> {
257        let tok = self.lookahead.take();
258        self.reload()?;
259        Ok(tok)
260    }
261
262    fn reload(&mut self) -> Result<()> {
263        if self.lookahead.is_none() && self.pos.offset < self.src.len() {
264            self.lookahead = self.next_token()?;
265        }
266        Ok(())
267    }
268
269    /// Peek ahead at the next token.
270    pub fn peek(&self) -> Option<&(Pos, Token)> {
271        self.lookahead.as_ref()
272    }
273
274    /// Are we at the end of the source input?
275    pub fn eof(&self) -> bool {
276        self.lookahead.is_none()
277    }
278
279    fn peek_byte(&self) -> Option<u8> {
280        self.lookahead_byte(0)
281    }
282
283    fn lookahead_byte(&self, n: usize) -> Option<u8> {
284        self.src.as_bytes().get(self.pos.offset + n).copied()
285    }
286}
287
288impl Token {
289    /// Is this an `Int` token?
290    pub fn is_int(&self) -> bool {
291        matches!(self, Token::Int(_))
292    }
293
294    /// Is this a `Sym` token?
295    pub fn is_sym(&self) -> bool {
296        matches!(self, Token::Symbol(_))
297    }
298}
299
300#[cfg(test)]
301mod test {
302    use super::*;
303
304    #[track_caller]
305    fn lex(src: &str) -> Vec<Token> {
306        let mut toks = vec![];
307        let mut lexer = Lexer::new(0, src).unwrap();
308        while let Some((_, tok)) = lexer.next().unwrap() {
309            toks.push(tok);
310        }
311        toks
312    }
313
314    #[test]
315    fn lexer_basic() {
316        assert_eq!(
317            lex(
318                ";; comment\n; another\r\n   \t(one two three (; block comment ;) 23 (; nested (; block ;) comment ;) -568  )\n"
319            ),
320            [
321                Token::LParen,
322                Token::Symbol("one".to_string()),
323                Token::Symbol("two".to_string()),
324                Token::Symbol("three".to_string()),
325                Token::Int(23),
326                Token::Int(-568),
327                Token::RParen
328            ]
329        );
330    }
331
332    #[test]
333    fn ends_with_sym() {
334        assert_eq!(lex("asdf"), [Token::Symbol("asdf".to_string())]);
335    }
336
337    #[test]
338    fn ends_with_num() {
339        assert_eq!(lex("23"), [Token::Int(23)]);
340    }
341
342    #[test]
343    fn weird_syms() {
344        assert_eq!(
345            lex("(+ [] => !! _test!;comment\n)"),
346            [
347                Token::LParen,
348                Token::Symbol("+".to_string()),
349                Token::Symbol("[]".to_string()),
350                Token::Symbol("=>".to_string()),
351                Token::Symbol("!!".to_string()),
352                Token::Symbol("_test!".to_string()),
353                Token::RParen,
354            ]
355        );
356    }
357
358    #[test]
359    fn integers() {
360        assert_eq!(
361            lex("0 1 -1"),
362            [Token::Int(0), Token::Int(1), Token::Int(-1)]
363        );
364
365        assert_eq!(
366            lex("340_282_366_920_938_463_463_374_607_431_768_211_455"),
367            [Token::Int(-1)]
368        );
369
370        assert_eq!(
371            lex("170_141_183_460_469_231_731_687_303_715_884_105_727"),
372            [Token::Int(i128::MAX)]
373        );
374
375        assert!(Lexer::new(0, "-170_141_183_460_469_231_731_687_303_715_884_105_728").is_err())
376    }
377}
cranelift_isle/lexer.rs

cranelift_isle/
lexer.rs
