toml_parser/lexer/
mod.rs

1//! Lex TOML tokens
2//!
3//! To get started, see [`Source::lex`][crate::Source::lex]
4
5#[cfg(test)]
6#[cfg(feature = "std")]
7mod test;
8mod token;
9
10#[cfg(feature = "alloc")]
11use alloc::vec::Vec;
12
13use winnow::stream::AsBStr as _;
14use winnow::stream::ContainsToken as _;
15use winnow::stream::FindSlice as _;
16use winnow::stream::Location;
17use winnow::stream::Stream as _;
18
19use crate::Span;
20
21pub use token::Token;
22pub use token::TokenKind;
23
24/// Lex TOML [tokens][Token]
25///
26/// To get started, see [`Source::lex`][crate::Source::lex]
27pub struct Lexer<'i> {
28    stream: Stream<'i>,
29    eof: bool,
30}
31
32impl<'i> Lexer<'i> {
33    pub(crate) fn new(input: &'i str) -> Self {
34        let mut stream = Stream::new(input);
35        if input.as_bytes().starts_with(BOM) {
36            let offset = BOM.len();
37            #[cfg(feature = "unsafe")] // SAFETY: only called when next character is ASCII
38            unsafe {
39                stream.next_slice_unchecked(offset)
40            };
41            #[cfg(not(feature = "unsafe"))]
42            stream.next_slice(offset);
43        }
44        Lexer { stream, eof: false }
45    }
46
47    #[cfg(feature = "alloc")]
48    pub fn into_vec(self) -> Vec<Token> {
49        let capacity = self.stream.len().div_ceil(4);
50        let mut vec = Vec::with_capacity(capacity);
51        vec.extend(self);
52        vec
53    }
54}
55
56impl Iterator for Lexer<'_> {
57    type Item = Token;
58
59    fn next(&mut self) -> Option<Self::Item> {
60        let Some(peek_byte) = self.stream.as_bstr().first() else {
61            if self.eof {
62                return None;
63            } else {
64                self.eof = true;
65                let start = self.stream.current_token_start();
66                let span = Span::new_unchecked(start, start);
67                return Some(Token::new(TokenKind::Eof, span));
68            }
69        };
70        Some(process_token(*peek_byte, &mut self.stream))
71    }
72}
73
74const BOM: &[u8] = b"\xEF\xBB\xBF";
75
76pub(crate) type Stream<'i> = winnow::stream::LocatingSlice<&'i str>;
77
78fn process_token(peek_byte: u8, stream: &mut Stream<'_>) -> Token {
79    let token = match peek_byte {
80        b'.' => lex_ascii_char(stream, TokenKind::Dot),
81        b'=' => lex_ascii_char(stream, TokenKind::Equals),
82        b',' => lex_ascii_char(stream, TokenKind::Comma),
83        b'[' => lex_ascii_char(stream, TokenKind::LeftSquareBracket),
84        b']' => lex_ascii_char(stream, TokenKind::RightSquareBracket),
85        b'{' => lex_ascii_char(stream, TokenKind::LeftCurlyBracket),
86        b'}' => lex_ascii_char(stream, TokenKind::RightCurlyBracket),
87        b' ' => lex_whitespace(stream),
88        b'\t' => lex_whitespace(stream),
89        b'#' => lex_comment(stream),
90        b'\r' => lex_crlf(stream),
91        b'\n' => lex_ascii_char(stream, TokenKind::Newline),
92        b'\'' => {
93            if stream.starts_with(ML_LITERAL_STRING_DELIM) {
94                lex_ml_literal_string(stream)
95            } else {
96                lex_literal_string(stream)
97            }
98        }
99        b'"' => {
100            if stream.starts_with(ML_BASIC_STRING_DELIM) {
101                lex_ml_basic_string(stream)
102            } else {
103                lex_basic_string(stream)
104            }
105        }
106        _ => lex_atom(stream),
107    };
108    token
109}
110
111/// Process an ASCII character token
112///
113/// # Safety
114///
115/// - `stream` must be UTF-8
116/// - `stream` must be non-empty
117/// - `stream[0]` must be ASCII
118fn lex_ascii_char(stream: &mut Stream<'_>, kind: TokenKind) -> Token {
119    debug_assert!(!stream.is_empty());
120    let start = stream.current_token_start();
121
122    let offset = 1; // an ascii character
123    #[cfg(feature = "unsafe")] // SAFETY: only called when next character is ASCII
124    unsafe {
125        stream.next_slice_unchecked(offset)
126    };
127    #[cfg(not(feature = "unsafe"))]
128    stream.next_slice(offset);
129
130    let end = stream.previous_token_end();
131    let span = Span::new_unchecked(start, end);
132    Token::new(kind, span)
133}
134
135/// Process Whitespace
136///
137/// ```abnf
138/// ;; Whitespace
139///
140/// ws = *wschar
141/// wschar =  %x20  ; Space
142/// wschar =/ %x09  ; Horizontal tab
143/// ```
144///
145/// # Safety
146///
147/// - `stream` must be UTF-8
148/// - `stream` must be non-empty
149fn lex_whitespace(stream: &mut Stream<'_>) -> Token {
150    debug_assert!(!stream.is_empty());
151    let start = stream.current_token_start();
152
153    let offset = stream
154        .as_bstr()
155        .offset_for(|b| !WSCHAR.contains_token(b))
156        .unwrap_or(stream.eof_offset());
157    #[cfg(feature = "unsafe")] // SAFETY: WSCHAR ensures `offset` will be at UTF-8 boundary
158    unsafe {
159        stream.next_slice_unchecked(offset)
160    };
161    #[cfg(not(feature = "unsafe"))]
162    stream.next_slice(offset);
163
164    let end = stream.previous_token_end();
165    let span = Span::new_unchecked(start, end);
166    Token::new(TokenKind::Whitespace, span)
167}
168
169/// ```abnf
170/// wschar =  %x20  ; Space
171/// wschar =/ %x09  ; Horizontal tab
172/// ```
173pub(crate) const WSCHAR: (u8, u8) = (b' ', b'\t');
174
175/// Process Comment
176///
177/// ```abnf
178/// ;; Comment
179///
180/// comment-start-symbol = %x23 ; #
181/// non-ascii = %x80-D7FF / %xE000-10FFFF
182/// non-eol = %x09 / %x20-7E / non-ascii
183///
184/// comment = comment-start-symbol *non-eol
185/// ```
186///
187/// # Safety
188///
189/// - `stream` must be UTF-8
190/// - `stream[0] == b'#'`
191fn lex_comment(stream: &mut Stream<'_>) -> Token {
192    let start = stream.current_token_start();
193
194    let offset = stream
195        .as_bytes()
196        .find_slice((b'\r', b'\n'))
197        .map(|s| s.start)
198        .unwrap_or_else(|| stream.eof_offset());
199    #[cfg(feature = "unsafe")] // SAFETY: newlines ensure `offset` is along UTF-8 boundary
200    unsafe {
201        stream.next_slice_unchecked(offset)
202    };
203    #[cfg(not(feature = "unsafe"))]
204    stream.next_slice(offset);
205
206    let end = stream.previous_token_end();
207    let span = Span::new_unchecked(start, end);
208    Token::new(TokenKind::Comment, span)
209}
210
211/// ```abnf
212/// comment-start-symbol = %x23 ; #
213/// ```
214pub(crate) const COMMENT_START_SYMBOL: u8 = b'#';
215
216/// Process Newline
217///
218/// ```abnf
219/// ;; Newline
220///
221/// newline =  %x0A     ; LF
222/// newline =/ %x0D.0A  ; CRLF
223/// ```
224///
225/// # Safety
226///
227/// - `stream` must be UTF-8
228/// - `stream[0] == b'\r'`
229fn lex_crlf(stream: &mut Stream<'_>) -> Token {
230    let start = stream.current_token_start();
231
232    let mut offset = '\r'.len_utf8();
233    let has_lf = stream.as_bstr().get(1) == Some(&b'\n');
234    if has_lf {
235        offset += '\n'.len_utf8();
236    }
237
238    #[cfg(feature = "unsafe")] // SAFETY: newlines ensure `offset` is along UTF-8 boundary
239    unsafe {
240        stream.next_slice_unchecked(offset)
241    };
242    #[cfg(not(feature = "unsafe"))]
243    stream.next_slice(offset);
244    let end = stream.previous_token_end();
245    let span = Span::new_unchecked(start, end);
246
247    Token::new(TokenKind::Newline, span)
248}
249
250/// Process literal string
251///
252/// ```abnf
253/// ;; Literal String
254///
255/// literal-string = apostrophe *literal-char apostrophe
256///
257/// apostrophe = %x27 ; ' apostrophe
258///
259/// literal-char = %x09 / %x20-26 / %x28-7E / non-ascii
260/// ```
261///
262/// # Safety
263///
264/// - `stream` must be UTF-8
265/// - `stream[0] == b'\''`
266fn lex_literal_string(stream: &mut Stream<'_>) -> Token {
267    let start = stream.current_token_start();
268
269    let offset = 1; // APOSTROPHE
270    #[cfg(feature = "unsafe")] // SAFETY: only called when next character is ASCII
271    unsafe {
272        stream.next_slice_unchecked(offset)
273    };
274    #[cfg(not(feature = "unsafe"))]
275    stream.next_slice(offset);
276
277    let offset = match stream.as_bstr().find_slice((APOSTROPHE, b'\n')) {
278        Some(span) => {
279            if stream.as_bstr()[span.start] == APOSTROPHE {
280                span.end
281            } else {
282                span.start
283            }
284        }
285        None => stream.eof_offset(),
286    };
287    #[cfg(feature = "unsafe")]
288    // SAFETY: `APOSTROPHE`/newline ensure `offset` is along UTF-8 boundary
289    unsafe {
290        stream.next_slice_unchecked(offset)
291    };
292    #[cfg(not(feature = "unsafe"))]
293    stream.next_slice(offset);
294
295    let end = stream.previous_token_end();
296    let span = Span::new_unchecked(start, end);
297    Token::new(TokenKind::LiteralString, span)
298}
299
300/// ```abnf
301/// apostrophe = %x27 ; ' apostrophe
302/// ```
303pub(crate) const APOSTROPHE: u8 = b'\'';
304
305/// Process multi-line literal string
306///
307/// ```abnf
308/// ;; Multiline Literal String
309///
310/// ml-literal-string = ml-literal-string-delim [ newline ] ml-literal-body
311///                     ml-literal-string-delim
312/// ml-literal-string-delim = 3apostrophe
313/// ml-literal-body = *mll-content *( mll-quotes 1*mll-content ) [ mll-quotes ]
314///
315/// mll-content = literal-char / newline
316/// mll-quotes = 1*2apostrophe
317/// ```
318///
319/// # Safety
320///
321/// - `stream` must be UTF-8
322/// - `stream.starts_with(ML_LITERAL_STRING_DELIM)`
323fn lex_ml_literal_string(stream: &mut Stream<'_>) -> Token {
324    let start = stream.current_token_start();
325
326    let offset = ML_LITERAL_STRING_DELIM.len();
327    #[cfg(feature = "unsafe")] // SAFETY: only called when next character is ASCII
328    unsafe {
329        stream.next_slice_unchecked(offset)
330    };
331    #[cfg(not(feature = "unsafe"))]
332    stream.next_slice(offset);
333
334    let offset = match stream.as_bstr().find_slice(ML_LITERAL_STRING_DELIM) {
335        Some(span) => span.end,
336        None => stream.eof_offset(),
337    };
338    #[cfg(feature = "unsafe")]
339    // SAFETY: `ML_LITERAL_STRING_DELIM` ensure `offset` is along UTF-8 boundary
340    unsafe {
341        stream.next_slice_unchecked(offset)
342    };
343    #[cfg(not(feature = "unsafe"))]
344    stream.next_slice(offset);
345
346    if stream.as_bstr().peek_token() == Some(APOSTROPHE) {
347        let offset = 1;
348        #[cfg(feature = "unsafe")] // SAFETY: `APOSTROPHE` ensure `offset` is along UTF-8 boundary
349        unsafe {
350            stream.next_slice_unchecked(offset)
351        };
352        #[cfg(not(feature = "unsafe"))]
353        stream.next_slice(offset);
354
355        if stream.as_bstr().peek_token() == Some(APOSTROPHE) {
356            let offset = 1;
357            #[cfg(feature = "unsafe")]
358            // SAFETY: `APOSTROPHE` ensure `offset` is along UTF-8 boundary
359            unsafe {
360                stream.next_slice_unchecked(offset)
361            };
362            #[cfg(not(feature = "unsafe"))]
363            stream.next_slice(offset);
364        }
365    }
366
367    let end = stream.previous_token_end();
368    let span = Span::new_unchecked(start, end);
369    Token::new(TokenKind::MlLiteralString, span)
370}
371
372/// ```abnf
373/// ml-literal-string-delim = 3apostrophe
374/// ```
375pub(crate) const ML_LITERAL_STRING_DELIM: &str = "'''";
376
377/// Process basic string
378///
379/// ```abnf
380/// ;; Basic String
381///
382/// basic-string = quotation-mark *basic-char quotation-mark
383///
384/// quotation-mark = %x22            ; "
385///
386/// basic-char = basic-unescaped / escaped
387/// basic-unescaped = wschar / %x21 / %x23-5B / %x5D-7E / non-ascii
388/// escaped = escape escape-seq-char
389///
390/// escape = %x5C                   ; \
391/// escape-seq-char =  %x22         ; "    quotation mark  U+0022
392/// escape-seq-char =/ %x5C         ; \    reverse solidus U+005C
393/// escape-seq-char =/ %x62         ; b    backspace       U+0008
394/// escape-seq-char =/ %x65         ; e    escape          U+001B
395/// escape-seq-char =/ %x66         ; f    form feed       U+000C
396/// escape-seq-char =/ %x6E         ; n    line feed       U+000A
397/// escape-seq-char =/ %x72         ; r    carriage return U+000D
398/// escape-seq-char =/ %x74         ; t    tab             U+0009
399/// escape-seq-char =/ %x78 2HEXDIG ; xHH                  U+00HH
400/// escape-seq-char =/ %x75 4HEXDIG ; uHHHH                U+HHHH
401/// escape-seq-char =/ %x55 8HEXDIG ; UHHHHHHHH            U+HHHHHHHH
402/// ```
403///
404/// # Safety
405///
406/// - `stream` must be UTF-8
407/// - `stream[0] == b'"'`
408fn lex_basic_string(stream: &mut Stream<'_>) -> Token {
409    let start = stream.current_token_start();
410
411    let offset = 1; // QUOTATION_MARK
412    #[cfg(feature = "unsafe")] // SAFETY: only called when next character is ASCII
413    unsafe {
414        stream.next_slice_unchecked(offset)
415    };
416    #[cfg(not(feature = "unsafe"))]
417    stream.next_slice(offset);
418
419    loop {
420        // newline is present for error recovery
421        match stream.as_bstr().find_slice((QUOTATION_MARK, ESCAPE, b'\n')) {
422            Some(span) => {
423                let found = stream.as_bstr()[span.start];
424                if found == QUOTATION_MARK {
425                    let offset = span.end;
426                    #[cfg(feature = "unsafe")]
427                    // SAFETY: `QUOTATION_MARK` ensure `offset` is along UTF-8 boundary
428                    unsafe {
429                        stream.next_slice_unchecked(offset)
430                    };
431                    #[cfg(not(feature = "unsafe"))]
432                    stream.next_slice(offset);
433                    break;
434                } else if found == ESCAPE {
435                    let offset = span.end;
436                    #[cfg(feature = "unsafe")]
437                    // SAFETY: `ESCAPE` / newline ensure `offset` is along UTF-8 boundary
438                    unsafe {
439                        stream.next_slice_unchecked(offset)
440                    };
441                    #[cfg(not(feature = "unsafe"))]
442                    stream.next_slice(offset);
443
444                    let peek = stream.as_bstr().peek_token();
445                    match peek {
446                        Some(ESCAPE) | Some(QUOTATION_MARK) => {
447                            let offset = 1; // ESCAPE / QUOTATION_MARK
448                            #[cfg(feature = "unsafe")]
449                            #[cfg(feature = "unsafe")]
450                            // SAFETY: `ESCAPE` / newline ensure `offset` is along UTF-8 boundary
451                            unsafe {
452                                stream.next_slice_unchecked(offset)
453                            };
454                            #[cfg(not(feature = "unsafe"))]
455                            stream.next_slice(offset);
456                        }
457                        _ => {}
458                    }
459                    continue;
460                } else if found == b'\n' {
461                    let offset = span.start;
462                    #[cfg(feature = "unsafe")]
463                    // SAFETY: newline ensure `offset` is along UTF-8 boundary
464                    unsafe {
465                        stream.next_slice_unchecked(offset)
466                    };
467                    #[cfg(not(feature = "unsafe"))]
468                    stream.next_slice(offset);
469                    break;
470                } else {
471                    unreachable!("found `{found}`");
472                }
473            }
474            None => {
475                stream.finish();
476                break;
477            }
478        }
479    }
480
481    let end = stream.previous_token_end();
482    let span = Span::new_unchecked(start, end);
483    Token::new(TokenKind::BasicString, span)
484}
485
486/// ```abnf
487/// quotation-mark = %x22            ; "
488/// ```
489pub(crate) const QUOTATION_MARK: u8 = b'"';
490
491/// ```abnf
492/// escape = %x5C                   ; \
493/// ```
494pub(crate) const ESCAPE: u8 = b'\\';
495
496/// Process multi-line basic string
497///
498/// ```abnf
499/// ;; Multiline Basic String
500///
501/// ml-basic-string = ml-basic-string-delim [ newline ] ml-basic-body
502///                   ml-basic-string-delim
503/// ml-basic-string-delim = 3quotation-mark
504/// ml-basic-body = *mlb-content *( mlb-quotes 1*mlb-content ) [ mlb-quotes ]
505///
506/// mlb-content = basic-char / newline / mlb-escaped-nl
507/// mlb-quotes = 1*2quotation-mark
508/// mlb-escaped-nl = escape ws newline *( wschar / newline )
509/// ```
510///
511/// # Safety
512///
513/// - `stream` must be UTF-8
514/// - `stream.starts_with(ML_BASIC_STRING_DELIM)`
515fn lex_ml_basic_string(stream: &mut Stream<'_>) -> Token {
516    let start = stream.current_token_start();
517
518    let offset = ML_BASIC_STRING_DELIM.len();
519    #[cfg(feature = "unsafe")] // SAFETY: only called when next character is ASCII
520    unsafe {
521        stream.next_slice_unchecked(offset)
522    };
523    #[cfg(not(feature = "unsafe"))]
524    stream.next_slice(offset);
525
526    loop {
527        // newline is present for error recovery
528        match stream.as_bstr().find_slice((ML_BASIC_STRING_DELIM, "\\")) {
529            Some(span) => {
530                let found = stream.as_bstr()[span.start];
531                if found == QUOTATION_MARK {
532                    let offset = span.end;
533                    #[cfg(feature = "unsafe")]
534                    // SAFETY: `QUOTATION_MARK` ensure `offset` is along UTF-8 boundary
535                    unsafe {
536                        stream.next_slice_unchecked(offset)
537                    };
538                    #[cfg(not(feature = "unsafe"))]
539                    stream.next_slice(offset);
540                    break;
541                } else if found == ESCAPE {
542                    let offset = span.end;
543                    #[cfg(feature = "unsafe")]
544                    // SAFETY: `ESCAPE` ensure `offset` is along UTF-8 boundary
545                    unsafe {
546                        stream.next_slice_unchecked(offset)
547                    };
548                    #[cfg(not(feature = "unsafe"))]
549                    stream.next_slice(offset);
550
551                    let peek = stream.as_bstr().peek_token();
552                    match peek {
553                        Some(ESCAPE) | Some(QUOTATION_MARK) => {
554                            let offset = 1; // ESCAPE / QUOTATION_MARK
555                            #[cfg(feature = "unsafe")]
556                            // SAFETY: `QUOTATION_MARK`/`ESCAPE` ensure `offset` is along UTF-8 boundary
557                            unsafe {
558                                stream.next_slice_unchecked(offset)
559                            };
560                            #[cfg(not(feature = "unsafe"))]
561                            stream.next_slice(offset);
562                        }
563                        _ => {}
564                    }
565                    continue;
566                } else {
567                    unreachable!("found `{found}`");
568                }
569            }
570            None => {
571                stream.finish();
572                break;
573            }
574        }
575    }
576    if stream.as_bstr().peek_token() == Some(QUOTATION_MARK) {
577        let offset = 1;
578        #[cfg(feature = "unsafe")]
579        // SAFETY: `QUOTATION_MARK` ensure `offset` is along UTF-8 boundary
580        unsafe {
581            stream.next_slice_unchecked(offset)
582        };
583        #[cfg(not(feature = "unsafe"))]
584        stream.next_slice(offset);
585        if stream.as_bstr().peek_token() == Some(QUOTATION_MARK) {
586            let offset = 1;
587            #[cfg(feature = "unsafe")]
588            // SAFETY: `QUOTATION_MARK` ensure `offset` is along UTF-8 boundary
589            unsafe {
590                stream.next_slice_unchecked(offset)
591            };
592            #[cfg(not(feature = "unsafe"))]
593            stream.next_slice(offset);
594        }
595    }
596
597    let end = stream.previous_token_end();
598    let span = Span::new_unchecked(start, end);
599    Token::new(TokenKind::MlBasicString, span)
600}
601
602/// ```abnf
603/// ml-basic-string-delim = 3quotation-mark
604/// ```
605pub(crate) const ML_BASIC_STRING_DELIM: &str = "\"\"\"";
606
607/// Process Atom
608///
609/// This is everything else
610///
611/// # Safety
612///
613/// - `stream` must be UTF-8
614/// - `stream` must be non-empty
615fn lex_atom(stream: &mut Stream<'_>) -> Token {
616    let start = stream.current_token_start();
617
618    // Intentionally leaves off quotes in case the opening quote was missing
619    const TOKEN_START: &[u8] = b".=,[]{} \t#\r\n";
620    let offset = stream
621        .as_bstr()
622        .offset_for(|b| TOKEN_START.contains_token(b))
623        .unwrap_or_else(|| stream.eof_offset());
624    #[cfg(feature = "unsafe")] // SAFETY: `TOKEN_START` ensure `offset` is along UTF-8 boundary
625    unsafe {
626        stream.next_slice_unchecked(offset)
627    };
628    #[cfg(not(feature = "unsafe"))]
629    stream.next_slice(offset);
630
631    let end = stream.previous_token_end();
632    let span = Span::new_unchecked(start, end);
633    Token::new(TokenKind::Atom, span)
634}
toml_parser/lexer/mod.rs

toml_parser/lexer/
mod.rs
