1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
|
mod commands;
pub(super) mod types;
use logos::Logos;
use syntax::latex::SyntaxKind;
use crate::SyntaxConfig;
use self::types::{CommandName, Token};
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct Lexer<'a> {
tokens: Vec<(Token, &'a str)>,
}
impl<'a> Lexer<'a> {
pub fn new(input: &'a str, config: &SyntaxConfig) -> Self {
let mut tokens = tokenize(input, config);
tokens.reverse();
Self { tokens }
}
pub fn peek(&self) -> Option<Token> {
self.tokens.last().map(|(kind, _)| *kind)
}
pub fn eat(&mut self) -> Option<(SyntaxKind, &'a str)> {
let (kind, text) = self.tokens.pop()?;
let kind = match kind {
Token::LineBreak => SyntaxKind::LINE_BREAK,
Token::Whitespace => SyntaxKind::WHITESPACE,
Token::LineComment => SyntaxKind::COMMENT,
Token::LCurly => SyntaxKind::L_CURLY,
Token::RCurly => SyntaxKind::R_CURLY,
Token::LBrack => SyntaxKind::L_BRACK,
Token::RBrack => SyntaxKind::R_BRACK,
Token::LParen => SyntaxKind::L_PAREN,
Token::RParen => SyntaxKind::R_PAREN,
Token::Comma => SyntaxKind::COMMA,
Token::Eq => SyntaxKind::EQUALITY_SIGN,
Token::Pipe => SyntaxKind::WORD,
Token::Word => SyntaxKind::WORD,
Token::Dollar => SyntaxKind::DOLLAR,
Token::CommandName(_) => SyntaxKind::COMMAND_NAME,
};
Some((kind, text))
}
}
fn tokenize<'a>(input: &'a str, config: &SyntaxConfig) -> Vec<(Token, &'a str)> {
let mut lexer = Token::lexer(input);
std::iter::from_fn(move || {
let kind = lexer.next()?.unwrap();
let text = lexer.slice();
Some((kind, text))
})
.map(|(kind, text)| {
if kind == Token::CommandName(CommandName::Generic) {
let name = commands::classify(&text[1..], config);
(Token::CommandName(name), text)
} else {
(kind, text)
}
})
.collect()
}
|