summaryrefslogtreecommitdiff
path: root/support/texlab/src/parser/latex/lexer.rs
blob: ee5b8dd328f63a5479e7ca8433a834b58294cf08 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
mod commands;
pub(super) mod types;

use logos::Logos;

use crate::syntax::latex::SyntaxKind;

use self::types::{CommandName, Token};

#[derive(Debug, PartialEq, Eq, Clone)]
pub struct Lexer<'a> {
    tokens: Vec<(Token, &'a str)>,
}

impl<'a> Lexer<'a> {
    pub fn new(input: &'a str) -> Self {
        let mut tokens = tokenize(input);
        tokens.reverse();
        Self { tokens }
    }

    pub fn peek(&self) -> Option<Token> {
        self.tokens.last().map(|(kind, _)| *kind)
    }

    pub fn eat(&mut self) -> Option<(SyntaxKind, &'a str)> {
        let (kind, text) = self.tokens.pop()?;
        let kind = match kind {
            Token::LineBreak => SyntaxKind::LINE_BREAK,
            Token::Whitespace => SyntaxKind::WHITESPACE,
            Token::LineComment => SyntaxKind::COMMENT,
            Token::LCurly => SyntaxKind::L_CURLY,
            Token::RCurly => SyntaxKind::R_CURLY,
            Token::LBrack => SyntaxKind::L_BRACK,
            Token::RBrack => SyntaxKind::R_BRACK,
            Token::LParen => SyntaxKind::L_PAREN,
            Token::RParen => SyntaxKind::R_PAREN,
            Token::Comma => SyntaxKind::COMMA,
            Token::Eq => SyntaxKind::EQUALITY_SIGN,
            Token::Word => SyntaxKind::WORD,
            Token::Dollar => SyntaxKind::DOLLAR,
            Token::CommandName(_) => SyntaxKind::COMMAND_NAME,
        };

        Some((kind, text))
    }
}

fn tokenize(input: &str) -> Vec<(Token, &str)> {
    let mut lexer = Token::lexer(input);
    std::iter::from_fn(move || {
        let kind = lexer.next()?;
        let text = lexer.slice();
        Some((kind, text))
    })
    .map(|(kind, text)| {
        if kind == Token::CommandName(CommandName::Generic) {
            let name = commands::classify(&text[1..]);
            (Token::CommandName(name), text)
        } else {
            (kind, text)
        }
    })
    .collect()
}