summaryrefslogtreecommitdiff
path: root/support/texlab/src/syntax/bibtex/lexer.rs
blob: dff111d6454a112092e3b7870a939a2a680c4174 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
use logos::Logos;

use super::kind::SyntaxKind;

#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, Logos)]
#[allow(non_camel_case_types)]
#[repr(u16)]
enum LogosToken {
    #[regex(r"\s+")]
    WHITESPACE = 2,

    #[regex(r"@[Pp][Rr][Ee][Aa][Mm][Bb][Ll][Ee]")]
    PREAMBLE_TYPE,

    #[regex(r"@[Ss][Tt][Rr][Ii][Nn][Gg]")]
    STRING_TYPE,

    #[regex(r"@[Cc][Oo][Mm][Mm][Ee][Nn][Tt]")]
    COMMENT_TYPE,

    #[regex(r"@[!\$\&\*\+\-\./:;<>\?@\[\]\\\^_`\|\~a-zA-Z][!\$\&\*\+\-\./:;<>\?@\[\]\\\^_`\|\~a-zA-Z0-9]*|@")]
    ENTRY_TYPE,

    #[regex(r#"[^\s\{\}\(\),#"=\\]+"#)]
    #[error]
    WORD,

    #[token("{")]
    L_CURLY,

    #[token("}")]
    R_CURLY,

    #[token("(")]
    L_PAREN,

    #[token(")")]
    R_PAREN,

    #[token(",")]
    COMMA,

    #[token("#")]
    HASH,

    #[token("\"")]
    QUOTE,

    #[token("=")]
    EQUALITY_SIGN,

    #[regex(r"\\([^\r\n]|[@a-zA-Z:_]+\*?)?")]
    COMMAND_NAME,
}

#[derive(Debug, PartialEq, Eq, Clone)]
pub struct Lexer<'a> {
    tokens: Vec<(SyntaxKind, &'a str)>,
}

impl<'a> Lexer<'a> {
    pub fn new(text: &'a str) -> Self {
        let mut tokens = Vec::new();
        let mut lexer = LogosToken::lexer(text);
        while let Some(kind) = lexer.next() {
            tokens.push((
                unsafe { std::mem::transmute::<LogosToken, SyntaxKind>(kind) },
                lexer.slice(),
            ));
        }
        tokens.reverse();
        Self { tokens }
    }

    pub fn peek(&self) -> Option<SyntaxKind> {
        self.tokens.last().map(|(kind, _)| *kind)
    }

    pub fn consume(&mut self) -> Option<(SyntaxKind, &'a str)> {
        self.tokens.pop()
    }
}