1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
|
use logos::Logos;
use super::kind::SyntaxKind;
#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, Logos)]
#[allow(non_camel_case_types)]
#[repr(u16)]
enum LogosToken {
#[regex(r"\s+")]
WHITESPACE = 2,
#[regex(r"@[Pp][Rr][Ee][Aa][Mm][Bb][Ll][Ee]")]
PREAMBLE_TYPE,
#[regex(r"@[Ss][Tt][Rr][Ii][Nn][Gg]")]
STRING_TYPE,
#[regex(r"@[Cc][Oo][Mm][Mm][Ee][Nn][Tt]")]
COMMENT_TYPE,
#[regex(r"@[!\$\&\*\+\-\./:;<>\?@\[\]\\\^_`\|\~a-zA-Z][!\$\&\*\+\-\./:;<>\?@\[\]\\\^_`\|\~a-zA-Z0-9]*|@")]
ENTRY_TYPE,
#[regex(r#"[^\s\{\}\(\),#"=\\]+"#)]
#[error]
WORD,
#[token("{")]
L_CURLY,
#[token("}")]
R_CURLY,
#[token("(")]
L_PAREN,
#[token(")")]
R_PAREN,
#[token(",")]
COMMA,
#[token("#")]
HASH,
#[token("\"")]
QUOTE,
#[token("=")]
EQUALITY_SIGN,
#[regex(r"\\([^\r\n]|[@a-zA-Z:_]+\*?)?")]
COMMAND_NAME,
}
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct Lexer<'a> {
tokens: Vec<(SyntaxKind, &'a str)>,
}
impl<'a> Lexer<'a> {
pub fn new(text: &'a str) -> Self {
let mut tokens = Vec::new();
let mut lexer = LogosToken::lexer(text);
while let Some(kind) = lexer.next() {
tokens.push((
unsafe { std::mem::transmute::<LogosToken, SyntaxKind>(kind) },
lexer.slice(),
));
}
tokens.reverse();
Self { tokens }
}
pub fn peek(&self) -> Option<SyntaxKind> {
self.tokens.last().map(|(kind, _)| *kind)
}
pub fn consume(&mut self) -> Option<(SyntaxKind, &'a str)> {
self.tokens.pop()
}
}
|