diff options
Diffstat (limited to 'support/texlab/src/syntax/bibtex/lexer.rs')
-rw-r--r-- | support/texlab/src/syntax/bibtex/lexer.rs | 176 |
1 files changed, 176 insertions, 0 deletions
diff --git a/support/texlab/src/syntax/bibtex/lexer.rs b/support/texlab/src/syntax/bibtex/lexer.rs new file mode 100644 index 0000000000..8278ef2631 --- /dev/null +++ b/support/texlab/src/syntax/bibtex/lexer.rs @@ -0,0 +1,176 @@ +use super::ast::{BibtexToken, BibtexTokenKind}; +use crate::syntax::text::CharStream; + +pub struct BibtexLexer<'a> { + stream: CharStream<'a>, +} + +impl<'a> BibtexLexer<'a> { + pub fn new(text: &'a str) -> Self { + BibtexLexer { + stream: CharStream::new(text), + } + } + + fn kind(&mut self) -> BibtexToken { + fn is_type_char(c: char) -> bool { + c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' + } + + self.stream.start_span(); + self.stream.next().unwrap(); + while self.stream.satifies(|c| is_type_char(*c)) { + self.stream.next(); + } + let span = self.stream.end_span(); + let kind = match span.text.to_lowercase().as_ref() { + "@preamble" => BibtexTokenKind::PreambleKind, + "@string" => BibtexTokenKind::StringKind, + _ => BibtexTokenKind::EntryKind, + }; + BibtexToken::new(span, kind) + } + + fn single_character(&mut self, kind: BibtexTokenKind) -> BibtexToken { + self.stream.start_span(); + self.stream.next(); + let span = self.stream.end_span(); + BibtexToken::new(span, kind) + } + + fn command(&mut self) -> BibtexToken { + let span = self.stream.command(); + BibtexToken::new(span, BibtexTokenKind::Command) + } + + fn word(&mut self) -> BibtexToken { + fn is_word_char(c: char) -> bool { + !c.is_whitespace() + && c != '@' + && c != '=' + && c != ',' + && c != '#' + && c != '"' + && c != '{' + && c != '}' + && c != '(' + && c != ')' + } + + self.stream.start_span(); + while self.stream.satifies(|c| is_word_char(*c)) { + self.stream.next(); + } + let span = self.stream.end_span(); + BibtexToken::new(span, BibtexTokenKind::Word) + } +} + +impl<'a> Iterator for BibtexLexer<'a> { + type Item = BibtexToken; + + fn next(&mut self) -> Option<BibtexToken> { + loop { + match self.stream.peek() { + Some('@') => return Some(self.kind()), + Some('=') => return Some(self.single_character(BibtexTokenKind::Assign)), + Some(',') => return Some(self.single_character(BibtexTokenKind::Comma)), + Some('#') => return Some(self.single_character(BibtexTokenKind::Concat)), + Some('"') => return Some(self.single_character(BibtexTokenKind::Quote)), + Some('{') => return Some(self.single_character(BibtexTokenKind::BeginBrace)), + Some('}') => return Some(self.single_character(BibtexTokenKind::EndBrace)), + Some('(') => return Some(self.single_character(BibtexTokenKind::BeginParen)), + Some(')') => return Some(self.single_character(BibtexTokenKind::EndParen)), + Some('\\') => return Some(self.command()), + Some(c) => { + if c.is_whitespace() { + self.stream.next(); + } else { + return Some(self.word()); + } + } + None => { + return None; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::syntax::text::Span; + use lsp_types::{Position, Range}; + + fn verify<'a>( + lexer: &mut BibtexLexer<'a>, + line: u64, + character: u64, + text: &str, + kind: BibtexTokenKind, + ) { + let start = Position::new(line, character); + let end = Position::new(line, character + text.chars().count() as u64); + let range = Range::new(start, end); + let span = Span::new(range, text.to_owned()); + let token = BibtexToken::new(span, kind); + assert_eq!(Some(token), lexer.next()); + } + + #[test] + fn test_word() { + let mut lexer = BibtexLexer::new("foo bar baz"); + verify(&mut lexer, 0, 0, "foo", BibtexTokenKind::Word); + verify(&mut lexer, 0, 4, "bar", BibtexTokenKind::Word); + verify(&mut lexer, 0, 8, "baz", BibtexTokenKind::Word); + assert_eq!(None, lexer.next()); + } + + #[test] + fn test_command() { + let mut lexer = BibtexLexer::new("\\foo\\bar@baz"); + verify(&mut lexer, 0, 0, "\\foo", BibtexTokenKind::Command); + verify(&mut lexer, 0, 4, "\\bar@baz", BibtexTokenKind::Command); + assert_eq!(None, lexer.next()); + } + + #[test] + fn test_escape_sequence() { + let mut lexer = BibtexLexer::new("\\foo*\n\\%\\**"); + verify(&mut lexer, 0, 0, "\\foo*", BibtexTokenKind::Command); + verify(&mut lexer, 1, 0, "\\%", BibtexTokenKind::Command); + verify(&mut lexer, 1, 2, "\\*", BibtexTokenKind::Command); + verify(&mut lexer, 1, 4, "*", BibtexTokenKind::Word); + assert_eq!(None, lexer.next()); + } + + #[test] + fn test_delimiter() { + let mut lexer = BibtexLexer::new("{}()\""); + verify(&mut lexer, 0, 0, "{", BibtexTokenKind::BeginBrace); + verify(&mut lexer, 0, 1, "}", BibtexTokenKind::EndBrace); + verify(&mut lexer, 0, 2, "(", BibtexTokenKind::BeginParen); + verify(&mut lexer, 0, 3, ")", BibtexTokenKind::EndParen); + verify(&mut lexer, 0, 4, "\"", BibtexTokenKind::Quote); + assert_eq!(None, lexer.next()); + } + + #[test] + fn test_kind() { + let mut lexer = BibtexLexer::new("@pReAmBlE\n@article\n@string"); + verify(&mut lexer, 0, 0, "@pReAmBlE", BibtexTokenKind::PreambleKind); + verify(&mut lexer, 1, 0, "@article", BibtexTokenKind::EntryKind); + verify(&mut lexer, 2, 0, "@string", BibtexTokenKind::StringKind); + assert_eq!(None, lexer.next()); + } + + #[test] + fn test_operator() { + let mut lexer = BibtexLexer::new("=,#"); + verify(&mut lexer, 0, 0, "=", BibtexTokenKind::Assign); + verify(&mut lexer, 0, 1, ",", BibtexTokenKind::Comma); + verify(&mut lexer, 0, 2, "#", BibtexTokenKind::Concat); + assert_eq!(None, lexer.next()); + } +} |