summaryrefslogtreecommitdiff
path: root/support/texlab/src/syntax/bibtex/lexer.rs
diff options
context:
space:
mode:
Diffstat (limited to 'support/texlab/src/syntax/bibtex/lexer.rs')
-rw-r--r--support/texlab/src/syntax/bibtex/lexer.rs176
1 files changed, 176 insertions, 0 deletions
diff --git a/support/texlab/src/syntax/bibtex/lexer.rs b/support/texlab/src/syntax/bibtex/lexer.rs
new file mode 100644
index 0000000000..8278ef2631
--- /dev/null
+++ b/support/texlab/src/syntax/bibtex/lexer.rs
@@ -0,0 +1,176 @@
+use super::ast::{BibtexToken, BibtexTokenKind};
+use crate::syntax::text::CharStream;
+
+pub struct BibtexLexer<'a> {
+ stream: CharStream<'a>,
+}
+
+impl<'a> BibtexLexer<'a> {
+ pub fn new(text: &'a str) -> Self {
+ BibtexLexer {
+ stream: CharStream::new(text),
+ }
+ }
+
+ fn kind(&mut self) -> BibtexToken {
+ fn is_type_char(c: char) -> bool {
+ c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z'
+ }
+
+ self.stream.start_span();
+ self.stream.next().unwrap();
+ while self.stream.satifies(|c| is_type_char(*c)) {
+ self.stream.next();
+ }
+ let span = self.stream.end_span();
+ let kind = match span.text.to_lowercase().as_ref() {
+ "@preamble" => BibtexTokenKind::PreambleKind,
+ "@string" => BibtexTokenKind::StringKind,
+ _ => BibtexTokenKind::EntryKind,
+ };
+ BibtexToken::new(span, kind)
+ }
+
+ fn single_character(&mut self, kind: BibtexTokenKind) -> BibtexToken {
+ self.stream.start_span();
+ self.stream.next();
+ let span = self.stream.end_span();
+ BibtexToken::new(span, kind)
+ }
+
+ fn command(&mut self) -> BibtexToken {
+ let span = self.stream.command();
+ BibtexToken::new(span, BibtexTokenKind::Command)
+ }
+
+ fn word(&mut self) -> BibtexToken {
+ fn is_word_char(c: char) -> bool {
+ !c.is_whitespace()
+ && c != '@'
+ && c != '='
+ && c != ','
+ && c != '#'
+ && c != '"'
+ && c != '{'
+ && c != '}'
+ && c != '('
+ && c != ')'
+ }
+
+ self.stream.start_span();
+ while self.stream.satifies(|c| is_word_char(*c)) {
+ self.stream.next();
+ }
+ let span = self.stream.end_span();
+ BibtexToken::new(span, BibtexTokenKind::Word)
+ }
+}
+
+impl<'a> Iterator for BibtexLexer<'a> {
+ type Item = BibtexToken;
+
+ fn next(&mut self) -> Option<BibtexToken> {
+ loop {
+ match self.stream.peek() {
+ Some('@') => return Some(self.kind()),
+ Some('=') => return Some(self.single_character(BibtexTokenKind::Assign)),
+ Some(',') => return Some(self.single_character(BibtexTokenKind::Comma)),
+ Some('#') => return Some(self.single_character(BibtexTokenKind::Concat)),
+ Some('"') => return Some(self.single_character(BibtexTokenKind::Quote)),
+ Some('{') => return Some(self.single_character(BibtexTokenKind::BeginBrace)),
+ Some('}') => return Some(self.single_character(BibtexTokenKind::EndBrace)),
+ Some('(') => return Some(self.single_character(BibtexTokenKind::BeginParen)),
+ Some(')') => return Some(self.single_character(BibtexTokenKind::EndParen)),
+ Some('\\') => return Some(self.command()),
+ Some(c) => {
+ if c.is_whitespace() {
+ self.stream.next();
+ } else {
+ return Some(self.word());
+ }
+ }
+ None => {
+ return None;
+ }
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::syntax::text::Span;
+ use lsp_types::{Position, Range};
+
+ fn verify<'a>(
+ lexer: &mut BibtexLexer<'a>,
+ line: u64,
+ character: u64,
+ text: &str,
+ kind: BibtexTokenKind,
+ ) {
+ let start = Position::new(line, character);
+ let end = Position::new(line, character + text.chars().count() as u64);
+ let range = Range::new(start, end);
+ let span = Span::new(range, text.to_owned());
+ let token = BibtexToken::new(span, kind);
+ assert_eq!(Some(token), lexer.next());
+ }
+
+ #[test]
+ fn test_word() {
+ let mut lexer = BibtexLexer::new("foo bar baz");
+ verify(&mut lexer, 0, 0, "foo", BibtexTokenKind::Word);
+ verify(&mut lexer, 0, 4, "bar", BibtexTokenKind::Word);
+ verify(&mut lexer, 0, 8, "baz", BibtexTokenKind::Word);
+ assert_eq!(None, lexer.next());
+ }
+
+ #[test]
+ fn test_command() {
+ let mut lexer = BibtexLexer::new("\\foo\\bar@baz");
+ verify(&mut lexer, 0, 0, "\\foo", BibtexTokenKind::Command);
+ verify(&mut lexer, 0, 4, "\\bar@baz", BibtexTokenKind::Command);
+ assert_eq!(None, lexer.next());
+ }
+
+ #[test]
+ fn test_escape_sequence() {
+ let mut lexer = BibtexLexer::new("\\foo*\n\\%\\**");
+ verify(&mut lexer, 0, 0, "\\foo*", BibtexTokenKind::Command);
+ verify(&mut lexer, 1, 0, "\\%", BibtexTokenKind::Command);
+ verify(&mut lexer, 1, 2, "\\*", BibtexTokenKind::Command);
+ verify(&mut lexer, 1, 4, "*", BibtexTokenKind::Word);
+ assert_eq!(None, lexer.next());
+ }
+
+ #[test]
+ fn test_delimiter() {
+ let mut lexer = BibtexLexer::new("{}()\"");
+ verify(&mut lexer, 0, 0, "{", BibtexTokenKind::BeginBrace);
+ verify(&mut lexer, 0, 1, "}", BibtexTokenKind::EndBrace);
+ verify(&mut lexer, 0, 2, "(", BibtexTokenKind::BeginParen);
+ verify(&mut lexer, 0, 3, ")", BibtexTokenKind::EndParen);
+ verify(&mut lexer, 0, 4, "\"", BibtexTokenKind::Quote);
+ assert_eq!(None, lexer.next());
+ }
+
+ #[test]
+ fn test_kind() {
+ let mut lexer = BibtexLexer::new("@pReAmBlE\n@article\n@string");
+ verify(&mut lexer, 0, 0, "@pReAmBlE", BibtexTokenKind::PreambleKind);
+ verify(&mut lexer, 1, 0, "@article", BibtexTokenKind::EntryKind);
+ verify(&mut lexer, 2, 0, "@string", BibtexTokenKind::StringKind);
+ assert_eq!(None, lexer.next());
+ }
+
+ #[test]
+ fn test_operator() {
+ let mut lexer = BibtexLexer::new("=,#");
+ verify(&mut lexer, 0, 0, "=", BibtexTokenKind::Assign);
+ verify(&mut lexer, 0, 1, ",", BibtexTokenKind::Comma);
+ verify(&mut lexer, 0, 2, "#", BibtexTokenKind::Concat);
+ assert_eq!(None, lexer.next());
+ }
+}