summaryrefslogtreecommitdiff
path: root/support/texlab/src/syntax/bibtex/lexer.rs
diff options
context:
space:
mode:
Diffstat (limited to 'support/texlab/src/syntax/bibtex/lexer.rs')
-rw-r--r--support/texlab/src/syntax/bibtex/lexer.rs135
1 files changed, 66 insertions, 69 deletions
diff --git a/support/texlab/src/syntax/bibtex/lexer.rs b/support/texlab/src/syntax/bibtex/lexer.rs
index 8278ef2631..e089b15864 100644
--- a/support/texlab/src/syntax/bibtex/lexer.rs
+++ b/support/texlab/src/syntax/bibtex/lexer.rs
@@ -1,18 +1,19 @@
-use super::ast::{BibtexToken, BibtexTokenKind};
+use super::ast::{Token, TokenKind};
use crate::syntax::text::CharStream;
-pub struct BibtexLexer<'a> {
+#[derive(Debug)]
+pub struct Lexer<'a> {
stream: CharStream<'a>,
}
-impl<'a> BibtexLexer<'a> {
+impl<'a> Lexer<'a> {
pub fn new(text: &'a str) -> Self {
- BibtexLexer {
+ Self {
stream: CharStream::new(text),
}
}
- fn kind(&mut self) -> BibtexToken {
+ fn kind(&mut self) -> Token {
fn is_type_char(c: char) -> bool {
c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z'
}
@@ -24,26 +25,26 @@ impl<'a> BibtexLexer<'a> {
}
let span = self.stream.end_span();
let kind = match span.text.to_lowercase().as_ref() {
- "@preamble" => BibtexTokenKind::PreambleKind,
- "@string" => BibtexTokenKind::StringKind,
- _ => BibtexTokenKind::EntryKind,
+ "@preamble" => TokenKind::PreambleKind,
+ "@string" => TokenKind::StringKind,
+ _ => TokenKind::EntryKind,
};
- BibtexToken::new(span, kind)
+ Token::new(span, kind)
}
- fn single_character(&mut self, kind: BibtexTokenKind) -> BibtexToken {
+ fn single_character(&mut self, kind: TokenKind) -> Token {
self.stream.start_span();
self.stream.next();
let span = self.stream.end_span();
- BibtexToken::new(span, kind)
+ Token::new(span, kind)
}
- fn command(&mut self) -> BibtexToken {
+ fn command(&mut self) -> Token {
let span = self.stream.command();
- BibtexToken::new(span, BibtexTokenKind::Command)
+ Token::new(span, TokenKind::Command)
}
- fn word(&mut self) -> BibtexToken {
+ fn word(&mut self) -> Token {
fn is_word_char(c: char) -> bool {
!c.is_whitespace()
&& c != '@'
@@ -62,25 +63,25 @@ impl<'a> BibtexLexer<'a> {
self.stream.next();
}
let span = self.stream.end_span();
- BibtexToken::new(span, BibtexTokenKind::Word)
+ Token::new(span, TokenKind::Word)
}
}
-impl<'a> Iterator for BibtexLexer<'a> {
- type Item = BibtexToken;
+impl<'a> Iterator for Lexer<'a> {
+ type Item = Token;
- fn next(&mut self) -> Option<BibtexToken> {
+ fn next(&mut self) -> Option<Token> {
loop {
match self.stream.peek() {
Some('@') => return Some(self.kind()),
- Some('=') => return Some(self.single_character(BibtexTokenKind::Assign)),
- Some(',') => return Some(self.single_character(BibtexTokenKind::Comma)),
- Some('#') => return Some(self.single_character(BibtexTokenKind::Concat)),
- Some('"') => return Some(self.single_character(BibtexTokenKind::Quote)),
- Some('{') => return Some(self.single_character(BibtexTokenKind::BeginBrace)),
- Some('}') => return Some(self.single_character(BibtexTokenKind::EndBrace)),
- Some('(') => return Some(self.single_character(BibtexTokenKind::BeginParen)),
- Some(')') => return Some(self.single_character(BibtexTokenKind::EndParen)),
+ Some('=') => return Some(self.single_character(TokenKind::Assign)),
+ Some(',') => return Some(self.single_character(TokenKind::Comma)),
+ Some('#') => return Some(self.single_character(TokenKind::Concat)),
+ Some('"') => return Some(self.single_character(TokenKind::Quote)),
+ Some('{') => return Some(self.single_character(TokenKind::BeginBrace)),
+ Some('}') => return Some(self.single_character(TokenKind::EndBrace)),
+ Some('(') => return Some(self.single_character(TokenKind::BeginParen)),
+ Some(')') => return Some(self.single_character(TokenKind::EndParen)),
Some('\\') => return Some(self.command()),
Some(c) => {
if c.is_whitespace() {
@@ -100,77 +101,73 @@ impl<'a> Iterator for BibtexLexer<'a> {
#[cfg(test)]
mod tests {
use super::*;
- use crate::syntax::text::Span;
- use lsp_types::{Position, Range};
-
- fn verify<'a>(
- lexer: &mut BibtexLexer<'a>,
- line: u64,
- character: u64,
- text: &str,
- kind: BibtexTokenKind,
- ) {
+ use crate::{
+ protocol::{Position, Range},
+ syntax::text::Span,
+ };
+
+ fn verify<'a>(lexer: &mut Lexer<'a>, line: u64, character: u64, text: &str, kind: TokenKind) {
let start = Position::new(line, character);
let end = Position::new(line, character + text.chars().count() as u64);
let range = Range::new(start, end);
let span = Span::new(range, text.to_owned());
- let token = BibtexToken::new(span, kind);
+ let token = Token::new(span, kind);
assert_eq!(Some(token), lexer.next());
}
#[test]
- fn test_word() {
- let mut lexer = BibtexLexer::new("foo bar baz");
- verify(&mut lexer, 0, 0, "foo", BibtexTokenKind::Word);
- verify(&mut lexer, 0, 4, "bar", BibtexTokenKind::Word);
- verify(&mut lexer, 0, 8, "baz", BibtexTokenKind::Word);
+ fn word() {
+ let mut lexer = Lexer::new("foo bar baz");
+ verify(&mut lexer, 0, 0, "foo", TokenKind::Word);
+ verify(&mut lexer, 0, 4, "bar", TokenKind::Word);
+ verify(&mut lexer, 0, 8, "baz", TokenKind::Word);
assert_eq!(None, lexer.next());
}
#[test]
- fn test_command() {
- let mut lexer = BibtexLexer::new("\\foo\\bar@baz");
- verify(&mut lexer, 0, 0, "\\foo", BibtexTokenKind::Command);
- verify(&mut lexer, 0, 4, "\\bar@baz", BibtexTokenKind::Command);
+ fn command() {
+ let mut lexer = Lexer::new("\\foo\\bar@baz");
+ verify(&mut lexer, 0, 0, "\\foo", TokenKind::Command);
+ verify(&mut lexer, 0, 4, "\\bar@baz", TokenKind::Command);
assert_eq!(None, lexer.next());
}
#[test]
- fn test_escape_sequence() {
- let mut lexer = BibtexLexer::new("\\foo*\n\\%\\**");
- verify(&mut lexer, 0, 0, "\\foo*", BibtexTokenKind::Command);
- verify(&mut lexer, 1, 0, "\\%", BibtexTokenKind::Command);
- verify(&mut lexer, 1, 2, "\\*", BibtexTokenKind::Command);
- verify(&mut lexer, 1, 4, "*", BibtexTokenKind::Word);
+ fn escape_sequence() {
+ let mut lexer = Lexer::new("\\foo*\n\\%\\**");
+ verify(&mut lexer, 0, 0, "\\foo*", TokenKind::Command);
+ verify(&mut lexer, 1, 0, "\\%", TokenKind::Command);
+ verify(&mut lexer, 1, 2, "\\*", TokenKind::Command);
+ verify(&mut lexer, 1, 4, "*", TokenKind::Word);
assert_eq!(None, lexer.next());
}
#[test]
- fn test_delimiter() {
- let mut lexer = BibtexLexer::new("{}()\"");
- verify(&mut lexer, 0, 0, "{", BibtexTokenKind::BeginBrace);
- verify(&mut lexer, 0, 1, "}", BibtexTokenKind::EndBrace);
- verify(&mut lexer, 0, 2, "(", BibtexTokenKind::BeginParen);
- verify(&mut lexer, 0, 3, ")", BibtexTokenKind::EndParen);
- verify(&mut lexer, 0, 4, "\"", BibtexTokenKind::Quote);
+ fn delimiter() {
+ let mut lexer = Lexer::new("{}()\"");
+ verify(&mut lexer, 0, 0, "{", TokenKind::BeginBrace);
+ verify(&mut lexer, 0, 1, "}", TokenKind::EndBrace);
+ verify(&mut lexer, 0, 2, "(", TokenKind::BeginParen);
+ verify(&mut lexer, 0, 3, ")", TokenKind::EndParen);
+ verify(&mut lexer, 0, 4, "\"", TokenKind::Quote);
assert_eq!(None, lexer.next());
}
#[test]
- fn test_kind() {
- let mut lexer = BibtexLexer::new("@pReAmBlE\n@article\n@string");
- verify(&mut lexer, 0, 0, "@pReAmBlE", BibtexTokenKind::PreambleKind);
- verify(&mut lexer, 1, 0, "@article", BibtexTokenKind::EntryKind);
- verify(&mut lexer, 2, 0, "@string", BibtexTokenKind::StringKind);
+ fn kind() {
+ let mut lexer = Lexer::new("@pReAmBlE\n@article\n@string");
+ verify(&mut lexer, 0, 0, "@pReAmBlE", TokenKind::PreambleKind);
+ verify(&mut lexer, 1, 0, "@article", TokenKind::EntryKind);
+ verify(&mut lexer, 2, 0, "@string", TokenKind::StringKind);
assert_eq!(None, lexer.next());
}
#[test]
- fn test_operator() {
- let mut lexer = BibtexLexer::new("=,#");
- verify(&mut lexer, 0, 0, "=", BibtexTokenKind::Assign);
- verify(&mut lexer, 0, 1, ",", BibtexTokenKind::Comma);
- verify(&mut lexer, 0, 2, "#", BibtexTokenKind::Concat);
+ fn operator() {
+ let mut lexer = Lexer::new("=,#");
+ verify(&mut lexer, 0, 0, "=", TokenKind::Assign);
+ verify(&mut lexer, 0, 1, ",", TokenKind::Comma);
+ verify(&mut lexer, 0, 2, "#", TokenKind::Concat);
assert_eq!(None, lexer.next());
}
}