From 745892fbddea56040139108277e728b53fd8fc11 Mon Sep 17 00:00:00 2001 From: Norbert Preining Date: Thu, 28 May 2020 03:03:21 +0000 Subject: CTAN sync 202005280303 --- support/texlab/src/syntax/bibtex/lexer.rs | 135 +++++++++++++++--------------- 1 file changed, 66 insertions(+), 69 deletions(-) (limited to 'support/texlab/src/syntax/bibtex/lexer.rs') diff --git a/support/texlab/src/syntax/bibtex/lexer.rs b/support/texlab/src/syntax/bibtex/lexer.rs index 8278ef2631..e089b15864 100644 --- a/support/texlab/src/syntax/bibtex/lexer.rs +++ b/support/texlab/src/syntax/bibtex/lexer.rs @@ -1,18 +1,19 @@ -use super::ast::{BibtexToken, BibtexTokenKind}; +use super::ast::{Token, TokenKind}; use crate::syntax::text::CharStream; -pub struct BibtexLexer<'a> { +#[derive(Debug)] +pub struct Lexer<'a> { stream: CharStream<'a>, } -impl<'a> BibtexLexer<'a> { +impl<'a> Lexer<'a> { pub fn new(text: &'a str) -> Self { - BibtexLexer { + Self { stream: CharStream::new(text), } } - fn kind(&mut self) -> BibtexToken { + fn kind(&mut self) -> Token { fn is_type_char(c: char) -> bool { c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' } @@ -24,26 +25,26 @@ impl<'a> BibtexLexer<'a> { } let span = self.stream.end_span(); let kind = match span.text.to_lowercase().as_ref() { - "@preamble" => BibtexTokenKind::PreambleKind, - "@string" => BibtexTokenKind::StringKind, - _ => BibtexTokenKind::EntryKind, + "@preamble" => TokenKind::PreambleKind, + "@string" => TokenKind::StringKind, + _ => TokenKind::EntryKind, }; - BibtexToken::new(span, kind) + Token::new(span, kind) } - fn single_character(&mut self, kind: BibtexTokenKind) -> BibtexToken { + fn single_character(&mut self, kind: TokenKind) -> Token { self.stream.start_span(); self.stream.next(); let span = self.stream.end_span(); - BibtexToken::new(span, kind) + Token::new(span, kind) } - fn command(&mut self) -> BibtexToken { + fn command(&mut self) -> Token { let span = self.stream.command(); - BibtexToken::new(span, BibtexTokenKind::Command) + Token::new(span, TokenKind::Command) } - fn word(&mut self) -> BibtexToken { + fn word(&mut self) -> Token { fn is_word_char(c: char) -> bool { !c.is_whitespace() && c != '@' @@ -62,25 +63,25 @@ impl<'a> BibtexLexer<'a> { self.stream.next(); } let span = self.stream.end_span(); - BibtexToken::new(span, BibtexTokenKind::Word) + Token::new(span, TokenKind::Word) } } -impl<'a> Iterator for BibtexLexer<'a> { - type Item = BibtexToken; +impl<'a> Iterator for Lexer<'a> { + type Item = Token; - fn next(&mut self) -> Option { + fn next(&mut self) -> Option { loop { match self.stream.peek() { Some('@') => return Some(self.kind()), - Some('=') => return Some(self.single_character(BibtexTokenKind::Assign)), - Some(',') => return Some(self.single_character(BibtexTokenKind::Comma)), - Some('#') => return Some(self.single_character(BibtexTokenKind::Concat)), - Some('"') => return Some(self.single_character(BibtexTokenKind::Quote)), - Some('{') => return Some(self.single_character(BibtexTokenKind::BeginBrace)), - Some('}') => return Some(self.single_character(BibtexTokenKind::EndBrace)), - Some('(') => return Some(self.single_character(BibtexTokenKind::BeginParen)), - Some(')') => return Some(self.single_character(BibtexTokenKind::EndParen)), + Some('=') => return Some(self.single_character(TokenKind::Assign)), + Some(',') => return Some(self.single_character(TokenKind::Comma)), + Some('#') => return Some(self.single_character(TokenKind::Concat)), + Some('"') => return Some(self.single_character(TokenKind::Quote)), + Some('{') => return Some(self.single_character(TokenKind::BeginBrace)), + Some('}') => return Some(self.single_character(TokenKind::EndBrace)), + Some('(') => return Some(self.single_character(TokenKind::BeginParen)), + Some(')') => return Some(self.single_character(TokenKind::EndParen)), Some('\\') => return Some(self.command()), Some(c) => { if c.is_whitespace() { @@ -100,77 +101,73 @@ impl<'a> Iterator for BibtexLexer<'a> { #[cfg(test)] mod tests { use super::*; - use crate::syntax::text::Span; - use lsp_types::{Position, Range}; - - fn verify<'a>( - lexer: &mut BibtexLexer<'a>, - line: u64, - character: u64, - text: &str, - kind: BibtexTokenKind, - ) { + use crate::{ + protocol::{Position, Range}, + syntax::text::Span, + }; + + fn verify<'a>(lexer: &mut Lexer<'a>, line: u64, character: u64, text: &str, kind: TokenKind) { let start = Position::new(line, character); let end = Position::new(line, character + text.chars().count() as u64); let range = Range::new(start, end); let span = Span::new(range, text.to_owned()); - let token = BibtexToken::new(span, kind); + let token = Token::new(span, kind); assert_eq!(Some(token), lexer.next()); } #[test] - fn test_word() { - let mut lexer = BibtexLexer::new("foo bar baz"); - verify(&mut lexer, 0, 0, "foo", BibtexTokenKind::Word); - verify(&mut lexer, 0, 4, "bar", BibtexTokenKind::Word); - verify(&mut lexer, 0, 8, "baz", BibtexTokenKind::Word); + fn word() { + let mut lexer = Lexer::new("foo bar baz"); + verify(&mut lexer, 0, 0, "foo", TokenKind::Word); + verify(&mut lexer, 0, 4, "bar", TokenKind::Word); + verify(&mut lexer, 0, 8, "baz", TokenKind::Word); assert_eq!(None, lexer.next()); } #[test] - fn test_command() { - let mut lexer = BibtexLexer::new("\\foo\\bar@baz"); - verify(&mut lexer, 0, 0, "\\foo", BibtexTokenKind::Command); - verify(&mut lexer, 0, 4, "\\bar@baz", BibtexTokenKind::Command); + fn command() { + let mut lexer = Lexer::new("\\foo\\bar@baz"); + verify(&mut lexer, 0, 0, "\\foo", TokenKind::Command); + verify(&mut lexer, 0, 4, "\\bar@baz", TokenKind::Command); assert_eq!(None, lexer.next()); } #[test] - fn test_escape_sequence() { - let mut lexer = BibtexLexer::new("\\foo*\n\\%\\**"); - verify(&mut lexer, 0, 0, "\\foo*", BibtexTokenKind::Command); - verify(&mut lexer, 1, 0, "\\%", BibtexTokenKind::Command); - verify(&mut lexer, 1, 2, "\\*", BibtexTokenKind::Command); - verify(&mut lexer, 1, 4, "*", BibtexTokenKind::Word); + fn escape_sequence() { + let mut lexer = Lexer::new("\\foo*\n\\%\\**"); + verify(&mut lexer, 0, 0, "\\foo*", TokenKind::Command); + verify(&mut lexer, 1, 0, "\\%", TokenKind::Command); + verify(&mut lexer, 1, 2, "\\*", TokenKind::Command); + verify(&mut lexer, 1, 4, "*", TokenKind::Word); assert_eq!(None, lexer.next()); } #[test] - fn test_delimiter() { - let mut lexer = BibtexLexer::new("{}()\""); - verify(&mut lexer, 0, 0, "{", BibtexTokenKind::BeginBrace); - verify(&mut lexer, 0, 1, "}", BibtexTokenKind::EndBrace); - verify(&mut lexer, 0, 2, "(", BibtexTokenKind::BeginParen); - verify(&mut lexer, 0, 3, ")", BibtexTokenKind::EndParen); - verify(&mut lexer, 0, 4, "\"", BibtexTokenKind::Quote); + fn delimiter() { + let mut lexer = Lexer::new("{}()\""); + verify(&mut lexer, 0, 0, "{", TokenKind::BeginBrace); + verify(&mut lexer, 0, 1, "}", TokenKind::EndBrace); + verify(&mut lexer, 0, 2, "(", TokenKind::BeginParen); + verify(&mut lexer, 0, 3, ")", TokenKind::EndParen); + verify(&mut lexer, 0, 4, "\"", TokenKind::Quote); assert_eq!(None, lexer.next()); } #[test] - fn test_kind() { - let mut lexer = BibtexLexer::new("@pReAmBlE\n@article\n@string"); - verify(&mut lexer, 0, 0, "@pReAmBlE", BibtexTokenKind::PreambleKind); - verify(&mut lexer, 1, 0, "@article", BibtexTokenKind::EntryKind); - verify(&mut lexer, 2, 0, "@string", BibtexTokenKind::StringKind); + fn kind() { + let mut lexer = Lexer::new("@pReAmBlE\n@article\n@string"); + verify(&mut lexer, 0, 0, "@pReAmBlE", TokenKind::PreambleKind); + verify(&mut lexer, 1, 0, "@article", TokenKind::EntryKind); + verify(&mut lexer, 2, 0, "@string", TokenKind::StringKind); assert_eq!(None, lexer.next()); } #[test] - fn test_operator() { - let mut lexer = BibtexLexer::new("=,#"); - verify(&mut lexer, 0, 0, "=", BibtexTokenKind::Assign); - verify(&mut lexer, 0, 1, ",", BibtexTokenKind::Comma); - verify(&mut lexer, 0, 2, "#", BibtexTokenKind::Concat); + fn operator() { + let mut lexer = Lexer::new("=,#"); + verify(&mut lexer, 0, 0, "=", TokenKind::Assign); + verify(&mut lexer, 0, 1, ",", TokenKind::Comma); + verify(&mut lexer, 0, 2, "#", TokenKind::Concat); assert_eq!(None, lexer.next()); } } -- cgit v1.2.3