diff options
Diffstat (limited to 'support/texlab/src/syntax/latex/lexer.rs')
-rw-r--r-- | support/texlab/src/syntax/latex/lexer.rs | 117 |
1 files changed, 57 insertions, 60 deletions
diff --git a/support/texlab/src/syntax/latex/lexer.rs b/support/texlab/src/syntax/latex/lexer.rs index f753d7afce..e14f1f6d85 100644 --- a/support/texlab/src/syntax/latex/lexer.rs +++ b/support/texlab/src/syntax/latex/lexer.rs @@ -1,40 +1,41 @@ -use super::ast::{LatexToken, LatexTokenKind}; +use super::ast::{Token, TokenKind}; use crate::syntax::text::CharStream; -pub struct LatexLexer<'a> { +#[derive(Debug)] +pub struct Lexer<'a> { stream: CharStream<'a>, } -impl<'a> LatexLexer<'a> { +impl<'a> Lexer<'a> { pub fn new(text: &'a str) -> Self { - LatexLexer { + Self { stream: CharStream::new(text), } } - fn single_char(&mut self, kind: LatexTokenKind) -> LatexToken { + fn single_char(&mut self, kind: TokenKind) -> Token { self.stream.start_span(); self.stream.next(); let span = self.stream.end_span(); - LatexToken::new(span, kind) + Token::new(span, kind) } - fn math(&mut self) -> LatexToken { + fn math(&mut self) -> Token { self.stream.start_span(); self.stream.next(); if self.stream.satifies(|c| *c == '$') { self.stream.next(); } let span = self.stream.end_span(); - LatexToken::new(span, LatexTokenKind::Math) + Token::new(span, TokenKind::Math) } - fn command(&mut self) -> LatexToken { + fn command(&mut self) -> Token { let span = self.stream.command(); - LatexToken::new(span, LatexTokenKind::Command) + Token::new(span, TokenKind::Command) } - fn word(&mut self) -> LatexToken { + fn word(&mut self) -> Token { self.stream.start_span(); self.stream.next(); while self.stream.satifies(|c| is_word_char(*c)) { @@ -42,36 +43,36 @@ impl<'a> LatexLexer<'a> { } let span = self.stream.end_span(); - LatexToken::new(span, LatexTokenKind::Word) + Token::new(span, TokenKind::Word) } } -impl<'a> Iterator for LatexLexer<'a> { - type Item = LatexToken; +impl<'a> Iterator for Lexer<'a> { + type Item = Token; - fn next(&mut self) -> Option<LatexToken> { + fn next(&mut self) -> Option<Token> { loop { match self.stream.peek() { Some('%') => { self.stream.skip_rest_of_line(); } Some('{') => { - return Some(self.single_char(LatexTokenKind::BeginGroup)); + return Some(self.single_char(TokenKind::BeginGroup)); } Some('}') => { - return Some(self.single_char(LatexTokenKind::EndGroup)); + return Some(self.single_char(TokenKind::EndGroup)); } Some('[') => { - return Some(self.single_char(LatexTokenKind::BeginOptions)); + return Some(self.single_char(TokenKind::BeginOptions)); } Some(']') => { - return Some(self.single_char(LatexTokenKind::EndOptions)); + return Some(self.single_char(TokenKind::EndOptions)); } Some('$') => { return Some(self.math()); } Some(',') => { - return Some(self.single_char(LatexTokenKind::Comma)); + return Some(self.single_char(TokenKind::Comma)); } Some('\\') => { return Some(self.command()); @@ -106,74 +107,70 @@ fn is_word_char(c: char) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::syntax::text::Span; - use lsp_types::{Position, Range}; - - fn verify<'a>( - lexer: &mut LatexLexer<'a>, - line: u64, - character: u64, - text: &str, - kind: LatexTokenKind, - ) { + use crate::{ + protocol::{Position, Range}, + syntax::text::Span, + }; + + fn verify<'a>(lexer: &mut Lexer<'a>, line: u64, character: u64, text: &str, kind: TokenKind) { let start = Position::new(line, character); let end = Position::new(line, character + text.chars().count() as u64); let range = Range::new(start, end); let span = Span::new(range, text.to_owned()); - let token = LatexToken::new(span, kind); + let token = Token::new(span, kind); assert_eq!(Some(token), lexer.next()); } #[test] - fn test_word() { - let mut lexer = LatexLexer::new("foo bar baz"); - verify(&mut lexer, 0, 0, "foo", LatexTokenKind::Word); - verify(&mut lexer, 0, 4, "bar", LatexTokenKind::Word); - verify(&mut lexer, 0, 8, "baz", LatexTokenKind::Word); + fn word() { + let mut lexer = Lexer::new("foo bar baz"); + verify(&mut lexer, 0, 0, "foo", TokenKind::Word); + verify(&mut lexer, 0, 4, "bar", TokenKind::Word); + verify(&mut lexer, 0, 8, "baz", TokenKind::Word); assert_eq!(None, lexer.next()); } #[test] - fn test_command() { - let mut lexer = LatexLexer::new("\\foo\\bar@baz\n\\foo*"); - verify(&mut lexer, 0, 0, "\\foo", LatexTokenKind::Command); - verify(&mut lexer, 0, 4, "\\bar@baz", LatexTokenKind::Command); - verify(&mut lexer, 1, 0, "\\foo*", LatexTokenKind::Command); + fn command() { + let mut lexer = Lexer::new("\\foo\\bar@baz\n\\foo*"); + verify(&mut lexer, 0, 0, "\\foo", TokenKind::Command); + verify(&mut lexer, 0, 4, "\\bar@baz", TokenKind::Command); + verify(&mut lexer, 1, 0, "\\foo*", TokenKind::Command); assert_eq!(None, lexer.next()); } #[test] - fn test_escape_sequence() { - let mut lexer = LatexLexer::new("\\%\\**"); - verify(&mut lexer, 0, 0, "\\%", LatexTokenKind::Command); - verify(&mut lexer, 0, 2, "\\*", LatexTokenKind::Command); - verify(&mut lexer, 0, 4, "*", LatexTokenKind::Word); + fn escape_sequence() { + let mut lexer = Lexer::new("\\%\\**"); + verify(&mut lexer, 0, 0, "\\%", TokenKind::Command); + verify(&mut lexer, 0, 2, "\\*", TokenKind::Command); + verify(&mut lexer, 0, 4, "*", TokenKind::Word); assert_eq!(None, lexer.next()); } #[test] - fn test_group_delimiter() { - let mut lexer = LatexLexer::new("{}[]"); - verify(&mut lexer, 0, 0, "{", LatexTokenKind::BeginGroup); - verify(&mut lexer, 0, 1, "}", LatexTokenKind::EndGroup); - verify(&mut lexer, 0, 2, "[", LatexTokenKind::BeginOptions); - verify(&mut lexer, 0, 3, "]", LatexTokenKind::EndOptions); + fn group_delimiter() { + let mut lexer = Lexer::new("{}[]"); + verify(&mut lexer, 0, 0, "{", TokenKind::BeginGroup); + verify(&mut lexer, 0, 1, "}", TokenKind::EndGroup); + verify(&mut lexer, 0, 2, "[", TokenKind::BeginOptions); + verify(&mut lexer, 0, 3, "]", TokenKind::EndOptions); assert_eq!(None, lexer.next()); } #[test] - fn test_math() { - let mut lexer = LatexLexer::new("$$ $ $"); - verify(&mut lexer, 0, 0, "$$", LatexTokenKind::Math); - verify(&mut lexer, 0, 3, "$", LatexTokenKind::Math); - verify(&mut lexer, 0, 5, "$", LatexTokenKind::Math); + fn math() { + let mut lexer = Lexer::new("$$ $ $"); + verify(&mut lexer, 0, 0, "$$", TokenKind::Math); + verify(&mut lexer, 0, 3, "$", TokenKind::Math); + verify(&mut lexer, 0, 5, "$", TokenKind::Math); assert_eq!(None, lexer.next()); } #[test] - fn test_line_comment() { - let mut lexer = LatexLexer::new(" %foo \nfoo"); - verify(&mut lexer, 1, 0, "foo", LatexTokenKind::Word); + fn line_comment() { + let mut lexer = Lexer::new(" %foo \nfoo"); + verify(&mut lexer, 1, 0, "foo", TokenKind::Word); assert_eq!(None, lexer.next()); } } |