summaryrefslogtreecommitdiff
path: root/support/texlab/src/syntax/latex/lexer.rs
diff options
context:
space:
mode:
Diffstat (limited to 'support/texlab/src/syntax/latex/lexer.rs')
-rw-r--r--support/texlab/src/syntax/latex/lexer.rs117
1 files changed, 57 insertions, 60 deletions
diff --git a/support/texlab/src/syntax/latex/lexer.rs b/support/texlab/src/syntax/latex/lexer.rs
index f753d7afce..e14f1f6d85 100644
--- a/support/texlab/src/syntax/latex/lexer.rs
+++ b/support/texlab/src/syntax/latex/lexer.rs
@@ -1,40 +1,41 @@
-use super::ast::{LatexToken, LatexTokenKind};
+use super::ast::{Token, TokenKind};
use crate::syntax::text::CharStream;
-pub struct LatexLexer<'a> {
+#[derive(Debug)]
+pub struct Lexer<'a> {
stream: CharStream<'a>,
}
-impl<'a> LatexLexer<'a> {
+impl<'a> Lexer<'a> {
pub fn new(text: &'a str) -> Self {
- LatexLexer {
+ Self {
stream: CharStream::new(text),
}
}
- fn single_char(&mut self, kind: LatexTokenKind) -> LatexToken {
+ fn single_char(&mut self, kind: TokenKind) -> Token {
self.stream.start_span();
self.stream.next();
let span = self.stream.end_span();
- LatexToken::new(span, kind)
+ Token::new(span, kind)
}
- fn math(&mut self) -> LatexToken {
+ fn math(&mut self) -> Token {
self.stream.start_span();
self.stream.next();
if self.stream.satifies(|c| *c == '$') {
self.stream.next();
}
let span = self.stream.end_span();
- LatexToken::new(span, LatexTokenKind::Math)
+ Token::new(span, TokenKind::Math)
}
- fn command(&mut self) -> LatexToken {
+ fn command(&mut self) -> Token {
let span = self.stream.command();
- LatexToken::new(span, LatexTokenKind::Command)
+ Token::new(span, TokenKind::Command)
}
- fn word(&mut self) -> LatexToken {
+ fn word(&mut self) -> Token {
self.stream.start_span();
self.stream.next();
while self.stream.satifies(|c| is_word_char(*c)) {
@@ -42,36 +43,36 @@ impl<'a> LatexLexer<'a> {
}
let span = self.stream.end_span();
- LatexToken::new(span, LatexTokenKind::Word)
+ Token::new(span, TokenKind::Word)
}
}
-impl<'a> Iterator for LatexLexer<'a> {
- type Item = LatexToken;
+impl<'a> Iterator for Lexer<'a> {
+ type Item = Token;
- fn next(&mut self) -> Option<LatexToken> {
+ fn next(&mut self) -> Option<Token> {
loop {
match self.stream.peek() {
Some('%') => {
self.stream.skip_rest_of_line();
}
Some('{') => {
- return Some(self.single_char(LatexTokenKind::BeginGroup));
+ return Some(self.single_char(TokenKind::BeginGroup));
}
Some('}') => {
- return Some(self.single_char(LatexTokenKind::EndGroup));
+ return Some(self.single_char(TokenKind::EndGroup));
}
Some('[') => {
- return Some(self.single_char(LatexTokenKind::BeginOptions));
+ return Some(self.single_char(TokenKind::BeginOptions));
}
Some(']') => {
- return Some(self.single_char(LatexTokenKind::EndOptions));
+ return Some(self.single_char(TokenKind::EndOptions));
}
Some('$') => {
return Some(self.math());
}
Some(',') => {
- return Some(self.single_char(LatexTokenKind::Comma));
+ return Some(self.single_char(TokenKind::Comma));
}
Some('\\') => {
return Some(self.command());
@@ -106,74 +107,70 @@ fn is_word_char(c: char) -> bool {
#[cfg(test)]
mod tests {
use super::*;
- use crate::syntax::text::Span;
- use lsp_types::{Position, Range};
-
- fn verify<'a>(
- lexer: &mut LatexLexer<'a>,
- line: u64,
- character: u64,
- text: &str,
- kind: LatexTokenKind,
- ) {
+ use crate::{
+ protocol::{Position, Range},
+ syntax::text::Span,
+ };
+
+ fn verify<'a>(lexer: &mut Lexer<'a>, line: u64, character: u64, text: &str, kind: TokenKind) {
let start = Position::new(line, character);
let end = Position::new(line, character + text.chars().count() as u64);
let range = Range::new(start, end);
let span = Span::new(range, text.to_owned());
- let token = LatexToken::new(span, kind);
+ let token = Token::new(span, kind);
assert_eq!(Some(token), lexer.next());
}
#[test]
- fn test_word() {
- let mut lexer = LatexLexer::new("foo bar baz");
- verify(&mut lexer, 0, 0, "foo", LatexTokenKind::Word);
- verify(&mut lexer, 0, 4, "bar", LatexTokenKind::Word);
- verify(&mut lexer, 0, 8, "baz", LatexTokenKind::Word);
+ fn word() {
+ let mut lexer = Lexer::new("foo bar baz");
+ verify(&mut lexer, 0, 0, "foo", TokenKind::Word);
+ verify(&mut lexer, 0, 4, "bar", TokenKind::Word);
+ verify(&mut lexer, 0, 8, "baz", TokenKind::Word);
assert_eq!(None, lexer.next());
}
#[test]
- fn test_command() {
- let mut lexer = LatexLexer::new("\\foo\\bar@baz\n\\foo*");
- verify(&mut lexer, 0, 0, "\\foo", LatexTokenKind::Command);
- verify(&mut lexer, 0, 4, "\\bar@baz", LatexTokenKind::Command);
- verify(&mut lexer, 1, 0, "\\foo*", LatexTokenKind::Command);
+ fn command() {
+ let mut lexer = Lexer::new("\\foo\\bar@baz\n\\foo*");
+ verify(&mut lexer, 0, 0, "\\foo", TokenKind::Command);
+ verify(&mut lexer, 0, 4, "\\bar@baz", TokenKind::Command);
+ verify(&mut lexer, 1, 0, "\\foo*", TokenKind::Command);
assert_eq!(None, lexer.next());
}
#[test]
- fn test_escape_sequence() {
- let mut lexer = LatexLexer::new("\\%\\**");
- verify(&mut lexer, 0, 0, "\\%", LatexTokenKind::Command);
- verify(&mut lexer, 0, 2, "\\*", LatexTokenKind::Command);
- verify(&mut lexer, 0, 4, "*", LatexTokenKind::Word);
+ fn escape_sequence() {
+ let mut lexer = Lexer::new("\\%\\**");
+ verify(&mut lexer, 0, 0, "\\%", TokenKind::Command);
+ verify(&mut lexer, 0, 2, "\\*", TokenKind::Command);
+ verify(&mut lexer, 0, 4, "*", TokenKind::Word);
assert_eq!(None, lexer.next());
}
#[test]
- fn test_group_delimiter() {
- let mut lexer = LatexLexer::new("{}[]");
- verify(&mut lexer, 0, 0, "{", LatexTokenKind::BeginGroup);
- verify(&mut lexer, 0, 1, "}", LatexTokenKind::EndGroup);
- verify(&mut lexer, 0, 2, "[", LatexTokenKind::BeginOptions);
- verify(&mut lexer, 0, 3, "]", LatexTokenKind::EndOptions);
+ fn group_delimiter() {
+ let mut lexer = Lexer::new("{}[]");
+ verify(&mut lexer, 0, 0, "{", TokenKind::BeginGroup);
+ verify(&mut lexer, 0, 1, "}", TokenKind::EndGroup);
+ verify(&mut lexer, 0, 2, "[", TokenKind::BeginOptions);
+ verify(&mut lexer, 0, 3, "]", TokenKind::EndOptions);
assert_eq!(None, lexer.next());
}
#[test]
- fn test_math() {
- let mut lexer = LatexLexer::new("$$ $ $");
- verify(&mut lexer, 0, 0, "$$", LatexTokenKind::Math);
- verify(&mut lexer, 0, 3, "$", LatexTokenKind::Math);
- verify(&mut lexer, 0, 5, "$", LatexTokenKind::Math);
+ fn math() {
+ let mut lexer = Lexer::new("$$ $ $");
+ verify(&mut lexer, 0, 0, "$$", TokenKind::Math);
+ verify(&mut lexer, 0, 3, "$", TokenKind::Math);
+ verify(&mut lexer, 0, 5, "$", TokenKind::Math);
assert_eq!(None, lexer.next());
}
#[test]
- fn test_line_comment() {
- let mut lexer = LatexLexer::new(" %foo \nfoo");
- verify(&mut lexer, 1, 0, "foo", LatexTokenKind::Word);
+ fn line_comment() {
+ let mut lexer = Lexer::new(" %foo \nfoo");
+ verify(&mut lexer, 1, 0, "foo", TokenKind::Word);
assert_eq!(None, lexer.next());
}
}