summaryrefslogtreecommitdiff
path: root/support/texlab/src/syntax/bibtex/parser.rs
diff options
context:
space:
mode:
Diffstat (limited to 'support/texlab/src/syntax/bibtex/parser.rs')
-rw-r--r--support/texlab/src/syntax/bibtex/parser.rs589
1 files changed, 214 insertions, 375 deletions
diff --git a/support/texlab/src/syntax/bibtex/parser.rs b/support/texlab/src/syntax/bibtex/parser.rs
index 6e8c4b3b59..4a8a429b18 100644
--- a/support/texlab/src/syntax/bibtex/parser.rs
+++ b/support/texlab/src/syntax/bibtex/parser.rs
@@ -1,425 +1,264 @@
-use cstree::GreenNodeBuilder;
+use logos::{Lexer, Logos};
+use rowan::{GreenNode, GreenNodeBuilder};
use super::{
- lexer::Lexer,
+ lexer::*,
SyntaxKind::{self, *},
- SyntaxNode,
};
-#[derive(Debug, Clone)]
-pub struct Parse {
- pub root: SyntaxNode,
-}
-
-struct Parser<'a> {
- lexer: Lexer<'a>,
- builder: GreenNodeBuilder<'static, 'static>,
-}
-
-impl<'a> Parser<'a> {
- pub fn new(lexer: Lexer<'a>) -> Self {
- Self {
- lexer,
- builder: GreenNodeBuilder::new(),
- }
- }
-
- fn peek(&self) -> Option<SyntaxKind> {
- self.lexer.peek()
- }
-
- fn eat(&mut self) {
- let (kind, text) = self.lexer.consume().unwrap();
- self.builder.token(kind.into(), text);
- }
-
- fn expect(&mut self, kind: SyntaxKind) {
- if self.peek() == Some(kind) {
- self.eat();
- self.trivia();
- } else {
- self.builder.token(MISSING.into(), "");
+pub fn parse(input: &str) -> GreenNode {
+ let mut ptr = TokenPtr {
+ builder: GreenNodeBuilder::new(),
+ lexer: RootToken::lexer(input),
+ token: None,
+ };
+
+ ptr.builder.start_node(ROOT.into());
+
+ while let Some(token) = ptr.current() {
+ match token {
+ RootToken::Preamble => ptr = preamble(ptr),
+ RootToken::String => ptr = string(ptr),
+ RootToken::Entry => ptr = entry(ptr),
+ RootToken::Comment | RootToken::Junk => ptr.bump(),
}
}
- pub fn parse(mut self) -> Parse {
- self.builder.start_node(ROOT.into());
- while let Some(kind) = self.peek() {
- match kind {
- PREAMBLE_TYPE => self.preamble(),
- STRING_TYPE => self.string(),
- COMMENT_TYPE => self.comment(),
- ENTRY_TYPE => self.entry(),
- _ => self.junk(),
- }
- }
- self.builder.finish_node();
- let (green, resolver) = self.builder.finish();
- Parse {
- root: SyntaxNode::new_root_with_resolver(green, resolver.unwrap()),
- }
- }
-
- fn trivia(&mut self) {
- while self.peek() == Some(WHITESPACE) {
- self.eat();
- }
- }
-
- fn junk(&mut self) {
- self.builder.start_node(JUNK.into());
- while self
- .lexer
- .peek()
- .filter(|&kind| {
- !matches!(
- kind,
- PREAMBLE_TYPE | STRING_TYPE | COMMENT_TYPE | ENTRY_TYPE,
- )
- })
- .is_some()
- {
- self.eat();
- }
- self.builder.finish_node();
- }
-
- fn left_delimiter_or_missing(&mut self) {
- if self
- .lexer
- .peek()
- .filter(|&kind| matches!(kind, L_CURLY | L_PAREN))
- .is_some()
- {
- self.eat();
- self.trivia();
- } else {
- self.builder.token(MISSING.into(), "");
- }
- }
-
- fn right_delimiter_or_missing(&mut self) {
- if self
- .lexer
- .peek()
- .filter(|&kind| matches!(kind, R_CURLY | R_PAREN))
- .is_some()
- {
- self.eat();
- self.trivia();
- } else {
- self.builder.token(MISSING.into(), "");
- }
- }
-
- fn value_or_missing(&mut self) {
- if self
- .lexer
- .peek()
- .filter(|&kind| matches!(kind, L_CURLY | QUOTE | WORD))
- .is_some()
- {
- self.value();
- } else {
- self.builder.token(MISSING.into(), "");
- }
- }
-
- fn preamble(&mut self) {
- self.builder.start_node(PREAMBLE.into());
- self.eat();
- self.trivia();
- self.left_delimiter_or_missing();
- self.value_or_missing();
- self.right_delimiter_or_missing();
- self.builder.finish_node();
- }
-
- fn string(&mut self) {
- self.builder.start_node(STRING.into());
- self.eat();
- self.trivia();
- self.left_delimiter_or_missing();
-
- if self.peek() != Some(WORD) {
- self.builder.token(MISSING.into(), "");
- self.builder.finish_node();
- return;
- }
- self.eat();
- self.trivia();
- self.expect(EQUALITY_SIGN);
-
- self.value_or_missing();
-
- self.right_delimiter_or_missing();
- self.builder.finish_node();
- }
-
- fn comment(&mut self) {
- self.builder.start_node(COMMENT.into());
- self.eat();
- self.builder.finish_node();
- }
-
- fn entry(&mut self) {
- self.builder.start_node(ENTRY.into());
- self.eat();
- self.trivia();
-
- self.left_delimiter_or_missing();
-
- if self.peek() != Some(WORD) {
- self.builder.token(MISSING.into(), "");
- self.builder.finish_node();
- return;
- }
- self.key();
-
- while let Some(kind) = self.peek() {
- match kind {
- WHITESPACE => self.eat(),
- WORD => self.field(),
- COMMA => self.eat(),
- _ => break,
- };
- }
-
- self.right_delimiter_or_missing();
-
- self.builder.finish_node();
- }
-
- fn key(&mut self) {
- self.builder.start_node(KEY.into());
- while self
- .peek()
- .filter(|&kind| matches!(kind, WORD | WHITESPACE))
- .is_some()
- {
- self.eat();
- }
- self.builder.finish_node();
- }
-
- fn field(&mut self) {
- self.builder.start_node(FIELD.into());
- self.eat();
- self.trivia();
+ ptr.builder.finish_node();
+ ptr.builder.finish()
+}
- if self.peek() == Some(EQUALITY_SIGN) {
- self.eat();
- self.trivia();
- } else {
- self.builder.token(MISSING.into(), "");
- }
+fn preamble(mut ptr: TokenPtr<RootToken>) -> TokenPtr<RootToken> {
+ ptr.builder.start_node(PREAMBLE.into());
+ ptr.bump();
+ let mut ptr = ptr.morph();
+ ptr.expect(BodyToken::Whitespace);
+ ptr.expect(BodyToken::LDelim);
+ ptr.expect(BodyToken::Whitespace);
+ ptr = value(ptr.morph()).morph();
+ ptr.expect(BodyToken::RDelim);
+ ptr.builder.finish_node();
+ ptr.morph()
+}
- if self
- .lexer
- .peek()
- .filter(|&kind| matches!(kind, L_CURLY | QUOTE | WORD))
- .is_some()
- {
- self.value();
- } else {
- self.builder.token(MISSING.into(), "");
- }
+fn string(mut ptr: TokenPtr<RootToken>) -> TokenPtr<RootToken> {
+ ptr.builder.start_node(STRING.into());
+ ptr.bump();
+ let mut ptr = ptr.morph();
+ ptr.expect(BodyToken::Whitespace);
+ ptr.expect(BodyToken::LDelim);
+ ptr.expect(BodyToken::Whitespace);
+ ptr.expect(BodyToken::Name);
+ ptr.expect(BodyToken::Whitespace);
+ ptr.expect(BodyToken::Eq);
+ ptr.expect(BodyToken::Whitespace);
+ ptr = value(ptr.morph()).morph();
+ ptr.expect(BodyToken::RDelim);
+ ptr.builder.finish_node();
+ ptr.morph()
+}
- self.builder.finish_node();
- }
+fn entry(mut ptr: TokenPtr<RootToken>) -> TokenPtr<RootToken> {
+ ptr.builder.start_node(ENTRY.into());
+ ptr.bump();
+ let mut ptr = ptr.morph();
+ ptr.expect(BodyToken::Whitespace);
+ ptr.expect(BodyToken::LDelim);
+ ptr.expect(BodyToken::Whitespace);
+ ptr.expect(BodyToken::Name);
+ ptr.expect(BodyToken::Whitespace);
+ ptr.expect(BodyToken::Comma);
+ ptr.expect(BodyToken::Whitespace);
+
+ while ptr.at(BodyToken::Name) {
+ ptr = field(ptr);
+ ptr.expect(BodyToken::Whitespace);
+ }
+
+ ptr.expect(BodyToken::RDelim);
+ ptr.builder.finish_node();
+ ptr.morph()
+}
- fn value(&mut self) {
- self.builder.start_node(VALUE.into());
- self.token();
- while let Some(kind) = self.peek() {
- match kind {
- WHITESPACE => self.eat(),
- L_CURLY | QUOTE | WORD => self.token(),
- HASH => self.eat(),
- _ => break,
- }
- }
- self.builder.finish_node();
- }
+fn field(mut ptr: TokenPtr<BodyToken>) -> TokenPtr<BodyToken> {
+ ptr.builder.start_node(FIELD.into());
+ ptr.bump();
+ ptr.expect(BodyToken::Whitespace);
+ ptr.expect(BodyToken::Eq);
+ ptr.expect(BodyToken::Whitespace);
+ ptr = value(ptr.morph()).morph();
+ ptr.expect(BodyToken::Whitespace);
+ ptr.expect(BodyToken::Comma);
+ ptr.builder.finish_node();
+ ptr
+}
- fn token(&mut self) {
- self.builder.start_node(TOKEN.into());
- match self.peek().unwrap() {
- L_CURLY => self.brace_group(),
- QUOTE => self.quote_group(),
- WORD => self.eat(),
- _ => unreachable!(),
+fn value(mut ptr: TokenPtr<ValueToken>) -> TokenPtr<ValueToken> {
+ let checkpoint = ptr.builder.checkpoint();
+ if let Some(token) = ptr.current() {
+ match token {
+ ValueToken::Whitespace => unreachable!(),
+ ValueToken::Pound | ValueToken::Comma | ValueToken::RCurly => return ptr,
+ ValueToken::Integer | ValueToken::Name => ptr = literal(ptr),
+ ValueToken::LCurly => ptr = curly_group(ptr.morph()).morph(),
+ ValueToken::Quote => ptr = quote_group(ptr.morph()).morph(),
};
- self.builder.finish_node();
- }
- fn brace_group(&mut self) {
- self.builder.start_node(BRACE_GROUP.into());
- self.eat();
-
- while let Some(kind) = self.peek() {
- match kind {
- WHITESPACE => self.eat(),
- PREAMBLE_TYPE => break,
- STRING_TYPE => break,
- COMMENT_TYPE => break,
- ENTRY_TYPE => break,
- WORD => self.eat(),
- L_CURLY => self.brace_group(),
- R_CURLY => break,
- L_PAREN => self.eat(),
- R_PAREN => self.eat(),
- COMMA => self.eat(),
- HASH => self.eat(),
- QUOTE => self.eat(),
- EQUALITY_SIGN => self.eat(),
- COMMAND_NAME => self.eat(),
- _ => unreachable!(),
- };
+ ptr.expect(ValueToken::Whitespace);
+ if ptr.at(ValueToken::Pound) {
+ ptr.builder.start_node_at(checkpoint, JOIN.into());
+ ptr.bump();
+ ptr.expect(ValueToken::Whitespace);
+ ptr = value(ptr);
+ ptr.builder.finish_node();
}
-
- self.expect(R_CURLY);
-
- self.builder.finish_node();
}
- fn quote_group(&mut self) {
- self.builder.start_node(QUOTE_GROUP.into());
- self.eat();
-
- while let Some(kind) = self.peek() {
- match kind {
- WHITESPACE => self.eat(),
- PREAMBLE_TYPE => break,
- STRING_TYPE => break,
- COMMENT_TYPE => break,
- ENTRY_TYPE => break,
- WORD => self.eat(),
- L_CURLY => self.brace_group(),
- R_CURLY => break,
- L_PAREN => self.eat(),
- R_PAREN => self.eat(),
- COMMA => self.eat(),
- HASH => self.eat(),
- QUOTE => break,
- EQUALITY_SIGN => self.eat(),
- COMMAND_NAME => self.eat(),
- _ => unreachable!(),
- };
- }
-
- self.expect(QUOTE);
- self.builder.finish_node();
- }
+ ptr
}
-pub fn parse(text: &str) -> Parse {
- Parser::new(Lexer::new(text)).parse()
+fn literal(mut ptr: TokenPtr<ValueToken>) -> TokenPtr<ValueToken> {
+ ptr.builder.start_node(LITERAL.into());
+ ptr.bump();
+ ptr.builder.finish_node();
+ ptr
}
-#[cfg(test)]
-mod tests {
- use insta::assert_debug_snapshot;
-
- use super::*;
-
- fn setup(text: &str) -> SyntaxNode {
- parse(&text.trim().replace("\r", "")).root
- }
-
- #[test]
- fn test_empty() {
- assert_debug_snapshot!(setup(r#""#));
+fn curly_group(mut ptr: TokenPtr<ContentToken>) -> TokenPtr<ContentToken> {
+ ptr.builder.start_node(CURLY_GROUP.into());
+ ptr.bump();
+ ptr.expect(ContentToken::Whitespace);
+
+ while let Some(token) = ptr.current() {
+ match token {
+ ContentToken::RCurly => break,
+ ContentToken::Whitespace
+ | ContentToken::Nbsp
+ | ContentToken::Comma
+ | ContentToken::Integer
+ | ContentToken::Word => ptr.bump(),
+ ContentToken::LCurly => ptr = curly_group(ptr),
+ ContentToken::Quote => ptr = quote_group(ptr),
+ ContentToken::AccentName => ptr = accent(ptr),
+ ContentToken::CommandName => ptr = command(ptr),
+ };
}
- #[test]
- fn test_junk() {
- assert_debug_snapshot!(setup(r#"Hello World!"#));
- }
+ ptr.expect(ContentToken::RCurly);
+ ptr.builder.finish_node();
+ ptr
+}
- #[test]
- fn test_preamble_complete() {
- assert_debug_snapshot!(setup(r#"@preamble{ "Hello World" }"#));
+fn quote_group(mut ptr: TokenPtr<ContentToken>) -> TokenPtr<ContentToken> {
+ ptr.builder.start_node(QUOTE_GROUP.into());
+ ptr.bump();
+ ptr.expect(ContentToken::Whitespace);
+
+ while let Some(token) = ptr.current() {
+ match token {
+ ContentToken::Quote => break,
+ ContentToken::Whitespace
+ | ContentToken::Nbsp
+ | ContentToken::Comma
+ | ContentToken::RCurly
+ | ContentToken::Integer
+ | ContentToken::Word => ptr.bump(),
+ ContentToken::LCurly => ptr = curly_group(ptr),
+ ContentToken::AccentName => ptr = accent(ptr),
+ ContentToken::CommandName => ptr = command(ptr),
+ };
}
- #[test]
- fn test_preamble_missing_end() {
- assert_debug_snapshot!(setup(r#"@preamble{ "Hello World" "#));
- }
+ ptr.expect(ContentToken::Quote);
+ ptr.builder.finish_node();
+ ptr
+}
- #[test]
- fn test_preamble_casing() {
- assert_debug_snapshot!(setup(r#"@preAmbLe{ "Hello World" }"#));
- }
+fn accent(mut ptr: TokenPtr<ContentToken>) -> TokenPtr<ContentToken> {
+ ptr.builder.start_node(ACCENT.into());
+ ptr.bump();
+ ptr.expect(ContentToken::Whitespace);
- #[test]
- fn test_string_complete() {
- assert_debug_snapshot!(setup(r#"@string{foo = {Hello World}}"#));
+ let group = ptr.at(ContentToken::LCurly);
+ if group {
+ ptr.expect(ContentToken::LCurly);
+ ptr.expect(ContentToken::Whitespace);
}
- #[test]
- fn test_string_incomplete() {
- assert_debug_snapshot!(setup(r#"@string{foo = {Hello World}"#));
- }
+ ptr.expect(ContentToken::Word);
- #[test]
- fn test_string_concatenation() {
- assert_debug_snapshot!(setup(
- r#"@string{foo = {Hello World}} @string{bar = foo # "!"}"#
- ));
+ if group {
+ ptr.expect(ContentToken::Whitespace);
+ ptr.expect(ContentToken::RCurly);
}
- #[test]
- fn test_string_casing() {
- assert_debug_snapshot!(setup(r#"@STRING{foo = "Hello World"}"#));
- }
+ ptr.builder.finish_node();
+ ptr
+}
- #[test]
- fn test_string_missing_quote() {
- assert_debug_snapshot!(setup(r#"@STRING{foo = "Hello World}"#));
- }
+fn command(mut ptr: TokenPtr<ContentToken>) -> TokenPtr<ContentToken> {
+ ptr.builder.start_node(COMMAND.into());
+ ptr.bump();
+ ptr.builder.finish_node();
+ ptr
+}
- #[test]
- fn test_entry_no_fields() {
- assert_debug_snapshot!(setup(r#"@article{foo,}"#));
- }
+struct TokenPtr<'a, T: Logos<'a>> {
+ builder: GreenNodeBuilder<'static>,
+ lexer: Lexer<'a, T>,
+ token: Option<(T, &'a str)>,
+}
- #[test]
- fn test_entry_no_fields_missing_comma() {
- assert_debug_snapshot!(setup(r#"@article{foo}"#));
+impl<'a, T> TokenPtr<'a, T>
+where
+ T: Logos<'a, Source = str> + Eq + Copy + Into<SyntaxKind>,
+ T::Extras: Default,
+{
+ pub fn morph<U>(mut self) -> TokenPtr<'a, U>
+ where
+ U: Logos<'a, Source = str> + Eq + Copy + Into<SyntaxKind>,
+ U::Extras: Default,
+ {
+ self.peek();
+ let start = self.lexer.span().start;
+ let input = &self.lexer.source()[start..];
+ TokenPtr {
+ builder: self.builder,
+ lexer: U::lexer(input),
+ token: None,
+ }
}
- #[test]
- fn test_entry_one_field() {
- assert_debug_snapshot!(setup(r#"@article{foo, author = {Foo Bar}}"#));
+ #[must_use]
+ pub fn at(&mut self, kind: T) -> bool {
+ self.peek().map_or(false, |(k, _)| k == kind)
}
- #[test]
- fn test_entry_one_field_number_key() {
- assert_debug_snapshot!(setup(r#"@article{foo2021, author = {Foo Bar}}"#));
+ #[must_use]
+ pub fn current(&mut self) -> Option<T> {
+ self.peek().map(|(k, _)| k)
}
- #[test]
- fn test_entry_one_field_trailing_comma() {
- assert_debug_snapshot!(setup(r#"@article{foo, author = {Foo Bar},}"#));
+ pub fn bump(&mut self) {
+ let (kind, text) = self.peek().unwrap();
+ self.token = None;
+ self.builder
+ .token(rowan::SyntaxKind::from(kind.into()), text);
}
- #[test]
- fn test_entry_two_fields() {
- assert_debug_snapshot!(setup(
- r#"@article{foo, author = {Foo Bar}, title = {Hello World}}"#
- ));
+ pub fn expect(&mut self, kind: T) {
+ if self.at(kind) {
+ self.bump();
+ }
}
- #[test]
- fn test_entry_two_fields_incomplete() {
- assert_debug_snapshot!(setup(r#"@article{foo, author = {Foo Bar}, t}"#));
- }
+ fn peek(&mut self) -> Option<(T, &'a str)> {
+ if self.token.is_none() {
+ let kind = self.lexer.next()?;
+ let text = self.lexer.slice();
+ self.token = Some((kind, text));
+ }
- #[test]
- fn test_entry_complete_parens() {
- assert_debug_snapshot!(setup(
- r#"@article(foo, author = {Foo Bar}, title = {Hello})"#
- ));
+ self.token
}
}