diff options
Diffstat (limited to 'support/texlab/src/syntax/bibtex/parser.rs')
-rw-r--r-- | support/texlab/src/syntax/bibtex/parser.rs | 589 |
1 files changed, 214 insertions, 375 deletions
diff --git a/support/texlab/src/syntax/bibtex/parser.rs b/support/texlab/src/syntax/bibtex/parser.rs index 6e8c4b3b59..4a8a429b18 100644 --- a/support/texlab/src/syntax/bibtex/parser.rs +++ b/support/texlab/src/syntax/bibtex/parser.rs @@ -1,425 +1,264 @@ -use cstree::GreenNodeBuilder; +use logos::{Lexer, Logos}; +use rowan::{GreenNode, GreenNodeBuilder}; use super::{ - lexer::Lexer, + lexer::*, SyntaxKind::{self, *}, - SyntaxNode, }; -#[derive(Debug, Clone)] -pub struct Parse { - pub root: SyntaxNode, -} - -struct Parser<'a> { - lexer: Lexer<'a>, - builder: GreenNodeBuilder<'static, 'static>, -} - -impl<'a> Parser<'a> { - pub fn new(lexer: Lexer<'a>) -> Self { - Self { - lexer, - builder: GreenNodeBuilder::new(), - } - } - - fn peek(&self) -> Option<SyntaxKind> { - self.lexer.peek() - } - - fn eat(&mut self) { - let (kind, text) = self.lexer.consume().unwrap(); - self.builder.token(kind.into(), text); - } - - fn expect(&mut self, kind: SyntaxKind) { - if self.peek() == Some(kind) { - self.eat(); - self.trivia(); - } else { - self.builder.token(MISSING.into(), ""); +pub fn parse(input: &str) -> GreenNode { + let mut ptr = TokenPtr { + builder: GreenNodeBuilder::new(), + lexer: RootToken::lexer(input), + token: None, + }; + + ptr.builder.start_node(ROOT.into()); + + while let Some(token) = ptr.current() { + match token { + RootToken::Preamble => ptr = preamble(ptr), + RootToken::String => ptr = string(ptr), + RootToken::Entry => ptr = entry(ptr), + RootToken::Comment | RootToken::Junk => ptr.bump(), } } - pub fn parse(mut self) -> Parse { - self.builder.start_node(ROOT.into()); - while let Some(kind) = self.peek() { - match kind { - PREAMBLE_TYPE => self.preamble(), - STRING_TYPE => self.string(), - COMMENT_TYPE => self.comment(), - ENTRY_TYPE => self.entry(), - _ => self.junk(), - } - } - self.builder.finish_node(); - let (green, resolver) = self.builder.finish(); - Parse { - root: SyntaxNode::new_root_with_resolver(green, resolver.unwrap()), - } - } - - fn trivia(&mut self) { - while self.peek() == Some(WHITESPACE) { - self.eat(); - } - } - - fn junk(&mut self) { - self.builder.start_node(JUNK.into()); - while self - .lexer - .peek() - .filter(|&kind| { - !matches!( - kind, - PREAMBLE_TYPE | STRING_TYPE | COMMENT_TYPE | ENTRY_TYPE, - ) - }) - .is_some() - { - self.eat(); - } - self.builder.finish_node(); - } - - fn left_delimiter_or_missing(&mut self) { - if self - .lexer - .peek() - .filter(|&kind| matches!(kind, L_CURLY | L_PAREN)) - .is_some() - { - self.eat(); - self.trivia(); - } else { - self.builder.token(MISSING.into(), ""); - } - } - - fn right_delimiter_or_missing(&mut self) { - if self - .lexer - .peek() - .filter(|&kind| matches!(kind, R_CURLY | R_PAREN)) - .is_some() - { - self.eat(); - self.trivia(); - } else { - self.builder.token(MISSING.into(), ""); - } - } - - fn value_or_missing(&mut self) { - if self - .lexer - .peek() - .filter(|&kind| matches!(kind, L_CURLY | QUOTE | WORD)) - .is_some() - { - self.value(); - } else { - self.builder.token(MISSING.into(), ""); - } - } - - fn preamble(&mut self) { - self.builder.start_node(PREAMBLE.into()); - self.eat(); - self.trivia(); - self.left_delimiter_or_missing(); - self.value_or_missing(); - self.right_delimiter_or_missing(); - self.builder.finish_node(); - } - - fn string(&mut self) { - self.builder.start_node(STRING.into()); - self.eat(); - self.trivia(); - self.left_delimiter_or_missing(); - - if self.peek() != Some(WORD) { - self.builder.token(MISSING.into(), ""); - self.builder.finish_node(); - return; - } - self.eat(); - self.trivia(); - self.expect(EQUALITY_SIGN); - - self.value_or_missing(); - - self.right_delimiter_or_missing(); - self.builder.finish_node(); - } - - fn comment(&mut self) { - self.builder.start_node(COMMENT.into()); - self.eat(); - self.builder.finish_node(); - } - - fn entry(&mut self) { - self.builder.start_node(ENTRY.into()); - self.eat(); - self.trivia(); - - self.left_delimiter_or_missing(); - - if self.peek() != Some(WORD) { - self.builder.token(MISSING.into(), ""); - self.builder.finish_node(); - return; - } - self.key(); - - while let Some(kind) = self.peek() { - match kind { - WHITESPACE => self.eat(), - WORD => self.field(), - COMMA => self.eat(), - _ => break, - }; - } - - self.right_delimiter_or_missing(); - - self.builder.finish_node(); - } - - fn key(&mut self) { - self.builder.start_node(KEY.into()); - while self - .peek() - .filter(|&kind| matches!(kind, WORD | WHITESPACE)) - .is_some() - { - self.eat(); - } - self.builder.finish_node(); - } - - fn field(&mut self) { - self.builder.start_node(FIELD.into()); - self.eat(); - self.trivia(); + ptr.builder.finish_node(); + ptr.builder.finish() +} - if self.peek() == Some(EQUALITY_SIGN) { - self.eat(); - self.trivia(); - } else { - self.builder.token(MISSING.into(), ""); - } +fn preamble(mut ptr: TokenPtr<RootToken>) -> TokenPtr<RootToken> { + ptr.builder.start_node(PREAMBLE.into()); + ptr.bump(); + let mut ptr = ptr.morph(); + ptr.expect(BodyToken::Whitespace); + ptr.expect(BodyToken::LDelim); + ptr.expect(BodyToken::Whitespace); + ptr = value(ptr.morph()).morph(); + ptr.expect(BodyToken::RDelim); + ptr.builder.finish_node(); + ptr.morph() +} - if self - .lexer - .peek() - .filter(|&kind| matches!(kind, L_CURLY | QUOTE | WORD)) - .is_some() - { - self.value(); - } else { - self.builder.token(MISSING.into(), ""); - } +fn string(mut ptr: TokenPtr<RootToken>) -> TokenPtr<RootToken> { + ptr.builder.start_node(STRING.into()); + ptr.bump(); + let mut ptr = ptr.morph(); + ptr.expect(BodyToken::Whitespace); + ptr.expect(BodyToken::LDelim); + ptr.expect(BodyToken::Whitespace); + ptr.expect(BodyToken::Name); + ptr.expect(BodyToken::Whitespace); + ptr.expect(BodyToken::Eq); + ptr.expect(BodyToken::Whitespace); + ptr = value(ptr.morph()).morph(); + ptr.expect(BodyToken::RDelim); + ptr.builder.finish_node(); + ptr.morph() +} - self.builder.finish_node(); - } +fn entry(mut ptr: TokenPtr<RootToken>) -> TokenPtr<RootToken> { + ptr.builder.start_node(ENTRY.into()); + ptr.bump(); + let mut ptr = ptr.morph(); + ptr.expect(BodyToken::Whitespace); + ptr.expect(BodyToken::LDelim); + ptr.expect(BodyToken::Whitespace); + ptr.expect(BodyToken::Name); + ptr.expect(BodyToken::Whitespace); + ptr.expect(BodyToken::Comma); + ptr.expect(BodyToken::Whitespace); + + while ptr.at(BodyToken::Name) { + ptr = field(ptr); + ptr.expect(BodyToken::Whitespace); + } + + ptr.expect(BodyToken::RDelim); + ptr.builder.finish_node(); + ptr.morph() +} - fn value(&mut self) { - self.builder.start_node(VALUE.into()); - self.token(); - while let Some(kind) = self.peek() { - match kind { - WHITESPACE => self.eat(), - L_CURLY | QUOTE | WORD => self.token(), - HASH => self.eat(), - _ => break, - } - } - self.builder.finish_node(); - } +fn field(mut ptr: TokenPtr<BodyToken>) -> TokenPtr<BodyToken> { + ptr.builder.start_node(FIELD.into()); + ptr.bump(); + ptr.expect(BodyToken::Whitespace); + ptr.expect(BodyToken::Eq); + ptr.expect(BodyToken::Whitespace); + ptr = value(ptr.morph()).morph(); + ptr.expect(BodyToken::Whitespace); + ptr.expect(BodyToken::Comma); + ptr.builder.finish_node(); + ptr +} - fn token(&mut self) { - self.builder.start_node(TOKEN.into()); - match self.peek().unwrap() { - L_CURLY => self.brace_group(), - QUOTE => self.quote_group(), - WORD => self.eat(), - _ => unreachable!(), +fn value(mut ptr: TokenPtr<ValueToken>) -> TokenPtr<ValueToken> { + let checkpoint = ptr.builder.checkpoint(); + if let Some(token) = ptr.current() { + match token { + ValueToken::Whitespace => unreachable!(), + ValueToken::Pound | ValueToken::Comma | ValueToken::RCurly => return ptr, + ValueToken::Integer | ValueToken::Name => ptr = literal(ptr), + ValueToken::LCurly => ptr = curly_group(ptr.morph()).morph(), + ValueToken::Quote => ptr = quote_group(ptr.morph()).morph(), }; - self.builder.finish_node(); - } - fn brace_group(&mut self) { - self.builder.start_node(BRACE_GROUP.into()); - self.eat(); - - while let Some(kind) = self.peek() { - match kind { - WHITESPACE => self.eat(), - PREAMBLE_TYPE => break, - STRING_TYPE => break, - COMMENT_TYPE => break, - ENTRY_TYPE => break, - WORD => self.eat(), - L_CURLY => self.brace_group(), - R_CURLY => break, - L_PAREN => self.eat(), - R_PAREN => self.eat(), - COMMA => self.eat(), - HASH => self.eat(), - QUOTE => self.eat(), - EQUALITY_SIGN => self.eat(), - COMMAND_NAME => self.eat(), - _ => unreachable!(), - }; + ptr.expect(ValueToken::Whitespace); + if ptr.at(ValueToken::Pound) { + ptr.builder.start_node_at(checkpoint, JOIN.into()); + ptr.bump(); + ptr.expect(ValueToken::Whitespace); + ptr = value(ptr); + ptr.builder.finish_node(); } - - self.expect(R_CURLY); - - self.builder.finish_node(); } - fn quote_group(&mut self) { - self.builder.start_node(QUOTE_GROUP.into()); - self.eat(); - - while let Some(kind) = self.peek() { - match kind { - WHITESPACE => self.eat(), - PREAMBLE_TYPE => break, - STRING_TYPE => break, - COMMENT_TYPE => break, - ENTRY_TYPE => break, - WORD => self.eat(), - L_CURLY => self.brace_group(), - R_CURLY => break, - L_PAREN => self.eat(), - R_PAREN => self.eat(), - COMMA => self.eat(), - HASH => self.eat(), - QUOTE => break, - EQUALITY_SIGN => self.eat(), - COMMAND_NAME => self.eat(), - _ => unreachable!(), - }; - } - - self.expect(QUOTE); - self.builder.finish_node(); - } + ptr } -pub fn parse(text: &str) -> Parse { - Parser::new(Lexer::new(text)).parse() +fn literal(mut ptr: TokenPtr<ValueToken>) -> TokenPtr<ValueToken> { + ptr.builder.start_node(LITERAL.into()); + ptr.bump(); + ptr.builder.finish_node(); + ptr } -#[cfg(test)] -mod tests { - use insta::assert_debug_snapshot; - - use super::*; - - fn setup(text: &str) -> SyntaxNode { - parse(&text.trim().replace("\r", "")).root - } - - #[test] - fn test_empty() { - assert_debug_snapshot!(setup(r#""#)); +fn curly_group(mut ptr: TokenPtr<ContentToken>) -> TokenPtr<ContentToken> { + ptr.builder.start_node(CURLY_GROUP.into()); + ptr.bump(); + ptr.expect(ContentToken::Whitespace); + + while let Some(token) = ptr.current() { + match token { + ContentToken::RCurly => break, + ContentToken::Whitespace + | ContentToken::Nbsp + | ContentToken::Comma + | ContentToken::Integer + | ContentToken::Word => ptr.bump(), + ContentToken::LCurly => ptr = curly_group(ptr), + ContentToken::Quote => ptr = quote_group(ptr), + ContentToken::AccentName => ptr = accent(ptr), + ContentToken::CommandName => ptr = command(ptr), + }; } - #[test] - fn test_junk() { - assert_debug_snapshot!(setup(r#"Hello World!"#)); - } + ptr.expect(ContentToken::RCurly); + ptr.builder.finish_node(); + ptr +} - #[test] - fn test_preamble_complete() { - assert_debug_snapshot!(setup(r#"@preamble{ "Hello World" }"#)); +fn quote_group(mut ptr: TokenPtr<ContentToken>) -> TokenPtr<ContentToken> { + ptr.builder.start_node(QUOTE_GROUP.into()); + ptr.bump(); + ptr.expect(ContentToken::Whitespace); + + while let Some(token) = ptr.current() { + match token { + ContentToken::Quote => break, + ContentToken::Whitespace + | ContentToken::Nbsp + | ContentToken::Comma + | ContentToken::RCurly + | ContentToken::Integer + | ContentToken::Word => ptr.bump(), + ContentToken::LCurly => ptr = curly_group(ptr), + ContentToken::AccentName => ptr = accent(ptr), + ContentToken::CommandName => ptr = command(ptr), + }; } - #[test] - fn test_preamble_missing_end() { - assert_debug_snapshot!(setup(r#"@preamble{ "Hello World" "#)); - } + ptr.expect(ContentToken::Quote); + ptr.builder.finish_node(); + ptr +} - #[test] - fn test_preamble_casing() { - assert_debug_snapshot!(setup(r#"@preAmbLe{ "Hello World" }"#)); - } +fn accent(mut ptr: TokenPtr<ContentToken>) -> TokenPtr<ContentToken> { + ptr.builder.start_node(ACCENT.into()); + ptr.bump(); + ptr.expect(ContentToken::Whitespace); - #[test] - fn test_string_complete() { - assert_debug_snapshot!(setup(r#"@string{foo = {Hello World}}"#)); + let group = ptr.at(ContentToken::LCurly); + if group { + ptr.expect(ContentToken::LCurly); + ptr.expect(ContentToken::Whitespace); } - #[test] - fn test_string_incomplete() { - assert_debug_snapshot!(setup(r#"@string{foo = {Hello World}"#)); - } + ptr.expect(ContentToken::Word); - #[test] - fn test_string_concatenation() { - assert_debug_snapshot!(setup( - r#"@string{foo = {Hello World}} @string{bar = foo # "!"}"# - )); + if group { + ptr.expect(ContentToken::Whitespace); + ptr.expect(ContentToken::RCurly); } - #[test] - fn test_string_casing() { - assert_debug_snapshot!(setup(r#"@STRING{foo = "Hello World"}"#)); - } + ptr.builder.finish_node(); + ptr +} - #[test] - fn test_string_missing_quote() { - assert_debug_snapshot!(setup(r#"@STRING{foo = "Hello World}"#)); - } +fn command(mut ptr: TokenPtr<ContentToken>) -> TokenPtr<ContentToken> { + ptr.builder.start_node(COMMAND.into()); + ptr.bump(); + ptr.builder.finish_node(); + ptr +} - #[test] - fn test_entry_no_fields() { - assert_debug_snapshot!(setup(r#"@article{foo,}"#)); - } +struct TokenPtr<'a, T: Logos<'a>> { + builder: GreenNodeBuilder<'static>, + lexer: Lexer<'a, T>, + token: Option<(T, &'a str)>, +} - #[test] - fn test_entry_no_fields_missing_comma() { - assert_debug_snapshot!(setup(r#"@article{foo}"#)); +impl<'a, T> TokenPtr<'a, T> +where + T: Logos<'a, Source = str> + Eq + Copy + Into<SyntaxKind>, + T::Extras: Default, +{ + pub fn morph<U>(mut self) -> TokenPtr<'a, U> + where + U: Logos<'a, Source = str> + Eq + Copy + Into<SyntaxKind>, + U::Extras: Default, + { + self.peek(); + let start = self.lexer.span().start; + let input = &self.lexer.source()[start..]; + TokenPtr { + builder: self.builder, + lexer: U::lexer(input), + token: None, + } } - #[test] - fn test_entry_one_field() { - assert_debug_snapshot!(setup(r#"@article{foo, author = {Foo Bar}}"#)); + #[must_use] + pub fn at(&mut self, kind: T) -> bool { + self.peek().map_or(false, |(k, _)| k == kind) } - #[test] - fn test_entry_one_field_number_key() { - assert_debug_snapshot!(setup(r#"@article{foo2021, author = {Foo Bar}}"#)); + #[must_use] + pub fn current(&mut self) -> Option<T> { + self.peek().map(|(k, _)| k) } - #[test] - fn test_entry_one_field_trailing_comma() { - assert_debug_snapshot!(setup(r#"@article{foo, author = {Foo Bar},}"#)); + pub fn bump(&mut self) { + let (kind, text) = self.peek().unwrap(); + self.token = None; + self.builder + .token(rowan::SyntaxKind::from(kind.into()), text); } - #[test] - fn test_entry_two_fields() { - assert_debug_snapshot!(setup( - r#"@article{foo, author = {Foo Bar}, title = {Hello World}}"# - )); + pub fn expect(&mut self, kind: T) { + if self.at(kind) { + self.bump(); + } } - #[test] - fn test_entry_two_fields_incomplete() { - assert_debug_snapshot!(setup(r#"@article{foo, author = {Foo Bar}, t}"#)); - } + fn peek(&mut self) -> Option<(T, &'a str)> { + if self.token.is_none() { + let kind = self.lexer.next()?; + let text = self.lexer.slice(); + self.token = Some((kind, text)); + } - #[test] - fn test_entry_complete_parens() { - assert_debug_snapshot!(setup( - r#"@article(foo, author = {Foo Bar}, title = {Hello})"# - )); + self.token } } |