diff options
author | Norbert Preining <norbert@preining.info> | 2020-05-28 03:03:21 +0000 |
---|---|---|
committer | Norbert Preining <norbert@preining.info> | 2020-05-28 03:03:21 +0000 |
commit | 745892fbddea56040139108277e728b53fd8fc11 (patch) | |
tree | 15e55e299b38fccdabce8364f1fbf4f02ef37994 /support/texlab/src/syntax/bibtex/parser.rs | |
parent | 195ca7e0c377d83455867bdd8e409d4e1cf024ea (diff) |
CTAN sync 202005280303
Diffstat (limited to 'support/texlab/src/syntax/bibtex/parser.rs')
-rw-r--r-- | support/texlab/src/syntax/bibtex/parser.rs | 372 |
1 files changed, 277 insertions, 95 deletions
diff --git a/support/texlab/src/syntax/bibtex/parser.rs b/support/texlab/src/syntax/bibtex/parser.rs index ffc33da545..b23903aeb3 100644 --- a/support/texlab/src/syntax/bibtex/parser.rs +++ b/support/texlab/src/syntax/bibtex/parser.rs @@ -1,197 +1,379 @@ use super::ast::*; +use crate::{ + protocol::{Range, RangeExt}, + syntax::text::SyntaxNode, +}; +use petgraph::graph::{Graph, NodeIndex}; use std::iter::Peekable; -pub struct BibtexParser<I: Iterator<Item = BibtexToken>> { +pub struct Parser<I: Iterator<Item = Token>> { + graph: Graph<Node, ()>, tokens: Peekable<I>, } -impl<I: Iterator<Item = BibtexToken>> BibtexParser<I> { +impl<I: Iterator<Item = Token>> Parser<I> { pub fn new(tokens: I) -> Self { - BibtexParser { + Self { tokens: tokens.peekable(), + graph: Graph::new(), } } - pub fn root(&mut self) -> BibtexRoot { + pub fn parse(mut self) -> Tree { let mut children = Vec::new(); + while let Some(ref token) = self.tokens.peek() { match token.kind { - BibtexTokenKind::PreambleKind => { - let preamble = Box::new(self.preamble()); - children.push(BibtexDeclaration::Preamble(preamble)); - } - BibtexTokenKind::StringKind => { - let string = Box::new(self.string()); - children.push(BibtexDeclaration::String(string)); - } - BibtexTokenKind::EntryKind => { - let entry = Box::new(self.entry()); - children.push(BibtexDeclaration::Entry(entry)); - } - _ => { - let comment = BibtexComment::new(self.tokens.next().unwrap()); - children.push(BibtexDeclaration::Comment(Box::new(comment))); - } + TokenKind::PreambleKind => children.push(self.preamble()), + TokenKind::StringKind => children.push(self.string()), + TokenKind::EntryKind => children.push(self.entry()), + _ => children.push(self.comment()), } } - BibtexRoot::new(children) + + let range = if children.is_empty() { + Range::new_simple(0, 0, 0, 0) + } else { + let start = self.graph[children[0]].start(); + let end = self.graph[children[children.len() - 1]].end(); + Range::new(start, end) + }; + + let root = self.graph.add_node(Node::Root(Root { range })); + self.connect(root, &children); + Tree { + graph: self.graph, + root, + } } - fn preamble(&mut self) -> BibtexPreamble { + fn preamble(&mut self) -> NodeIndex { let ty = self.tokens.next().unwrap(); - let left = self.expect2(BibtexTokenKind::BeginBrace, BibtexTokenKind::BeginParen); + let left = self.expect2(TokenKind::BeginBrace, TokenKind::BeginParen); if left.is_none() { - return BibtexPreamble::new(ty, None, None, None); + return self.graph.add_node(Node::Preamble(Box::new(Preamble { + range: ty.range(), + ty, + left: None, + right: None, + }))); } if !self.can_match_content() { - return BibtexPreamble::new(ty, left, None, None); + return self.graph.add_node(Node::Preamble(Box::new(Preamble { + range: Range::new(ty.start(), left.as_ref().unwrap().end()), + ty, + left, + right: None, + }))); } + let content = self.content(); - let right = self.expect2(BibtexTokenKind::EndBrace, BibtexTokenKind::EndParen); - BibtexPreamble::new(ty, left, Some(content), right) + let right = self.expect2(TokenKind::EndBrace, TokenKind::EndParen); + let end = right + .as_ref() + .map(Token::end) + .unwrap_or_else(|| self.graph[content].end()); + + let parent = self.graph.add_node(Node::Preamble(Box::new(Preamble { + range: Range::new(ty.start(), end), + ty, + left, + right, + }))); + self.graph.add_edge(parent, content, ()); + parent } - fn string(&mut self) -> BibtexString { + fn string(&mut self) -> NodeIndex { let ty = self.tokens.next().unwrap(); - let left = self.expect2(BibtexTokenKind::BeginBrace, BibtexTokenKind::BeginParen); + let left = self.expect2(TokenKind::BeginBrace, TokenKind::BeginParen); if left.is_none() { - return BibtexString::new(ty, None, None, None, None, None); + return self.graph.add_node(Node::String(Box::new(String { + range: ty.range(), + ty, + left: None, + name: None, + assign: None, + right: None, + }))); } - let name = self.expect1(BibtexTokenKind::Word); + let name = self.expect1(TokenKind::Word); if name.is_none() { - return BibtexString::new(ty, left, None, None, None, None); + return self.graph.add_node(Node::String(Box::new(String { + range: Range::new(ty.start(), left.as_ref().unwrap().end()), + ty, + left, + name: None, + assign: None, + right: None, + }))); } - let assign = self.expect1(BibtexTokenKind::Assign); + let assign = self.expect1(TokenKind::Assign); if assign.is_none() { - return BibtexString::new(ty, left, name, None, None, None); + return self.graph.add_node(Node::String(Box::new(String { + range: Range::new(ty.start(), name.as_ref().unwrap().end()), + ty, + left, + name, + assign: None, + right: None, + }))); } if !self.can_match_content() { - return BibtexString::new(ty, left, name, assign, None, None); + return self.graph.add_node(Node::String(Box::new(String { + range: Range::new(ty.start(), assign.as_ref().unwrap().end()), + ty, + left, + name, + assign, + right: None, + }))); } let value = self.content(); - let right = self.expect2(BibtexTokenKind::EndBrace, BibtexTokenKind::EndParen); - BibtexString::new(ty, left, name, assign, Some(value), right) + let right = self.expect2(TokenKind::EndBrace, TokenKind::EndParen); + let end = right + .as_ref() + .map(Token::end) + .unwrap_or_else(|| self.graph[value].end()); + + let parent = self.graph.add_node(Node::String(Box::new(String { + range: Range::new(ty.start(), end), + ty, + left, + name, + assign, + right, + }))); + self.graph.add_edge(parent, value, ()); + parent } - fn entry(&mut self) -> BibtexEntry { + fn entry(&mut self) -> NodeIndex { let ty = self.tokens.next().unwrap(); - let left = self.expect2(BibtexTokenKind::BeginBrace, BibtexTokenKind::BeginParen); + let left = self.expect2(TokenKind::BeginBrace, TokenKind::BeginParen); if left.is_none() { - return BibtexEntry::new(ty, None, None, None, Vec::new(), None); + return self.graph.add_node(Node::Entry(Box::new(Entry { + range: ty.range(), + ty, + left: None, + key: None, + comma: None, + right: None, + }))); } - let name = self.expect1(BibtexTokenKind::Word); - if name.is_none() { - return BibtexEntry::new(ty, left, None, None, Vec::new(), None); + let key = self.expect1(TokenKind::Word); + if key.is_none() { + return self.graph.add_node(Node::Entry(Box::new(Entry { + range: Range::new(ty.start(), left.as_ref().unwrap().end()), + ty, + left, + key: None, + comma: None, + right: None, + }))); } - let comma = self.expect1(BibtexTokenKind::Comma); + let comma = self.expect1(TokenKind::Comma); if comma.is_none() { - return BibtexEntry::new(ty, left, name, None, Vec::new(), None); + return self.graph.add_node(Node::Entry(Box::new(Entry { + range: Range::new(ty.start(), key.as_ref().unwrap().end()), + ty, + left, + key, + comma: None, + right: None, + }))); } let mut fields = Vec::new(); - while self.next_of_kind(BibtexTokenKind::Word) { + while self.next_of_kind(TokenKind::Word) { fields.push(self.field()); } - let right = self.expect2(BibtexTokenKind::EndBrace, BibtexTokenKind::EndParen); - BibtexEntry::new(ty, left, name, comma, fields, right) + let right = self.expect2(TokenKind::EndBrace, TokenKind::EndParen); + + let end = right + .as_ref() + .map(Token::end) + .or_else(|| fields.last().map(|field| self.graph[*field].end())) + .unwrap_or_else(|| comma.as_ref().unwrap().end()); + let parent = self.graph.add_node(Node::Entry(Box::new(Entry { + range: Range::new(ty.start(), end), + ty, + left, + key, + comma, + right, + }))); + self.connect(parent, &fields); + parent } - fn field(&mut self) -> BibtexField { + fn comment(&mut self) -> NodeIndex { + let token = self.tokens.next().unwrap(); + self.graph.add_node(Node::Comment(Comment { token })) + } + + fn field(&mut self) -> NodeIndex { let name = self.tokens.next().unwrap(); - let assign = self.expect1(BibtexTokenKind::Assign); + let assign = self.expect1(TokenKind::Assign); if assign.is_none() { - return BibtexField::new(name, None, None, None); + return self.graph.add_node(Node::Field(Box::new(Field { + range: name.range(), + name, + assign: None, + comma: None, + }))); } if !self.can_match_content() { - return BibtexField::new(name, assign, None, None); + return self.graph.add_node(Node::Field(Box::new(Field { + range: Range::new(name.start(), assign.as_ref().unwrap().end()), + name, + assign, + comma: None, + }))); } + let content = self.content(); - let comma = self.expect1(BibtexTokenKind::Comma); - BibtexField::new(name, assign, Some(content), comma) + let comma = self.expect1(TokenKind::Comma); + + let end = comma + .as_ref() + .map(Token::end) + .unwrap_or_else(|| self.graph[content].end()); + let parent = self.graph.add_node(Node::Field(Box::new(Field { + range: Range::new(name.start(), end), + name, + assign, + comma, + }))); + self.graph.add_edge(parent, content, ()); + parent } - fn content(&mut self) -> BibtexContent { + fn content(&mut self) -> NodeIndex { let token = self.tokens.next().unwrap(); let left = match token.kind { - BibtexTokenKind::PreambleKind - | BibtexTokenKind::StringKind - | BibtexTokenKind::EntryKind - | BibtexTokenKind::Word - | BibtexTokenKind::Assign - | BibtexTokenKind::Comma - | BibtexTokenKind::BeginParen - | BibtexTokenKind::EndParen => BibtexContent::Word(BibtexWord::new(token)), - BibtexTokenKind::Command => BibtexContent::Command(BibtexCommand::new(token)), - BibtexTokenKind::Quote => { + TokenKind::PreambleKind + | TokenKind::StringKind + | TokenKind::EntryKind + | TokenKind::Word + | TokenKind::Assign + | TokenKind::Comma + | TokenKind::BeginParen + | TokenKind::EndParen => self.graph.add_node(Node::Word(Word { token })), + TokenKind::Command => self.graph.add_node(Node::Command(Command { token })), + TokenKind::Quote => { let mut children = Vec::new(); while self.can_match_content() { - if self.next_of_kind(BibtexTokenKind::Quote) { + if self.next_of_kind(TokenKind::Quote) { break; } children.push(self.content()); } - let right = self.expect1(BibtexTokenKind::Quote); - BibtexContent::QuotedContent(BibtexQuotedContent::new(token, children, right)) + let right = self.expect1(TokenKind::Quote); + + let end = right + .as_ref() + .map(Token::end) + .or_else(|| children.last().map(|child| self.graph[*child].end())) + .unwrap_or_else(|| token.end()); + let parent = self.graph.add_node(Node::QuotedContent(QuotedContent { + range: Range::new(token.start(), end), + left: token, + right, + })); + self.connect(parent, &children); + parent } - BibtexTokenKind::BeginBrace => { + TokenKind::BeginBrace => { let mut children = Vec::new(); while self.can_match_content() { children.push(self.content()); } - let right = self.expect1(BibtexTokenKind::EndBrace); - BibtexContent::BracedContent(BibtexBracedContent::new(token, children, right)) + let right = self.expect1(TokenKind::EndBrace); + + let end = right + .as_ref() + .map(Token::end) + .or_else(|| children.last().map(|child| self.graph[*child].end())) + .unwrap_or_else(|| token.end()); + let parent = self.graph.add_node(Node::BracedContent(BracedContent { + range: Range::new(token.start(), end), + left: token, + right, + })); + self.connect(parent, &children); + parent } _ => unreachable!(), }; - if let Some(operator) = self.expect1(BibtexTokenKind::Concat) { - let right = if self.can_match_content() { - Some(self.content()) - } else { - None - }; - BibtexContent::Concat(Box::new(BibtexConcat::new(left, operator, right))) - } else { - left + + match self.expect1(TokenKind::Concat) { + Some(operator) => { + if self.can_match_content() { + let right = self.content(); + let parent = self.graph.add_node(Node::Concat(Concat { + range: Range::new(self.graph[left].start(), self.graph[right].end()), + operator, + })); + self.graph.add_edge(parent, left, ()); + self.graph.add_edge(parent, right, ()); + parent + } else { + let parent = self.graph.add_node(Node::Concat(Concat { + range: Range::new(self.graph[left].start(), operator.end()), + operator, + })); + self.graph.add_edge(parent, left, ()); + parent + } + } + None => left, + } + } + + fn connect(&mut self, parent: NodeIndex, children: &[NodeIndex]) { + for child in children { + self.graph.add_edge(parent, *child, ()); } } fn can_match_content(&mut self) -> bool { if let Some(ref token) = self.tokens.peek() { match token.kind { - BibtexTokenKind::PreambleKind - | BibtexTokenKind::StringKind - | BibtexTokenKind::EntryKind - | BibtexTokenKind::Word - | BibtexTokenKind::Command - | BibtexTokenKind::Assign - | BibtexTokenKind::Comma - | BibtexTokenKind::Quote - | BibtexTokenKind::BeginBrace - | BibtexTokenKind::BeginParen - | BibtexTokenKind::EndParen => true, - BibtexTokenKind::Concat | BibtexTokenKind::EndBrace => false, + TokenKind::PreambleKind + | TokenKind::StringKind + | TokenKind::EntryKind + | TokenKind::Word + | TokenKind::Command + | TokenKind::Assign + | TokenKind::Comma + | TokenKind::Quote + | TokenKind::BeginBrace + | TokenKind::BeginParen + | TokenKind::EndParen => true, + TokenKind::Concat | TokenKind::EndBrace => false, } } else { false } } - fn expect1(&mut self, kind: BibtexTokenKind) -> Option<BibtexToken> { + fn expect1(&mut self, kind: TokenKind) -> Option<Token> { if let Some(ref token) = self.tokens.peek() { if token.kind == kind { return self.tokens.next(); @@ -200,7 +382,7 @@ impl<I: Iterator<Item = BibtexToken>> BibtexParser<I> { None } - fn expect2(&mut self, kind1: BibtexTokenKind, kind2: BibtexTokenKind) -> Option<BibtexToken> { + fn expect2(&mut self, kind1: TokenKind, kind2: TokenKind) -> Option<Token> { if let Some(ref token) = self.tokens.peek() { if token.kind == kind1 || token.kind == kind2 { return self.tokens.next(); @@ -209,7 +391,7 @@ impl<I: Iterator<Item = BibtexToken>> BibtexParser<I> { None } - fn next_of_kind(&mut self, kind: BibtexTokenKind) -> bool { + fn next_of_kind(&mut self, kind: TokenKind) -> bool { if let Some(token) = self.tokens.peek() { token.kind == kind } else { |