diff options
Diffstat (limited to 'support/texlab/src/syntax/latex')
25 files changed, 1301 insertions, 389 deletions
diff --git a/support/texlab/src/syntax/latex/analysis.rs b/support/texlab/src/syntax/latex/analysis.rs index 2849ade1be..02d9b315ba 100644 --- a/support/texlab/src/syntax/latex/analysis.rs +++ b/support/texlab/src/syntax/latex/analysis.rs @@ -2,6 +2,7 @@ mod command; mod distro_file; mod environment; mod explicit_link; +mod graphics_path; mod implicit_link; mod label_name; mod label_number; @@ -15,6 +16,7 @@ use self::{ command::{analyze_command, analyze_command_definition}, environment::analyze_begin, explicit_link::{analyze_import, analyze_include}, + graphics_path::analyze_graphics_path, implicit_link::analyze_implicit_links, label_name::analyze_label_name, label_number::analyze_label_number, @@ -24,14 +26,15 @@ use self::{ pub fn analyze(context: &mut LatexAnalyzerContext, root: &latex::SyntaxNode) { analyze_implicit_links(context); for node in root.descendants() { - analyze_command(context, node) - .or_else(|| analyze_command_definition(context, node)) - .or_else(|| analyze_begin(context, node)) - .or_else(|| analyze_include(context, node)) - .or_else(|| analyze_import(context, node)) - .or_else(|| analyze_label_name(context, node)) - .or_else(|| analyze_label_number(context, node)) - .or_else(|| analyze_theorem_definition(context, node)); + analyze_command(context, node.clone()) + .or_else(|| analyze_command_definition(context, node.clone())) + .or_else(|| analyze_begin(context, node.clone())) + .or_else(|| analyze_include(context, node.clone())) + .or_else(|| analyze_import(context, node.clone())) + .or_else(|| analyze_label_name(context, node.clone())) + .or_else(|| analyze_label_number(context, node.clone())) + .or_else(|| analyze_theorem_definition(context, node.clone())) + .or_else(|| analyze_graphics_path(context, node)); } context.extras.has_document_environment = context.extras.environment_names.contains("document"); } diff --git a/support/texlab/src/syntax/latex/analysis/command.rs b/support/texlab/src/syntax/latex/analysis/command.rs index b8b3f0ef9a..60aceef725 100644 --- a/support/texlab/src/syntax/latex/analysis/command.rs +++ b/support/texlab/src/syntax/latex/analysis/command.rs @@ -1,8 +1,10 @@ -use crate::syntax::{latex, CstNode}; +use rowan::ast::AstNode; + +use crate::syntax::latex; use super::LatexAnalyzerContext; -pub fn analyze_command(context: &mut LatexAnalyzerContext, node: &latex::SyntaxNode) -> Option<()> { +pub fn analyze_command(context: &mut LatexAnalyzerContext, node: latex::SyntaxNode) -> Option<()> { let command = latex::GenericCommand::cast(node)?; context .extras @@ -13,7 +15,7 @@ pub fn analyze_command(context: &mut LatexAnalyzerContext, node: &latex::SyntaxN pub fn analyze_command_definition( context: &mut LatexAnalyzerContext, - node: &latex::SyntaxNode, + node: latex::SyntaxNode, ) -> Option<()> { let definition = latex::CommandDefinition::cast(node)?; context diff --git a/support/texlab/src/syntax/latex/analysis/distro_file.rs b/support/texlab/src/syntax/latex/analysis/distro_file.rs index 4c747ce760..f278c33627 100644 --- a/support/texlab/src/syntax/latex/analysis/distro_file.rs +++ b/support/texlab/src/syntax/latex/analysis/distro_file.rs @@ -1,6 +1,8 @@ -use crate::{distro::Resolver, Uri}; +use lsp_types::Url; -pub fn resolve_distro_file(resolver: &Resolver, stem: &str, extensions: &[&str]) -> Option<Uri> { +use crate::distro::Resolver; + +pub fn resolve_distro_file(resolver: &Resolver, stem: &str, extensions: &[&str]) -> Option<Url> { let mut document = resolver.files_by_name.get(stem); for extension in extensions { document = document.or_else(|| { @@ -8,7 +10,7 @@ pub fn resolve_distro_file(resolver: &Resolver, stem: &str, extensions: &[&str]) resolver.files_by_name.get(full_name.as_str()) }); } - document.and_then(|path| Uri::from_file_path(path).ok()) + document.and_then(|path| Url::from_file_path(path).ok()) } #[cfg(test)] @@ -31,12 +33,12 @@ mod tests { assert_eq!( resolve_distro_file(&resolver, "foo", &["tex"]), - Some(Uri::from_file_path("C:/distro/foo.tex").unwrap()) + Some(Url::from_file_path("C:/distro/foo.tex").unwrap()) ); assert_eq!( resolve_distro_file(&resolver, "foo", &["sty"]), - Some(Uri::from_file_path("C:/distro/foo.sty").unwrap()) + Some(Url::from_file_path("C:/distro/foo.sty").unwrap()) ); assert_eq!(resolve_distro_file(&resolver, "foo", &["cls"]), None); @@ -58,12 +60,12 @@ mod tests { assert_eq!( resolve_distro_file(&resolver, "foo", &["tex"]), - Some(Uri::from_file_path("/distro/foo.tex").unwrap()) + Some(Url::from_file_path("/distro/foo.tex").unwrap()) ); assert_eq!( resolve_distro_file(&resolver, "foo", &["sty"]), - Some(Uri::from_file_path("/distro/foo.sty").unwrap()) + Some(Url::from_file_path("/distro/foo.sty").unwrap()) ); assert_eq!(resolve_distro_file(&resolver, "foo", &["cls"]), None); diff --git a/support/texlab/src/syntax/latex/analysis/environment.rs b/support/texlab/src/syntax/latex/analysis/environment.rs index 7cda0c126b..e99483f706 100644 --- a/support/texlab/src/syntax/latex/analysis/environment.rs +++ b/support/texlab/src/syntax/latex/analysis/environment.rs @@ -1,11 +1,13 @@ -use crate::syntax::{latex, CstNode}; +use rowan::ast::AstNode; + +use crate::syntax::latex; use super::LatexAnalyzerContext; -pub fn analyze_begin(context: &mut LatexAnalyzerContext, node: &latex::SyntaxNode) -> Option<()> { +pub fn analyze_begin(context: &mut LatexAnalyzerContext, node: latex::SyntaxNode) -> Option<()> { let begin = latex::Begin::cast(node)?; let name = begin.name()?.key()?.to_string(); let extras = &mut context.extras; - extras.environment_names.insert(name.into()); + extras.environment_names.insert(name); Some(()) } diff --git a/support/texlab/src/syntax/latex/analysis/explicit_link.rs b/support/texlab/src/syntax/latex/analysis/explicit_link.rs index 48f8d6b5ee..77e2cfb696 100644 --- a/support/texlab/src/syntax/latex/analysis/explicit_link.rs +++ b/support/texlab/src/syntax/latex/analysis/explicit_link.rs @@ -1,12 +1,14 @@ use std::sync::Arc; -use crate::syntax::{latex, CstNode}; +use rowan::ast::AstNode; + +use crate::syntax::latex; use super::{ distro_file::resolve_distro_file, ExplicitLink, ExplicitLinkKind, LatexAnalyzerContext, }; -pub fn analyze_include(context: &mut LatexAnalyzerContext, node: &latex::SyntaxNode) -> Option<()> { +pub fn analyze_include(context: &mut LatexAnalyzerContext, node: latex::SyntaxNode) -> Option<()> { let include = latex::Include::cast(node)?; let kind = match include.syntax().kind() { latex::LATEX_INCLUDE => ExplicitLinkKind::Latex, @@ -25,20 +27,20 @@ pub fn analyze_include(context: &mut LatexAnalyzerContext, node: &latex::SyntaxN for path in include.path_list()?.keys() { let stem = path.to_string(); - let mut targets = vec![Arc::new(context.base_uri.join(&stem).ok()?.into())]; + let mut targets = vec![Arc::new(context.base_uri.join(&stem).ok()?)]; for extension in extensions { let path = format!("{}.{}", stem, extension); - targets.push(Arc::new(context.base_uri.join(&path).ok()?.into())); + targets.push(Arc::new(context.base_uri.join(&path).ok()?)); } - resolve_distro_file(&context.inner.resolver.lock().unwrap(), &stem, extensions) + resolve_distro_file(&context.environment.resolver, &stem, extensions) .into_iter() .for_each(|target| targets.push(Arc::new(target))); context.extras.explicit_links.push(ExplicitLink { kind, stem: stem.into(), - stem_range: path.small_range(), + stem_range: latex::small_range(&path), targets, }); } @@ -46,25 +48,23 @@ pub fn analyze_include(context: &mut LatexAnalyzerContext, node: &latex::SyntaxN Some(()) } -pub fn analyze_import(context: &mut LatexAnalyzerContext, node: &latex::SyntaxNode) -> Option<()> { +pub fn analyze_import(context: &mut LatexAnalyzerContext, node: latex::SyntaxNode) -> Option<()> { let import = latex::Import::cast(node)?; let mut targets = Vec::new(); let directory = match import.directory().and_then(|dir| dir.key()) { - Some(dir) => Arc::new(context.base_uri.join(&dir.to_string()).ok()?.into()), + Some(dir) => Arc::new(context.base_uri.join(&dir.to_string()).ok()?), None => Arc::clone(&context.base_uri), }; let file = import.file()?.key()?; let stem = file.to_string(); - targets.push(Arc::new(directory.join(&stem).ok()?.into())); - targets.push(Arc::new( - directory.join(&format!("{}.tex", stem)).ok()?.into(), - )); + targets.push(Arc::new(directory.join(&stem).ok()?)); + targets.push(Arc::new(directory.join(&format!("{}.tex", stem)).ok()?)); context.extras.explicit_links.push(ExplicitLink { stem: stem.into(), - stem_range: file.small_range(), + stem_range: latex::small_range(&file), targets, kind: ExplicitLinkKind::Latex, }); diff --git a/support/texlab/src/syntax/latex/analysis/graphics_path.rs b/support/texlab/src/syntax/latex/analysis/graphics_path.rs new file mode 100644 index 0000000000..477094b841 --- /dev/null +++ b/support/texlab/src/syntax/latex/analysis/graphics_path.rs @@ -0,0 +1,21 @@ +use rowan::ast::AstNode; + +use crate::syntax::latex; + +use super::LatexAnalyzerContext; + +pub fn analyze_graphics_path( + context: &mut LatexAnalyzerContext, + node: latex::SyntaxNode, +) -> Option<()> { + let definition = latex::GraphicsPath::cast(node)?; + for path in definition + .path_list() + .filter_map(|group| group.key()) + .map(|path| path.to_string()) + { + context.extras.graphics_paths.insert(path); + } + + Some(()) +} diff --git a/support/texlab/src/syntax/latex/analysis/implicit_link.rs b/support/texlab/src/syntax/latex/analysis/implicit_link.rs index 74055b3e31..bba3acfb80 100644 --- a/support/texlab/src/syntax/latex/analysis/implicit_link.rs +++ b/support/texlab/src/syntax/latex/analysis/implicit_link.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use crate::Uri; +use lsp_types::Url; use super::LatexAnalyzerContext; @@ -10,32 +10,41 @@ pub fn analyze_implicit_links(context: &mut LatexAnalyzerContext) { context.extras.implicit_links.pdf = find_by_extension(context, "pdf").unwrap_or_default(); } -fn find_by_extension(context: &LatexAnalyzerContext, extension: &str) -> Option<Vec<Arc<Uri>>> { - let mut targets: Vec<Arc<Uri>> = Vec::new(); - targets.push(Arc::new(context.document_uri.with_extension(extension)?)); +fn find_by_extension(context: &LatexAnalyzerContext, extension: &str) -> Option<Vec<Arc<Url>>> { + let mut targets = vec![Arc::new(with_extension(&context.document_uri, extension)?)]; if context.document_uri.scheme() == "file" { let file_path = context.document_uri.to_file_path().ok()?; let file_stem = file_path.file_stem()?; let aux_name = format!("{}.{}", file_stem.to_str()?, extension); - let options = context.inner.options.read().unwrap(); + let options = &context.environment.options; if let Some(root_dir) = options.root_directory.as_ref() { let path = context - .inner + .environment .current_directory .join(root_dir) .join(&aux_name); - targets.push(Arc::new(Uri::from_file_path(path).ok()?)); + targets.push(Arc::new(Url::from_file_path(path).ok()?)); } if let Some(build_dir) = options.aux_directory.as_ref() { let path = context - .inner + .environment .current_directory .join(build_dir) .join(&aux_name); - targets.push(Arc::new(Uri::from_file_path(path).ok()?)); + targets.push(Arc::new(Url::from_file_path(path).ok()?)); } } Some(targets) } + +fn with_extension(uri: &Url, extension: &str) -> Option<Url> { + let file_name = uri.path_segments()?.last()?; + let file_stem = file_name + .rfind('.') + .map(|i| &file_name[..i]) + .unwrap_or(file_name); + + uri.join(&format!("{}.{}", file_stem, extension)).ok() +} diff --git a/support/texlab/src/syntax/latex/analysis/label_name.rs b/support/texlab/src/syntax/latex/analysis/label_name.rs index e5ff6829cd..1e3339db5e 100644 --- a/support/texlab/src/syntax/latex/analysis/label_name.rs +++ b/support/texlab/src/syntax/latex/analysis/label_name.rs @@ -1,27 +1,28 @@ use latex::LabelReferenceRange; +use rowan::ast::AstNode; -use crate::syntax::{latex, CstNode}; +use crate::syntax::latex; use super::{LabelName, LatexAnalyzerContext}; pub fn analyze_label_name( context: &mut LatexAnalyzerContext, - node: &latex::SyntaxNode, + node: latex::SyntaxNode, ) -> Option<()> { - analyze_label_definition_name(context, node) - .or_else(|| analyze_label_reference_name(context, node)) + analyze_label_definition_name(context, node.clone()) + .or_else(|| analyze_label_reference_name(context, node.clone())) .or_else(|| analyze_label_reference_range_name(context, node)) } fn analyze_label_definition_name( context: &mut LatexAnalyzerContext, - node: &latex::SyntaxNode, + node: latex::SyntaxNode, ) -> Option<()> { let label = latex::LabelDefinition::cast(node)?; let name = label.name()?.key()?; context.extras.label_names.push(LabelName { text: name.to_string().into(), - range: name.small_range(), + range: latex::small_range(&name), is_definition: true, }); Some(()) @@ -29,13 +30,13 @@ fn analyze_label_definition_name( fn analyze_label_reference_name( context: &mut LatexAnalyzerContext, - node: &latex::SyntaxNode, + node: latex::SyntaxNode, ) -> Option<()> { let label = latex::LabelReference::cast(node)?; for name in label.name_list()?.keys() { context.extras.label_names.push(LabelName { text: name.to_string().into(), - range: name.small_range(), + range: latex::small_range(&name), is_definition: false, }); } @@ -44,13 +45,13 @@ fn analyze_label_reference_name( fn analyze_label_reference_range_name( context: &mut LatexAnalyzerContext, - node: &latex::SyntaxNode, + node: latex::SyntaxNode, ) -> Option<()> { let label = LabelReferenceRange::cast(node)?; if let Some(name1) = label.from().and_then(|name| name.key()) { context.extras.label_names.push(LabelName { text: name1.to_string().into(), - range: name1.small_range(), + range: latex::small_range(&name1), is_definition: false, }); } @@ -58,7 +59,7 @@ fn analyze_label_reference_range_name( if let Some(name2) = label.to().and_then(|name| name.key()) { context.extras.label_names.push(LabelName { text: name2.to_string().into(), - range: name2.small_range(), + range: latex::small_range(&name2), is_definition: false, }); } diff --git a/support/texlab/src/syntax/latex/analysis/label_number.rs b/support/texlab/src/syntax/latex/analysis/label_number.rs index 93ad09523f..61ad348680 100644 --- a/support/texlab/src/syntax/latex/analysis/label_number.rs +++ b/support/texlab/src/syntax/latex/analysis/label_number.rs @@ -1,10 +1,12 @@ -use crate::syntax::{latex, CstNode}; +use rowan::ast::AstNode; + +use crate::syntax::latex; use super::LatexAnalyzerContext; pub fn analyze_label_number( context: &mut LatexAnalyzerContext, - node: &latex::SyntaxNode, + node: latex::SyntaxNode, ) -> Option<()> { let number = latex::LabelNumber::cast(node)?; let name = number.name()?.key()?.to_string(); diff --git a/support/texlab/src/syntax/latex/analysis/theorem.rs b/support/texlab/src/syntax/latex/analysis/theorem.rs index 50236e24a5..70047362d1 100644 --- a/support/texlab/src/syntax/latex/analysis/theorem.rs +++ b/support/texlab/src/syntax/latex/analysis/theorem.rs @@ -1,13 +1,12 @@ -use crate::syntax::{ - latex::{self, HasCurly}, - CstNode, -}; +use rowan::ast::AstNode; + +use crate::syntax::latex::{self, HasCurly}; use super::{LatexAnalyzerContext, TheoremEnvironment}; pub fn analyze_theorem_definition( context: &mut LatexAnalyzerContext, - node: &latex::SyntaxNode, + node: latex::SyntaxNode, ) -> Option<()> { let theorem = latex::TheoremDefinition::cast(node)?; let name = theorem.name()?.key()?.to_string(); diff --git a/support/texlab/src/syntax/latex/analysis/types.rs b/support/texlab/src/syntax/latex/analysis/types.rs index bd2789b3b7..7fb938e3d2 100644 --- a/support/texlab/src/syntax/latex/analysis/types.rs +++ b/support/texlab/src/syntax/latex/analysis/types.rs @@ -1,16 +1,17 @@ use std::sync::Arc; -use cstree::TextRange; +use lsp_types::Url; +use rowan::TextRange; use rustc_hash::{FxHashMap, FxHashSet}; use smol_str::SmolStr; -use crate::{ServerContext, Uri}; +use crate::Environment; #[derive(Debug)] -pub struct LatexAnalyzerContext { - pub inner: Arc<ServerContext>, - pub document_uri: Arc<Uri>, - pub base_uri: Arc<Uri>, +pub struct LatexAnalyzerContext<'a> { + pub environment: &'a Environment, + pub document_uri: Arc<Url>, + pub base_uri: Arc<Url>, pub extras: Extras, } @@ -24,13 +25,14 @@ pub struct Extras { pub label_names: Vec<LabelName>, pub label_numbers_by_name: FxHashMap<String, String>, pub theorem_environments: Vec<TheoremEnvironment>, + pub graphics_paths: FxHashSet<String>, } #[derive(Debug, PartialEq, Eq, Clone, Default, Hash)] pub struct ImplicitLinks { - pub aux: Vec<Arc<Uri>>, - pub log: Vec<Arc<Uri>>, - pub pdf: Vec<Arc<Uri>>, + pub aux: Vec<Arc<Url>>, + pub log: Vec<Arc<Url>>, + pub pdf: Vec<Arc<Url>>, } #[derive(Debug, PartialEq, Eq, Clone, Copy, PartialOrd, Ord, Hash)] @@ -45,7 +47,7 @@ pub enum ExplicitLinkKind { pub struct ExplicitLink { pub stem: SmolStr, pub stem_range: TextRange, - pub targets: Vec<Arc<Uri>>, + pub targets: Vec<Arc<Url>>, pub kind: ExplicitLinkKind, } diff --git a/support/texlab/src/syntax/latex/cst.rs b/support/texlab/src/syntax/latex/cst.rs index 80c3375304..06c187213d 100644 --- a/support/texlab/src/syntax/latex/cst.rs +++ b/support/texlab/src/syntax/latex/cst.rs @@ -1,20 +1,43 @@ -use cstree::TextRange; use itertools::{EitherOrBoth, Itertools}; +use rowan::{ast::AstNode, TextRange}; + +use super::{ + LatexLanguage, + SyntaxKind::{self, *}, + SyntaxNode, SyntaxToken, +}; + +pub fn small_range(node: &dyn AstNode<Language = LatexLanguage>) -> TextRange { + let full_range = node.syntax().text_range(); + let start = full_range.start(); + let mut token = node.syntax().last_token(); + while let Some(current) = token { + if !matches!(current.kind(), LINE_BREAK | WHITESPACE | COMMENT) { + return TextRange::new(start, current.text_range().end()); + } + token = current.prev_token(); + } -use crate::syntax::CstNode; - -use super::{Language, SyntaxKind::*, SyntaxNode, SyntaxToken}; + TextRange::new(start, start) +} macro_rules! cst_node { ($name:ident, $($kind:pat),+) => { - #[derive(Clone, Copy)] + #[derive(Clone)] #[repr(transparent)] - pub struct $name<'a>(&'a SyntaxNode); + pub struct $name(SyntaxNode); + + impl AstNode for $name { + type Language = LatexLanguage; - impl<'a> CstNode<'a> for $name<'a> { - type Lang = Language; + fn can_cast(kind: SyntaxKind) -> bool { + match kind { + $($kind => true,)+ + _ => false, + } + } - fn cast(node: &'a cstree::ResolvedNode<Self::Lang>) -> Option<Self> + fn cast(node: SyntaxNode) -> Option<Self> where Self: Sized, { @@ -24,30 +47,17 @@ macro_rules! cst_node { } } - fn syntax(&self) -> &'a cstree::ResolvedNode<Self::Lang> { + fn syntax(&self) -> &SyntaxNode { &self.0 } - - fn small_range(&self) -> TextRange { - let full_range = self.syntax().text_range(); - let start = full_range.start(); - let mut token = self.syntax().last_token(); - while let Some(current) = token { - if !matches!(current.kind(), LINE_BREAK | WHITESPACE | COMMENT) { - return TextRange::new(start, current.text_range().end()); - } - token = current.prev_token(); - } - TextRange::new(start, start) - } } }; } cst_node!(Text, TEXT); -impl<'a> Text<'a> { - pub fn words(&self) -> impl Iterator<Item = &'a SyntaxToken> { +impl Text { + pub fn words(&self) -> impl Iterator<Item = SyntaxToken> { self.syntax() .children_with_tokens() .filter_map(|node| node.into_token()) @@ -55,15 +65,15 @@ impl<'a> Text<'a> { } } -pub trait HasCurly<'a>: CstNode<'a, Lang = Language> { - fn left_curly(&self) -> Option<&'a SyntaxToken> { +pub trait HasCurly: AstNode<Language = LatexLanguage> { + fn left_curly(&self) -> Option<SyntaxToken> { self.syntax() .children_with_tokens() .filter_map(|node| node.into_token()) .find(|node| node.kind() == L_CURLY) } - fn right_curly(&self) -> Option<&'a SyntaxToken> { + fn right_curly(&self) -> Option<SyntaxToken> { self.syntax() .children_with_tokens() .filter_map(|node| node.into_token()) @@ -91,19 +101,19 @@ pub trait HasCurly<'a>: CstNode<'a, Lang = Language> { cst_node!(CurlyGroup, CURLY_GROUP); -impl<'a> HasCurly<'a> for CurlyGroup<'a> {} +impl HasCurly for CurlyGroup {} -impl<'a> CurlyGroup<'a> {} +impl CurlyGroup {} -pub trait HasBrack<'a>: CstNode<'a, Lang = Language> { - fn left_brack(&self) -> Option<&'a SyntaxToken> { +pub trait HasBrack: AstNode<Language = LatexLanguage> { + fn left_brack(&self) -> Option<SyntaxToken> { self.syntax() .children_with_tokens() .filter_map(|node| node.into_token()) .find(|node| node.kind() == L_BRACK) } - fn right_brack(&self) -> Option<&'a SyntaxToken> { + fn right_brack(&self) -> Option<SyntaxToken> { self.syntax() .children_with_tokens() .filter_map(|node| node.into_token()) @@ -131,19 +141,19 @@ pub trait HasBrack<'a>: CstNode<'a, Lang = Language> { cst_node!(BrackGroup, BRACK_GROUP); -impl<'a> BrackGroup<'a> {} +impl BrackGroup {} -impl<'a> HasBrack<'a> for BrackGroup<'a> {} +impl HasBrack for BrackGroup {} -pub trait HasParen<'a>: CstNode<'a, Lang = Language> { - fn left_paren(&self) -> Option<&'a SyntaxToken> { +pub trait HasParen: AstNode<Language = LatexLanguage> { + fn left_paren(&self) -> Option<SyntaxToken> { self.syntax() .children_with_tokens() .filter_map(|node| node.into_token()) .find(|node| node.kind() == L_PAREN) } - fn right_paren(&self) -> Option<&'a SyntaxToken> { + fn right_paren(&self) -> Option<SyntaxToken> { self.syntax() .children_with_tokens() .filter_map(|node| node.into_token()) @@ -153,62 +163,62 @@ pub trait HasParen<'a>: CstNode<'a, Lang = Language> { cst_node!(ParenGroup, PAREN_GROUP); -impl<'a> HasParen<'a> for ParenGroup<'a> {} +impl HasParen for ParenGroup {} cst_node!(MixedGroup, MIXED_GROUP); -impl<'a> MixedGroup<'a> { - pub fn left_delim(&self) -> Option<&'a SyntaxToken> { +impl MixedGroup { + pub fn left_delim(&self) -> Option<SyntaxToken> { self.syntax() .children_with_tokens() .filter_map(|node| node.into_token()) - .find(|node| matches!(node.kind().into(), L_BRACK | L_PAREN)) + .find(|node| matches!(node.kind(), L_BRACK | L_PAREN)) } - pub fn right_delim(&self) -> Option<&'a SyntaxToken> { + pub fn right_delim(&self) -> Option<SyntaxToken> { self.syntax() .children_with_tokens() .filter_map(|node| node.into_token()) - .find(|node| matches!(node.kind().into(), R_BRACK | R_PAREN)) + .find(|node| matches!(node.kind(), R_BRACK | R_PAREN)) } } cst_node!(CurlyGroupWord, CURLY_GROUP_WORD); -impl<'a> HasCurly<'a> for CurlyGroupWord<'a> {} +impl HasCurly for CurlyGroupWord {} -impl<'a> CurlyGroupWord<'a> { - pub fn key(&self) -> Option<Key<'a>> { +impl CurlyGroupWord { + pub fn key(&self) -> Option<Key> { self.syntax().children().find_map(Key::cast) } } cst_node!(BrackGroupWord, BRACK_GROUP_WORD); -impl<'a> HasBrack<'a> for BrackGroupWord<'a> {} +impl HasBrack for BrackGroupWord {} -impl<'a> BrackGroupWord<'a> { - pub fn key(&self) -> Option<Key<'a>> { +impl BrackGroupWord { + pub fn key(&self) -> Option<Key> { self.syntax().children().find_map(Key::cast) } } cst_node!(CurlyGroupWordList, CURLY_GROUP_WORD_LIST); -impl<'a> HasCurly<'a> for CurlyGroupWordList<'a> {} +impl HasCurly for CurlyGroupWordList {} -impl<'a> CurlyGroupWordList<'a> { - pub fn keys(&self) -> impl Iterator<Item = Key<'a>> { +impl CurlyGroupWordList { + pub fn keys(&self) -> impl Iterator<Item = Key> { self.syntax().children().filter_map(Key::cast) } } cst_node!(CurlyGroupCommand, CURLY_GROUP_COMMAND); -impl<'a> HasCurly<'a> for CurlyGroupCommand<'a> {} +impl HasCurly for CurlyGroupCommand {} -impl<'a> CurlyGroupCommand<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl CurlyGroupCommand { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax() .children_with_tokens() .filter_map(|node| node.into_token()) @@ -218,8 +228,8 @@ impl<'a> CurlyGroupCommand<'a> { cst_node!(Key, KEY); -impl<'a> Key<'a> { - pub fn words(&self) -> impl Iterator<Item = &'a SyntaxToken> { +impl Key { + pub fn words(&self) -> impl Iterator<Item = SyntaxToken> { self.syntax() .children_with_tokens() .filter_map(|node| node.into_token()) @@ -227,7 +237,7 @@ impl<'a> Key<'a> { } } -impl<'a> PartialEq for Key<'a> { +impl PartialEq for Key { fn eq(&self, other: &Self) -> bool { self.words() .zip_longest(other.words()) @@ -238,11 +248,18 @@ impl<'a> PartialEq for Key<'a> { } } -impl<'a> Eq for Key<'a> {} +impl Eq for Key {} -impl<'a> ToString for Key<'a> { +impl ToString for Key { fn to_string(&self) -> String { - self.words().map(|word| word.text()).join(" ") + let mut buf = String::new(); + for word in self.words() { + buf.push_str(word.text()); + buf.push(' '); + } + + buf.pop().unwrap(); + buf } } @@ -250,48 +267,48 @@ cst_node!(Value, VALUE); cst_node!(KeyValuePair, KEY_VALUE_PAIR); -impl<'a> KeyValuePair<'a> { - pub fn key(&self) -> Option<Key<'a>> { +impl KeyValuePair { + pub fn key(&self) -> Option<Key> { self.syntax().children().find_map(Key::cast) } - pub fn value(&self) -> Option<Value<'a>> { + pub fn value(&self) -> Option<Value> { self.syntax().children().find_map(Value::cast) } } cst_node!(KeyValueBody, KEY_VALUE_BODY); -impl<'a> KeyValueBody<'a> { - pub fn pairs(&self) -> impl Iterator<Item = KeyValuePair<'a>> { +impl KeyValueBody { + pub fn pairs(&self) -> impl Iterator<Item = KeyValuePair> { self.syntax().children().filter_map(KeyValuePair::cast) } } -pub trait HasKeyValueBody<'a>: CstNode<'a, Lang = Language> { - fn body(&self) -> Option<KeyValueBody<'a>> { +pub trait HasKeyValueBody: AstNode<Language = LatexLanguage> { + fn body(&self) -> Option<KeyValueBody> { self.syntax().children().find_map(KeyValueBody::cast) } } cst_node!(CurlyGroupKeyValue, CURLY_GROUP_KEY_VALUE); -impl<'a> HasCurly<'a> for CurlyGroupKeyValue<'a> {} +impl HasCurly for CurlyGroupKeyValue {} -impl<'a> HasKeyValueBody<'a> for CurlyGroupKeyValue<'a> {} +impl HasKeyValueBody for CurlyGroupKeyValue {} cst_node!(BrackGroupKeyValue, BRACK_GROUP_KEY_VALUE); -impl<'a> HasBrack<'a> for BrackGroupKeyValue<'a> {} +impl HasBrack for BrackGroupKeyValue {} -impl<'a> HasKeyValueBody<'a> for BrackGroupKeyValue<'a> {} +impl HasKeyValueBody for BrackGroupKeyValue {} cst_node!(Formula, FORMULA); cst_node!(GenericCommand, GENERIC_COMMAND); -impl<'a> GenericCommand<'a> { - pub fn name(&self) -> Option<&'a SyntaxToken> { +impl GenericCommand { + pub fn name(&self) -> Option<SyntaxToken> { self.syntax() .children_with_tokens() .filter_map(|node| node.into_token()) @@ -303,40 +320,40 @@ cst_node!(Equation, EQUATION); cst_node!(Begin, BEGIN); -impl<'a> Begin<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl Begin { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn name(&self) -> Option<CurlyGroupWord<'a>> { + pub fn name(&self) -> Option<CurlyGroupWord> { self.syntax().children().find_map(CurlyGroupWord::cast) } - pub fn options(&self) -> Option<BrackGroup<'a>> { + pub fn options(&self) -> Option<BrackGroup> { self.syntax().children().find_map(BrackGroup::cast) } } cst_node!(End, END); -impl<'a> End<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl End { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn name(&self) -> Option<CurlyGroupWord<'a>> { + pub fn name(&self) -> Option<CurlyGroupWord> { self.syntax().children().find_map(CurlyGroupWord::cast) } } cst_node!(Environment, ENVIRONMENT); -impl<'a> Environment<'a> { - pub fn begin(&self) -> Option<Begin<'a>> { +impl Environment { + pub fn begin(&self) -> Option<Begin> { self.syntax().children().find_map(Begin::cast) } - pub fn end(&self) -> Option<End<'a>> { + pub fn end(&self) -> Option<End> { self.syntax().children().find_map(End::cast) } } @@ -352,60 +369,60 @@ cst_node!( SUBPARAGRAPH ); -impl<'a> Section<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl Section { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn name(&self) -> Option<CurlyGroup<'a>> { + pub fn name(&self) -> Option<CurlyGroup> { self.syntax().children().find_map(CurlyGroup::cast) } } cst_node!(EnumItem, ENUM_ITEM); -impl<'a> EnumItem<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl EnumItem { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn label(&self) -> Option<BrackGroup<'a>> { + pub fn label(&self) -> Option<BrackGroup> { self.syntax().children().find_map(BrackGroup::cast) } } cst_node!(Caption, CAPTION); -impl<'a> Caption<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl Caption { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn short(&self) -> Option<BrackGroup<'a>> { + pub fn short(&self) -> Option<BrackGroup> { self.syntax().children().find_map(BrackGroup::cast) } - pub fn long(&self) -> Option<CurlyGroup<'a>> { + pub fn long(&self) -> Option<CurlyGroup> { self.syntax().children().find_map(CurlyGroup::cast) } } cst_node!(Citation, CITATION); -impl<'a> Citation<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl Citation { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn prenote(&self) -> Option<BrackGroup<'a>> { + pub fn prenote(&self) -> Option<BrackGroup> { self.syntax().children().find_map(BrackGroup::cast) } - pub fn postnote(&self) -> Option<BrackGroup<'a>> { + pub fn postnote(&self) -> Option<BrackGroup> { self.syntax().children().filter_map(BrackGroup::cast).nth(1) } - pub fn key_list(&self) -> Option<CurlyGroupWordList<'a>> { + pub fn key_list(&self) -> Option<CurlyGroupWordList> { self.syntax().children().find_map(CurlyGroupWordList::cast) } } @@ -423,28 +440,28 @@ cst_node!( VERBATIM_INCLUDE ); -impl<'a> Include<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl Include { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn path_list(&self) -> Option<CurlyGroupWordList<'a>> { + pub fn path_list(&self) -> Option<CurlyGroupWordList> { self.syntax().children().find_map(CurlyGroupWordList::cast) } } cst_node!(Import, IMPORT); -impl<'a> Import<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl Import { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn directory(&self) -> Option<CurlyGroupWord<'a>> { + pub fn directory(&self) -> Option<CurlyGroupWord> { self.syntax().children().find_map(CurlyGroupWord::cast) } - pub fn file(&self) -> Option<CurlyGroupWord<'a>> { + pub fn file(&self) -> Option<CurlyGroupWord> { self.syntax() .children() .filter_map(CurlyGroupWord::cast) @@ -454,40 +471,40 @@ impl<'a> Import<'a> { cst_node!(LabelDefinition, LABEL_DEFINITION); -impl<'a> LabelDefinition<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl LabelDefinition { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn name(&self) -> Option<CurlyGroupWord<'a>> { + pub fn name(&self) -> Option<CurlyGroupWord> { self.syntax().children().find_map(CurlyGroupWord::cast) } } cst_node!(LabelReference, LABEL_REFERENCE); -impl<'a> LabelReference<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl LabelReference { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn name_list(&self) -> Option<CurlyGroupWordList<'a>> { + pub fn name_list(&self) -> Option<CurlyGroupWordList> { self.syntax().children().find_map(CurlyGroupWordList::cast) } } cst_node!(LabelReferenceRange, LABEL_REFERENCE_RANGE); -impl<'a> LabelReferenceRange<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl LabelReferenceRange { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn from(&self) -> Option<CurlyGroupWord<'a>> { + pub fn from(&self) -> Option<CurlyGroupWord> { self.syntax().children().find_map(CurlyGroupWord::cast) } - pub fn to(&self) -> Option<CurlyGroupWord<'a>> { + pub fn to(&self) -> Option<CurlyGroupWord> { self.syntax() .children() .filter_map(CurlyGroupWord::cast) @@ -497,163 +514,175 @@ impl<'a> LabelReferenceRange<'a> { cst_node!(LabelNumber, LABEL_NUMBER); -impl<'a> LabelNumber<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl LabelNumber { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn name(&self) -> Option<CurlyGroupWord<'a>> { + pub fn name(&self) -> Option<CurlyGroupWord> { self.syntax().children().find_map(CurlyGroupWord::cast) } - pub fn text(&self) -> Option<CurlyGroup<'a>> { + pub fn text(&self) -> Option<CurlyGroup> { self.syntax().children().find_map(CurlyGroup::cast) } } cst_node!(TheoremDefinition, THEOREM_DEFINITION); -impl<'a> TheoremDefinition<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl TheoremDefinition { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn name(&self) -> Option<CurlyGroupWord<'a>> { + pub fn name(&self) -> Option<CurlyGroupWord> { self.syntax().children().find_map(CurlyGroupWord::cast) } - pub fn description(&self) -> Option<CurlyGroup<'a>> { + pub fn description(&self) -> Option<CurlyGroup> { self.syntax().children().find_map(CurlyGroup::cast) } } cst_node!(CommandDefinition, COMMAND_DEFINITION, MATH_OPERATOR); -impl<'a> CommandDefinition<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl CommandDefinition { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn name(&self) -> Option<CurlyGroupCommand<'a>> { + pub fn name(&self) -> Option<CurlyGroupCommand> { self.syntax().children().find_map(CurlyGroupCommand::cast) } - pub fn implementation(&self) -> Option<CurlyGroup<'a>> { + pub fn implementation(&self) -> Option<CurlyGroup> { self.syntax().children().find_map(CurlyGroup::cast) } } cst_node!(AcronymReference, ACRONYM_REFERENCE); -impl<'a> AcronymReference<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl AcronymReference { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } } cst_node!(AcronymDefinition, ACRONYM_DEFINITION); -impl<'a> AcronymDefinition<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl AcronymDefinition { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn name(&self) -> Option<CurlyGroupWord<'a>> { + pub fn name(&self) -> Option<CurlyGroupWord> { self.syntax().children().find_map(CurlyGroupWord::cast) } } cst_node!(AcronymDeclaration, ACRONYM_DECLARATION); -impl<'a> AcronymDeclaration<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl AcronymDeclaration { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn name(&self) -> Option<CurlyGroupWord<'a>> { + pub fn name(&self) -> Option<CurlyGroupWord> { self.syntax().children().find_map(CurlyGroupWord::cast) } } cst_node!(ColorDefinition, COLOR_DEFINITION); -impl<'a> ColorDefinition<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl ColorDefinition { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn name(&self) -> Option<CurlyGroupWord<'a>> { + pub fn name(&self) -> Option<CurlyGroupWord> { self.syntax().children().find_map(CurlyGroupWord::cast) } - pub fn model(&self) -> Option<CurlyGroupWord<'a>> { + pub fn model(&self) -> Option<CurlyGroupWord> { self.syntax() .children() .filter_map(CurlyGroupWord::cast) .nth(1) } - pub fn spec(&self) -> Option<CurlyGroup<'a>> { + pub fn spec(&self) -> Option<CurlyGroup> { self.syntax().children().find_map(CurlyGroup::cast) } } cst_node!(ColorSetDefinition, COLOR_SET_DEFINITION); -impl<'a> ColorSetDefinition<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl ColorSetDefinition { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn model_list(&self) -> Option<CurlyGroupWordList<'a>> { + pub fn model_list(&self) -> Option<CurlyGroupWordList> { self.syntax().children().find_map(CurlyGroupWordList::cast) } } cst_node!(ColorReference, COLOR_REFERENCE); -impl<'a> ColorReference<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl ColorReference { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn name(&self) -> Option<CurlyGroupWord<'a>> { + pub fn name(&self) -> Option<CurlyGroupWord> { self.syntax().children().find_map(CurlyGroupWord::cast) } } cst_node!(GlossaryEntryReference, GLOSSARY_ENTRY_REFERENCE); -impl<'a> GlossaryEntryReference<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl GlossaryEntryReference { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn name(&self) -> Option<CurlyGroupWord<'a>> { + pub fn name(&self) -> Option<CurlyGroupWord> { self.syntax().children().find_map(CurlyGroupWord::cast) } } cst_node!(GlossaryEntryDefinition, GLOSSARY_ENTRY_DEFINITION); -impl<'a> GlossaryEntryDefinition<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl GlossaryEntryDefinition { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn name(&self) -> Option<CurlyGroupWord<'a>> { + pub fn name(&self) -> Option<CurlyGroupWord> { self.syntax().children().find_map(CurlyGroupWord::cast) } } cst_node!(TikzLibraryImport, TIKZ_LIBRARY_IMPORT); -impl<'a> TikzLibraryImport<'a> { - pub fn command(&self) -> Option<&'a SyntaxToken> { +impl TikzLibraryImport { + pub fn command(&self) -> Option<SyntaxToken> { self.syntax().first_token() } - pub fn name_list(&self) -> Option<CurlyGroupWordList<'a>> { + pub fn name_list(&self) -> Option<CurlyGroupWordList> { self.syntax().children().find_map(CurlyGroupWordList::cast) } } + +cst_node!(GraphicsPath, GRAPHICS_PATH); + +impl GraphicsPath { + pub fn command(&self) -> Option<SyntaxToken> { + self.syntax().first_token() + } + + pub fn path_list(&self) -> impl Iterator<Item = CurlyGroupWord> { + self.syntax().children().filter_map(CurlyGroupWord::cast) + } +} diff --git a/support/texlab/src/syntax/latex/kind.rs b/support/texlab/src/syntax/latex/kind.rs index 410cd4fef7..81e0ea3588 100644 --- a/support/texlab/src/syntax/latex/kind.rs +++ b/support/texlab/src/syntax/latex/kind.rs @@ -8,6 +8,7 @@ pub enum SyntaxKind { LINE_BREAK, WHITESPACE, COMMENT, + VERBATIM, L_CURLY, R_CURLY, L_BRACK, @@ -59,7 +60,10 @@ pub enum SyntaxKind { COLOR_DEFINITION_NAME, COLOR_SET_DEFINITION_NAME, TIKZ_LIBRARY_IMPORT_NAME, - ENVIRONMENT_DEFINIITION_NAME, + ENVIRONMENT_DEFINITION_NAME, + BEGIN_BLOCK_COMMENT_NAME, + END_BLOCK_COMMENT_NAME, + GRAPHICS_PATH_NAME, PREAMBLE, TEXT, @@ -120,6 +124,8 @@ pub enum SyntaxKind { COLOR_SET_DEFINITION, TIKZ_LIBRARY_IMPORT, ENVIRONMENT_DEFINITION, + GRAPHICS_PATH, + BLOCK_COMMENT, ROOT, } @@ -173,7 +179,7 @@ impl SyntaxKind { } } -impl From<SyntaxKind> for cstree::SyntaxKind { +impl From<SyntaxKind> for rowan::SyntaxKind { fn from(kind: SyntaxKind) -> Self { Self(kind as u16) } diff --git a/support/texlab/src/syntax/latex/lexer.rs b/support/texlab/src/syntax/latex/lexer.rs index 4ea11bf377..39196375b1 100644 --- a/support/texlab/src/syntax/latex/lexer.rs +++ b/support/texlab/src/syntax/latex/lexer.rs @@ -5,172 +5,413 @@ use super::kind::SyntaxKind; #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, Logos)] #[allow(non_camel_case_types)] #[repr(u16)] -enum Token { +enum RootToken { #[regex(r"[\r\n]+", priority = 2)] - LINE_BREAK = 2, + LineBreak = 2, #[regex(r"\s+", priority = 1)] - WHITESPACE, + Whitespace, #[regex(r"%[^\r\n]*")] - COMMENT, + LineComment, #[token("{")] - L_CURLY, + LCurly, #[token("}")] - R_CURLY, + RCurly, #[token("[")] - L_BRACK, + LBrack, #[token("]")] - R_BRACK, + RBrack, #[token("(")] - L_PAREN, + LParen, #[token(")")] - R_PAREN, + RParen, #[token(",")] - COMMA, + Comma, #[token("=")] - EQUALITY_SIGN, + Eq, #[regex(r"[^\s\\%\{\},\$\[\]\(\)=]+")] #[error] - WORD, + Word, #[regex(r"\$\$?")] - DOLLAR, + Dollar, #[regex(r"\\([^\r\n]|[@a-zA-Z:_]+\*?)?")] - GENERIC_COMMAND_NAME, + CommandName, - #[regex(r"\\begin")] - BEGIN_ENVIRONMENT_NAME, + #[token("\\iffalse")] + BeginBlockComment, - #[regex(r"\\end")] - END_ENVIRONMENT_NAME, - - #[regex(r"\\\[")] - BEGIN_EQUATION_NAME, - - #[regex(r"\\\]")] - END_EQUATION_NAME, - - #[regex(r"\\part\*?")] - PART_NAME, - - #[regex(r"\\chapter\*?")] - CHAPTER_NAME, - - #[regex(r"\\section\*?")] - SECTION_NAME, - - #[regex(r"\\subsection\*?")] - SUBSECTION_NAME, - - #[regex(r"\\subsubsection\*?")] - SUBSUBSECTION_NAME, - - #[regex(r"\\paragraph\*?")] - PARAGRAPH_NAME, - - #[regex(r"\\subparagraph\*?")] - SUBPARAGRAPH_NAME, - - #[regex(r"\\item")] - ENUM_ITEM_NAME, - - #[regex(r"\\caption")] - CAPTION_NAME, - - #[regex(r"\\cite|\\cite\*|\\Cite|\\nocite|\\citet|\\citep|\\citet\*|\\citep\*|\\citeauthor|\\citeauthor\*|\\Citeauthor|\\Citeauthor\*|\\citetitle|\\citetitle\*|\\citeyear|\\citeyear\*|\\citedate|\\citedate\*|\\citeurl|\\fullcite|\\citeyearpar|\\citealt|\\citealp|\\citetext|\\parencite|\\parencite\*|\\Parencite|\\footcite|\\footfullcite|\\footcitetext|\\textcite|\\Textcite|\\smartcite|\\Smartcite|\\supercite|\\autocite|\\Autocite|\\autocite\*|\\Autocite\*|\\volcite|\\Volcite|\\pvolcite|\\Pvolcite|\\fvolcite|\\ftvolcite|\\svolcite|\\Svolcite|\\tvolcite|\\Tvolcite|\\avolcite|\\Avolcite|\\notecite|\\notecite|\\pnotecite|\\Pnotecite|\\fnotecite|\\citeA|\\citeA\*")] - CITATION_NAME, - - #[regex(r"\\usepackage|\\RequirePackage")] - PACKAGE_INCLUDE_NAME, - - #[regex(r"\\documentclass")] - CLASS_INCLUDE_NAME, - - #[regex(r"\\include|\\subfileinclude|\\input|\\subfile")] - LATEX_INCLUDE_NAME, - - #[regex(r"\\addbibresource")] - BIBLATEX_INCLUDE_NAME, - - #[regex(r"\\bibliography")] - BIBTEX_INCLUDE_NAME, - - #[regex(r"\\includegraphics")] - GRAPHICS_INCLUDE_NAME, - - #[regex(r"\\includesvg")] - SVG_INCLUDE_NAME, - - #[regex(r"\\includeinkscape")] - INKSCAPE_INCLUDE_NAME, - - #[regex(r"\\verbatiminput|\\VerbatimInput")] - VERBATIM_INCLUDE_NAME, - - #[regex(r"\\import|\\subimport|\\inputfrom|\\subimportfrom|\\includefrom|\\subincludefrom")] - IMPORT_NAME, - - #[regex(r"\\label")] - LABEL_DEFINITION_NAME, - - #[regex(r"\\ref|\\vref|\\Vref|\\autoref|\\pageref|\\cref|\\Cref|\\cref*|\\Cref*|\\namecref|\\nameCref|\\lcnamecref|\\namecrefs|\\nameCrefs|\\lcnamecrefs|\\labelcref|\\labelcpageref|\\eqref")] - LABEL_REFERENCE_NAME, - - #[regex(r"\\crefrange\*?|\\Crefrange\*?")] - LABEL_REFERENCE_RANGE_NAME, - - #[regex(r"\\newlabel")] - LABEL_NUMBER_NAME, - - #[regex(r"\\newcommand\*?|\\renewcommand|\\DeclareRobustCommand")] - COMMAND_DEFINITION_NAME, - - #[regex(r"\\DeclareMathOperator\*?")] - MATH_OPERATOR_NAME, - - #[regex(r"\\newglossaryentry")] - GLOSSARY_ENTRY_DEFINITION_NAME, - - #[regex(r"\\gls|\\Gls|\\GLS|\\glspl|\\Glspl|\\GLSpl|\\glsdisp|\\glslink|\\glstext|\\Glstext|\\GLStext|\\glsfirst|\\Glsfirst|\\GLSfirst|\\glsplural|\\Glsplural|\\GLSplural|\\glsfirstplural|\\Glsfirstplural|\\GLSfirstplural|\\glsname|\\Glsname|\\GLSname|\\glssymbol|\\Glssymbol|\\glsdesc|\\Glsdesc|\\GLSdesc|\\glsuseri|\\Glsuseri|\\GLSuseri|\\glsuserii|\\Glsuserii|\\GLSuserii|\\glsuseriii|\\Glsuseriii|\\GLSuseriii|\\glsuseriv|\\Glsuseriv|\\GLSuseriv|\\glsuserv|\\Glsuserv|\\GLSuserv|\\glsuservi|\\Glsuservi|\\GLSuservi")] - GLOSSARY_ENTRY_REFERENCE_NAME, - - #[regex(r"\\newacronym")] - ACRONYM_DEFINITION_NAME, - - #[regex(r"\\DeclareAcronym")] - ACRONYM_DECLARATION_NAME, - - #[regex(r"\\acrshort|\\Acrshort|\\ACRshort|\\acrshortpl|\\Acrshortpl|\\ACRshortpl|\\acrlong|\\Acrlong|\\ACRlong|\\acrlongpl|\\Acrlongpl|\\ACRlongpl|\\acrfull|\\Acrfull|\\ACRfull|\\acrfullpl|\\Acrfullpl|\\ACRfullpl|\\acs|\\Acs|\\acsp|\\Acsp|\\acl|\\Acl|\\aclp|\\Aclp|\\acf|\\Acf|\\acfp|\\Acfp|\\ac|\\Ac|\\acp|\\glsentrylong|\\Glsentrylong|\\glsentrylongpl|\\Glsentrylongpl|\\glsentryshort|\\Glsentryshort|\\glsentryshortpl|\\Glsentryshortpl|\\glsentryfullpl|\\Glsentryfullpl")] - ACRONYM_REFERENCE_NAME, + #[token("\\begin{asy}")] + #[token("\\begin{verbatim}")] + #[token("\\begin{lstlisting}")] + #[token("\\begin{minted}")] + #[token("\\begin{pycode}")] + BeginVerbatimEnvironment, +} - #[regex(r"\\newtheorem|\\declaretheorem")] - THEOREM_DEFINITION_NAME, +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, Logos)] +enum CommandNameToken { + #[token("\\begin")] + BeginEnvironment, + + #[token("\\end")] + EndEnvironment, + + #[token("\\[")] + BeginEquation, + + #[token("\\]")] + EndEquation, + + #[token("\\part")] + #[token("\\part*")] + Part, + + #[token("\\chapter")] + #[token("\\chapter*")] + Chapter, + + #[token("\\section")] + #[token("\\section*")] + Section, + + #[token("\\subsection")] + #[token("\\subsection*")] + Subsection, + + #[token("\\subsubsection")] + #[token("\\subsubsection*")] + Subsubsection, + + #[token("\\paragraph")] + #[token("\\paragraph*")] + Paragraph, + + #[token("\\subparagraph")] + #[token("\\subparagraph*")] + Subparagraph, + + #[token("\\item")] + EnumItem, + + #[token("\\caption")] + Caption, + + #[token("\\cite")] + #[token("\\cite*")] + #[token("\\Cite")] + #[token("\\nocite")] + #[token("\\citet")] + #[token("\\citet*")] + #[token("\\citep")] + #[token("\\citep*")] + #[token("\\citeauthor")] + #[token("\\citeauthor*")] + #[token("\\Citeauthor")] + #[token("\\Citeauthor*")] + #[token("\\citetitle")] + #[token("\\citetitle*")] + #[token("\\citeyear")] + #[token("\\citeyear*")] + #[token("\\citedate")] + #[token("\\citedate*")] + #[token("\\citeurl")] + #[token("\\fullcite")] + #[token("\\citeyearpar")] + #[token("\\citealt")] + #[token("\\citealp")] + #[token("\\citetext")] + #[token("\\parencite")] + #[token("\\parencite*")] + #[token("\\Parencite")] + #[token("\\footcite")] + #[token("\\footfullcite")] + #[token("\\footcitetext")] + #[token("\\textcite")] + #[token("\\Textcite")] + #[token("\\smartcite")] + #[token("\\supercite")] + #[token("\\autocite")] + #[token("\\autocite*")] + #[token("\\Autocite")] + #[token("\\Autocite*")] + #[token("\\volcite")] + #[token("\\Volcite")] + #[token("\\pvolcite")] + #[token("\\Pvolcite")] + #[token("\\fvolcite")] + #[token("\\ftvolcite")] + #[token("\\svolcite")] + #[token("\\Svolcite")] + #[token("\\tvolcite")] + #[token("\\Tvolcite")] + #[token("\\avolcite")] + #[token("\\Avolcite")] + #[token("\\notecite")] + #[token("\\pnotecite")] + #[token("\\Pnotecite")] + #[token("\\fnotecite")] + #[token("\\citeA")] + #[token("\\citeA*")] + Citation, + + #[token("\\usepackage")] + #[token("\\RequirePackage")] + PackageInclude, + + #[token("\\documentclass")] + ClassInclude, + + #[token("\\include")] + #[token("\\subfileinclude")] + #[token("\\input")] + #[token("\\subfile")] + LatexInclude, + + #[token("\\addbibresource")] + BiblatexInclude, + + #[token("\\bibliography")] + BibtexInclude, + + #[token("\\includegraphics")] + GraphicsInclude, + + #[token("\\includesvg")] + SvgInclude, + + #[token("\\includeinkscape")] + InkscapeInclude, + + #[token("\\verbatiminput")] + #[token("\\VerbatimInput")] + VerbatimInclude, + + #[token("\\import")] + #[token("\\subimport")] + #[token("\\inputfrom")] + #[token("\\subinputfrom")] + #[token("\\includefrom")] + #[token("\\subincludefrom")] + Import, + + #[token("\\label")] + LabelDefinition, + + #[token("\\ref")] + #[token("\\vref")] + #[token("\\Vref")] + #[token("\\autoref")] + #[token("\\pageref")] + #[token("\\cref")] + #[token("\\cref*")] + #[token("\\Cref")] + #[token("\\Cref*")] + #[token("\\namecref")] + #[token("\\nameCref")] + #[token("\\lcnamecref")] + #[token("\\namecrefs")] + #[token("\\nameCrefs")] + #[token("\\lcnamecrefs")] + #[token("\\labelcref")] + #[token("\\labelcpageref")] + #[token("\\eqref")] + LabelReference, + + #[token("\\crefrange")] + #[token("\\crefrange*")] + #[token("\\Crefrange")] + #[token("\\Crefrange*")] + LabelReferenceRange, + + #[token("\\newlabel")] + LabelNumber, + + #[token("\\newcommand")] + #[token("\\newcommand*")] + #[token("\\renewcommand")] + #[token("\\renewcommand*")] + #[token("\\DeclareRobustCommand")] + #[token("\\DeclareRobustCommand*")] + CommandDefinition, + + #[token("\\DeclareMathOperator")] + #[token("\\DeclareMathOperator*")] + MathOperator, + + #[token("\\newglossaryentry")] + GlossaryEntryDefinition, + + #[token("\\gls")] + #[token("\\Gls")] + #[token("\\GLS")] + #[token("\\glspl")] + #[token("\\Glspl")] + #[token("\\GLSpl")] + #[token("\\glsdisp")] + #[token("\\glslink")] + #[token("\\glstext")] + #[token("\\Glstext")] + #[token("\\GLStext")] + #[token("\\glsfirst")] + #[token("\\Glsfirst")] + #[token("\\GLSfirst")] + #[token("\\glsplural")] + #[token("\\Glsplural")] + #[token("\\GLSplural")] + #[token("\\glsfirstplural")] + #[token("\\Glsfirstplural")] + #[token("\\GLSfirstplural")] + #[token("\\glsname")] + #[token("\\Glsname")] + #[token("\\GLSname")] + #[token("\\glssymbol")] + #[token("\\Glssymbol")] + #[token("\\glsdesc")] + #[token("\\Glsdesc")] + #[token("\\GLSdesc")] + #[token("\\glsuseri")] + #[token("\\Glsuseri")] + #[token("\\GLSuseri")] + #[token("\\glsuserii")] + #[token("\\Glsuserii")] + #[token("\\glsuseriii")] + #[token("\\glsuseriv")] + #[token("\\Glsuseriv")] + #[token("\\GLSuseriv")] + #[token("\\glsuserv")] + #[token("\\Glsuserv")] + #[token("\\GLSuserv")] + #[token("\\glsuservi")] + #[token("\\Glsuservi")] + #[token("\\GLSuservi")] + GlossaryEntryReference, + + #[token("\\newacronym")] + AcronymDefinition, + + #[token("\\DeclareAcronym")] + AcronymDeclaration, + + #[token("\\acrshort")] + #[token("\\Acrshort")] + #[token("\\ACRshort")] + #[token("\\acrshortpl")] + #[token("\\Acrshortpl")] + #[token("\\ACRshortpl")] + #[token("\\acrlong")] + #[token("\\Acrlong")] + #[token("\\ACRlong")] + #[token("\\acrlongpl")] + #[token("\\Acrlongpl")] + #[token("\\ACRlongpl")] + #[token("\\acrfull")] + #[token("\\Acrfull")] + #[token("\\ACRfull")] + #[token("\\acrfullpl")] + #[token("\\Acrfullpl")] + #[token("\\ACRfullpl")] + #[token("\\acs")] + #[token("\\Acs")] + #[token("\\acsp")] + #[token("\\Acsp")] + #[token("\\acl")] + #[token("\\Acl")] + #[token("\\aclp")] + #[token("\\Aclp")] + #[token("\\acf")] + #[token("\\Acf")] + #[token("\\acfp")] + #[token("\\Acfp")] + #[token("\\ac")] + #[token("\\Ac")] + #[token("\\acp")] + #[token("\\glsentrylong")] + #[token("\\Glsentrylong")] + #[token("\\glsentrylongpl")] + #[token("\\Glsentrylongpl")] + #[token("\\glsentryshort")] + #[token("\\Glsentryshort")] + #[token("\\glsentryshortpl")] + #[token("\\Glsentryshortpl")] + #[token("\\glsentryfullpl")] + #[token("\\Glsentryfullpl")] + AcronymReference, + + #[token("\\newtheorem")] + #[token("\\newtheorem*")] + #[token("\\declaretheorem")] + #[token("\\declaretheorem*")] + TheoremDefinition, + + #[token("\\color")] + #[token("\\colorbox")] + #[token("\\textcolor")] + #[token("\\pagecolor")] + ColorReference, + + #[token("\\definecolor")] + ColorDefinition, + + #[token("\\definecolorset")] + ColorSetDefinition, + + #[token("\\usepgflibrary")] + #[token("\\usetikzlibrary")] + TikzLibraryImport, + + #[token("\\newenvironment")] + #[token("\\newenvironment*")] + #[token("\\renewenvironment")] + #[token("\\renewenvironment*")] + EnvironmentDefinition, + + #[token("\\graphicspath")] + GraphicsPath, + + #[token("\\fi")] + EndBlockComment, - #[regex(r"\\color|\\colorbox|\\textcolor|\\pagecolor")] - COLOR_REFERENCE_NAME, + #[regex(r"\\([^\r\n]|[@a-zA-Z:_]+\*?)?")] + #[error] + Generic, +} - #[regex(r"\\definecolor")] - COLOR_DEFINITION_NAME, +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, Logos)] +enum BlockCommentToken { + #[token("\\fi")] + End, - #[regex(r"\\definecolorset")] - COLOR_SET_DEFINITION_NAME, + #[regex(r"\\([^\r\n]|[@a-zA-Z:_]+\*?)?")] + #[regex(r"[^\\]+")] + #[error] + Verbatim, +} - #[regex(r"\\usepgflibrary|\\usetikzlibrary")] - TIKZ_LIBRARY_IMPORT_NAME, +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, Logos)] +enum VerbatimEnvironmentToken { + #[token("\\end{asy}")] + #[token("\\end{verbatim}")] + #[token("\\end{lstlisting}")] + #[token("\\end{minted}")] + #[token("\\end{pycode}")] + End, - #[regex(r"\\newenvironment|\\newenvironment*|\\renewenvironment|\\renewenvironment*")] - ENVIRONMENT_DEFINITION_NAME, + #[regex(r"\\([^\r\n]|[@a-zA-Z:_]+\*?)?")] + #[regex(r"[^\\]+")] + #[error] + Verbatim, } #[derive(Debug, PartialEq, Eq, Clone)] @@ -179,15 +420,9 @@ pub struct Lexer<'a> { } impl<'a> Lexer<'a> { - pub fn new(text: &'a str) -> Self { + pub fn new(input: &'a str) -> Self { let mut tokens = Vec::new(); - let mut lexer = Token::lexer(text); - while let Some(kind) = lexer.next() { - tokens.push(( - unsafe { std::mem::transmute::<Token, SyntaxKind>(kind) }, - lexer.slice(), - )); - } + tokenize(input, &mut tokens); tokens.reverse(); Self { tokens } } @@ -201,6 +436,171 @@ impl<'a> Lexer<'a> { } } +fn tokenize<'a>(input: &'a str, tokens: &mut Vec<(SyntaxKind, &'a str)>) { + let mut lexer = RootToken::lexer(input); + while let Some(kind) = lexer.next() { + let text = lexer.slice(); + match kind { + RootToken::LineBreak => { + tokens.push((SyntaxKind::LINE_BREAK, text)); + } + RootToken::Whitespace => { + tokens.push((SyntaxKind::WHITESPACE, text)); + } + RootToken::LineComment => { + tokens.push((SyntaxKind::COMMENT, text)); + } + RootToken::LCurly => { + tokens.push((SyntaxKind::L_CURLY, text)); + } + RootToken::RCurly => { + tokens.push((SyntaxKind::R_CURLY, text)); + } + RootToken::LBrack => { + tokens.push((SyntaxKind::L_BRACK, text)); + } + RootToken::RBrack => { + tokens.push((SyntaxKind::R_BRACK, text)); + } + RootToken::LParen => { + tokens.push((SyntaxKind::L_PAREN, text)); + } + RootToken::RParen => { + tokens.push((SyntaxKind::R_PAREN, text)); + } + RootToken::Comma => { + tokens.push((SyntaxKind::COMMA, text)); + } + RootToken::Eq => { + tokens.push((SyntaxKind::EQUALITY_SIGN, text)); + } + RootToken::Word => { + tokens.push((SyntaxKind::WORD, text)); + } + RootToken::Dollar => { + tokens.push((SyntaxKind::DOLLAR, text)); + } + RootToken::CommandName => { + let kind = tokenize_command_name(text); + tokens.push((kind, text)); + } + RootToken::BeginBlockComment => { + tokens.push((SyntaxKind::BEGIN_BLOCK_COMMENT_NAME, text)); + let end = lexer.span().end; + lexer = RootToken::lexer(tokenize_block_comment(&lexer.source()[end..], tokens)); + } + RootToken::BeginVerbatimEnvironment => { + tokens.push((SyntaxKind::BEGIN_ENVIRONMENT_NAME, "\\begin")); + tokens.push((SyntaxKind::L_CURLY, "{")); + tokens.push((SyntaxKind::WORD, &text["\\begin{".len()..text.len() - 1])); + tokens.push((SyntaxKind::R_CURLY, "}")); + let end = lexer.span().end; + lexer = RootToken::lexer(tokenize_verbatim_environment( + &lexer.source()[end..], + tokens, + )); + } + } + } +} + +fn tokenize_command_name(text: &str) -> SyntaxKind { + let mut lexer = CommandNameToken::lexer(text); + match lexer.next().unwrap() { + CommandNameToken::BeginEnvironment => SyntaxKind::BEGIN_ENVIRONMENT_NAME, + CommandNameToken::EndEnvironment => SyntaxKind::END_ENVIRONMENT_NAME, + CommandNameToken::BeginEquation => SyntaxKind::BEGIN_EQUATION_NAME, + CommandNameToken::EndEquation => SyntaxKind::END_EQUATION_NAME, + CommandNameToken::Part => SyntaxKind::PART_NAME, + CommandNameToken::Chapter => SyntaxKind::CHAPTER_NAME, + CommandNameToken::Section => SyntaxKind::SECTION_NAME, + CommandNameToken::Subsection => SyntaxKind::SUBSECTION_NAME, + CommandNameToken::Subsubsection => SyntaxKind::SUBSUBSECTION_NAME, + CommandNameToken::Paragraph => SyntaxKind::PARAGRAPH_NAME, + CommandNameToken::Subparagraph => SyntaxKind::SUBPARAGRAPH_NAME, + CommandNameToken::EnumItem => SyntaxKind::ENUM_ITEM_NAME, + CommandNameToken::Caption => SyntaxKind::CAPTION_NAME, + CommandNameToken::Citation => SyntaxKind::CITATION_NAME, + CommandNameToken::PackageInclude => SyntaxKind::PACKAGE_INCLUDE_NAME, + CommandNameToken::ClassInclude => SyntaxKind::CLASS_INCLUDE_NAME, + CommandNameToken::LatexInclude => SyntaxKind::LATEX_INCLUDE_NAME, + CommandNameToken::BiblatexInclude => SyntaxKind::BIBLATEX_INCLUDE_NAME, + CommandNameToken::BibtexInclude => SyntaxKind::BIBTEX_INCLUDE_NAME, + CommandNameToken::GraphicsInclude => SyntaxKind::GRAPHICS_INCLUDE_NAME, + CommandNameToken::SvgInclude => SyntaxKind::SVG_INCLUDE_NAME, + CommandNameToken::InkscapeInclude => SyntaxKind::INKSCAPE_INCLUDE_NAME, + CommandNameToken::VerbatimInclude => SyntaxKind::VERBATIM_INCLUDE_NAME, + CommandNameToken::Import => SyntaxKind::IMPORT_NAME, + CommandNameToken::LabelDefinition => SyntaxKind::LABEL_DEFINITION_NAME, + CommandNameToken::LabelReference => SyntaxKind::LABEL_REFERENCE_NAME, + CommandNameToken::LabelReferenceRange => SyntaxKind::LABEL_REFERENCE_RANGE_NAME, + CommandNameToken::LabelNumber => SyntaxKind::LABEL_NUMBER_NAME, + CommandNameToken::CommandDefinition => SyntaxKind::COMMAND_DEFINITION_NAME, + CommandNameToken::MathOperator => SyntaxKind::MATH_OPERATOR_NAME, + CommandNameToken::GlossaryEntryDefinition => SyntaxKind::GLOSSARY_ENTRY_DEFINITION_NAME, + CommandNameToken::GlossaryEntryReference => SyntaxKind::GLOSSARY_ENTRY_REFERENCE_NAME, + CommandNameToken::AcronymDefinition => SyntaxKind::ACRONYM_DEFINITION_NAME, + CommandNameToken::AcronymDeclaration => SyntaxKind::ACRONYM_DECLARATION_NAME, + CommandNameToken::AcronymReference => SyntaxKind::ACRONYM_REFERENCE_NAME, + CommandNameToken::TheoremDefinition => SyntaxKind::THEOREM_DEFINITION_NAME, + CommandNameToken::ColorReference => SyntaxKind::COLOR_REFERENCE_NAME, + CommandNameToken::ColorDefinition => SyntaxKind::COLOR_DEFINITION_NAME, + CommandNameToken::ColorSetDefinition => SyntaxKind::COLOR_SET_DEFINITION_NAME, + CommandNameToken::TikzLibraryImport => SyntaxKind::TIKZ_LIBRARY_IMPORT_NAME, + CommandNameToken::EnvironmentDefinition => SyntaxKind::ENVIRONMENT_DEFINITION_NAME, + CommandNameToken::EndBlockComment => SyntaxKind::END_BLOCK_COMMENT_NAME, + CommandNameToken::GraphicsPath => SyntaxKind::GRAPHICS_PATH_NAME, + CommandNameToken::Generic => SyntaxKind::GENERIC_COMMAND_NAME, + } +} + +fn tokenize_block_comment<'a>(input: &'a str, tokens: &mut Vec<(SyntaxKind, &'a str)>) -> &'a str { + let mut lexer = BlockCommentToken::lexer(input); + let mut end = 0; + while let Some(kind) = lexer.next() { + match kind { + BlockCommentToken::Verbatim => { + end = lexer.span().end; + } + BlockCommentToken::End => { + end = lexer.span().start; + break; + } + }; + } + + if end > 0 { + tokens.push((SyntaxKind::VERBATIM, &input[..end])); + } + + &input[end..] +} + +fn tokenize_verbatim_environment<'a>( + input: &'a str, + tokens: &mut Vec<(SyntaxKind, &'a str)>, +) -> &'a str { + let mut lexer = VerbatimEnvironmentToken::lexer(input); + let mut end = 0; + while let Some(kind) = lexer.next() { + match kind { + VerbatimEnvironmentToken::Verbatim => { + end = lexer.span().end; + } + VerbatimEnvironmentToken::End => { + end = lexer.span().start; + break; + } + }; + } + + if end > 0 { + tokens.push((SyntaxKind::VERBATIM, &input[..end])); + } + + &input[end..] +} + #[cfg(test)] mod tests { use insta::assert_debug_snapshot; @@ -257,4 +657,18 @@ mod tests { fn test_line_break() { assert_debug_snapshot!(verify("hello\nworld")); } + + #[test] + fn test_block_comment() { + assert_debug_snapshot!(verify("Foo\\iffalse\n\\Bar{Baz}\n\\fi\\Qux")); + } + + #[test] + fn test_asymptote() { + assert_debug_snapshot!(verify( + r#"\begin{asy} + printf("Hello World\n"); +\end{asy}"# + )); + } } diff --git a/support/texlab/src/syntax/latex/parser.rs b/support/texlab/src/syntax/latex/parser.rs index fcca6771a2..db9462874b 100644 --- a/support/texlab/src/syntax/latex/parser.rs +++ b/support/texlab/src/syntax/latex/parser.rs @@ -1,14 +1,13 @@ -use cstree::GreenNodeBuilder; +use rowan::{GreenNode, GreenNodeBuilder}; use super::{ lexer::Lexer, SyntaxKind::{self, *}, - SyntaxNode, }; #[derive(Clone)] pub struct Parse { - pub root: SyntaxNode, + pub green: GreenNode, } #[derive(Debug, Clone, Copy)] @@ -29,7 +28,7 @@ impl Default for ParserContext { #[derive(Debug)] struct Parser<'a> { lexer: Lexer<'a>, - builder: GreenNodeBuilder<'static, 'static>, + builder: GreenNodeBuilder<'static>, } impl<'a> Parser<'a> { @@ -88,15 +87,13 @@ impl<'a> Parser<'a> { self.content(ParserContext::default()); } self.builder.finish_node(); - let (green_node, interner) = self.builder.finish(); - Parse { - root: SyntaxNode::new_root_with_resolver(green_node, interner.unwrap()), - } + let green = self.builder.finish(); + Parse { green } } fn content(&mut self, context: ParserContext) { match self.peek().unwrap() { - LINE_BREAK | WHITESPACE | COMMENT => self.eat(), + LINE_BREAK | WHITESPACE | COMMENT | VERBATIM => self.eat(), L_CURLY if context.allow_environment => self.curly_group(), L_CURLY => self.curly_group_without_environments(), L_BRACK | L_PAREN => self.mixed_group(), @@ -151,7 +148,10 @@ impl<'a> Parser<'a> { COLOR_DEFINITION_NAME => self.color_definition(), COLOR_SET_DEFINITION_NAME => self.color_set_definition(), TIKZ_LIBRARY_IMPORT_NAME => self.tikz_library_import(), - ENVIRONMENT_DEFINIITION_NAME => self.environment_definition(), + ENVIRONMENT_DEFINITION_NAME => self.environment_definition(), + BEGIN_BLOCK_COMMENT_NAME => self.block_comment(), + END_BLOCK_COMMENT_NAME => self.generic_command(), + GRAPHICS_PATH_NAME => self.graphics_path(), _ => unreachable!(), } } @@ -186,6 +186,21 @@ impl<'a> Parser<'a> { self.builder.finish_node(); } + fn curly_group_impl(&mut self) { + self.builder.start_node(CURLY_GROUP.into()); + self.eat(); + while let Some(kind) = self.peek() { + match kind { + R_CURLY => break, + BEGIN_ENVIRONMENT_NAME => self.begin(), + END_ENVIRONMENT_NAME => self.end(), + _ => self.content(ParserContext::default()), + }; + } + self.expect(R_CURLY); + self.builder.finish_node(); + } + fn curly_group_without_environments(&mut self) { self.builder.start_node(CURLY_GROUP.into()); self.eat(); @@ -350,10 +365,15 @@ impl<'a> Parser<'a> { fn value(&mut self) { self.builder.start_node(VALUE.into()); - self.content(ParserContext { - allow_environment: true, - allow_comma: false, - }); + while let Some(kind) = self.lexer.peek() { + match kind { + COMMA | R_BRACK | R_CURLY => break, + _ => self.content(ParserContext { + allow_environment: true, + allow_comma: false, + }), + }; + } self.builder.finish_node(); } @@ -760,6 +780,23 @@ impl<'a> Parser<'a> { self.builder.finish_node(); } + fn block_comment(&mut self) { + self.builder.start_node(BLOCK_COMMENT.into()); + self.eat(); + + if self.peek() == Some(VERBATIM) { + self.eat(); + } + + if self.peek() == Some(END_BLOCK_COMMENT_NAME) { + self.eat(); + } else { + self.builder.token(MISSING.into(), ""); + } + + self.builder.finish_node(); + } + fn caption(&mut self) { self.builder.start_node(CAPTION.into()); self.eat(); @@ -806,7 +843,7 @@ impl<'a> Parser<'a> { } if self.lexer.peek() == Some(L_CURLY) { - self.curly_group_word_list(); + self.curly_group_path_list(); } else { self.builder.token(MISSING.into(), ""); } @@ -814,6 +851,57 @@ impl<'a> Parser<'a> { self.builder.finish_node(); } + fn curly_group_path(&mut self) { + self.builder.start_node(CURLY_GROUP_WORD.into()); + self.eat(); + while matches!(self.peek(), Some(WORD)) { + self.path(); + } + + self.expect(R_CURLY); + self.builder.finish_node(); + } + + fn curly_group_path_list(&mut self) { + self.builder.start_node(CURLY_GROUP_WORD_LIST.into()); + self.eat(); + + while self + .peek() + .filter(|&kind| { + matches!( + kind, + LINE_BREAK | WHITESPACE | COMMENT | WORD | COMMA | EQUALITY_SIGN + ) + }) + .is_some() + { + if self.peek() == Some(WORD) { + self.path(); + } else { + self.eat(); + } + } + + self.expect(R_CURLY); + self.builder.finish_node(); + } + + fn path(&mut self) { + self.builder.start_node(KEY.into()); + self.eat(); + while self + .peek() + .filter(|&kind| matches!(kind, WHITESPACE | COMMENT | WORD | EQUALITY_SIGN)) + .is_some() + { + self.eat(); + } + + self.trivia(); + self.builder.finish_node(); + } + fn package_include(&mut self) { self.generic_include(PACKAGE_INCLUDE, true); } @@ -944,7 +1032,7 @@ impl<'a> Parser<'a> { } if self.lexer.peek() == Some(L_CURLY) { - self.curly_group(); + self.curly_group_impl(); } else { self.builder.token(MISSING.into(), ""); } @@ -964,7 +1052,7 @@ impl<'a> Parser<'a> { } if self.lexer.peek() == Some(L_CURLY) { - self.curly_group(); + self.curly_group_impl(); } else { self.builder.token(MISSING.into(), ""); } @@ -1195,6 +1283,9 @@ impl<'a> Parser<'a> { if self.lexer.peek() == Some(L_BRACK) { self.brack_group_word(); + if self.lexer.peek() == Some(L_BRACK) { + self.brack_group(); + } } for _ in 0..2 { @@ -1207,6 +1298,26 @@ impl<'a> Parser<'a> { self.builder.finish_node(); } + + fn graphics_path(&mut self) { + self.builder.start_node(GRAPHICS_PATH.into()); + self.eat(); + self.trivia(); + + if self.lexer.peek() == Some(L_CURLY) { + self.eat(); + self.trivia(); + + while matches!(self.lexer.peek(), Some(L_CURLY)) { + self.curly_group_path(); + } + } else { + self.builder.token(MISSING.into(), ""); + } + + self.expect(R_CURLY); + self.builder.finish_node(); + } } pub fn parse(text: &str) -> Parse { @@ -1217,10 +1328,12 @@ pub fn parse(text: &str) -> Parse { mod tests { use insta::assert_debug_snapshot; + use crate::syntax::latex; + use super::*; - fn setup(text: &str) -> SyntaxNode { - parse(&text.trim().replace("\r", "")).root + fn setup(text: &str) -> latex::SyntaxNode { + latex::SyntaxNode::new_root(parse(&text.trim().replace('\r', "")).green) } #[test] @@ -1460,6 +1573,11 @@ mod tests { } #[test] + fn test_latex_include_equality_sign() { + assert_debug_snapshot!(setup(r#"\include{foo=bar}"#)); + } + + #[test] fn test_latex_input_simple() { assert_debug_snapshot!(setup(r#"\input{foo/bar.tex}"#)); } @@ -1490,6 +1608,11 @@ mod tests { } #[test] + fn test_graphics_include_complicated_options() { + assert_debug_snapshot!(setup(r#"\includegraphics[width=0.5\textwidth]{}"#)); + } + + #[test] fn test_svg_include_simple() { assert_debug_snapshot!(setup(r#"\includesvg{foo/bar.svg}"#)); } @@ -1595,6 +1718,13 @@ mod tests { } #[test] + fn test_command_definition_with_begin() { + assert_debug_snapshot!(setup( + r#"\newcommand{\CVSubHeadingListStart}{\begin{itemize}[leftmargin=0.5cm, label={}]}"# + )); + } + + #[test] fn test_math_operator_simple() { assert_debug_snapshot!(setup(r#"\DeclareMathOperator{\foo}{foo}"#)); } @@ -1717,6 +1847,11 @@ mod tests { } #[test] + fn test_environment_definition_optional_arg() { + assert_debug_snapshot!(setup(r#"\newenvironment{foo}[1][default]{begin}{end}"#)); + } + + #[test] fn test_acronym_declaration() { assert_debug_snapshot!(setup( r#"\DeclareAcronym{eg}{short = e.g,long = for example,tag = abbrev}"# @@ -1727,4 +1862,33 @@ mod tests { fn test_label_definition_line_break() { assert_debug_snapshot!(setup("\\label{hello\nworld}")); } + + #[test] + fn test_block_comments() { + assert_debug_snapshot!(setup( + r#"Foo +\iffalse +Test1 +\fi +Bar +\iffalse +\fii +\fi +Baz"# + )); + } + + #[test] + fn test_asymptote() { + assert_debug_snapshot!(setup( + r#"\begin{asy} + printf("Hello World\n"); +\end{asy}"# + )); + } + + #[test] + fn test_graphics_path() { + assert_debug_snapshot!(setup(r#"\graphicspath{{../figures/}}"#)); + } } diff --git a/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__lexer__tests__asymptote.snap b/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__lexer__tests__asymptote.snap new file mode 100644 index 0000000000..12f0f3ca63 --- /dev/null +++ b/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__lexer__tests__asymptote.snap @@ -0,0 +1,43 @@ +--- +source: src/syntax/latex/lexer.rs +assertion_line: 664 +expression: "verify(r#\"\\begin{asy}\n printf(\"Hello World\\n\");\n\\end{asy}\"#)" +--- +[ + ( + BEGIN_ENVIRONMENT_NAME, + "\\begin", + ), + ( + L_CURLY, + "{", + ), + ( + WORD, + "asy", + ), + ( + R_CURLY, + "}", + ), + ( + VERBATIM, + "\n printf(\"Hello World\\n\");\n", + ), + ( + END_ENVIRONMENT_NAME, + "\\end", + ), + ( + L_CURLY, + "{", + ), + ( + WORD, + "asy", + ), + ( + R_CURLY, + "}", + ), +] diff --git a/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__lexer__tests__block_comment.snap b/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__lexer__tests__block_comment.snap new file mode 100644 index 0000000000..5ced043aaf --- /dev/null +++ b/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__lexer__tests__block_comment.snap @@ -0,0 +1,27 @@ +--- +source: src/syntax/latex/lexer.rs +assertion_line: 601 +expression: "verify(\"Foo\\\\iffalse\\n\\\\Bar{Baz}\\n\\\\fi\\\\Qux\")" +--- +[ + ( + WORD, + "Foo", + ), + ( + BEGIN_BLOCK_COMMENT_NAME, + "\\iffalse", + ), + ( + VERBATIM, + "\n\\Bar{Baz}\n", + ), + ( + END_BLOCK_COMMENT_NAME, + "\\fi", + ), + ( + GENERIC_COMMAND_NAME, + "\\Qux", + ), +] diff --git a/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__asymptote.snap b/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__asymptote.snap new file mode 100644 index 0000000000..a735d71e73 --- /dev/null +++ b/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__asymptote.snap @@ -0,0 +1,24 @@ +--- +source: src/syntax/latex/parser.rs +assertion_line: 1829 +expression: "setup(r#\"\\begin{asy}\n printf(\"Hello World\\n\");\n\\end{asy}\"#)" +--- +ROOT@0..50 + PREAMBLE@0..50 + ENVIRONMENT@0..50 + BEGIN@0..11 + BEGIN_ENVIRONMENT_NAME@0..6 "\\begin" + CURLY_GROUP_WORD@6..11 + L_CURLY@6..7 "{" + KEY@7..10 + WORD@7..10 "asy" + R_CURLY@10..11 "}" + VERBATIM@11..41 "\n printf(\"Hello Wo ..." + END@41..50 + END_ENVIRONMENT_NAME@41..45 "\\end" + CURLY_GROUP_WORD@45..50 + L_CURLY@45..46 "{" + KEY@46..49 + WORD@46..49 "asy" + R_CURLY@49..50 "}" + diff --git a/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__block_comments.snap b/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__block_comments.snap new file mode 100644 index 0000000000..355a93ebad --- /dev/null +++ b/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__block_comments.snap @@ -0,0 +1,26 @@ +--- +source: src/syntax/latex/parser.rs +assertion_line: 1814 +expression: "setup(r#\"Foo\n\\iffalse\nTest1\n\\fi\nBar\n\\iffalse\n\\fii\n\\fi\nBaz\"#)" +--- +ROOT@0..48 + PREAMBLE@0..48 + TEXT@0..4 + WORD@0..3 "Foo" + LINE_BREAK@3..4 "\n" + BLOCK_COMMENT@4..22 + BEGIN_BLOCK_COMMENT_NAME@4..12 "\\iffalse" + VERBATIM@12..19 "\nTest1\n" + END_BLOCK_COMMENT_NAME@19..22 "\\fi" + LINE_BREAK@22..23 "\n" + TEXT@23..27 + WORD@23..26 "Bar" + LINE_BREAK@26..27 "\n" + BLOCK_COMMENT@27..44 + BEGIN_BLOCK_COMMENT_NAME@27..35 "\\iffalse" + VERBATIM@35..41 "\n\\fii\n" + END_BLOCK_COMMENT_NAME@41..44 "\\fi" + LINE_BREAK@44..45 "\n" + TEXT@45..48 + WORD@45..48 "Baz" + diff --git a/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__command_definition_with_begin.snap b/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__command_definition_with_begin.snap new file mode 100644 index 0000000000..6b7724c0aa --- /dev/null +++ b/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__command_definition_with_begin.snap @@ -0,0 +1,39 @@ +--- +source: src/syntax/latex/parser.rs +assertion_line: 1723 +expression: "setup(r#\"\\newcommand{\\CVSubHeadingListStart}{\\begin{itemize}[leftmargin=0.5cm, label={}]}\"#)" +--- +ROOT@0..80 + PREAMBLE@0..80 + COMMAND_DEFINITION@0..80 + COMMAND_DEFINITION_NAME@0..11 "\\newcommand" + CURLY_GROUP_COMMAND@11..35 + L_CURLY@11..12 "{" + GENERIC_COMMAND_NAME@12..34 "\\CVSubHeadingListStart" + R_CURLY@34..35 "}" + CURLY_GROUP@35..80 + L_CURLY@35..36 "{" + BEGIN@36..79 + BEGIN_ENVIRONMENT_NAME@36..42 "\\begin" + CURLY_GROUP_WORD@42..51 + L_CURLY@42..43 "{" + KEY@43..50 + WORD@43..50 "itemize" + R_CURLY@50..51 "}" + BRACK_GROUP@51..79 + L_BRACK@51..52 "[" + TEXT@52..62 + WORD@52..62 "leftmargin" + EQUALITY_SIGN@62..63 "=" + TEXT@63..75 + WORD@63..68 "0.5cm" + COMMA@68..69 "," + WHITESPACE@69..70 " " + WORD@70..75 "label" + EQUALITY_SIGN@75..76 "=" + CURLY_GROUP@76..78 + L_CURLY@76..77 "{" + R_CURLY@77..78 "}" + R_BRACK@78..79 "]" + R_CURLY@79..80 "}" + diff --git a/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__environment_definition.snap b/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__environment_definition.snap index 982d3b8667..5d977db814 100644 --- a/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__environment_definition.snap +++ b/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__environment_definition.snap @@ -1,12 +1,12 @@ --- source: src/syntax/latex/parser.rs +assertion_line: 1792 expression: "setup(r#\"\\newenvironment{bar}[1]{\\begin{foo}}{\\end{foo}}\"#)" - --- ROOT@0..47 PREAMBLE@0..47 ENVIRONMENT_DEFINITION@0..47 - ENVIRONMENT_DEFINIITION_NAME@0..15 "\\newenvironment" + ENVIRONMENT_DEFINITION_NAME@0..15 "\\newenvironment" CURLY_GROUP_WORD@15..20 L_CURLY@15..16 "{" KEY@16..19 diff --git a/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__environment_definition_optional_arg.snap b/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__environment_definition_optional_arg.snap new file mode 100644 index 0000000000..1a064fefe3 --- /dev/null +++ b/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__environment_definition_optional_arg.snap @@ -0,0 +1,35 @@ +--- +source: src/syntax/latex/parser.rs +assertion_line: 1797 +expression: "setup(r#\"\\newenvironment{foo}[1][default]{begin}{end}\"#)" +--- +ROOT@0..44 + PREAMBLE@0..44 + ENVIRONMENT_DEFINITION@0..44 + ENVIRONMENT_DEFINITION_NAME@0..15 "\\newenvironment" + CURLY_GROUP_WORD@15..20 + L_CURLY@15..16 "{" + KEY@16..19 + WORD@16..19 "foo" + R_CURLY@19..20 "}" + BRACK_GROUP_WORD@20..23 + L_BRACK@20..21 "[" + KEY@21..22 + WORD@21..22 "1" + R_BRACK@22..23 "]" + BRACK_GROUP@23..32 + L_BRACK@23..24 "[" + TEXT@24..31 + WORD@24..31 "default" + R_BRACK@31..32 "]" + CURLY_GROUP@32..39 + L_CURLY@32..33 "{" + TEXT@33..38 + WORD@33..38 "begin" + R_CURLY@38..39 "}" + CURLY_GROUP@39..44 + L_CURLY@39..40 "{" + TEXT@40..43 + WORD@40..43 "end" + R_CURLY@43..44 "}" + diff --git a/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__graphics_include_complicated_options.snap b/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__graphics_include_complicated_options.snap new file mode 100644 index 0000000000..a79c7f925e --- /dev/null +++ b/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__graphics_include_complicated_options.snap @@ -0,0 +1,27 @@ +--- +source: src/syntax/latex/parser.rs +assertion_line: 1547 +expression: "setup(r#\"\\includegraphics[width=0.5\\textwidth]{}\"#)" + +--- +ROOT@0..39 + PREAMBLE@0..39 + GRAPHICS_INCLUDE@0..39 + GRAPHICS_INCLUDE_NAME@0..16 "\\includegraphics" + BRACK_GROUP_KEY_VALUE@16..37 + L_BRACK@16..17 "[" + KEY_VALUE_BODY@17..36 + KEY_VALUE_PAIR@17..36 + KEY@17..22 + WORD@17..22 "width" + EQUALITY_SIGN@22..23 "=" + VALUE@23..36 + TEXT@23..26 + WORD@23..26 "0.5" + GENERIC_COMMAND@26..36 + GENERIC_COMMAND_NAME@26..36 "\\textwidth" + R_BRACK@36..37 "]" + CURLY_GROUP_WORD_LIST@37..39 + L_CURLY@37..38 "{" + R_CURLY@38..39 "}" + diff --git a/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__graphics_path.snap b/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__graphics_path.snap new file mode 100644 index 0000000000..622a0fc869 --- /dev/null +++ b/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__graphics_path.snap @@ -0,0 +1,17 @@ +--- +source: src/syntax/latex/parser.rs +assertion_line: 1871 +expression: "setup(r#\"\\graphicspath{{../figures/}}\"#)" +--- +ROOT@0..28 + PREAMBLE@0..28 + GRAPHICS_PATH@0..28 + GRAPHICS_PATH_NAME@0..13 "\\graphicspath" + L_CURLY@13..14 "{" + CURLY_GROUP_WORD@14..27 + L_CURLY@14..15 "{" + KEY@15..26 + WORD@15..26 "../figures/" + R_CURLY@26..27 "}" + R_CURLY@27..28 "}" + diff --git a/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__latex_include_equality_sign.snap b/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__latex_include_equality_sign.snap new file mode 100644 index 0000000000..c6cfb5dda3 --- /dev/null +++ b/support/texlab/src/syntax/latex/snapshots/texlab__syntax__latex__parser__tests__latex_include_equality_sign.snap @@ -0,0 +1,18 @@ +--- +source: src/syntax/latex/parser.rs +assertion_line: 1504 +expression: "setup(r#\"\\include{foo=bar}\"#)" + +--- +ROOT@0..17 + PREAMBLE@0..17 + LATEX_INCLUDE@0..17 + LATEX_INCLUDE_NAME@0..8 "\\include" + CURLY_GROUP_WORD_LIST@8..17 + L_CURLY@8..9 "{" + KEY@9..16 + WORD@9..12 "foo" + EQUALITY_SIGN@12..13 "=" + WORD@13..16 "bar" + R_CURLY@16..17 "}" + |