summaryrefslogtreecommitdiff
path: root/support/texlab/src/syntax/latex/lexer.rs
diff options
context:
space:
mode:
Diffstat (limited to 'support/texlab/src/syntax/latex/lexer.rs')
-rw-r--r--support/texlab/src/syntax/latex/lexer.rs690
1 files changed, 552 insertions, 138 deletions
diff --git a/support/texlab/src/syntax/latex/lexer.rs b/support/texlab/src/syntax/latex/lexer.rs
index 4ea11bf377..39196375b1 100644
--- a/support/texlab/src/syntax/latex/lexer.rs
+++ b/support/texlab/src/syntax/latex/lexer.rs
@@ -5,172 +5,413 @@ use super::kind::SyntaxKind;
#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, Logos)]
#[allow(non_camel_case_types)]
#[repr(u16)]
-enum Token {
+enum RootToken {
#[regex(r"[\r\n]+", priority = 2)]
- LINE_BREAK = 2,
+ LineBreak = 2,
#[regex(r"\s+", priority = 1)]
- WHITESPACE,
+ Whitespace,
#[regex(r"%[^\r\n]*")]
- COMMENT,
+ LineComment,
#[token("{")]
- L_CURLY,
+ LCurly,
#[token("}")]
- R_CURLY,
+ RCurly,
#[token("[")]
- L_BRACK,
+ LBrack,
#[token("]")]
- R_BRACK,
+ RBrack,
#[token("(")]
- L_PAREN,
+ LParen,
#[token(")")]
- R_PAREN,
+ RParen,
#[token(",")]
- COMMA,
+ Comma,
#[token("=")]
- EQUALITY_SIGN,
+ Eq,
#[regex(r"[^\s\\%\{\},\$\[\]\(\)=]+")]
#[error]
- WORD,
+ Word,
#[regex(r"\$\$?")]
- DOLLAR,
+ Dollar,
#[regex(r"\\([^\r\n]|[@a-zA-Z:_]+\*?)?")]
- GENERIC_COMMAND_NAME,
+ CommandName,
- #[regex(r"\\begin")]
- BEGIN_ENVIRONMENT_NAME,
+ #[token("\\iffalse")]
+ BeginBlockComment,
- #[regex(r"\\end")]
- END_ENVIRONMENT_NAME,
-
- #[regex(r"\\\[")]
- BEGIN_EQUATION_NAME,
-
- #[regex(r"\\\]")]
- END_EQUATION_NAME,
-
- #[regex(r"\\part\*?")]
- PART_NAME,
-
- #[regex(r"\\chapter\*?")]
- CHAPTER_NAME,
-
- #[regex(r"\\section\*?")]
- SECTION_NAME,
-
- #[regex(r"\\subsection\*?")]
- SUBSECTION_NAME,
-
- #[regex(r"\\subsubsection\*?")]
- SUBSUBSECTION_NAME,
-
- #[regex(r"\\paragraph\*?")]
- PARAGRAPH_NAME,
-
- #[regex(r"\\subparagraph\*?")]
- SUBPARAGRAPH_NAME,
-
- #[regex(r"\\item")]
- ENUM_ITEM_NAME,
-
- #[regex(r"\\caption")]
- CAPTION_NAME,
-
- #[regex(r"\\cite|\\cite\*|\\Cite|\\nocite|\\citet|\\citep|\\citet\*|\\citep\*|\\citeauthor|\\citeauthor\*|\\Citeauthor|\\Citeauthor\*|\\citetitle|\\citetitle\*|\\citeyear|\\citeyear\*|\\citedate|\\citedate\*|\\citeurl|\\fullcite|\\citeyearpar|\\citealt|\\citealp|\\citetext|\\parencite|\\parencite\*|\\Parencite|\\footcite|\\footfullcite|\\footcitetext|\\textcite|\\Textcite|\\smartcite|\\Smartcite|\\supercite|\\autocite|\\Autocite|\\autocite\*|\\Autocite\*|\\volcite|\\Volcite|\\pvolcite|\\Pvolcite|\\fvolcite|\\ftvolcite|\\svolcite|\\Svolcite|\\tvolcite|\\Tvolcite|\\avolcite|\\Avolcite|\\notecite|\\notecite|\\pnotecite|\\Pnotecite|\\fnotecite|\\citeA|\\citeA\*")]
- CITATION_NAME,
-
- #[regex(r"\\usepackage|\\RequirePackage")]
- PACKAGE_INCLUDE_NAME,
-
- #[regex(r"\\documentclass")]
- CLASS_INCLUDE_NAME,
-
- #[regex(r"\\include|\\subfileinclude|\\input|\\subfile")]
- LATEX_INCLUDE_NAME,
-
- #[regex(r"\\addbibresource")]
- BIBLATEX_INCLUDE_NAME,
-
- #[regex(r"\\bibliography")]
- BIBTEX_INCLUDE_NAME,
-
- #[regex(r"\\includegraphics")]
- GRAPHICS_INCLUDE_NAME,
-
- #[regex(r"\\includesvg")]
- SVG_INCLUDE_NAME,
-
- #[regex(r"\\includeinkscape")]
- INKSCAPE_INCLUDE_NAME,
-
- #[regex(r"\\verbatiminput|\\VerbatimInput")]
- VERBATIM_INCLUDE_NAME,
-
- #[regex(r"\\import|\\subimport|\\inputfrom|\\subimportfrom|\\includefrom|\\subincludefrom")]
- IMPORT_NAME,
-
- #[regex(r"\\label")]
- LABEL_DEFINITION_NAME,
-
- #[regex(r"\\ref|\\vref|\\Vref|\\autoref|\\pageref|\\cref|\\Cref|\\cref*|\\Cref*|\\namecref|\\nameCref|\\lcnamecref|\\namecrefs|\\nameCrefs|\\lcnamecrefs|\\labelcref|\\labelcpageref|\\eqref")]
- LABEL_REFERENCE_NAME,
-
- #[regex(r"\\crefrange\*?|\\Crefrange\*?")]
- LABEL_REFERENCE_RANGE_NAME,
-
- #[regex(r"\\newlabel")]
- LABEL_NUMBER_NAME,
-
- #[regex(r"\\newcommand\*?|\\renewcommand|\\DeclareRobustCommand")]
- COMMAND_DEFINITION_NAME,
-
- #[regex(r"\\DeclareMathOperator\*?")]
- MATH_OPERATOR_NAME,
-
- #[regex(r"\\newglossaryentry")]
- GLOSSARY_ENTRY_DEFINITION_NAME,
-
- #[regex(r"\\gls|\\Gls|\\GLS|\\glspl|\\Glspl|\\GLSpl|\\glsdisp|\\glslink|\\glstext|\\Glstext|\\GLStext|\\glsfirst|\\Glsfirst|\\GLSfirst|\\glsplural|\\Glsplural|\\GLSplural|\\glsfirstplural|\\Glsfirstplural|\\GLSfirstplural|\\glsname|\\Glsname|\\GLSname|\\glssymbol|\\Glssymbol|\\glsdesc|\\Glsdesc|\\GLSdesc|\\glsuseri|\\Glsuseri|\\GLSuseri|\\glsuserii|\\Glsuserii|\\GLSuserii|\\glsuseriii|\\Glsuseriii|\\GLSuseriii|\\glsuseriv|\\Glsuseriv|\\GLSuseriv|\\glsuserv|\\Glsuserv|\\GLSuserv|\\glsuservi|\\Glsuservi|\\GLSuservi")]
- GLOSSARY_ENTRY_REFERENCE_NAME,
-
- #[regex(r"\\newacronym")]
- ACRONYM_DEFINITION_NAME,
-
- #[regex(r"\\DeclareAcronym")]
- ACRONYM_DECLARATION_NAME,
-
- #[regex(r"\\acrshort|\\Acrshort|\\ACRshort|\\acrshortpl|\\Acrshortpl|\\ACRshortpl|\\acrlong|\\Acrlong|\\ACRlong|\\acrlongpl|\\Acrlongpl|\\ACRlongpl|\\acrfull|\\Acrfull|\\ACRfull|\\acrfullpl|\\Acrfullpl|\\ACRfullpl|\\acs|\\Acs|\\acsp|\\Acsp|\\acl|\\Acl|\\aclp|\\Aclp|\\acf|\\Acf|\\acfp|\\Acfp|\\ac|\\Ac|\\acp|\\glsentrylong|\\Glsentrylong|\\glsentrylongpl|\\Glsentrylongpl|\\glsentryshort|\\Glsentryshort|\\glsentryshortpl|\\Glsentryshortpl|\\glsentryfullpl|\\Glsentryfullpl")]
- ACRONYM_REFERENCE_NAME,
+ #[token("\\begin{asy}")]
+ #[token("\\begin{verbatim}")]
+ #[token("\\begin{lstlisting}")]
+ #[token("\\begin{minted}")]
+ #[token("\\begin{pycode}")]
+ BeginVerbatimEnvironment,
+}
- #[regex(r"\\newtheorem|\\declaretheorem")]
- THEOREM_DEFINITION_NAME,
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, Logos)]
+enum CommandNameToken {
+ #[token("\\begin")]
+ BeginEnvironment,
+
+ #[token("\\end")]
+ EndEnvironment,
+
+ #[token("\\[")]
+ BeginEquation,
+
+ #[token("\\]")]
+ EndEquation,
+
+ #[token("\\part")]
+ #[token("\\part*")]
+ Part,
+
+ #[token("\\chapter")]
+ #[token("\\chapter*")]
+ Chapter,
+
+ #[token("\\section")]
+ #[token("\\section*")]
+ Section,
+
+ #[token("\\subsection")]
+ #[token("\\subsection*")]
+ Subsection,
+
+ #[token("\\subsubsection")]
+ #[token("\\subsubsection*")]
+ Subsubsection,
+
+ #[token("\\paragraph")]
+ #[token("\\paragraph*")]
+ Paragraph,
+
+ #[token("\\subparagraph")]
+ #[token("\\subparagraph*")]
+ Subparagraph,
+
+ #[token("\\item")]
+ EnumItem,
+
+ #[token("\\caption")]
+ Caption,
+
+ #[token("\\cite")]
+ #[token("\\cite*")]
+ #[token("\\Cite")]
+ #[token("\\nocite")]
+ #[token("\\citet")]
+ #[token("\\citet*")]
+ #[token("\\citep")]
+ #[token("\\citep*")]
+ #[token("\\citeauthor")]
+ #[token("\\citeauthor*")]
+ #[token("\\Citeauthor")]
+ #[token("\\Citeauthor*")]
+ #[token("\\citetitle")]
+ #[token("\\citetitle*")]
+ #[token("\\citeyear")]
+ #[token("\\citeyear*")]
+ #[token("\\citedate")]
+ #[token("\\citedate*")]
+ #[token("\\citeurl")]
+ #[token("\\fullcite")]
+ #[token("\\citeyearpar")]
+ #[token("\\citealt")]
+ #[token("\\citealp")]
+ #[token("\\citetext")]
+ #[token("\\parencite")]
+ #[token("\\parencite*")]
+ #[token("\\Parencite")]
+ #[token("\\footcite")]
+ #[token("\\footfullcite")]
+ #[token("\\footcitetext")]
+ #[token("\\textcite")]
+ #[token("\\Textcite")]
+ #[token("\\smartcite")]
+ #[token("\\supercite")]
+ #[token("\\autocite")]
+ #[token("\\autocite*")]
+ #[token("\\Autocite")]
+ #[token("\\Autocite*")]
+ #[token("\\volcite")]
+ #[token("\\Volcite")]
+ #[token("\\pvolcite")]
+ #[token("\\Pvolcite")]
+ #[token("\\fvolcite")]
+ #[token("\\ftvolcite")]
+ #[token("\\svolcite")]
+ #[token("\\Svolcite")]
+ #[token("\\tvolcite")]
+ #[token("\\Tvolcite")]
+ #[token("\\avolcite")]
+ #[token("\\Avolcite")]
+ #[token("\\notecite")]
+ #[token("\\pnotecite")]
+ #[token("\\Pnotecite")]
+ #[token("\\fnotecite")]
+ #[token("\\citeA")]
+ #[token("\\citeA*")]
+ Citation,
+
+ #[token("\\usepackage")]
+ #[token("\\RequirePackage")]
+ PackageInclude,
+
+ #[token("\\documentclass")]
+ ClassInclude,
+
+ #[token("\\include")]
+ #[token("\\subfileinclude")]
+ #[token("\\input")]
+ #[token("\\subfile")]
+ LatexInclude,
+
+ #[token("\\addbibresource")]
+ BiblatexInclude,
+
+ #[token("\\bibliography")]
+ BibtexInclude,
+
+ #[token("\\includegraphics")]
+ GraphicsInclude,
+
+ #[token("\\includesvg")]
+ SvgInclude,
+
+ #[token("\\includeinkscape")]
+ InkscapeInclude,
+
+ #[token("\\verbatiminput")]
+ #[token("\\VerbatimInput")]
+ VerbatimInclude,
+
+ #[token("\\import")]
+ #[token("\\subimport")]
+ #[token("\\inputfrom")]
+ #[token("\\subinputfrom")]
+ #[token("\\includefrom")]
+ #[token("\\subincludefrom")]
+ Import,
+
+ #[token("\\label")]
+ LabelDefinition,
+
+ #[token("\\ref")]
+ #[token("\\vref")]
+ #[token("\\Vref")]
+ #[token("\\autoref")]
+ #[token("\\pageref")]
+ #[token("\\cref")]
+ #[token("\\cref*")]
+ #[token("\\Cref")]
+ #[token("\\Cref*")]
+ #[token("\\namecref")]
+ #[token("\\nameCref")]
+ #[token("\\lcnamecref")]
+ #[token("\\namecrefs")]
+ #[token("\\nameCrefs")]
+ #[token("\\lcnamecrefs")]
+ #[token("\\labelcref")]
+ #[token("\\labelcpageref")]
+ #[token("\\eqref")]
+ LabelReference,
+
+ #[token("\\crefrange")]
+ #[token("\\crefrange*")]
+ #[token("\\Crefrange")]
+ #[token("\\Crefrange*")]
+ LabelReferenceRange,
+
+ #[token("\\newlabel")]
+ LabelNumber,
+
+ #[token("\\newcommand")]
+ #[token("\\newcommand*")]
+ #[token("\\renewcommand")]
+ #[token("\\renewcommand*")]
+ #[token("\\DeclareRobustCommand")]
+ #[token("\\DeclareRobustCommand*")]
+ CommandDefinition,
+
+ #[token("\\DeclareMathOperator")]
+ #[token("\\DeclareMathOperator*")]
+ MathOperator,
+
+ #[token("\\newglossaryentry")]
+ GlossaryEntryDefinition,
+
+ #[token("\\gls")]
+ #[token("\\Gls")]
+ #[token("\\GLS")]
+ #[token("\\glspl")]
+ #[token("\\Glspl")]
+ #[token("\\GLSpl")]
+ #[token("\\glsdisp")]
+ #[token("\\glslink")]
+ #[token("\\glstext")]
+ #[token("\\Glstext")]
+ #[token("\\GLStext")]
+ #[token("\\glsfirst")]
+ #[token("\\Glsfirst")]
+ #[token("\\GLSfirst")]
+ #[token("\\glsplural")]
+ #[token("\\Glsplural")]
+ #[token("\\GLSplural")]
+ #[token("\\glsfirstplural")]
+ #[token("\\Glsfirstplural")]
+ #[token("\\GLSfirstplural")]
+ #[token("\\glsname")]
+ #[token("\\Glsname")]
+ #[token("\\GLSname")]
+ #[token("\\glssymbol")]
+ #[token("\\Glssymbol")]
+ #[token("\\glsdesc")]
+ #[token("\\Glsdesc")]
+ #[token("\\GLSdesc")]
+ #[token("\\glsuseri")]
+ #[token("\\Glsuseri")]
+ #[token("\\GLSuseri")]
+ #[token("\\glsuserii")]
+ #[token("\\Glsuserii")]
+ #[token("\\glsuseriii")]
+ #[token("\\glsuseriv")]
+ #[token("\\Glsuseriv")]
+ #[token("\\GLSuseriv")]
+ #[token("\\glsuserv")]
+ #[token("\\Glsuserv")]
+ #[token("\\GLSuserv")]
+ #[token("\\glsuservi")]
+ #[token("\\Glsuservi")]
+ #[token("\\GLSuservi")]
+ GlossaryEntryReference,
+
+ #[token("\\newacronym")]
+ AcronymDefinition,
+
+ #[token("\\DeclareAcronym")]
+ AcronymDeclaration,
+
+ #[token("\\acrshort")]
+ #[token("\\Acrshort")]
+ #[token("\\ACRshort")]
+ #[token("\\acrshortpl")]
+ #[token("\\Acrshortpl")]
+ #[token("\\ACRshortpl")]
+ #[token("\\acrlong")]
+ #[token("\\Acrlong")]
+ #[token("\\ACRlong")]
+ #[token("\\acrlongpl")]
+ #[token("\\Acrlongpl")]
+ #[token("\\ACRlongpl")]
+ #[token("\\acrfull")]
+ #[token("\\Acrfull")]
+ #[token("\\ACRfull")]
+ #[token("\\acrfullpl")]
+ #[token("\\Acrfullpl")]
+ #[token("\\ACRfullpl")]
+ #[token("\\acs")]
+ #[token("\\Acs")]
+ #[token("\\acsp")]
+ #[token("\\Acsp")]
+ #[token("\\acl")]
+ #[token("\\Acl")]
+ #[token("\\aclp")]
+ #[token("\\Aclp")]
+ #[token("\\acf")]
+ #[token("\\Acf")]
+ #[token("\\acfp")]
+ #[token("\\Acfp")]
+ #[token("\\ac")]
+ #[token("\\Ac")]
+ #[token("\\acp")]
+ #[token("\\glsentrylong")]
+ #[token("\\Glsentrylong")]
+ #[token("\\glsentrylongpl")]
+ #[token("\\Glsentrylongpl")]
+ #[token("\\glsentryshort")]
+ #[token("\\Glsentryshort")]
+ #[token("\\glsentryshortpl")]
+ #[token("\\Glsentryshortpl")]
+ #[token("\\glsentryfullpl")]
+ #[token("\\Glsentryfullpl")]
+ AcronymReference,
+
+ #[token("\\newtheorem")]
+ #[token("\\newtheorem*")]
+ #[token("\\declaretheorem")]
+ #[token("\\declaretheorem*")]
+ TheoremDefinition,
+
+ #[token("\\color")]
+ #[token("\\colorbox")]
+ #[token("\\textcolor")]
+ #[token("\\pagecolor")]
+ ColorReference,
+
+ #[token("\\definecolor")]
+ ColorDefinition,
+
+ #[token("\\definecolorset")]
+ ColorSetDefinition,
+
+ #[token("\\usepgflibrary")]
+ #[token("\\usetikzlibrary")]
+ TikzLibraryImport,
+
+ #[token("\\newenvironment")]
+ #[token("\\newenvironment*")]
+ #[token("\\renewenvironment")]
+ #[token("\\renewenvironment*")]
+ EnvironmentDefinition,
+
+ #[token("\\graphicspath")]
+ GraphicsPath,
+
+ #[token("\\fi")]
+ EndBlockComment,
- #[regex(r"\\color|\\colorbox|\\textcolor|\\pagecolor")]
- COLOR_REFERENCE_NAME,
+ #[regex(r"\\([^\r\n]|[@a-zA-Z:_]+\*?)?")]
+ #[error]
+ Generic,
+}
- #[regex(r"\\definecolor")]
- COLOR_DEFINITION_NAME,
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, Logos)]
+enum BlockCommentToken {
+ #[token("\\fi")]
+ End,
- #[regex(r"\\definecolorset")]
- COLOR_SET_DEFINITION_NAME,
+ #[regex(r"\\([^\r\n]|[@a-zA-Z:_]+\*?)?")]
+ #[regex(r"[^\\]+")]
+ #[error]
+ Verbatim,
+}
- #[regex(r"\\usepgflibrary|\\usetikzlibrary")]
- TIKZ_LIBRARY_IMPORT_NAME,
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, PartialOrd, Ord, Logos)]
+enum VerbatimEnvironmentToken {
+ #[token("\\end{asy}")]
+ #[token("\\end{verbatim}")]
+ #[token("\\end{lstlisting}")]
+ #[token("\\end{minted}")]
+ #[token("\\end{pycode}")]
+ End,
- #[regex(r"\\newenvironment|\\newenvironment*|\\renewenvironment|\\renewenvironment*")]
- ENVIRONMENT_DEFINITION_NAME,
+ #[regex(r"\\([^\r\n]|[@a-zA-Z:_]+\*?)?")]
+ #[regex(r"[^\\]+")]
+ #[error]
+ Verbatim,
}
#[derive(Debug, PartialEq, Eq, Clone)]
@@ -179,15 +420,9 @@ pub struct Lexer<'a> {
}
impl<'a> Lexer<'a> {
- pub fn new(text: &'a str) -> Self {
+ pub fn new(input: &'a str) -> Self {
let mut tokens = Vec::new();
- let mut lexer = Token::lexer(text);
- while let Some(kind) = lexer.next() {
- tokens.push((
- unsafe { std::mem::transmute::<Token, SyntaxKind>(kind) },
- lexer.slice(),
- ));
- }
+ tokenize(input, &mut tokens);
tokens.reverse();
Self { tokens }
}
@@ -201,6 +436,171 @@ impl<'a> Lexer<'a> {
}
}
+fn tokenize<'a>(input: &'a str, tokens: &mut Vec<(SyntaxKind, &'a str)>) {
+ let mut lexer = RootToken::lexer(input);
+ while let Some(kind) = lexer.next() {
+ let text = lexer.slice();
+ match kind {
+ RootToken::LineBreak => {
+ tokens.push((SyntaxKind::LINE_BREAK, text));
+ }
+ RootToken::Whitespace => {
+ tokens.push((SyntaxKind::WHITESPACE, text));
+ }
+ RootToken::LineComment => {
+ tokens.push((SyntaxKind::COMMENT, text));
+ }
+ RootToken::LCurly => {
+ tokens.push((SyntaxKind::L_CURLY, text));
+ }
+ RootToken::RCurly => {
+ tokens.push((SyntaxKind::R_CURLY, text));
+ }
+ RootToken::LBrack => {
+ tokens.push((SyntaxKind::L_BRACK, text));
+ }
+ RootToken::RBrack => {
+ tokens.push((SyntaxKind::R_BRACK, text));
+ }
+ RootToken::LParen => {
+ tokens.push((SyntaxKind::L_PAREN, text));
+ }
+ RootToken::RParen => {
+ tokens.push((SyntaxKind::R_PAREN, text));
+ }
+ RootToken::Comma => {
+ tokens.push((SyntaxKind::COMMA, text));
+ }
+ RootToken::Eq => {
+ tokens.push((SyntaxKind::EQUALITY_SIGN, text));
+ }
+ RootToken::Word => {
+ tokens.push((SyntaxKind::WORD, text));
+ }
+ RootToken::Dollar => {
+ tokens.push((SyntaxKind::DOLLAR, text));
+ }
+ RootToken::CommandName => {
+ let kind = tokenize_command_name(text);
+ tokens.push((kind, text));
+ }
+ RootToken::BeginBlockComment => {
+ tokens.push((SyntaxKind::BEGIN_BLOCK_COMMENT_NAME, text));
+ let end = lexer.span().end;
+ lexer = RootToken::lexer(tokenize_block_comment(&lexer.source()[end..], tokens));
+ }
+ RootToken::BeginVerbatimEnvironment => {
+ tokens.push((SyntaxKind::BEGIN_ENVIRONMENT_NAME, "\\begin"));
+ tokens.push((SyntaxKind::L_CURLY, "{"));
+ tokens.push((SyntaxKind::WORD, &text["\\begin{".len()..text.len() - 1]));
+ tokens.push((SyntaxKind::R_CURLY, "}"));
+ let end = lexer.span().end;
+ lexer = RootToken::lexer(tokenize_verbatim_environment(
+ &lexer.source()[end..],
+ tokens,
+ ));
+ }
+ }
+ }
+}
+
+fn tokenize_command_name(text: &str) -> SyntaxKind {
+ let mut lexer = CommandNameToken::lexer(text);
+ match lexer.next().unwrap() {
+ CommandNameToken::BeginEnvironment => SyntaxKind::BEGIN_ENVIRONMENT_NAME,
+ CommandNameToken::EndEnvironment => SyntaxKind::END_ENVIRONMENT_NAME,
+ CommandNameToken::BeginEquation => SyntaxKind::BEGIN_EQUATION_NAME,
+ CommandNameToken::EndEquation => SyntaxKind::END_EQUATION_NAME,
+ CommandNameToken::Part => SyntaxKind::PART_NAME,
+ CommandNameToken::Chapter => SyntaxKind::CHAPTER_NAME,
+ CommandNameToken::Section => SyntaxKind::SECTION_NAME,
+ CommandNameToken::Subsection => SyntaxKind::SUBSECTION_NAME,
+ CommandNameToken::Subsubsection => SyntaxKind::SUBSUBSECTION_NAME,
+ CommandNameToken::Paragraph => SyntaxKind::PARAGRAPH_NAME,
+ CommandNameToken::Subparagraph => SyntaxKind::SUBPARAGRAPH_NAME,
+ CommandNameToken::EnumItem => SyntaxKind::ENUM_ITEM_NAME,
+ CommandNameToken::Caption => SyntaxKind::CAPTION_NAME,
+ CommandNameToken::Citation => SyntaxKind::CITATION_NAME,
+ CommandNameToken::PackageInclude => SyntaxKind::PACKAGE_INCLUDE_NAME,
+ CommandNameToken::ClassInclude => SyntaxKind::CLASS_INCLUDE_NAME,
+ CommandNameToken::LatexInclude => SyntaxKind::LATEX_INCLUDE_NAME,
+ CommandNameToken::BiblatexInclude => SyntaxKind::BIBLATEX_INCLUDE_NAME,
+ CommandNameToken::BibtexInclude => SyntaxKind::BIBTEX_INCLUDE_NAME,
+ CommandNameToken::GraphicsInclude => SyntaxKind::GRAPHICS_INCLUDE_NAME,
+ CommandNameToken::SvgInclude => SyntaxKind::SVG_INCLUDE_NAME,
+ CommandNameToken::InkscapeInclude => SyntaxKind::INKSCAPE_INCLUDE_NAME,
+ CommandNameToken::VerbatimInclude => SyntaxKind::VERBATIM_INCLUDE_NAME,
+ CommandNameToken::Import => SyntaxKind::IMPORT_NAME,
+ CommandNameToken::LabelDefinition => SyntaxKind::LABEL_DEFINITION_NAME,
+ CommandNameToken::LabelReference => SyntaxKind::LABEL_REFERENCE_NAME,
+ CommandNameToken::LabelReferenceRange => SyntaxKind::LABEL_REFERENCE_RANGE_NAME,
+ CommandNameToken::LabelNumber => SyntaxKind::LABEL_NUMBER_NAME,
+ CommandNameToken::CommandDefinition => SyntaxKind::COMMAND_DEFINITION_NAME,
+ CommandNameToken::MathOperator => SyntaxKind::MATH_OPERATOR_NAME,
+ CommandNameToken::GlossaryEntryDefinition => SyntaxKind::GLOSSARY_ENTRY_DEFINITION_NAME,
+ CommandNameToken::GlossaryEntryReference => SyntaxKind::GLOSSARY_ENTRY_REFERENCE_NAME,
+ CommandNameToken::AcronymDefinition => SyntaxKind::ACRONYM_DEFINITION_NAME,
+ CommandNameToken::AcronymDeclaration => SyntaxKind::ACRONYM_DECLARATION_NAME,
+ CommandNameToken::AcronymReference => SyntaxKind::ACRONYM_REFERENCE_NAME,
+ CommandNameToken::TheoremDefinition => SyntaxKind::THEOREM_DEFINITION_NAME,
+ CommandNameToken::ColorReference => SyntaxKind::COLOR_REFERENCE_NAME,
+ CommandNameToken::ColorDefinition => SyntaxKind::COLOR_DEFINITION_NAME,
+ CommandNameToken::ColorSetDefinition => SyntaxKind::COLOR_SET_DEFINITION_NAME,
+ CommandNameToken::TikzLibraryImport => SyntaxKind::TIKZ_LIBRARY_IMPORT_NAME,
+ CommandNameToken::EnvironmentDefinition => SyntaxKind::ENVIRONMENT_DEFINITION_NAME,
+ CommandNameToken::EndBlockComment => SyntaxKind::END_BLOCK_COMMENT_NAME,
+ CommandNameToken::GraphicsPath => SyntaxKind::GRAPHICS_PATH_NAME,
+ CommandNameToken::Generic => SyntaxKind::GENERIC_COMMAND_NAME,
+ }
+}
+
+fn tokenize_block_comment<'a>(input: &'a str, tokens: &mut Vec<(SyntaxKind, &'a str)>) -> &'a str {
+ let mut lexer = BlockCommentToken::lexer(input);
+ let mut end = 0;
+ while let Some(kind) = lexer.next() {
+ match kind {
+ BlockCommentToken::Verbatim => {
+ end = lexer.span().end;
+ }
+ BlockCommentToken::End => {
+ end = lexer.span().start;
+ break;
+ }
+ };
+ }
+
+ if end > 0 {
+ tokens.push((SyntaxKind::VERBATIM, &input[..end]));
+ }
+
+ &input[end..]
+}
+
+fn tokenize_verbatim_environment<'a>(
+ input: &'a str,
+ tokens: &mut Vec<(SyntaxKind, &'a str)>,
+) -> &'a str {
+ let mut lexer = VerbatimEnvironmentToken::lexer(input);
+ let mut end = 0;
+ while let Some(kind) = lexer.next() {
+ match kind {
+ VerbatimEnvironmentToken::Verbatim => {
+ end = lexer.span().end;
+ }
+ VerbatimEnvironmentToken::End => {
+ end = lexer.span().start;
+ break;
+ }
+ };
+ }
+
+ if end > 0 {
+ tokens.push((SyntaxKind::VERBATIM, &input[..end]));
+ }
+
+ &input[end..]
+}
+
#[cfg(test)]
mod tests {
use insta::assert_debug_snapshot;
@@ -257,4 +657,18 @@ mod tests {
fn test_line_break() {
assert_debug_snapshot!(verify("hello\nworld"));
}
+
+ #[test]
+ fn test_block_comment() {
+ assert_debug_snapshot!(verify("Foo\\iffalse\n\\Bar{Baz}\n\\fi\\Qux"));
+ }
+
+ #[test]
+ fn test_asymptote() {
+ assert_debug_snapshot!(verify(
+ r#"\begin{asy}
+ printf("Hello World\n");
+\end{asy}"#
+ ));
+ }
}