diff options
Diffstat (limited to 'Master/texmf-dist/source/latex/l3experimental')
8 files changed, 4576 insertions, 3594 deletions
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3dt/l3dt.dtx b/Master/texmf-dist/source/latex/l3experimental/l3dt/l3dt.dtx index e5063618228..0fd090d9ef7 100644 --- a/Master/texmf-dist/source/latex/l3experimental/l3dt/l3dt.dtx +++ b/Master/texmf-dist/source/latex/l3experimental/l3dt/l3dt.dtx @@ -36,7 +36,7 @@ % %<*driver|package> \RequirePackage{expl3} -\GetIdInfo$Id: l3dt.dtx 3251 2012-01-28 09:42:44Z joseph $ +\GetIdInfo$Id: l3dt.dtx 3459 2012-02-26 08:58:53Z joseph $ {L3 Experimental data tables} %</driver|package> %<*driver> @@ -500,15 +500,15 @@ % \cs{q_dt_header} \\ % \cs{q_dt_row} \\ % \meta{row$_0$} \\ -% \cs{q_dt} \meta{key$_0$} \cs{q_dt} \Arg{data$_{\0,\0}$} \\ -% \cs{q_dt} \meta{key$_1$} \cs{q_dt} \Arg{data$_{\0,\1}$} \\ +% \cs{q_dt} \meta{key$_0$} \cs{q_dt} \Arg{data$_{0,0}$} \\ +% \cs{q_dt} \meta{key$_1$} \cs{q_dt} \Arg{data$_{0,1}$} \\ % \ldots \\ % \cs{q_dt} \\ % \cs{q_nil} \\ % \cs{q_dt_row} \\ % \meta{row$_2$} \\ -% \cs{q_dt} \meta{key$_0$} \cs{q_dt} \Arg{data$_{\1,\0}$} \\ -% \cs{q_dt} \meta{key$_1$} \cs{q_dt} \Arg{data$_{\1,\1}$} \\ +% \cs{q_dt} \meta{key$_0$} \cs{q_dt} \Arg{data$_{1,0}$} \\ +% \cs{q_dt} \meta{key$_1$} \cs{q_dt} \Arg{data$_{1,1}$} \\ % \ldots \\ % \cs{q_dt} \\ % \cs{q_nil} \\ diff --git a/Master/texmf-dist/source/latex/l3experimental/l3sort/l3sort.dtx b/Master/texmf-dist/source/latex/l3experimental/l3sort/l3sort.dtx index 65a95dbee29..0d80aebcc51 100644 --- a/Master/texmf-dist/source/latex/l3experimental/l3sort/l3sort.dtx +++ b/Master/texmf-dist/source/latex/l3experimental/l3sort/l3sort.dtx @@ -1,5 +1,5 @@ % \iffalse -% +% %% File l3sort.dtx (C) Copyright 2012 The LaTeX3 Project %% %% It may be distributed and/or modified under the conditions of the @@ -36,7 +36,7 @@ % %<*driver|package> \RequirePackage{expl3} -\GetIdInfo$Id: l3sort.dtx 3295 2012-02-02 09:30:14Z joseph $ +\GetIdInfo$Id: l3sort.dtx 3467 2012-02-26 15:39:35Z bruno $ {L3 Experimental sorting functions} %</driver|package> %<*driver> diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx index d2968338f46..887884d7c3a 100644 --- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx @@ -35,7 +35,7 @@ % %<*driver|package> \RequirePackage{expl3} -\GetIdInfo$Id: l3regex.dtx 3326 2012-02-07 00:39:44Z bruno $ +\GetIdInfo$Id: l3regex.dtx 3463 2012-02-26 15:05:14Z bruno $ {L3 Experimental Regular Expressions} %</driver|package> %<*driver> @@ -76,7 +76,8 @@ % The \pkg{l3regex} package provides regular expression testing, % extraction of submatches, splitting, and replacement, all acting % on token lists. The syntax of regular expressions is mostly a subset -% of the PCRE syntax (and very close to POSIX), with some additions +% of the \textsc{pcre} syntax (and very close to \textsc{posix}), +% with some additions % due to the fact that \TeX{} manipulates tokens rather than characters. % For performance reasons, only a limited set of features are implemented. % Notably, back-references are not supported. @@ -180,18 +181,25 @@ % \item[\\S] Any token not matched by |\s|. % \item[\\V] Any token not matched by |\v|. % \item[\\W] Any token not matched by |\w|. -% \item[\\N] Any token not matched by |\n|. % \end{l3regex-syntax} % Of those, |.|, |\D|, |\H|, |\N|, |\S|, |\V|, and |\W| will match arbitrary % control sequences. % -% Character classes match exactly one character in the subject string. +% Character classes match exactly one token in the subject. % \begin{l3regex-syntax} % \item[{[\ldots{}]}] Positive character class. % Matches any of the specified tokens. % \item[{[\char`\^\ldots{}]}] Negative character class. % Matches any token other than the specified characters. -% \item[{[x-y]}] Range (can be used with escaped characters). +% \item[{x-y}] Within a character class, this denotes a range (can be +% used with escaped characters). +% \item[{[:\meta{name}:]}] Within a character class (one more set of +% brackets), this denotes the \textsc{posix} character class +% \meta{name}, which can be \texttt{alnum}, \texttt{alpha}, +% \texttt{ascii}, \texttt{blank}, \texttt{cntrl}, \texttt{digit}, +% \texttt{graph}, \texttt{lower}, \texttt{print}, \texttt{punct}, +% \texttt{space}, \texttt{upper}, \texttt{word}, or \texttt{xdigit}. +% \item[{[:\char`\^\meta{name}:]}] Negative \textsc{posix} character class. % \end{l3regex-syntax} % For instance, |[a-oq-z\cC.]| matches any lowercase latin letter % except |p|, as well as control sequences (see below for a description @@ -264,25 +272,29 @@ % \item[\\c\Arg{regex}] A control sequence whose csname matches the % \meta{regex}, anchored at the beginning and end, so that |\c{begin}| % matches exactly \cs{begin}, and nothing else. -% \item[\\cX\meta{character or class}] Matches a token with category -% code |X| (any of |CBEMTPUDSLOA|) if it is also matched by the -% \meta{character or class}. For instance, |\cL[A-Z]| matches -% uppercase letters of category code letter, |\cC.| matches any -% control sequence, and |\cO\d| matches digits of category other. -% \item[{\\c[XYZ]\meta{character or class}}] Matches a token with -% category |X|, |Y|, or |Z| (each being any of |CBEMTPUDSLOA|), -% if it is also matched by the \meta{character or class}. For instance, -% |\c[LSO].| matches tokens of category letter, space, or other. -% \item[{\\c[\char`\^XYZ]\meta{character or class}}] Matches a token with -% category different from |X|, |Y|, or |Z| (each being any of -% |CBEMTPUDSLOA|), if it is also matched by the \meta{character or -% class}. For instance, |\c[LSO].| matches tokens of category -% letter, space, or other. +% \item[\\cX] Applies to the next object, which can be a character, +% character property, class, or group, and forces this object to +% only match tokens with category |X| (any of |CBEMTPUDSLOA|. For +% instance, |\cL[A-Z\d]| matches uppercase letters and digits of +% category code letter, |\cC.| matches any control sequence, and +% |\cO(abc)| matches |abc| where each character has category other. +% \item[{\\c[XYZ]}] Applies to the next object, and forces it to only +% match tokens with category |X|, |Y|, or |Z| (each being any of +% |CBEMTPUDSLOA|). For instance, |\c[LSO](..)| matches two tokens of +% category letter, space, or other. +% \item[{\\c[\char`\^XYZ]}] Applies to the next object and prevents it +% from matching any token with category |X|, |Y|, or |Z| (each being +% any of |CBEMTPUDSLOA|). For instance, |\c[^O]\d| matches digits +% which have any category different from other. % \end{l3regex-syntax} % The category code tests can be used inside classes; for instance, % |[\cO\d \c[LO][A-F]]| matches what \TeX{} considers as hexadecimal % digits, namely digits with category other, or uppercase letters from -% |A| to |F| with category either letter or other. +% |A| to |F| with category either letter or other. Within a group +% affected by a category code test, the outer test can be overridden by +% a nested test: for instance, |\cL(ab\cO\*cd)| matches |ab*cd| where +% all characters are of category letter, except |*| which has category +% other. % % The |\u| escape sequence allows to insert the contents of a token list % directly into a regular expression or a replacement, avoiding the need @@ -292,37 +304,50 @@ % argument once, in effect performing \cs{tl_to_str:v}. Quantifiers are % not supported directly: use a group. % -% Options can be set with |(?|\meta{option}|)| and -% unset with |(?-|\meta{option}|)|. Options are local -% to the group in which they are set, and revert to their -% previous setting upon reaching the closing parenthesis. -% For instance, in \verb"(?i)a(b(?-i)c|d)e", the |i| option -% applies to the letters |a|, |b| and |e|. -% \begin{l3regex-syntax} -% \item[(?i) \textrm{and} (?-i)] Toggle to a case -% insensitive/sensitive mode. This only applies to ascii letters -% (mapping \texttt{A}--\texttt{Z} to \texttt{a}--\texttt{z}). -% For instance, |(?i)[Y-\\]| matches the characters |Y|, |Z|, |[|, -% |\|, and the lower case letters |y| and |z|, while |(?i)[^aeiou]| -% matches any character which is not a vowel. -% \end{l3regex-syntax} -% -% In character classes, only |^|, |-|, |]|, |\| and spaces are special, -% and should be escaped. Other non-alphanumeric characters can -% still be escaped without harm. The escape sequences |\d|, -% |\D|, etc. are also supported in character classes. -% If the first character is |^|, then the meaning of the character -% class is inverted. Ranges of characters can be expressed using -% |-|, for instance, |[\D 0-5]| is equivalent to |[^6-9]|. +% The option |(?i)| makes the match case insensitive (identifying +% \texttt{A}--\texttt{Z} with \texttt{a}--\texttt{z}; no Unicode support +% yet). This applies until the end of the group in which it appears, and +% can be reverted using |(?-i)|. For instance, in +% \verb"(?i)(a(?-i)b|c)d", the letters |a| and |d| are affected by the +% |i| option. Characters within ranges and classes are affected +% individually: |(?i)[Y-\\]| is equivalent to |[YZ\[\\yz]|, and +% |(?i)[^aeiou]| matches any character which is not a vowel. Neither +% character properties, nor |\c{...}| nor |\u{...}| are affected by the +% |i| option. +% +% In character classes, only |[|, |^|, |-|, |]|, |\| and spaces are +% special, and should be escaped. Other non-alphanumeric characters can +% still be escaped without harm. Any escape sequence which matches a +% single character (|\d|, |\D|, \emph{etc.}) is supported in character +% classes. If the first character is |^|, then +% the meaning of the character class is inverted. Ranges of characters +% can be expressed using |-|, for instance, |[\D 0-5]| and |[^6-9]| are +% equivalent. % % Capturing groups are a means of extracting information about the % match. Parenthesized groups are labelled in the order of their % opening parenthesis, starting at $1$. The contents of those groups % corresponding to the \enquote{best} match (leftmost longest) -% can be extracted and stored in a sequence of strings using for +% can be extracted and stored in a sequence of token lists using for % instance \cs{regex_extract_once:nnNTF}. % -% \subsection{Syntax of in the replacement text} +% The |\K| escape sequence resets the beginning of the match to the +% current position in the token list. This only affects what is reported +% as the full match. For instance, +% \begin{verbatim} +% \regex_extract_all:nnN { a \K . } { a123aaxyz } \l_foo_seq +% \end{verbatim} +% results in \cs{l_foo_seq} containing the items |{1}| and |{a}|: the +% true matches are |{a1}| and |{aa}|, but they are trimmed by the use of +% |\K|. The |\K| command does not affect capturing groups: for instance, +% \begin{verbatim} +% \regex_extract_once:nnN { (. \K c)+ \d } { acbc3 } \l_foo_seq +% \end{verbatim} +% results in \cs{l_foo_seq} containing the items |{c3}| and |{bc}|: the +% true match is |{acbc3}|, with first submatch |{bc}|, but |\K| resets +% the beginning of the match to the last position where it appears. +% +% \subsection{Syntax of the replacement text} % % Most of the features described in regular expressions do not make sense % within the replacement text. Escaped characters are supported as inside @@ -349,15 +374,15 @@ % |\0|, |\1| \emph{etc.} % \end{l3regex-syntax} % -% \subsection{Precompiling regular expressions} +% \subsection{Pre-compiling regular expressions} % % If a regular expression is to be used several times, % it is better to compile it once rather than doing it -% each time the regular expression is used. The precompiled +% each time the regular expression is used. The compiled % regular expression is stored in a variable. All % of the \pkg{l3regex} module's functions can be given their % regular expression argument either as an explicit string -% or as a precompiled regular expression. +% or as a compiled regular expression. % % \begin{function}{\regex_new:N} % \begin{syntax} @@ -372,7 +397,7 @@ % \begin{syntax} % \cs{regex_set:Nn} \meta{regex~var} \Arg{regex} % \end{syntax} -% Stores a precompiled version of the \meta{regular expression} +% Stores a compiled version of the \meta{regular expression} % in the \meta{regex~var}. For instance, this function can be used % as % \begin{verbatim} @@ -380,15 +405,32 @@ % \regex_set:Nn \l_my_regex { my\ (simple\ )? reg(ex|ular\ expression) } % \end{verbatim} % The assignment is local for \cs{regex_set:Nn} and global for -% \cs{regex_gset:Nn}. Use \cs{regex_const:Nn} for precompiled expressions +% \cs{regex_gset:Nn}. Use \cs{regex_const:Nn} for compiled expressions % which will never change. % \end{function} % +% \begin{function}{\regex_show:n, \regex_show:N} +% \begin{syntax} +% \cs{regex_show:n} \Arg{regex} +% \end{syntax} +% Shows how \pkg{l3regex} interprets the \meta{regex}. For instance, +% \cs{regex_show:n} \verb+{\A X|Y}+ shows +% \begin{verbatim} +% +-branch +% anchor at start (\A) +% char code 88 +% +-branch +% char code 89 +% \end{verbatim} +% indicating that the anchor |\A| only applies to the first branch: +% the second branch is not anchored to the beginning of the match. +% \end{function} +% % \subsection{Matching} % % All regular expression functions are available in both |:n| and |:N| % variants. The former require a \enquote{standard} regular expression, -% while the later require a precompiled expression as generated by +% while the later require a compiled expression as generated by % \cs{regex_(g)set:Nn}. % % \begin{function}[TF]{\regex_match:nn, \regex_match:Nn} @@ -416,7 +458,7 @@ % starts again from the character following the last character % of the previous match, until reaching the end of the token list. % Infinite loops are prevented in the case where the regular expression -% can match an empty string: then we count one match between each +% can match an empty token list: then we count one match between each % pair of characters. % For instance, % \begin{verbatim} @@ -514,8 +556,8 @@ % Searches for the \meta{regular expression} in the \meta{token list} % and replaces the first match with the \meta{replacement}. The result % is assigned locally to \meta{tl~var}. In the \meta{replacement}, -% |\0| represents the full match, |\1| represent the contents -% of the first capturing group, |\2| of the second, \emph{etc.} +% |\0| represents the full match, |\1| represent the contents of the +% first capturing group, |\2| of the second, \emph{etc.} % \end{function} % % \begin{function}[TF]{\regex_replace_all:nnN, \regex_replace_all:NnN} @@ -523,97 +565,133 @@ % \cs{regex_replace_all:nnN} \Arg{regular expression} \Arg{replacement} \meta{tl~var} % \cs{regex_replace_all:nnNTF} \Arg{regular expression} \Arg{replacement} \meta{tl~var} \Arg{true code} \Arg{false code} % \end{syntax} -% Replaces all occurrences of the \cs{regular expression} -% in the \meta{token list} by the \meta{replacement}, where -% |\0| represents the full match, |\1| -% represent the contents of the first capturing group, -% |\2| of the second, \emph{etc.} Every match -% is treated independently, and matches cannot overlap. -% The result is assigned locally to \meta{tl~var}. +% Replaces all occurrences of the \cs{regular expression} in the +% \meta{token list} by the \meta{replacement}, where |\0| represents +% the full match, |\1| represent the contents of the first capturing +% group, |\2| of the second, \emph{etc.} Every match is treated +% independently, and matches cannot overlap. The result is assigned +% locally to \meta{tl~var}. % \end{function} % % \subsection{Bugs, misfeatures, future work, and other possibilities} % % The following need to be done now. % \begin{itemize} +% \item Change user function names! % \item Clean up the use of messages. -% \item Code comments. -% \item Rewrite the documentation in a clearer way. +% \item Rewrite the documentation in a more ordered way, perhaps add a +% \textsc{bnf}? +% \end{itemize} +% +% Additional error-checking to come. +% \begin{itemize} +% \item Detect that a trailing |\c|\meta{category} is an invalid regex. +% \item Currently, |a{\x34}| is recognized as |a{4}|. +% \item Cleaner error reporting in the replacement phase. +% \item Add tracing information. +% \item Detect attempts to use back-references. +% \item Test for the maximum register \cs{c_max_register_int}. +% \item Find out whether the fact that |\W| and friends match the +% end-marker leads to bugs. Possibly update \cs{regex_item_reverse:n}. +% \item Enforce that |\cC| can only be followed by a match-all dot. % \end{itemize} % % Code improvements to come. % \begin{itemize} -% \item \cs{regex_show:N} to show how a given regular expression is -% interpreted. -% \item Test for the maximum register \cs{c_max_register_int}. -% \item Use \tn{dimen} registers rather than \cs{l_regex_nesting_tl} -% to build \cs{regex_nesting:n}. -% \item Reduce the number of epsilon-transitions in alternatives. -% \item Move the \enquote{reconstruction} part of \pkg{l3regex} -% to \pkg{l3tl-analysis}. -% \item Optimize regexes for csnames when the regex is a simple string. -% \item Optimize simple strings: use less states (|abcade| should give -% two states, for |abc| and |ade|). [Does that really make sense?] -% \item Optimize groups with no alternative. -% \item Optimize the use of \cs{prg_stepwise_...} functions. -% \item Decide what |\c{\c{...}}| should do in the replacement text. -% \item Fix the |\c{\cBx}| bug in replacement text. +% \item Change \tn{skip} to \tn{dimen} for the array of active +% threads, and shift the array of submatch informations so that it +% starts at \tn{skip}$0$. +% \item Optimize |\c{abc}| for matching a specific control sequence. +% \item Only build \c{...} once. +% \item Use \tn{skip} for the left and right state stacks when +% compiling a regex. +% \item Should \cs{regex_action_free_group:n} only be used for greedy +% |{n,}| quantifier? (I think not.) +% \item Quantifiers for |\u| and assertions. +% \item Improve digit grabbing for the |\g| escape in replacement. +% Allow arbitrary integer expressions for all those numbers? +% \item When matching, keep track of an explicit stack of +% \texttt{current_state} and \texttt{current_submatches}. +% \item If possible, when a state is reused by the same thread, kill +% other subthreads. +% \item Use \tn{dimen} registers rather than \cs{l_regex_balance_tl} +% to build \cs{regex_replacement_balance_one_match:n}. +% \item Reduce the number of epsilon-transitions in alternatives. +% \item Optimize simple strings: use less states (|abcade| should give +% two states, for |abc| and |ade|). [Does that really make sense?] +% \item Optimize groups with no alternative. +% \item Optimize states with a single \cs{regex_action_free:n}. +% \item Optimize the use of \cs{regex_action_success:} by inserting it +% in state $2$ directly instead of having an extra transition. +% \item Optimize the use of \cs{prg_stepwise_...} functions. +% \item Groups don't capture within regexes for csnames; optimize and +% document. +% \item Decide and document what |\c{\c{...}}| should do in the +% replacement text, similar questions for |\u|. +% \item Better \enquote{show} for anchors, properties, and catcode tests. +% \item Does |\K| really need a new state for itself? +% \item When compiling, use a boolean \texttt{in_cs} and less magic +% numbers. +% \item Instead of checking whether the character is special or +% alphanumeric using its character code, check if it is special in +% regexes with \cs{cs_if_exist} tests. % \end{itemize} % % The following features are likely to be implemented at some point % in the future. % \begin{itemize} -% \item General look-ahead/behind assertions. -% \item Regex matching on external files. -% \item Conditional subpatterns with look ahead/behind: \enquote{if -% what follows is [\ldots{}], then [\ldots{}]}. -% \item |(*..)| and |(?..)| sequences to set some options -% (partially implemented). -% \item |\K| for resetting the beginning of the match. -% \item UTF-8 mode for pdf\TeX{}. -% \item Newline conventions are not done. -% In particular, we should have an option for |.| not to match newlines. -% Also, |\A| should differ from |^|, and |\Z|, |\z| and |$| should -% differ. -% \item Unicode properties: |\p{..}| and |\P{..}|; -% |\X| which should match any \enquote{extended} Unicode sequence. -% This requires to manipulate a lot of data, hence a lot of optimization -% ahead. +% \item Allow |\cL(abc)| in replacement text. +% \item General look-ahead/behind assertions. +% \item Regex matching on external files. +% \item Conditional subpatterns with look ahead/behind: \enquote{if +% what follows is [\ldots{}], then [\ldots{}]}. +% \item |(*..)| and |(?..)| sequences to set some options. +% \item UTF-8 mode for pdf\TeX{}. +% \item Newline conventions are not done. +% In particular, we should have an option for |.| not to match newlines. +% Also, |\A| should differ from |^|, and |\Z|, |\z| and |$| should +% differ. +% \item Unicode properties: |\p{..}| and |\P{..}|; +% |\X| which should match any \enquote{extended} Unicode sequence. +% This requires to manipulate a lot of data, probably using tree-boxes. % \end{itemize} % -% The following features of PCRE or Perl will probably not be implemented. +% The following features of \textsc{pcre} or Perl will probably not be +% implemented. % \begin{itemize} -% \item |\ddd|, matching the character with code \texttt{ddd} in octal; -% \item POSIX character classes |[:alpha:]| \emph{etc.}, this is redundant; -% \item Callout with |(?C...)|, we cannot run arbitrary user code during -% the matching, because the regex code uses registers in an unsafe way; -% \item Conditional subpatterns (other than with a look-ahead -% or look-behind condition): this is non-regular, isn't it? -% \item Named subpatterns: \TeX{} programmers have lived so far without -% any need for named macro parameters. +% \item |\ddd|, matching the character with octal code \texttt{ddd}; +% \item Callout with |(?C...)|, we cannot run arbitrary user code +% during the matching, because the regex code uses registers in an +% unsafe way; +% \item Conditional subpatterns (other than with a look-ahead or +% look-behind condition): this is non-regular, isn't it? +% \item Named subpatterns: \TeX{} programmers have lived so far +% without any need for named macro parameters. % \end{itemize} % -% The following features of PCRE or perl will definitely not be implemented. +% The following features of \textsc{pcre} or Perl will definitely not be +% implemented. % \begin{itemize} -% \item |\cx|, similar to \TeX{}'s own |\^^x|; -% \item Comments: \TeX{} already has its own system for comments. -% \item |\Q...\E| escaping: this would require to read the argument -% verbatim, which is not in the scope of this module. -% \item Atomic grouping, possessive quantifiers: those tools, mostly -% meant to fix catastrophic backtracking, are unnecessary in a -% non-backtracking algorithm, and difficult to implement. -% \item Subroutine calls: this syntactic sugar is difficult to include -% in a non-backtracking algorithm, in particular because the -% corresponding group should be treated as atomic. Also, we cannot -% afford to run user code within the regular expression matching, -% because of our \enquote{misuse} of registers. -% \item Recursion: this is a non-regular feature. -% \item Back-references: non-regular feature, this requires backtracking, -% which is prohibitively slow. -% \item Backtracking control verbs: intrinsically tied to backtracking. -% \item |\C| single byte in UTF-8 mode: Xe\TeX{} and Lua\TeX{} serve -% us characters directly, and splitting those into bytes is tricky, -% encoding dependent, and most likely not useful anyways. +% \item |\cx|, similar to \TeX{}'s own |\^^x|; +% \item Comments: \TeX{} already has its own system for comments. +% \item |\Q...\E| escaping: this would require to read the argument +% verbatim, which is not in the scope of this module. +% \item Atomic grouping, possessive quantifiers: those tools, mostly +% meant to fix catastrophic backtracking, are unnecessary in a +% non-backtracking algorithm, and difficult to implement. +% \item Subroutine calls: this syntactic sugar is difficult to include +% in a non-backtracking algorithm, in particular because the +% corresponding group should be treated as atomic. Also, we cannot +% afford to run user code within the regular expression matching, +% because of our \enquote{misuse} of registers. +% \item Recursion: this is a non-regular feature. +% \item Back-references: non-regular feature, this requires +% backtracking, which is prohibitively slow. +% \item Backtracking control verbs: intrinsically tied to +% backtracking. +% \item |\C| single byte in UTF-8 mode: Xe\TeX{} and Lua\TeX{} serve +% us characters directly, and splitting those into bytes is tricky, +% encoding dependent, and most likely not useful anyways. % \end{itemize} % % \end{documentation} @@ -626,7 +704,7 @@ % \begin{macrocode} \ProvidesExplPackage {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription} -\RequirePackage{l3str, l3tl-analysis, l3flag} +\RequirePackage{l3tl-build, l3tl-analysis, l3flag, l3str} % \end{macrocode} % % \subsection{Plan of attack} @@ -638,36 +716,42 @@ % code tends to run slowly, and we must use faster, albeit slightly more % restrictive, techniques, coming from automata theory. % -% Given a regular expression of $n$ characters, we build a -% non-deterministic finite automaton (NFA) with roughly $n$ states, -% which accepts precisely those token lists matching that regular -% expression. Then loop through the query token list one token (one -% \enquote{index}) at a time, exploring in parallel every possible path -% through the NFA. When performing this matching, we keep track of an -% array of the states currently \enquote{active}, which are to be -% considered in order when the next token is read. +% Given a regular expression of $n$ characters, we do the following: +% \begin{itemize} +% \item (Compiling.) Analyse the regex, finding invalid input, and +% convert it to an internal representation. +% \item (Building.) Convert the compiled regex to a non-deterministic +% finite automaton (\textsc{nfa}) with roughly $n$ states which +% accepts precisely token lists matching that regex. +% \item (Matching.) Loop through the query token list one token (one +% \enquote{position}) at a time, exploring in parallel every +% possible path (\enquote{active thread}) through the \textsc{nfa}, +% considering active threads in an order determined by the +% quantifiers' greediness. +% \end{itemize} % % We use the following vocabulary in the code comments (and in variable % names). % \begin{itemize} % \item \emph{Group}: index of the capturing group, $-1$ for % non-capturing groups. -% \item \emph{Index}: each token in the query is labelled by an integer -% \meta{index}, with $\cs{l_regex_min_index_int} - 1 \leq \meta{index} -% \leq \cs{l_regex_max_index_int}$. The lowest and highest indexes -% correspond to imaginary begin and end markers (with inacessible -% category code and character code). +% \item \emph{Position}: each token in the query is labelled by an +% integer \meta{position}, with $\texttt{min_pos} - 1 \leq +% \meta{position} \leq \texttt{max_pos}$. The lowest and highest +% positions correspond to imaginary begin and end markers (with +% inacessible category code and character code). % \item \emph{Query}: the token list to which we apply the regular % expression. -% \item \emph{State}: each state of the NFA is labelled by an integer -% \meta{state} with $0 \leq \meta{state} < -% \cs{l_regex_max_state_int}$. -% \item \emph{Active state}: state of the NFA that is reached when -% reading the query string for the matching. Those states are -% ordered according to the greediness of quantifiers. +% \item \emph{State}: each state of the \textsc{nfa} is labelled by an +% integer \meta{state} with $\texttt{min_state} \leq \meta{state} < +% \texttt{max_state}$. +% \item \emph{Active thread}: state of the \textsc{nfa} that is reached +% when reading the query token list for the matching. Those threads +% are ordered according to the greediness of quantifiers. % \item \emph{Step}: used when matching, starts at $0$, incremented % every time a character is read, and is not reset when searching -% for repeated matches. +% for repeated matches. The integer \cs{l_regex_step_int} is a +% unique id for all the steps of the matching algorithm. % \end{itemize} % % To acheive a good performance, we abuse \TeX{}'s registers in two @@ -676,764 +760,120 @@ % functions. And we store integers in \tn{dimen} registers in scaled % points (\texttt{sp}), using \TeX{}'s implicit conversion from % dimensions to integers in some contexts. Specifically, the registers -% are used as follows. When building, +% are used as follows. When compiling, \tn{toks} registers are used +% under the hood by functions from the \pkg{l3tl-build} module. When +% building, % \begin{itemize} -% \item \tn{toks}\meta{state} has two parts separated by \cs{s_stop}: -% a property list which holds the submatch information, followed by -% the tests and actions to perform in the \meta{state} of the NFA. -% \item \tn{skip}$i$ has the form \meta{group} \texttt{plus} +% \item \tn{toks}\meta{state} holds the tests and actions to perform +% in the \meta{state} of the \textsc{nfa}. +% \item (Not implemented yet.) +% \tn{skip}$i$ has the form \meta{group id} \texttt{plus} % \meta{left state} \texttt{minus} \meta{right state}. -% ^^A todo: not done yet. % \end{itemize} % When matching, % \begin{itemize} % \item \tn{dimen}\meta{state} is equal to the last \meta{step} in % which the \meta{state} was active. -% \item \tn{skip}$i$ holds the \meta{state} number corresponding to -% the $i$-th active \meta{state} (smaller $i$ is higher precendence) -% and has no shrink or stretch components. -% \item \tn{toks}\meta{index} holds \meta{tokens} which \texttt{o}- -% and \texttt{x}-expand to the \meta{index}-th token in the query. +% \item (Currently, we use \tn{skip} instead of \tn{dimen}.) +% \tn{dimen}\meta{thread}, with $\texttt{min_active} \leq +% \meta{thread} < \texttt{max_active}$, is equal to the +% \meta{state} in which the \meta{thread} currently is. The +% \meta{threads} or ordered starting from the best to the least +% preferred. +% \item \tn{toks}\meta{thread} holds the submatch information for the +% \meta{thread}, as the contents of a property list. +% \item \tn{muskip}\meta{position} holds as its main and stretch +% components the character and category code of the token at this +% \meta{position} in the query. +% \item \tn{toks}\meta{position} holds \meta{tokens} which \texttt{o}- +% and \texttt{x}-expand to the \meta{position}-th token in the query. +% \item \tn{skip} registers hold the value of end-points of all +% submatches as would be extracted by the \cs{regex_extract} +% functions. Since smaller \tn{skip} registers are used, the minimum +% index is twice \texttt{max_state}, and the used registers go up to +% \cs{l_regex_submatch_int}. They are organized in blocks of +% \texttt{capturing_group}, each block corresponding to one match +% with all its submatches stored in consecutive \tn{skip}s. % \end{itemize} % \tn{count} registers are not abused, which means that we can safely -% use named integers in this module. +% use named integers in this module. Note that \tn{box} registers are +% not abused either; maybe we could leverage those for some purpose. +% +% The code is structured as follows. Variables are introduced in the +% relevant section. First we present some generic helper functions. Then +% comes the code for compiling a regular expression, and for showing the +% result of the compilation. The building phase converts a compiled +% regex to \textsc{nfa} states, and the automaton is run by the code in +% the following section. The only remaining brick is parsing the +% replacement text and performing the replacement. We are then ready for +% all the user functions. Finally, messages, and a little bit of tracing +% code. % -% The code is structured as follows. Variables, constants and various -% helper functions are introduced first, to limit the clutter in later -% parts. Then functions pertaining to parsing the regular expression are -% defined: that part is rather long because of the many bells and -% whistles of the regex notation. The next subsection takes care of -% running the NFA. Thereafter, we give tools for the replacements. -% Finally, user functions. -% -% \subsection{Constants and variables} +% \subsection{Helpers} % -% \begin{macro}{\regex_tmp:w} -% \begin{variable} -% {\l_regex_internal_a_int, \l_regex_internal_b_int, \l_regex_internal_c_int} -% \begin{variable}{\l_regex_internal_a_tl, \l_regex_internal_b_tl} -% \begin{variable}{\g_regex_internal_tl} -% Temporary variables used for various purposes. +% \begin{macro}[aux]{\tl_to_str:V} +% A variant we need for the |\u| escape in the replacement text. % \begin{macrocode} -\cs_new:Npn \regex_tmp:w { } -\tl_new:N \l_regex_internal_a_tl -\tl_new:N \l_regex_internal_b_tl -\int_new:N \l_regex_internal_a_int -\int_new:N \l_regex_internal_b_int -\int_new:N \l_regex_internal_c_int -\tl_new:N \g_regex_internal_tl +\cs_generate_variant:Nn \tl_to_str:n { V } % \end{macrocode} -% \end{variable} -% \end{variable} -% \end{variable} % \end{macro} % -% \subsubsection{Character properties} +% \subsubsection{Constants and variables} % -% \begin{variable} -% { -% \c_regex_d_tl, \c_regex_D_tl, \c_regex_h_tl, \c_regex_H_tl, -% \c_regex_s_tl, \c_regex_S_tl, \c_regex_v_tl, \c_regex_V_tl, -% \c_regex_w_tl, \c_regex_W_tl, \c_regex_N_tl -% } -% -% These constant token lists encode which characters are recognized by -% |\d|, |\D|, |\w|, \emph{etc.} in regular expressions. Namely, -% |\d=[0-9]|, |\w=[0-9A-Z_a-z]|, \verb*+\s=[\ \^^I\^^J\^^L\^^M]+, -% \verb*+\h=[\ \^^I]+, |\v=[\^^J-\^^M]|, and the upper case -% counterparts match anything that the lower case does not match. The -% order in which the various tests appear is optimized for usual -% mostly lower case letter text. -% \begin{macrocode} -\tl_const:Nn \c_regex_d_tl - { \regex_item_range:nn \c_forty_eight { 57 } } % 0--9 -\tl_const:Nn \c_regex_h_tl - { - \regex_item_equal:n \c_thirty_two % space - \regex_item_equal:n \c_nine % tab - } -\tl_const:Nn \c_regex_s_tl - { - \regex_item_equal:n \c_thirty_two % space - \regex_item_equal:n \c_nine % tab - \regex_item_equal:n \c_ten % lf - \regex_item_equal:n \c_twelve % ff - \regex_item_equal:n \c_thirteen % cr - } -\tl_const:Nn \c_regex_v_tl - { \regex_item_range:nn \c_ten \c_thirteen } % lf, vtab, ff, cr -\tl_const:Nn \c_regex_w_tl - { - \regex_item_range:nn \c_ninety_seven { 122 } % a--z - \regex_item_range:nn \c_sixty_five { 90 } % A--Z - \regex_item_range:nn \c_forty_eight { 57 } % 0--9 - \regex_item_equal:n { 95 } % _ - } -\tl_const:Nn \c_regex_D_tl - { \c_regex_d_tl \regex_break_point:TF { } \regex_break_true:w } -\tl_const:Nn \c_regex_H_tl - { \c_regex_h_tl \regex_break_point:TF { } \regex_break_true:w } -\tl_const:Nn \c_regex_S_tl - { \c_regex_s_tl \regex_break_point:TF { } \regex_break_true:w } -\tl_const:Nn \c_regex_V_tl - { \c_regex_v_tl \regex_break_point:TF { } \regex_break_true:w } -\tl_const:Nn \c_regex_W_tl - { \c_regex_w_tl \regex_break_point:TF { } \regex_break_true:w } -\tl_const:Nn \c_regex_N_tl - { - \regex_item_equal:n \c_ten - \regex_break_point:TF { } { \regex_break_true:w } - } -% \end{macrocode} -% \end{variable} -% -% \begin{variable}{\c_regex_._tl} -% The dot meta-character matches any character, except the end marker, -% whose character code is $-2$. -% \begin{macrocode} -\tl_const:cn { c_regex_._tl } - { - \if_num:w \l_regex_current_char_int > - \c_two - \exp_after:wN \regex_break_true:w - \fi: - } -% \end{macrocode} -% \end{variable} -% -% \subsubsection{Variables used while building} -% -% \begin{variable}{\l_regex_build_mode_int} -% While building, ten modes are recognized, labelled $-63$, $-23$, -% $-6$, $-2$, $0$, $2$, $3$, $6$, $23$, $63$. See -% section~\ref{sec:regex-modes}. +% \begin{macro}[aux]{\regex_tmp:w} +% Temporary function used for various short-term purposes. % \begin{macrocode} -\int_new:N \l_regex_build_mode_int -% \end{macrocode} -% \end{variable} -% -% \begin{variable}{\l_regex_max_state_int} -% \begin{variable}{\l_regex_left_state_int, \l_regex_right_state_int} -% The last state that was allocated is $\cs{l_regex_max_state_int}-1$, -% so that \cs{l_regex_max_state_int} always points to a free state. -% The starting state and end state of the last group (which any -% quantifier would act on) are stored as -% \cs{l_regex_left/right_state_int}. In almost all cases, the left -% and right pointers only differ by~$1$. -% \begin{macrocode} -\int_new:N \l_regex_max_state_int -\int_new:N \l_regex_left_state_int -\int_new:N \l_regex_right_state_int -% \end{macrocode} -% \end{variable} -% \end{variable} -% -% \begin{variable}{\l_regex_left_state_seq, \l_regex_right_state_seq} -% Alternatives are implemented by branching from a state into the -% various choices, then merging those into another state. We store -% information about those states in two sequences. -% \begin{macrocode} -\seq_new:N \l_regex_left_state_seq -\seq_new:N \l_regex_right_state_seq -% \end{macrocode} -% \end{variable} -% -% \begin{variable}{\l_regex_end_group_seq} -% \begin{variable}{\l_regex_end_alternation_seq} -% These sequences hold actions to be performed at the end of a group, -% and at the end of each branch of the alternation, respectively. -% Currently, \cs{l_regex_end_group_seq} is used to keep track of -% letter case, and \cs{l_regex_end_alternation_seq} is used -% for \verb"(?|...)" groups. -% \begin{macrocode} -\seq_new:N \l_regex_end_group_seq -\seq_new:N \l_regex_end_alternation_seq -% \end{macrocode} -% \end{variable} -% \end{variable} -% -% \begin{variable}{\l_regex_capturing_group_int} -% \begin{variable}{\l_regex_capturing_group_seq} -% \begin{variable}{\l_regex_capturing_group_max_int} -% \cs{l_regex_capturing_group_int} is the ID number of the current -% capturing group, starting at $0$ for a group enclosing the full -% regular expression, and counting in the order of their left parenthesis. -% This number is used when a branch of the alternation ends. -% Capturing groups can be arbitrarily nested, and we keep track of -% the stack of ID numbers in \cs{l_regex_capturing_group_seq}. -% The \texttt{max_int} variable is used in the case of the special -% groups \verb+(?|...|...)+, which reset the capturing group number in -% each alternative: at the end of the group, we must use a number -% larger than every capturing group appearing in any alternative. -% \begin{macrocode} -\int_new:N \l_regex_capturing_group_int -\seq_new:N \l_regex_capturing_group_seq -\int_new:N \l_regex_capturing_group_max_int -% \end{macrocode} -% \end{variable} -% \end{variable} -% \end{variable} -% -% \begin{variable}{\l_regex_one_or_group_tl} -% When looking for quantifiers, this variable holds either -% \enquote{one} or \enquote{group} depending on whether the -% object to which the quantifier applies matches one character -% (\emph{i.e.}, is a character or character class), or is a group. -% \begin{macrocode} -\tl_new:N \l_regex_one_or_group_tl -% \end{macrocode} -% \end{variable} -% -% \begin{variable}{\l_regex_catcodes_int, \l_regex_catcodes_default_int} -% \begin{variable} -% { -% \c_regex_catcode_C_int, \c_regex_catcode_B_int, \c_regex_catcode_E_int, -% \c_regex_catcode_M_int, \c_regex_catcode_T_int, \c_regex_catcode_P_int, -% \c_regex_catcode_U_int, \c_regex_catcode_D_int, \c_regex_catcode_S_int, -% \c_regex_catcode_L_int, \c_regex_catcode_O_int, \c_regex_catcode_A_int -% } -% \begin{variable}{\c_regex_catcodes_all_int} -% We wish to allow constructions such as |\c[^BE](..\cL[a-z]..)|, -% matching two tokens which are neither a begin-group nor an end-group -% token, followed by a token of category letter and character code in -% |[a-z]|, followed by two more tokens which are neither begin-group -% nor end-group tokens. For this to work, we need to keep track of -% lists of allowed category codes: \cs{l_regex_catcodes_int} and -% \cs{l_regex_catcodes_default_int} are bitmaps, sums of $4^c$, for -% all allowed catcodes $c$. The latter is local to each capturing -% group, and we reset \cs{l_regex_catcodes_int} to that value after -% each character or class, changing it only when encountering a |\c| -% escape. Errata: the \cs{l_regex_catcodes_default_int} is not used -% yet, because the idea of having \verb+\cL(...|...)+ act on the whole -% contents of the group is not yet implemented. -% \begin{macrocode} -\int_new:N \l_regex_catcodes_int -\int_new:N \l_regex_catcodes_default_int -\int_const:Nn \c_regex_catcode_C_int { "1 } -\int_const:Nn \c_regex_catcode_B_int { "4 } -\int_const:Nn \c_regex_catcode_E_int { "10 } -\int_const:Nn \c_regex_catcode_M_int { "40 } -\int_const:Nn \c_regex_catcode_T_int { "100 } -\int_const:Nn \c_regex_catcode_P_int { "1000 } -\int_const:Nn \c_regex_catcode_U_int { "4000 } -\int_const:Nn \c_regex_catcode_D_int { "10000 } -\int_const:Nn \c_regex_catcode_S_int { "100000 } -\int_const:Nn \c_regex_catcode_L_int { "400000 } -\int_const:Nn \c_regex_catcode_O_int { "1000000 } -\int_const:Nn \c_regex_catcode_A_int { "4000000 } -\int_const:Nn \c_regex_catcodes_all_int { "5515155 } -% \end{macrocode} -% \end{variable} -% \end{variable} -% \end{variable} -% -% \begin{variable}{\l_regex_catcodes_bool} -% Controls whether the bitmap of category codes built should be -% inverted or not. -% \begin{macrocode} -\bool_new:N \l_regex_catcodes_bool -% \end{macrocode} -% \end{variable} -% -% \begin{variable}{\l_regex_tests_bool,\l_regex_tests_tl} -% The tests which should be performed on an item of the token list are -% stored in \cs{l_regex_tests_tl}. In the case of character classes, -% \cs{l_regex_tests_bool} is \texttt{false} for negative character -% classes. In the other case, \cs{l_regex_tests_tl} is directly -% converted to a state of the NFA. The two variables are cleared -% (boolean set to \texttt{true}) after looking for a quantifier for -% the class or single character or property. It would seem cleaner to -% clear them instead before grabbing the next character or class, but -% that doesn't interact correctly with constructions like |\cL[...]| -% where |\cL| inserts some tests in \cs{l_regex_tests_tl}, which -% should not be eliminated before finding the class. -% \begin{macrocode} -\bool_new:N \l_regex_tests_bool -\tl_new:N \l_regex_tests_tl +\cs_new:Npn \regex_tmp:w { } % \end{macrocode} -% \end{variable} +% \end{macro} % % \begin{variable} % { -% \l_regex_tests_saved_bool, -% \l_regex_tests_saved_tl, -% \l_regex_catcodes_saved_int +% \l_regex_internal_a_tl, \l_regex_internal_b_tl, +% \l_regex_internal_a_int, \l_regex_internal_b_int, +% \l_regex_internal_c_int, \l_regex_internal_bool, +% \l_regex_internal_seq, \g_regex_internal_tl, % } -% In nested class, which can only occur as |[\cA[...]]|, the variables -% \cs{l_regex_tests_bool}, \cs{l_regex_tests_tl}, and -% \cs{l_regex_catcodes_int} from the outer class must be saved while -% processing the inner class. -% \begin{macrocode} -\bool_new:N \l_regex_tests_saved_bool -\tl_new:N \l_regex_tests_saved_tl -\int_new:N \l_regex_catcodes_saved_int -% \end{macrocode} -% \end{variable} -% -% \subsubsection{Variables used when matching} -% -% \begin{variable}{\l_regex_min_index_int} -% \begin{variable}{\l_regex_max_index_int} -% The tokens in the query are indexed from \cs{l_regex_min_index_int} -% for the first to $\cs{l_regex_max_index_int}-1$ for the last, and -% their information is stored in \tn{muskip} and \tn{toks} registers -% with those numbers. We don't start from $0$ because the \tn{toks} -% registers with low numbers are used to hold the states of the NFA. -% \begin{macrocode} -\int_new:N \l_regex_min_index_int -\int_new:N \l_regex_max_index_int -% \end{macrocode} -% \end{variable} -% \end{variable} -% -% \begin{variable}{\l_regex_nesting_int} -% The first phase when matching is to go once through the query token -% list and store the information for each token as \tn{muskip} and -% \tn{toks} registers. During this phase, \cs{l_regex_nesting_int} -% counts the balance of begin-group and end-group character tokens -% which appear before a given point in the string, and is stored as -% the shrink component of the \tn{muskip} registers. This integer is -% also used to keep track of the balance in the replacement text. -% \begin{macrocode} -\int_new:N \l_regex_nesting_int -% \end{macrocode} -% \end{variable} -% -% \begin{variable}{\l_regex_current_index_int} -% \begin{variable}{\l_regex_start_index_int} -% \begin{variable}{\l_regex_success_index_int} -% While reading through the query token list, -% \cs{l_regex_current_index_int} is the position in the token list, -% starting at \cs{l_regex_min_index_int} for the first token. -% Each match begins at the position given by -% \cs{l_regex_start_index_int}. Whenever an execution thread succeeds, -% the corresponding index is stored into \cs{l_regex_success_index_int}, -% which will be the next starting index (except in the case of empty -% matches). -% \begin{macrocode} -\int_new:N \l_regex_current_index_int -\int_new:N \l_regex_start_index_int -\int_new:N \l_regex_success_index_int -% \end{macrocode} -% \end{variable} -% \end{variable} -% \end{variable} -% -% \begin{variable}{\l_regex_current_char_int} -% \begin{variable}{\l_regex_current_catcode_int} -% \begin{variable}{\l_regex_current_token_tl} -% \begin{variable}{\l_regex_last_char_int} -% \begin{variable}{\l_regex_case_changed_char_int} -% The character and category codes of the token at the current -% position; \meta{tokens} which \texttt{o}- and \texttt{x}-expand to -% the token (as provided by \cs{tl_analysis:n}); the character code of -% the token at the previous position; and the character code of the -% result of changing the case of the current token -% (|A-Z|$\leftrightarrow$|a-z|). This last integer is only computed if -% the \enquote{case insensitive} option |(?i)| is used in the regex. -% The \cs{l_regex_current_char_int} variable is also used in various -% other phases to hold a character code. -% \begin{macrocode} -\int_new:N \l_regex_current_char_int -\int_new:N \l_regex_current_catcode_int -\tl_new:N \l_regex_current_token_tl -\int_new:N \l_regex_last_char_int -\int_new:N \l_regex_case_changed_char_int -% \end{macrocode} -% \end{variable} -% \end{variable} -% \end{variable} -% \end{variable} -% \end{variable} -% -% \begin{variable}{\l_regex_caseless_bool} -% True if caseless matching is used within the regular expression. -% This controls whether \cs{l_regex_case_changed_char_int} is computed. -% \begin{macrocode} -\bool_new:N \l_regex_caseless_bool -% \end{macrocode} -% \end{variable} -% -% \begin{variable}{\l_regex_current_state_int} -% For every character in the token list, each of the active states is -% considered in turn. -% The variable \cs{l_regex_current_state_int} holds the state -% of the NFA which is currently considered: transitions are then -% given as shifts relative to the current state. -% In some cases of groups with quantifiers, -% \cs{l_regex_current_state_int} is shifted to a fake value for -% transitions to point to the correct states. -% \begin{macrocode} -\int_new:N \l_regex_current_state_int -% \end{macrocode} -% \end{variable} -% -% \begin{variable}{\l_regex_current_submatches_prop} -% \begin{variable}{\l_regex_success_submatches_prop} -% The submatches for the thread which lies at the -% \cs{l_regex_current_state_int} are stored in a property -% list variable. This property list is stored by -% \cs{regex_action_cost:n} into the \tn{toks} register -% for the target state of the transition. When a thread -% succeeds, this property list is copied to -% \cs{l_regex_success_submatches_prop} and only the last -% sucessful thread will remain there. -% \begin{macrocode} -\prop_new:N \l_regex_current_submatches_prop -\prop_new:N \l_regex_success_submatches_prop -% \end{macrocode} -% \end{variable} -% \end{variable} -% -% \begin{variable}{\l_regex_step_int} -% In the case of repeated matches, \cs{l_regex_current_index_int} -% is reset to the end-position of the previous match. In contrast, -% \cs{l_regex_step_int} is simply incremented to provide -% a unique number for each iteration of the matching loop. This -% is handy to attach each set of submatch information to a given -% iteration (and automatically discard it when it corresponds to -% a past iteration). -% \begin{macrocode} -\int_new:N \l_regex_step_int -% \end{macrocode} -% \end{variable} -% -% \begin{variable}{\l_regex_max_active_int} -% All the currently active states are kept in order of precedence -% in the \tn{skip} registers, which for our purpose serve as an array: -% the $i$th item of the array is \tn{skip}$i$. The largest index used -% after treating the previous character is \cs{l_regex_max_active_int}. -% At the start of every step, the whole array is unpacked, so that the -% space can immediately be reused, and \cs{l_regex_max_active_int} is -% reset to zero, effectively clearing the array. -% \begin{macrocode} -\int_new:N \l_regex_max_active_int -% \end{macrocode} -% \end{variable} -% -% \begin{variable}{\l_regex_every_match_tl} -% Every time a match is found, this token list is used. For single -% matching, the token list is empty. For multiple matching, the token -% list is set to repeat the matching. -% \begin{macrocode} -\tl_new:N \l_regex_every_match_tl -% \end{macrocode} -% \end{variable} -% -% \begin{variable}{\l_regex_fresh_thread_bool} -% \begin{variable}{\l_regex_success_empty_bool} -% \begin{macro}{\regex_if_two_empty_matches:F} -% When doing multiple matches, we need to avoid infinite loops where -% each iteration matches the same empty token list. When an empty -% token list is matched, the next successful match of the same empty -% token list is suppressed. We detect empty matches by setting -% \cs{l_regex_fresh_thread_bool} to \texttt{true} for threads which -% directly come from the start of the regular expression, and testing -% that boolean whenever a thread succeeds. The function -% \cs{regex_if_two_empty_matches:F} is redefined at every match -% attempt, depending on whether the previous match was empty or not: -% if it was, then the function must cancel a purported success if it -% is empty and at the same spot as the previous match; otherwise, we -% definitely don't have two identical empty matches, so the function -% is \cs{use:n}. -% \begin{macrocode} -\bool_new:N \l_regex_fresh_thread_bool -\bool_new:N \l_regex_success_empty_bool -\cs_new_eq:NN \regex_if_two_empty_matches:F \use:n -% \end{macrocode} -% \end{macro} -% \end{variable} -% \end{variable} -% -% \begin{variable}{\g_regex_success_bool} -% \begin{variable}{\l_regex_saved_success_bool} -% \begin{variable}{\l_regex_success_match_bool} -% The boolean \cs{g_regex_success_bool} is true if there was at least -% one successful match, and \cs{l_regex_success_match_bool} is true if -% the current match attempt was successful. The variable -% \cs{g_regex_success_bool} is the only global variable in this whole -% module. When nesting \cs{regex} functions internally, the value of -% \cs{g_regex_success_bool} is saved into -% \cs{l_regex_saved_success_bool}, which is local, hence not affected -% by the changes due to inner regex functions. +% Temporary variables used for various purposes. % \begin{macrocode} -\bool_new:N \g_regex_success_bool -\bool_new:N \l_regex_saved_success_bool -\bool_new:N \l_regex_success_match_bool +\tl_new:N \l_regex_internal_a_tl +\tl_new:N \l_regex_internal_b_tl +\int_new:N \l_regex_internal_a_int +\int_new:N \l_regex_internal_b_int +\int_new:N \l_regex_internal_c_int +\bool_new:N \l_regex_internal_bool +\seq_new:N \l_regex_internal_seq +\tl_new:N \g_regex_internal_tl % \end{macrocode} % \end{variable} -% \end{variable} -% \end{variable} -% -% \subsubsection{Regular expression variables} % % \begin{variable}{\c_regex_no_match_regex} % This regular expression matches nothing, but is still a valid -% regular expression. It is used as the initial value for regular +% regular expression. We could use a failing assertion, but I went for +% an empty class. It is used as the initial value for regular % expressions declared using \cs{regex_new:N}. % \begin{macrocode} \tl_const:Nn \c_regex_no_match_regex { - \regex_nfa:Nw \c_regex_no_match_regex - \l_regex_max_state_int = \c_one - \l_regex_capturing_group_int = \c_zero - \tex_toks:D \c_zero { \s_stop } - \s_stop - } -% \end{macrocode} -% \end{variable} -% -% \begin{macro}{\regex_new:N} -% As for many data types deriving from the \texttt{tl} data type, -% creating a new one is simply a matter of checking that it does not -% exist yet, and setting it equal to a default value. -% \begin{macrocode} -\cs_new_protected:Npn \regex_new:N #1 - { - \chk_if_free_cs:N #1 - \cs_gset_eq:NN #1 \c_regex_no_match_regex + \regex_branch:n + { \regex_class:NnnnN \c_true_bool { } { 1 } { 0 } \c_true_bool } } % \end{macrocode} -% \end{macro} -% -% \begin{variable}{\l_regex_internal_regex} -% This holds a temporary pre-compiled regular expression -% when matching a control sequence name. -% \begin{macrocode} -\regex_new:N \l_regex_internal_regex -% \end{macrocode} -% \end{variable} -% -% \subsubsection{Other variables} -%^^A todo: move? -%^^A todo: run with the check-declarations option. -% -% \begin{variable}{\l_regex_replacement_csnames_int} -% The behaviour of closing braces inside a replacement text depends on -% whether a sequences |\c{| or |\u{| has been encountered. The number -% of \enquote{open} such sequences that should be closed by |}| is -% stored in \cs{l_regex_replacement_csnames_int}, and decreased by -% $1$ by each |}|. -% \begin{macrocode} -\int_new:N \l_regex_replacement_csnames_int -% \end{macrocode} -% \end{variable} -% -% \begin{variable}{\l_regex_replacement_int} -% \begin{macrocode} -\int_new:N \l_regex_replacement_int -% \end{macrocode} -% \end{variable} -% -% \begin{variable}{\l_regex_submatch_int} -% \begin{variable}{\l_regex_submatch_start_int} -% \begin{macrocode} -\int_new:N \l_regex_submatch_int -\int_new:N \l_regex_submatch_start_int -% \end{macrocode} -% \end{variable} -% \end{variable} -% -% \begin{variable}{\l_regex_match_count_int} -% The number of matches found so far is stored -% in \cs{l_regex_match_count_int}. This is only used -% in the \cs{regex_count:nnN} functions. -% \begin{macrocode} -\int_new:N \l_regex_match_count_int -% \end{macrocode} % \end{variable} % -% \begin{variable}{regex_begin, regex_end} -% Those flags are raised to indicate extra begin-group -% or end-group tokens when extracting submatches. +% \begin{variable}{\l_regex_balance_int} +% The first thing we do when matching is to go once through the query +% token list and store the information for each token as \tn{muskip} +% and \tn{toks} registers. During this phase, \cs{l_regex_balance_int} +% counts the balance of begin-group and end-group character tokens +% which appear before a given point in the token list, and we store it +% as the shrink component of each \tn{muskip} register. This variable +% is also used to keep track of the balance in the replacement text. % \begin{macrocode} -\flag_new:n { regex_begin } -\flag_new:n { regex_end } +\int_new:N \l_regex_balance_int % \end{macrocode} % \end{variable} % -% \subsection{Helpers} -% -% \subsubsection{Toks} -% -% Two unrelated sets of functions for manipulating \tn{toks} registers. -% -% \begin{macro}[int]{\regex_toks_put_left:Nx} -% \begin{macro}[int]{\regex_toks_put_right:Nx} -% During the building phase, every \tn{toks} register starts with -% \cs{s_stop}, and we wish to add \texttt{x}-expanded material to -% those registers. The expansion is done \enquote{by hand} for -% optimization (these operations are used quite a lot). When adding -% material to the left, we define \cs{regex_tmp:w} to remove the -% \cs{s_stop} marker and put it back to the left of the new material. -% \begin{macrocode} -\cs_new_protected:Npn \regex_toks_put_left:Nx #1#2 - { - \cs_set_nopar:Npx \regex_tmp:w \s_stop { \s_stop #2 } - \tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN - { \exp_after:wN \regex_tmp:w \tex_the:D \tex_toks:D #1 } - } -\cs_new_protected:Npn \regex_toks_put_right:Nx #1#2 - { - \cs_set_nopar:Npx \regex_tmp:w {#2} - \tex_toks:D #1 \exp_after:wN - { \tex_the:D \tex_toks:D \exp_after:wN #1 \regex_tmp:w } - } -% \end{macrocode} -% \end{macro} -% \end{macro} -% -% \begin{macro}[int, rEXP]{\regex_toks_range:nn} -% \begin{macro}[aux, rEXP]{\regex_toks_range:ww} -% Some non-expandable functions store pieces of their result in -% \tn{toks} registers. Those pieces are concatenated using -% \cs{regex_toks_range:nn}, which expects two integers, \meta{start} -% and \meta{end}, and expands to the contents of the \tn{toks} -% registers from \meta{start} (inclusive) to \meta{end} -% (exclusive). This correctly expands to nothing if \meta{start} -% $\geq$ \meta{end}. -% \begin{macrocode} -\cs_new:Npn \regex_toks_range:nn #1#2 - { - \exp_after:wN \regex_toks_range:ww - \int_use:N \int_eval:w #1 \exp_after:wN ; - \int_use:N \int_eval:w #2 ; - \prg_break_point:n { } - } -\cs_new:Npn \regex_toks_range:ww #1 ; #2 ; - { - \if_num:w #1 < #2 \exp_stop_f: - \else: - \exp_after:wN \prg_map_break: - \fi: - \tex_the:D \tex_toks:D #1 \exp_stop_f: - \exp_after:wN \regex_toks_range:ww - \int_use:N \int_eval:w #1 + \c_one ; #2 ; - } -% \end{macrocode} -% \end{macro} -% \end{macro} -% -% On the one hand, when performing the matching, the \tn{toks} registers -% hold submatch information, followed by the instruction for a given -% state of the NFA. The two parts are separated by \cs{s_stop}. On the -% other hand, we provide functions to unpack the contents from a range -% of \tn{toks} within an \texttt{x}-expanding assignment. -% -% When it is time to extract submatches from the token list, the various -% tokens are stored in \tn{toks} registers numbered from -% \cs{l_regex_min_index_int} inclusive to \cs{l_regex_max_index_int} -% exclusive. Furthermore, the \tn{skip} registers hold\ldots{} ^^A ?? -% -% \begin{macro}[int]{\regex_query_submatch:nn} -% \begin{macro}[int]{\regex_query_submatch:w} -%^^A ?? -% \begin{macrocode} -\cs_new:Npn \regex_query_submatch:nn #1#2 - { - \if_num:w #1 < \l_regex_capturing_group_int - \exp_after:wN \regex_query_submatch:w - \int_use:N \int_eval:w #1 + #2 ; - \fi: - } -\cs_new:Npn \regex_query_submatch:w #1 ; - { - \regex_toks_range:nn - { \tex_skip:D #1 } - { \etex_gluestretch:D \tex_skip:D #1 } - } -% \end{macrocode} -% \end{macro} -% \end{macro} -% -% \subsubsection{Sequences} -% -% \begin{macro}[int]{\regex_seq_pop_int:NN} -% \begin{macro}[int]{\regex_seq_get_int:NN} -% \begin{macro}[int]{\regex_seq_push_int:NN} -% When building the regular expression, we keep track of some integers -% (pointers to various states) without help from \TeX{}'s grouping. -% Here are variants of \cs{seq_pop:NN} and \cs{seq_get:NN} which -% assign using \cs{int_set:Nn} rather than \cs{tl_set:Nn}. -% \begin{macrocode} -\cs_new_protected:Npn \regex_seq_pop_int:NN #1#2 - { - \seq_pop:NN #1 \l_regex_internal_a_tl - \int_set:Nn #2 \l_regex_internal_a_tl - } -\cs_new_protected:Npn \regex_seq_get_int:NN #1#2 - { - \seq_get:NN #1 \l_regex_internal_a_tl - \int_set:Nn #2 \l_regex_internal_a_tl - } -\cs_new_protected:Npn \regex_seq_push_int:NN #1#2 - { \seq_push:No #1 { \int_use:N #2 } } -% \end{macrocode} -% \end{macro} -% \end{macro} -% \end{macro} -% -% \begin{macro}[int]{\regex_seq_pop_use:N} -% \begin{macro}[int]{\regex_seq_get_use:N} -% When building the regular expression, some settings are kept -% local to capturing groups without any help from \TeX{}'s grouping. -% This is done \enquote{by hand}, in sequences whose items should -% be run immediately. -% \begin{macrocode} -\cs_new_protected:Npn \regex_seq_pop_use:N #1 - { - \seq_pop:NN #1 \l_regex_internal_a_tl - \l_regex_internal_a_tl - } -\cs_new_protected:Npn \regex_seq_get_use:N #1 - { - \seq_get:NN #1 \l_regex_internal_a_tl - \l_regex_internal_a_tl - } -% \end{macrocode} -% \end{macro} -% \end{macro} -% -% \subsubsection{Grabbing digits} -% -% \begin{macro}[int]{\regex_get_digits:nw} -% \begin{macro}[aux]{\regex_get_digits_loop:N,\regex_get_digits_end:w} -% When parsing the \texttt{\{} quantifier, we need a tool to grab -% digits until reaching the first non-digit. This -% \cs{regex_get_digits:nw} function places whatever digits it found as -% a brace group after |#1|. -% \begin{macrocode} -\cs_new_protected:Npn \regex_get_digits:nw #1 - { - \tex_afterassignment:D \regex_tmp:w - \cs_set_nopar:Npx \regex_tmp:w - { - \exp_not:n {#1} - { \if_false: } } \fi: - \regex_get_digits_aux:NN - } -\cs_new:Npn \regex_get_digits_aux:NN #1#2 - { - \if_meaning:w \regex_build_raw:N #1 - \if_num:w 9 < 1 \exp_not:N #2 \exp_stop_f: - #2 - \else: - \regex_get_digits_end:w #1 #2 - \fi: - \else: - \regex_get_digits_end:w #1 #2 - \fi: - \regex_get_digits_aux:NN - } -\cs_new:Npn \regex_get_digits_end:w #1 \fi: #2 \regex_get_digits_aux:NN - { - \fi: #2 - \if_false: { { \fi: } } - #1 - } -% \end{macrocode} -% \end{macro} -% \end{macro} -% % \subsubsection{Testing characters} % % \begin{macro}[int]{\regex_break_point:TF} @@ -1458,9 +898,23 @@ % \end{macro} % \end{macro} % -% \begin{macro}[int]{\regex_item_caseful_equal:n} -% \begin{macro}[int]{\regex_item_caseful_range:nn} -% \begin{macro}[int]{\regex_item_caseful_geq:n} +% \begin{macro}[int]{\regex_item_reverse:n} +% This function makes showing regular expressions easier, and lets us +% define |\D| in terms of |\d| for instance. There is a subtelty: the +% end of the query is marked by $-2$, and will thus match |\D| and +% other negated properties; this case is catched by another part of +% the code. +% \begin{macrocode} +\cs_new_protected:Npn \regex_item_reverse:n #1 + { + #1 + \regex_break_point:TF { } \regex_break_true:w + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int] +% {\regex_item_caseful_equal:n, \regex_item_caseful_range:nn} % Simple comparisons triggering \cs{regex_break_true:w} when true. % \begin{macrocode} \cs_new_protected:Npn \regex_item_caseful_equal:n #1 @@ -1477,29 +931,24 @@ \fi: \fi: } -\cs_new_protected:Npn \regex_item_caseful_geq:n #1 - { - \reverse_if:N \if_num:w #1 > \l_regex_current_char_int - \exp_after:wN \regex_break_true:w - \fi: - } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}[int]{\regex_item_caseless_equal:n} -% \begin{macro}[int]{\regex_item_caseless_range:nn} -% \begin{macro}[int]{\regex_item_caseless_geq:n} -% For caseless matching, we perform the test both on -% \cs{l_regex_current_char_int} and on -% \cs{l_regex_case_changed_char_int}. +% \begin{macro}[int] +% {\regex_item_caseless_equal:n, \regex_item_caseless_range:nn} +% For caseless matching, we perform the test both on the +% \texttt{current_char} and on the \texttt{case_changed_char}. Before +% doing the second set of tests, we make sure that +% \texttt{case_changed_char} has been computed. % \begin{macrocode} \cs_new_protected:Npn \regex_item_caseless_equal:n #1 { \if_num:w #1 = \l_regex_current_char_int \exp_after:wN \regex_break_true:w \fi: + \if_num:w \l_regex_case_changed_char_int = \c_max_int + \regex_compute_case_changed_char: + \fi: \if_num:w #1 = \l_regex_case_changed_char_int \exp_after:wN \regex_break_true:w \fi: @@ -1511,64 +960,57 @@ \exp_after:wN \exp_after:wN \exp_after:wN \regex_break_true:w \fi: \fi: + \if_num:w \l_regex_case_changed_char_int = \c_max_int + \regex_compute_case_changed_char: + \fi: \reverse_if:N \if_num:w #1 > \l_regex_case_changed_char_int \reverse_if:N \if_num:w #2 < \l_regex_case_changed_char_int \exp_after:wN \exp_after:wN \exp_after:wN \regex_break_true:w \fi: \fi: } -\cs_new_protected:Npn \regex_item_caseless_geq:n #1 - { - \reverse_if:N \if_num:w #1 > \l_regex_current_char_int - \exp_after:wN \regex_break_true:w - \fi: - \reverse_if:N \if_num:w #1 > \l_regex_case_changed_char_int - \exp_after:wN \regex_break_true:w - \fi: - } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}[int]{\regex_item_equal:n} -% \begin{macro}[int]{\regex_item_range:nn} -% \begin{macro}[int]{\regex_item_geq:n} -% By default, matching takes the letter case into account. -% Note that those functions are not protected: -% they will expand at the building phase, hard-coding which -% states take care of caseless versus caseful matching. +% \begin{macro}[int]{\regex_compute_case_changed_char:} +% This function is called when \cs{l_regex_case_changed_char_int} has +% not yet been computed (or rather, when it is set to the marker value +% \cs{c_max_int}). If the current character code is in the range +% $[65,90]$ (upper-case), then add $32$, making it lowercase. If it is +% in the lower-case letter range $[97,122]$, subtract $32$. % \begin{macrocode} -\cs_new:Npn \regex_item_equal:n { \regex_item_caseful_equal:n } -\cs_new:Npn \regex_item_range:nn { \regex_item_caseful_range:nn } -\cs_new:Npn \regex_item_geq:n { \regex_item_caseful_geq:n } +\cs_new_protected_nopar:Npn \regex_compute_case_changed_char: + { + \int_set_eq:NN \l_regex_case_changed_char_int \l_regex_current_char_int + \if_num:w \l_regex_current_char_int < \c_ninety_one + \if_num:w \l_regex_current_char_int < \c_sixty_five + \else: + \int_add:Nn \l_regex_case_changed_char_int { \c_thirty_two } + \fi: + \else: + \if_num:w \l_regex_current_char_int < \c_one_hundred_twenty_three + \if_num:w \l_regex_current_char_int < \c_ninety_seven + \else: + \int_sub:Nn \l_regex_case_changed_char_int { \c_thirty_two } + \fi: + \fi: + \fi: + } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}[int]{\regex_build_caseless:,\regex_build_caseful:} -% Switch between caseful and caseless matching. -% This is only done during the building phase. +% \begin{macro}[int, EXP]{\regex_item_equal:n, \regex_item_range:nn} +% Those must always be defined to expand to a \texttt{caseful} +% (default) or \texttt{caseless} version, and not be protected: they +% must expand when compiling, to hard-code which tests are caseless or +% caseful. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_caseless: - { - \bool_set_true:N \l_regex_caseless_bool - \cs_set:Npn \regex_item_equal:n { \regex_item_caseless_equal:n } - \cs_set:Npn \regex_item_range:nn { \regex_item_caseless_range:nn } - \cs_set:Npn \regex_item_geq:n { \regex_item_caseless_geq:n } - } -\cs_new_protected_nopar:Npn \regex_build_caseful: - { - \bool_set_false:N \l_regex_caseless_bool - \cs_set:Npn \regex_item_equal:n { \regex_item_caseful_equal:n } - \cs_set:Npn \regex_item_range:nn { \regex_item_caseful_range:nn } - \cs_set:Npn \regex_item_geq:n { \regex_item_caseful_geq:n } - } +\cs_new_eq:NN \regex_item_equal:n ? +\cs_new_eq:NN \regex_item_range:nn ? % \end{macrocode} % \end{macro} % -% \begin{macro}[int]{\regex_item_catcode:nT} +% \begin{macro}[int]{\regex_item_catcode:nT, \regex_item_catcode_reverse:nT} % \begin{macro}[aux]{\regex_item_catcode_aux:} % The argument is a sum of powers of $4$ with exponents given by the % allowed category codes (between $0$ and $13$). Dividing by a given @@ -1594,39 +1036,172 @@ \exp_after:wN \use_none:n \fi: } +\cs_new_protected:Npn \regex_item_catcode_reverse:nT #1#2 + { \regex_item_catcode:nT {#1} { \regex_item_reverse:n {#2} } } % \end{macrocode} % \end{macro} % \end{macro} % +% \begin{macro}[int]{\regex_item_exact:nn, \regex_item_exact_cs:c} +% This matches an exact \meta{category}-\meta{character code} pair, or +% an exact control sequence. +% \begin{macrocode} +\cs_new_protected:Npn \regex_item_exact:nn #1#2 + { + \if_num:w #1 = \l_regex_current_catcode_int + \if_num:w #2 = \l_regex_current_char_int + \exp_after:wN \exp_after:wN \exp_after:wN \regex_break_true:w + \fi: + \fi: + } +\cs_new_protected:Npn \regex_item_exact_cs:c #1 + { + \int_compare:nNnTF \l_regex_current_catcode_int = \c_zero + { + \str_if_eq:xxTF + { + \exp_after:wN \exp_after:wN \exp_after:wN \cs_to_str:N + \tex_the:D \tex_toks:D \l_regex_current_pos_int + } + { #1 } + { \regex_break_true:w } { } + } + { } + } +% \end{macrocode} +% \end{macro} +% % \begin{macro}[int]{\regex_item_cs:n} -% Match a control sequence (the argument is a pre-compiled regex). +% Match a control sequence (the argument is a compiled regex). % First test the catcode of the current token to be zero. % Then perform the matching test, and break if the csname % indeed matches. The three \cs{exp_after:wN} expand the contents -% of \cs{l_regex_current_token_tl} (of the form \cs{exp_not:n} +% of the \tn{toks}\meta{current position} (of the form \cs{exp_not:n} % \Arg{control sequence}) to \meta{control sequence}. % \begin{macrocode} \cs_new_protected:Npn \regex_item_cs:n #1 { \int_compare:nNnT \l_regex_current_catcode_int = \c_zero { - \tl_set:Nn \l_regex_internal_regex {#1} - \bool_set_eq:NN \l_regex_saved_success_bool \g_regex_success_bool - \exp_args:NNx \regex_match:NnTF \l_regex_internal_regex - { - \exp_after:wN \exp_after:wN - \exp_after:wN \cs_to_str:N \l_regex_current_token_tl - } - { - \bool_gset_eq:NN \g_regex_success_bool \l_regex_saved_success_bool - \regex_break_true:w - } - { \bool_gset_eq:NN \g_regex_success_bool \l_regex_saved_success_bool } + \group_begin: + \regex_single_match: + \regex_disable_submatches: + \regex_build_for_cs:n {#1} + \bool_set_eq:NN \l_regex_saved_success_bool \g_regex_success_bool + \exp_args:Nx \regex_match:n + { + \exp_after:wN \exp_after:wN + \exp_after:wN \cs_to_str:N + \tex_the:D \tex_toks:D \l_regex_current_pos_int + } + \if_meaning:w \c_true_bool \g_regex_success_bool + \group_insert_after:N \regex_break_true:w + \fi: + \bool_gset_eq:NN \g_regex_success_bool \l_regex_saved_success_bool + \group_end: } } % \end{macrocode} % \end{macro} % +% \subsubsection{Character property tests} +% +% \begin{macro}[aux] +% { +% \regex_prop_d:, \regex_prop_h:, \regex_prop_s:, +% \regex_prop_v:, \regex_prop_w:, \regex_prop_N: +% } +% Character property tests for |\d|, |\W|, \emph{etc.} These character +% properties are not affected by the |(?i)| option. The characters +% recognized by each one are as follows: |\d=[0-9]|, +% |\w=[0-9A-Z_a-z]|, \verb*+\s=[\ \^^I\^^J\^^L\^^M]+, +% \verb*+\h=[\ \^^I]+, |\v=[\^^J-\^^M]|, and the upper case +% counterparts match anything that the lower case does not match. The +% order in which the various tests appear is optimized for usual +% mostly lower case letter text. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \regex_prop_d: + { \regex_item_caseful_range:nn \c_forty_eight { 57 } } % 0--9 +\cs_new_protected_nopar:Npn \regex_prop_h: + { + \regex_item_caseful_equal:n \c_thirty_two % space + \regex_item_caseful_equal:n \c_nine % tab + } +\cs_new_protected_nopar:Npn \regex_prop_s: + { + \regex_item_caseful_equal:n \c_thirty_two % space + \regex_item_caseful_equal:n \c_nine % tab + \regex_item_caseful_equal:n \c_ten % lf + \regex_item_caseful_equal:n \c_twelve % ff + \regex_item_caseful_equal:n \c_thirteen % cr + } +\cs_new_protected_nopar:Npn \regex_prop_v: + { \regex_item_caseful_range:nn \c_ten \c_thirteen } % lf, vtab, ff, cr +\cs_new_protected_nopar:Npn \regex_prop_w: + { + \regex_item_caseful_range:nn \c_ninety_seven { 122 } % a--z + \regex_item_caseful_range:nn \c_sixty_five { 90 } % A--Z + \regex_item_caseful_range:nn \c_forty_eight { 57 } % 0--9 + \regex_item_caseful_equal:n { 95 } % _ + } +\cs_new_protected_nopar:Npn \regex_prop_N: + { \regex_item_reverse:n { \regex_item_caseful_equal:n \c_ten } } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux] +% { +% \regex_posix_alnum:, \regex_posix_alpha:, \regex_posix_ascii:, +% \regex_posix_blank:, \regex_posix_cntrl:, \regex_posix_digit:, +% \regex_posix_graph:, \regex_posix_lower:, \regex_posix_print:, +% \regex_posix_punct:, \regex_posix_space:, \regex_posix_upper:, +% \regex_posix_word: , \regex_posix_xdigit: +% } +% \textsc{posix} properties. No surprise. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \regex_posix_alnum: + { \regex_posix_alpha: \regex_posix_digit: } +\cs_new_protected_nopar:Npn \regex_posix_alpha: + { \regex_posix_lower: \regex_posix_upper: } +\cs_new_protected_nopar:Npn \regex_posix_ascii: + { \regex_item_caseful_range:nn \c_zero \c_one_hundred_twenty_seven } +\cs_new_eq:NN \regex_posix_blank: \regex_prop_h: +\cs_new_protected_nopar:Npn \regex_posix_cntrl: + { + \regex_item_caseful_range:nn \c_zero { 31 } + \regex_item_caseful_equal:n \c_one_hundred_twenty_seven + } +\cs_new_eq:NN \regex_posix_digit: \regex_prop_d: +\cs_new_protected_nopar:Npn \regex_posix_graph: + { \regex_item_caseful_range:nn { 33 } { 126 } } +\cs_new_protected_nopar:Npn \regex_posix_lower: + { \regex_item_caseful_range:nn \c_ninety_seven { 122 } } +\cs_new_protected_nopar:Npn \regex_posix_print: + { \regex_item_caseful_range:nn \c_thirty_two { 126 } } +\cs_new_protected_nopar:Npn \regex_posix_punct: + { + \regex_item_caseful_range:nn { 33 } { 47 } + \regex_item_caseful_range:nn { 58 } { 64 } + \regex_item_caseful_range:nn { 91 } { 96 } + \regex_item_caseful_range:nn { 123 } { 126 } + } +\cs_new_protected_nopar:Npn \regex_posix_space: + { + \regex_item_caseful_equal:n \c_thirty_two + \regex_item_caseful_range:nn \c_nine \c_thirteen + } +\cs_new_protected_nopar:Npn \regex_posix_upper: + { \regex_item_caseful_range:nn \c_sixty_five { 90 } } +\cs_new_eq:NN \regex_posix_word: \regex_prop_w: +\cs_new_protected_nopar:Npn \regex_posix_xdigit: + { + \regex_posix_digit: + \regex_item_caseful_range:nn \c_sixty_five { 70 } + \regex_item_caseful_range:nn \c_ninety_seven { 102 } + } +% \end{macrocode} +% \end{macro} +% % \subsubsection{Simple character escape} % % Before actually parsing the regular expression or the replacement @@ -1644,53 +1219,56 @@ % \meta{inline~3} \Arg{token list} The \meta{token list} is converted to % a string, then read from left to right, interpreting backslashes as % escaping the next character. Unescaped characters are fed to the -% function \meta{fn1}, and escaped characters are fed to the function -% \meta{fn2} within an \texttt{x}-expansion context (typically those +% function \meta{inline~1}, and escaped characters are fed to the function +% \meta{inline~2} within an \texttt{x}-expansion context (typically those % functions perform some tests on their argument to decide how to output % them). The escape sequences |\a|, |\e|, |\f|, |\n|, |\r|, |\t| and % |\x| are recognized, and those are replaced by the corresponding -% character, then fed to \meta{fn3}. The result is then left in the -% input stream. +% character, then fed to \meta{inline~3}. The result is then left in the +% input stream. Spaces are ignored unless escaped. % -% The idea is to feed unescaped characters to one function, -% escaped characters to another, and feed |\a|, |\e|, |\f|, -% |\n|, |\r|, |\t| and |\x| converted to the appropriate -% character to a third function. Spaces are ignored unless -% escaped. +% The conversion is mostly done within an \texttt{x}-expanding +% assignment, except for the |\x| escape sequence, which is not amenable +% to that in general. For this, we use the general framework of +% \cs{tl_set_build:Nw}. % % \begin{macro}[int]{\regex_escape_use:nnnn} -% \begin{macro}[aux]{\regex_escape_loop:N} -% \begin{macro}[aux]+\regex_escape_\:w+ -% Go through |#4| once, applying |#1|, |#2|, or |#3| to each character -% (after de-escaping it), then leave the result in the input -% stream. Most of the work is done within an \texttt{x}-expanding -% assignment, but the |\x| escape sequence cannot be done in that -% way. Therefore, we interrupt the assignment at each |\x| escape -% sequence, store the partial result in a \tn{toks} register, and at -% the end unpack all of the \tn{toks} registers to the left of the -% last chunk of \cs{l_regex_internal_a_tl}. +% The result is built in \cs{l_regex_internal_a_tl}, which is then +% left in the input stream. Go through |#4| once, applying |#1|, +% |#2|, or |#3| as relevant to each character (after de-escaping +% it). Note that we cannot replace \cs{tl_se:Nx} and +% \cs{tl_build_one:o} by a single call to \cs{tl_build_one:x}, because +% the \texttt{x}-expanding assignment is interrupted by |\x|. % \begin{macrocode} \cs_new_protected:Npn \regex_escape_use:nnnn #1#2#3#4 { - \group_begin: +%<trace> \trace_push:nnn { regex } { 1 } { regex_escape_use:nnnn } + \tl_set_build:Nw \l_regex_internal_a_tl \cs_set_nopar:Npn \regex_escape_unescaped:N ##1 { #1 } \cs_set_nopar:Npn \regex_escape_escaped:N ##1 { #2 } \cs_set_nopar:Npn \regex_escape_raw:N ##1 { #3 } \int_set:Nn \tex_escapechar:D { 92 } - \str_gset_other:Nn \g_regex_internal_tl {#4} - \int_zero:N \l_regex_internal_a_int - \tl_set:Nx \l_regex_internal_a_tl + \str_gset_other:Nn \g_regex_internal_tl { #4 } + \tl_set:Nx \l_regex_internal_b_tl { \exp_after:wN \regex_escape_loop:N \g_regex_internal_tl { break } \prg_break_point:n { } } - \use:x - { - \group_end: - \regex_toks_range:nn \c_zero \l_regex_internal_a_int - \exp_not:o \l_regex_internal_a_tl - } + \tl_build_one:o \l_regex_internal_b_tl + \tl_build_end: +%<trace> \trace_pop:nnn { regex } { 1 } { regex_escape_use:nnnn } + \l_regex_internal_a_tl } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_escape_loop:N} +% \begin{macro}[aux]+\regex_escape_\:w+ +% \cs{regex_escape_loop:N} reads one character: if it is special +% (space, backslash, or end-marker), perform the associated action, +% otherwise it is simply an unescaped character. After a backslash, +% the same is done, but unknown characters are \enquote{escaped}. +% \begin{macrocode} \cs_new:Npn \regex_escape_loop:N #1 { \cs_if_exist_use:cF { regex_escape_\token_to_str:N #1:w } @@ -1707,11 +1285,9 @@ % \end{macrocode} % \end{macro} % \end{macro} -% \end{macro} % -% \begin{macro}[aux]{\regex_escape_unescaped:N} -% \begin{macro}[aux]{\regex_escape_escaped:N} -% \begin{macro}[aux]{\regex_escape_raw:N} +% \begin{macro}[aux] +% {\regex_escape_unescaped:N, \regex_escape_escaped:N, \regex_escape_raw:N} % Those functions are never called before being given a new meaning, % so their definitions here don't matter. % \begin{macrocode} @@ -1720,20 +1296,18 @@ \cs_new_eq:NN \regex_escape_raw:N ? % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}[aux]+\regex_escape_\q_recursion_tail:w+ -% \begin{macro}[aux]+\regex_escape_/\q_recursion_tail:w+ -% \begin{macro}[aux]+\regex_escape_ :w+ % \begin{macro}[aux] % { +% \regex_escape_break:w, \regex_escape_/break:w, % \regex_escape_/a:w, \regex_escape_/e:w, \regex_escape_/f:w, % \regex_escape_/n:w, \regex_escape_/r:w, \regex_escape_/t:w % } -% The loop is ended upon seeing \cs{q_recursion_tail}. -% Spaces are ignored, and |\a|, |\e|, |\f|, |\n|, |\r|, |\t| take -% their meaning here. +% \begin{macro}[aux]+\regex_escape_ :w+ +% The loop is ended upon seeing the end-marker +% \enquote{\texttt{break}}, with an error if the string ended in a +% backslash. Spaces are ignored, and |\a|, |\e|, |\f|, |\n|, |\r|, +% |\t| take their meaning here. % \begin{macrocode} \cs_new_eq:NN \regex_escape_break:w \prg_map_break: \cs_new_nopar:cpn { regex_escape_/break:w } @@ -1758,126 +1332,149 @@ % \end{macrocode} % \end{macro} % \end{macro} -% \end{macro} -% \end{macro} % % \begin{macro}[aux]{\regex_escape_/x:w} -% \begin{macro}[aux]{\regex_escape_x_test:N} -% \begin{macro}[aux]{\regex_escape_x_unbraced_i:N} -% \begin{macro}[aux]{\regex_escape_x_unbraced_ii:N} -% \begin{macro}[aux]{\regex_escape_x_braced_loop:N} -% \begin{macro}[aux]{\regex_escape_x_braced_end:N} -% \begin{macro}[aux]{\regex_escape_x_end:} -% When |\x| is encountered, interrupt the assignment, -% and distinguish the cases of a braced or unbraced syntax. -% In the braced case, collect arbitrarily many hexadecimal digits, -% building the number in \cs{l_regex_current_char_int} (using -% \cs{str_aux_hexadecimal_use:NTF}), and check that -% the run of digits was stopped by a closing brace. -% In the unbraced case, collect up to two hexadecimal digits, -% possibly less, building the character number in \cs{l_regex_current_char_int}. -% In both cases, once all digits have been collected, use -% the \TeX{} primitive \tn{lowercase} to produce that character, -% and use an \cs{if_false:} trick to restart the assignment. -% \begin{macrocode} -\cs_new_nopar:cpn { regex_escape_/x:w } \regex_escape_loop:N +% \begin{macro}[aux]{\regex_escape_x_end:w, \regex_escape_x_large:n} +% When |\x| is encountered, \cs{regex_escape_x_test:N} is responsible +% for grabbing some hexadecimal digits, and feeding the result to +% \cs{regex_escape_x_end:w}. If the number is $<256$, then it is +% turned into a byte and fed to \cs{regex_escape_raw:N}. Otherwise, +% interrupt the assignment, and either produce an error, or use a +% standard \tn{lowercase} trick depending on the precise value. +% \begin{macrocode} +\cs_new:cpn { regex_escape_/x:w } \regex_escape_loop:N { - \if_false: { \fi: } - \tex_toks:D \l_regex_internal_a_int - \exp_after:wN { \l_regex_internal_a_tl } - \int_incr:N \l_regex_internal_a_int - \int_zero:N \l_regex_current_char_int - \regex_escape_x_test:N + \exp_after:wN \regex_escape_x_end:w + \int_value:w "0 \regex_escape_x_test:N } -\cs_new_protected:Npx \regex_escape_x_test:N #1 +\cs_new:Npn \regex_escape_x_end:w #1 ; { - \exp_not:N \token_if_eq_charcode:NNTF \c_space_token #1 - { \exp_not:N \regex_escape_x_test:N } + \int_compare:nNnTF {#1} < \c_two_hundred_fifty_six { - \l_regex_current_char_int = "0 - \exp_not:N \token_if_eq_charcode:NNTF \c_lbrace_str #1 - { \exp_not:N \regex_escape_x_braced_loop:N } - { \exp_not:N \regex_escape_x_unbraced_i:N #1 } + \exp_last_unbraced:Nf \regex_escape_raw:N + { \str_output_byte:n {#1} } } + { \regex_escape_x_large:n {#1} } } -\cs_new:Npn \regex_escape_x_unbraced_i:N #1 - { - \str_aux_hexadecimal_use:NTF #1 - { \regex_escape_x_unbraced_ii:N } - { \exp_stop_f: \regex_escape_x_end: #1 } - } -\cs_new:Npn \regex_escape_x_unbraced_ii:N #1 +\group_begin: + \char_set_catcode_other:n { 0 } + \cs_new:Npn \regex_escape_x_large:n #1 + { + \if_false: { \fi: } + \tl_build_one:o \l_regex_internal_b_tl + \int_compare:nNnTF {#1} > \c_max_char_int + { + \msg_kernel_error:nnx { regex } { x-overflow } {#1} + \tl_set:Nx \l_regex_internal_b_tl + { \if_false: } \fi: \regex_escape_loop:N + } + { + \char_set_lccode:nn { \c_zero } {#1} + \tl_to_lowercase:n + { + \tl_set:Nx \l_regex_internal_b_tl + { \if_false: } \fi: + \regex_escape_raw:N ^^@ + \regex_escape_loop:N + } + } + } +\group_end: +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[aux]{\regex_escape_x_test:N} +% Find out whether the first character is a left brace (allowing any +% number of hexadecimal digits), or not (allowing up to two +% hexadecimal digits). We need to check for the end-of-string marker. +% Eventually, call either \cs{regex_escape_x_loop:N} or +% \cs{regex_escape_x_ii:N}. +% \begin{macrocode} +\cs_new:Npn \regex_escape_x_test:N #1 { - \token_if_eq_charcode:NNTF \c_space_token #1 - { \regex_escape_x_unbraced_ii:N } + \str_if_eq:xxTF {#1} { break } { ; } { - \str_aux_hexadecimal_use:NTF #1 - { \exp_stop_f: \regex_escape_x_end: } - { \exp_stop_f: \regex_escape_x_end: #1 } + \if_charcode:w \c_space_token #1 + \exp_after:wN \regex_escape_x_test:N + \else: + \exp_after:wN \regex_escape_x_test_ii:N + \exp_after:wN #1 + \fi: } } -\cs_new:Npn \regex_escape_x_braced_loop:N #1 +\cs_new:Npn \regex_escape_x_test_ii:N #1 + { + \if_charcode:w \c_lbrace_str #1 + \exp_after:wN \regex_escape_x_loop:N + \else: + \str_aux_hexadecimal_use:NTF #1 + { \exp_after:wN \regex_escape_x_ii:N } + { ; \exp_after:wN \regex_escape_loop:N \exp_after:wN #1 } + \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_escape_x_ii:N} +% This looks for the second digit in the unbraced case. +% \begin{macrocode} +\cs_new:Npn \regex_escape_x_ii:N #1 { - \token_if_eq_charcode:NNTF \c_space_token #1 - { \regex_escape_x_braced_loop:N } + \str_if_eq:xxTF {#1} { break } { ; } { \str_aux_hexadecimal_use:NTF #1 - { \regex_escape_x_braced_loop:N } - { \exp_stop_f: \regex_escape_x_braced_end:N #1 } + { ; \regex_escape_loop:N } + { ; \regex_escape_loop:N #1 } } } -\cs_new_protected:Npx \regex_escape_x_braced_end:N #1 +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_escape_x_loop:N} +% Grab hexadecimal digits, skip spaces, and at the end, check that +% there is a right brace, otherwise raise an error outside the +% assignment. +% \begin{macrocode} +\cs_new:Npn \regex_escape_x_loop:N #1 { - \exp_not:N \token_if_eq_charcode:NNTF \c_rbrace_str #1 - { \exp_not:N \regex_escape_x_end: } + \str_aux_hexadecimal_use:NTF #1 + { \regex_escape_x_loop:N } { - \msg_kernel_error:nnx { regex } { x-missing-rbrace } - { \int_use:N \l_regex_current_char_int } - \exp_not:N \regex_escape_x_end: #1 + \token_if_eq_charcode:NNTF \c_space_token #1 + { \regex_escape_x_loop:N } + { + ; + \exp_after:wN \token_if_eq_charcode:NNTF \c_rbrace_str #1 + { \regex_escape_loop:N } + { + \if_false: { \fi: } + \tl_build_one:o \l_regex_internal_b_tl + \msg_kernel_error:nn { regex } { x-missing-rbrace } {#1} + \tl_set:Nx \l_regex_internal_b_tl + { \if_false: } \fi: \regex_escape_loop:N #1 + } + } } } -\group_begin: - \char_set_catcode_other:N \^^@ - \cs_new_protected_nopar:Npn \regex_escape_x_end: - { - \if_num:w \l_regex_current_char_int > \c_max_char_int - \msg_kernel_error:nnx { regex } { x-overflow } - { \int_use:N \l_regex_current_char_int } - \exp_after:wN \use:n - \else: - \tex_lccode:D \c_zero \l_regex_current_char_int - \exp_after:wN \tl_to_lowercase:n - \fi: - { - \tl_set:Nx \l_regex_internal_a_tl - { \if_false: } \fi: - \regex_escape_raw:N ^^@ - \regex_escape_loop:N - } - } -\group_end: % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}[EXP,aux]{\regex_char_if_alphanumeric:NTF} -% \begin{macro}[EXP,aux]{\regex_char_if_special:NTF} -% These two tests are used in the first pass when parsing a -% regular expression. That pass is responsible for finding -% escaped and non-escaped characters, and recognizing which -% ones have special meanings and which should be interpreted -% as \enquote{raw} characters. Namely, +% \begin{macro}[EXP, aux] +% {\regex_char_if_alphanumeric:NTF, \regex_char_if_special:NTF} +% These two tests are used in the first pass when parsing a regular +% expression. That pass is responsible for finding escaped and +% non-escaped characters, and recognizing which ones have special +% meanings and which should be interpreted as \enquote{raw} +% characters. Namely, % \begin{itemize} -% \item alphanumerics are \enquote{raw} if they are not escaped, -% and may have a special meaning when escaped; -% \item non-alphanumeric printable ascii characters are \enquote{raw} -% if they are escaped, and may have a special meaning when not escaped; -% \item characters other than printable ascii are always \enquote{raw}. +% \item alphanumerics are \enquote{raw} if they are not escaped, and +% may have a special meaning when escaped; +% \item non-alphanumeric printable ascii characters are +% \enquote{raw} if they are escaped, and may have a special +% meaning when not escaped; +% \item characters other than printable ascii are always +% \enquote{raw}. % \end{itemize} % The code is ugly, and highly based on magic numbers and the ascii % codes of characters. This is mostly unavoidable for performance @@ -1933,15 +1530,178 @@ } % \end{macrocode} % \end{macro} +% +% \subsection{Compiling} +% +% A regular expression starts its life as a string of characters. In +% this section, we convert it to internal instructions, resulting in a +% \enquote{compiled} regular expression. This compiled expression is +% then turned into states of an automaton in the building +% phase. Compiled regular expressions consist of the following: +% \begin{itemize} +% \item \cs{regex_class:NnnnN} \meta{boolean} \Arg{tests} \Arg{min} +% \Arg{more} \meta{lazyness} +% \item \cs{regex_group:nnnN} \Arg{branches} \Arg{min} \Arg{more} +% \meta{lazyness}, also \cs{regex_group_no_capture:nnnN} and +% \cs{regex_group_resetting:nnnN} with the same syntax. +% \item \cs{regex_branch:n} \Arg{contents} +% \item \cs{regex_command_K:} +% \item \cs{regex_assertion:Nn} \meta{boolean} \Arg{assertion test}, +% where the \meta{assertion test} is \cs{regex_b_test:} or +% |{|\cs{regex_anchor:N} \meta{integer}|}| +% \end{itemize} +% Tests can be the following: +% \begin{itemize} +% \item \cs{regex_item_caseful_equal:n} \Arg{char code} +% \item \cs{regex_item_caseless_equal:n} \Arg{char code} +% \item \cs{regex_item_caseful_range:nn} \Arg{min} \Arg{max} +% \item \cs{regex_item_caseless_range:nn} \Arg{min} \Arg{max} +% \item \cs{regex_item_catcode:nT} \Arg{catcode bitmap} \Arg{tests} +% \item \cs{regex_item_catcode_reverse:nT} \Arg{catcode bitmap} \Arg{tests} +% \item \cs{regex_item_reverse:n} \Arg{tests} +% \item \cs{regex_item_exact:nn} \Arg{catcode} \Arg{char code} +% \item \cs{regex_item_exact_cs:c} \Arg{csname} +% \item \cs{regex_item_cs:n} \Arg{compiled regex} +% \end{itemize} +% +% \subsubsection{Variables used when compiling} +% +% \begin{variable}{\l_regex_group_level_int} +% We make sure to open the same number of groups as we close. +% \begin{macrocode} +\int_new:N \l_regex_group_level_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_regex_mode_int} +% While compiling, ten modes are recognized, labelled $-63$, $-23$, +% $-6$, $-2$, $0$, $2$, $3$, $6$, $23$, $63$. See +% section~\ref{sec:regex-modes}. +% \begin{macrocode} +\int_new:N \l_regex_mode_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_regex_catcodes_int, \l_regex_default_catcodes_int} +% \begin{variable}{\l_regex_catcodes_bool} +% We wish to allow constructions such as |\c[^BE](..\cL[a-z]..)|, +% where the outer catcode test applies to the whole group, but is +% superseeded by the inner catcode test. For this to work, we need to +% keep track of lists of allowed category codes: +% \cs{l_regex_catcodes_int} and \cs{l_regex_default_catcodes_int} are +% bitmaps, sums of $4^c$, for all allowed catcodes $c$. The latter is +% local to each capturing group, and we reset +% \cs{l_regex_catcodes_int} to that value after each character or +% class, changing it only when encountering a |\c| escape. The boolean +% records whether the list of categories of a catcode test has to be +% inverted: compare |\c[^BE]| and |\c[BE]|. +% \begin{macrocode} +\int_new:N \l_regex_catcodes_int +\int_new:N \l_regex_default_catcodes_int +\bool_new:N \l_regex_catcodes_bool +% \end{macrocode} +% \end{variable} +% \end{variable} +% +% \begin{variable} +% { +% \c_regex_catcode_C_int, \c_regex_catcode_B_int, \c_regex_catcode_E_int, +% \c_regex_catcode_M_int, \c_regex_catcode_T_int, \c_regex_catcode_P_int, +% \c_regex_catcode_U_int, \c_regex_catcode_D_int, \c_regex_catcode_S_int, +% \c_regex_catcode_L_int, \c_regex_catcode_O_int, \c_regex_catcode_A_int +% } +% \begin{variable}{\c_regex_all_catcodes_int} +% Constants: $4^c$ for each category, and the sum of all powers of $4$. +% \begin{macrocode} +\int_const:Nn \c_regex_catcode_C_int { "1 } +\int_const:Nn \c_regex_catcode_B_int { "4 } +\int_const:Nn \c_regex_catcode_E_int { "10 } +\int_const:Nn \c_regex_catcode_M_int { "40 } +\int_const:Nn \c_regex_catcode_T_int { "100 } +\int_const:Nn \c_regex_catcode_P_int { "1000 } +\int_const:Nn \c_regex_catcode_U_int { "4000 } +\int_const:Nn \c_regex_catcode_D_int { "10000 } +\int_const:Nn \c_regex_catcode_S_int { "100000 } +\int_const:Nn \c_regex_catcode_L_int { "400000 } +\int_const:Nn \c_regex_catcode_O_int { "1000000 } +\int_const:Nn \c_regex_catcode_A_int { "4000000 } +\int_const:Nn \c_regex_all_catcodes_int { "5515155 } +% \end{macrocode} +% \end{variable} +% \end{variable} +% +% \begin{variable}{\l_regex_internal_regex} +% The compilation step stores its result in this variable. +% \begin{macrocode} +\cs_new_eq:NN \l_regex_internal_regex \c_regex_no_match_regex +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_regex_show_prefix_seq} +% This sequence holds the prefix that makes up the line displayed to +% the user. The various items must be removed from the right, which is +% tricky with a token list, hence we use a sequence. +% \begin{macrocode} +\seq_new:N \l_regex_show_prefix_seq +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_regex_show_lines_int} +% A hack. To know whether a given class has a single item in it or +% not, we count the number of lines when showing the class. +% \begin{macrocode} +\int_new:N \l_regex_show_lines_int +% \end{macrocode} +% \end{variable} +% +% \subsubsection{Generic helpers used when compiling} +% +% \begin{macro}[int]{\regex_get_digits:NTFw} +% \begin{macro}[aux, rEXP]{\regex_get_digits_loop:w} +% If followed by some raw digits, collect them one by one in the +% integer variable |#1|, and take the \texttt{true} branch. Otherwise, +% take the \texttt{false} branch. +% \begin{macrocode} +\cs_new_protected:Npn \regex_get_digits:NTFw #1#2#3#4#5 + { + \regex_if_raw_digit:NNTF #4 #5 + { #1 = #5 \regex_get_digits_loop:nw {#2} } + { #3 #4 #5 } + } +\cs_new:Npn \regex_get_digits_loop:nw #1#2#3 + { + \regex_if_raw_digit:NNTF #2 #3 + { #3 \regex_get_digits_loop:nw {#1} } + { \scan_stop: #1 #2 #3 } + } +% \end{macrocode} +% \end{macro} % \end{macro} % -% \subsection{Building} +% \begin{macro}[aux, EXP]{\regex_if_raw_digit:NNTF} +% Test used when grabbing digits for the |{m,n}| quantifier. +% It only accepts non-escaped digits. +% \begin{macrocode} +\prg_new_conditional:Npnn \regex_if_raw_digit:NN #1#2 { TF } + { + \if_meaning:w \regex_compile_raw:N #1 + \if_int_compare:w \c_one < 1 #2 \exp_stop_f: + \prg_return_true: + \else: + \prg_return_false: + \fi: + \else: + \prg_return_false: + \fi: + } +% \end{macrocode} +% \end{macro} % -% \subsubsection{Build mode} +% \subsubsection{Mode} % \label{sec:regex-modes} % -% When building the NFA corresponding to a given regex, we can be in -% ten distinct modes, which we label by some magic numbers: +% When compiling the \textsc{nfa} corresponding to a given regex string, +% we can be in ten distinct modes, which we label by some magic numbers: % \begin{itemize} % \item[-6] |[\c{...}]| control sequence in a class, % \item[-2] |\c{...}| control sequence, @@ -1955,8 +1715,8 @@ % \item[63] |[\c[...]]| class inside mode $6$. % \end{itemize} % This list is exhaustive, because |\c| escape sequences cannot be -% nested, and character classes cannot be nested directly. The choice -% of numbers is such as to optimize the most useful tests, and make +% nested, and character classes cannot be nested directly. The choice of +% numbers is such as to optimize the most useful tests, and make % transitions from one mode to another as simple as possible. % \begin{itemize} % \item Even modes mean that we are not directly in a character class. @@ -1980,15 +1740,15 @@ % any mode. % \end{itemize} % -% \begin{macro}[int, EXP]{\regex_build_if_in_class:TF} -% Test whether we are currently in a character class (at the -% inner-most level of nesting). There, many escape sequences are not -% recognized, and special characters are normal. Also, for every raw -% character, we must look ahead for a possible raw dash. +% \begin{macro}[int, EXP]{\regex_if_in_class:TF} +% Test whether we are directly in a character class (at the innermost +% level of nesting). There, many escape sequences are not recognized, +% and special characters are normal. Also, for every raw character, we +% must look ahead for a possible raw dash. % \begin{macrocode} -\cs_new_nopar:Npn \regex_build_if_in_class:TF +\cs_new_nopar:Npn \regex_if_in_class:TF { - \if_int_odd:w \l_regex_build_mode_int + \if_int_odd:w \l_regex_mode_int \exp_after:wN \use_i:nn \else: \exp_after:wN \use_ii:nn @@ -1997,16 +1757,35 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}{\regex_build_if_assertions_forbidden:TF} +% \begin{macro}[int, EXP]{\regex_if_in_cs:TF} +% Right braces are special only directly inside control sequences (at +% the inner-most level of nesting, not counting groups). +% \begin{macrocode} +\cs_new_nopar:Npn \regex_if_in_cs:TF + { + \if_int_odd:w \l_regex_mode_int + \exp_after:wN \use_ii:nn + \else: + \if_int_compare:w \l_regex_mode_int < \c_zero + \exp_after:wN \exp_after:wN \exp_after:wN \use_i:nn + \else: + \exp_after:wN \exp_after:wN \exp_after:wN \use_ii:nn + \fi: + \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int, EXP]{\regex_if_in_class_or_catcode:TF} % Assertions are only allowed in modes $0$, $-2$, and $-6$, % \emph{i.e.}, even, non-positive modes. % \begin{macrocode} -\cs_new_nopar:Npn \regex_build_if_assertions_forbidden:TF +\cs_new_nopar:Npn \regex_if_in_class_or_catcode:TF { - \if_int_odd:w \l_regex_build_mode_int + \if_int_odd:w \l_regex_mode_int \exp_after:wN \use_i:nn \else: - \if_int_compare:w \l_regex_build_mode_int > \c_zero + \if_int_compare:w \l_regex_mode_int > \c_zero \exp_after:wN \exp_after:wN \exp_after:wN \use_i:nn \else: \exp_after:wN \exp_after:wN \exp_after:wN \use_ii:nn @@ -2016,295 +1795,457 @@ % \end{macrocode} % \end{macro} % -% \subsubsection{Helpers for building an NFA} -% -% \begin{macro}[int]{\regex_build_new_state:} -% Add a new state to the NFA. At the end of the building phase, we -% want every \tn{toks} register to start with \cs{s_stop}, hence -% initialize the new register appropriately. Then update -% the left, right, and max \meta{states}. +% \begin{macro}[int, EXP]{\regex_if_within_catcode:TF} +% This test takes the true branch if we are in a catcode test, either +% immediately following it (modes $2$ and $6$) or in a class on which +% it applies (modes $23$ and $63$). This is used to tweak how left +% brackets behave in modes $2$ and $6$. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_new_state: +\cs_new_nopar:Npn \regex_if_within_catcode:TF { - \tex_toks:D \l_regex_max_state_int { \s_stop } - \int_set_eq:NN \l_regex_left_state_int \l_regex_right_state_int - \int_set_eq:NN \l_regex_right_state_int \l_regex_max_state_int - \int_incr:N \l_regex_max_state_int + \if_int_compare:w \l_regex_mode_int > \c_zero + \exp_after:wN \use_i:nn + \else: + \exp_after:wN \use_ii:nn + \fi: } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_transition:NN} -% \begin{macro}[aux]{\regex_build_transitions:NNNN} -% These functions create a new state, and put one or two transitions -% starting from the old current state. +% \begin{macro}[aux]{\regex_chk_c_allowed:T} +% The |\c| escape sequence is only allowed in modes $0$ and $3$, +% \emph{i.e.}, not within any other |\c| escape sequence. % \begin{macrocode} -\cs_new_protected:Npn \regex_build_transition:NN #1#2 +\cs_new_protected:Npn \regex_chk_c_allowed:T { - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int - { #1 { \int_eval:n { #2 - \l_regex_left_state_int } } } + \if_num:w \l_regex_mode_int = \c_zero + \exp_after:wN \use:n + \else: + \if_num:w \l_regex_mode_int = \c_three + \exp_after:wN \exp_after:wN \exp_after:wN \use:n + \else: + \msg_kernel_error:nn { regex } { c-bad-mode } + \exp_after:wN \exp_after:wN \exp_after:wN \use_none:n + \fi: + \fi: } -\cs_new_protected:Npn \regex_build_transitions:NNNN #1#2#3#4 +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_mode_quit_c:} +% This function changes the mode as it is needed just after a catcode +% test. +% \begin{macrocode} +\cs_new_protected:Npn \regex_mode_quit_c: { - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int - { - #1 { \int_eval:n { #2 - \l_regex_left_state_int } } - #3 { \int_eval:n { #4 - \l_regex_left_state_int } } - } + \if_num:w \l_regex_mode_int = \c_two + \l_regex_mode_int = \c_zero + \else: + \if_num:w \l_regex_mode_int = \c_six + \l_regex_mode_int = \c_three + \fi: + \fi: } % \end{macrocode} % \end{macro} -% \end{macro} % -% \subsubsection{From regex to NFA: framework} +% \subsubsection{Framework} % -% \begin{macro}[int]{\regex_build:w, \regex_build_end:} -% \begin{macro}[int]{\regex_build_exit_class:, \regex_build_exit_groups:} -% There are two situations where we want to convert a regular -% expression given as a string to an NFA. The first case is when the -% regex is given directly by the user, as an \texttt{n} argument to -% one of the \pkg{l3regex} functions, and the second case is when -% building the regex given as an argument of the |\c{...}| escape -% sequence within another regular expression. In both cases, some -% variables must be reset before starting to build the NFA, using -% \cs{regex_build:w}. Once building has ended (for instance when -% encountering the closing brace in |\c{...}|), we make sure to close -% any dangling class or open group, then add -% \cs{regex_action_success:} to make the match successful if it -% reaches the last state. The capturing group number is incremented: -% then its value tells us how many capturing group there are, -% including $0$. +% \begin{macro}[int]{\regex_compile:w, \regex_compile_end:} +% Used when compiling a user regex or a regex for the |\c{...}| escape +% sequence within another regex. Start building a token list within a +% group (with \texttt{x}-expansion at the outset), and set a few +% variables (group level, catcodes), then start the first branch. At +% the end, make sure there are no dangling classes nor groups, close +% the last branch: we are done building \cs{l_regex_internal_regex}. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build:w - { - \int_set_eq:NN \l_regex_catcodes_default_int \c_regex_catcodes_all_int - \int_set_eq:NN \l_regex_catcodes_int \l_regex_catcodes_default_int - \int_set_eq:NN \l_regex_capturing_group_int \c_zero - \seq_clear:N \l_regex_capturing_group_seq - \tl_clear:N \l_regex_tests_tl - \bool_set_true:N \l_regex_tests_bool - \int_zero:N \l_regex_max_state_int - \regex_build_new_state: - } -\cs_new_protected_nopar:Npn \regex_build_end: +\cs_new_protected_nopar:Npn \regex_compile:w { - \regex_build_exit_class: - \regex_build_exit_groups: - \regex_toks_put_right:Nx \l_regex_right_state_int - { \regex_action_success: } - \int_incr:N \l_regex_capturing_group_int + \tl_set_build_x:Nw \l_regex_internal_regex + \int_zero:N \l_regex_group_level_int + \int_set_eq:NN \l_regex_default_catcodes_int \c_regex_all_catcodes_int + \int_set_eq:NN \l_regex_catcodes_int \l_regex_default_catcodes_int + \cs_set_nopar:Npn \regex_item_equal:n { \regex_item_caseful_equal:n } + \cs_set_nopar:Npn \regex_item_range:nn { \regex_item_caseful_range:nn } + \tl_build_one:n { \regex_branch:n { \if_false: } \fi: } } -\cs_new_protected_nopar:Npn \regex_build_exit_class: +\cs_new_protected_nopar:Npn \regex_compile_end: { - \regex_build_if_in_class:TF - { - \msg_kernel_error:nn { regex } { missing-rbrack } - \use:c { regex_build_]: } - } - { } - } -\cs_new_protected_nopar:Npn \regex_build_exit_groups: - { - \seq_if_empty:NF \l_regex_capturing_group_seq - { + \regex_if_in_class:TF + { + \msg_kernel_error:nn { regex } { missing-rbrack } + \use:c { regex_compile_]: } + \prg_do_nothing: \prg_do_nothing: + } + { } + \if_num:w \l_regex_group_level_int > \c_zero \msg_kernel_error:nnx { regex } { missing-rparen } - { \seq_length:N \l_regex_capturing_group_seq } + { \int_use:N \l_regex_group_level_int } \prg_replicate:nn - { \seq_length:N \l_regex_capturing_group_seq } - { \regex_build_close_aux: \regex_build_group_: } - } + { \l_regex_group_level_int } + { + \tl_build_one:n + { + \if_false: { \fi: } + \if_false: { \fi: } { 1 } { 0 } \c_true_bool + } + \tl_build_end: + \tl_build_one:o \l_regex_internal_regex + } + \fi: + \tl_build_one:n { \if_false: { \fi: } } + \tl_build_end: } % \end{macrocode} % \end{macro} -% \end{macro} % -% \begin{macro}[int]{\regex_build:n} -% Setup some variables with \cs{regex_build:w}, and additionally set -% the build mode to $0$: we are neither in a class nor a |\c| -% construction. Initiate the regular expression by a wildcard: the -% search is unanchored and should be tried at every index in the token -% list. We surround the regular expression by parentheses with -% \cs{regex_build_open_aux:} and \cs{regex_build_close_aux:}, so that -% any alternative appears within a group (\emph{e.g.}, \verb"ab|cd" -% will now be within the external group); this also forms a capturing -% group (labelled $0$), which gives us the whole match as the $0$-th -% submatch. The regular expression itself is parsed by using the -% generic framework of \cs{regex_escape_use:nnnn} to recognize special -% characters, escaped ones, and escape sequences such as |\n| or -% |\x{02f}|. This results in successive \cs{regex_build_raw:N}, -% \cs{regex_build_escaped:N}, \cs{regex_build_special:N} followed by -% their arguments. The trailing \cs{prg_do_nothing:} ensure that the -% look-ahead done by some of the operations is harmless. Finally, -% \cs{regex_build_end:} adds the finishing code (checking that -% parentheses are properly nested, for instance). +% \begin{macro}[int]{\regex_compile:n} +% The compilation is done between \cs{regex_compile:w} and +% \cs{regex_compile_end:}, starting in mode~$0$. Then +% \cs{regex_escape_use:nnnn} distinguishes special characters, escaped +% alphanumerics, and raw characters, interpreting |\a|, |\x| and other +% sequences. The $4$ trailing \cs{prg_do_nothing:} are needed because +% some functions defined later look up to $4$ tokens ahead. Before +% ending, make sure that any |\c{...}| is properly closed. % \begin{macrocode} -\cs_new_protected:Npn \regex_build:n #1 +\cs_new_protected:Npn \regex_compile:n #1 { - \regex_build:w - \int_set_eq:NN \l_regex_build_mode_int \c_zero - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int - { \regex_action_start_wildcard: } - \regex_build_open_aux: - \regex_escape_use:nnnn - { - \regex_char_if_special:NTF ##1 - \regex_build_special:N \regex_build_raw:N ##1 - } - { - \regex_char_if_alphanumeric:NTF ##1 - \regex_build_escaped:N \regex_build_raw:N ##1 - } - { \regex_build_raw:N ##1 } - { #1 } - \prg_do_nothing: \prg_do_nothing: - \prg_do_nothing: \prg_do_nothing: - \int_compare:nNnT \l_regex_build_mode_int < \c_zero + \regex_compile:w + \int_set:Nn \tex_escapechar:D { 92 } + \int_set_eq:NN \l_regex_mode_int \c_zero + \regex_escape_use:nnnn + { + \regex_char_if_special:NTF ##1 + \regex_compile_special:N \regex_compile_raw:N ##1 + } + { + \regex_char_if_alphanumeric:NTF ##1 + \regex_compile_escaped:N \regex_compile_raw:N ##1 + } + { \regex_compile_raw:N ##1 } + { #1 } + \prg_do_nothing: \prg_do_nothing: + \prg_do_nothing: \prg_do_nothing: + \int_compare:nNnT \l_regex_mode_int < \c_zero { \msg_kernel_error:nn { regex } { c-missing-rbrace } - \exp_after:wN \regex_build_special:N \c_rbrace_str + \regex_compile_end: + \regex_compile_one:x + { \regex_item_cs:n { \exp_not:o \l_regex_internal_regex } } + \prg_do_nothing: \prg_do_nothing: + \prg_do_nothing: \prg_do_nothing: } - \seq_put_right:Nn \l_regex_capturing_group_seq {0} - \regex_build_close_aux: \regex_build_group_: - \regex_build_end: + \regex_compile_end: } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_special:N} -% \begin{macro}[aux]{\regex_build_escaped:N} +% \begin{macro}[int]{\regex_compile_escaped:N, \regex_compile_special:N} % If the special character or escaped alphanumeric has a particular % meaning in regexes, the corresponding function is used. Otherwise, % it is interpreted as a raw character. We distinguish special % characters from escaped alphanumeric characters because they behave % differently when appearing as an end-point of a range. % \begin{macrocode} -\cs_new_protected:Npn \regex_build_special:N #1 +\cs_new_protected:Npn \regex_compile_special:N #1 { - \cs_if_exist_use:cF { regex_build_#1: } - { \regex_build_raw:N #1 } + \cs_if_exist_use:cF { regex_compile_#1: } + { \regex_compile_raw:N #1 } } -\cs_new_protected:Npn \regex_build_escaped:N #1 +\cs_new_protected:Npn \regex_compile_escaped:N #1 { - \cs_if_exist_use:cF { regex_build_/#1: } - { \regex_build_raw:N #1 } + \cs_if_exist_use:cF { regex_compile_/#1: } + { \regex_compile_raw:N #1 } } % \end{macrocode} % \end{macro} -% \end{macro} % -% \begin{macro}[int]{\regex_build_one:n, \regex_build_one:x} -% In a class, add the argument to the current class. Outside a class, -% this argument is the only test, and we look for quantifiers. +% \begin{macro}[int]{\regex_compile_one:x} +% This is used after finding one \enquote{test}, such as |\d|, or a +% raw character. If that followed a catcode test (\emph{e.g.}, |\cL|), +% then restore the mode. If we are not in a class, then the test is +% \enquote{standalone}, and we need to add \cs{regex_class:NnnnN} and +% search for quantifiers. In any case, insert the test, possibly +% together with a catcode test if appropriate. % \begin{macrocode} -\cs_new_protected:Npn \regex_build_one:n #1 - { \regex_build_one:x { \exp_not:n {#1} } } -\cs_new_protected:Npn \regex_build_one:x #1 +\cs_new_protected:Npn \regex_compile_one:x #1 { - \tl_put_right:Nx \l_regex_tests_tl + \regex_mode_quit_c: + \regex_if_in_class:TF { } + { + \tl_build_one:n + { \regex_class:NnnnN \c_true_bool { \if_false: } \fi: } + } + \tl_build_one:x { - \if_num:w \l_regex_catcodes_int < \c_regex_catcodes_all_int + \if_num:w \l_regex_catcodes_int < \c_regex_all_catcodes_int \regex_item_catcode:nT { \int_use:N \l_regex_catcodes_int } + { \exp_not:N \exp_not:n {#1} } \else: - \exp_after:wN \use:n + \exp_not:N \exp_not:n {#1} \fi: - {#1} } - \int_set_eq:NN \l_regex_catcodes_int \l_regex_catcodes_default_int - \if_num:w \l_regex_build_mode_int = \c_two - \l_regex_build_mode_int = \c_zero - \else: - \if_num:w \l_regex_build_mode_int = \c_six - \l_regex_build_mode_int = \c_three - \fi: - \fi: - \if_int_odd:w \l_regex_build_mode_int \else: - \exp_after:wN \regex_build_one_quantifier: - \fi: + \int_set_eq:NN \l_regex_catcodes_int \l_regex_default_catcodes_int + \regex_if_in_class:TF { } { \regex_compile_quantifier:w } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int] +% {\regex_compile_abort_tokens:n, \regex_compile_abort_tokens:x} +% This function places the collected tokens back in the input stream, +% each as a raw character. Spaces are not preserved. +% \begin{macrocode} +\cs_new_protected:Npn \regex_compile_abort_tokens:n #1 + { + \use:x + { + \exp_args:No \tl_map_function:nN { \tl_to_str:n {#1} } + \regex_compile_raw:N + } + } +\cs_generate_variant:Nn \regex_compile_abort_tokens:n { x } +% \end{macrocode} +% \end{macro} +% +% \subsubsection{Quantifers} +% +% \begin{macro}[int]{\regex_compile_quantifier:w} +% This looks ahead and finds any quantifier (special character equal +% to either of \texttt{?+*\{}). +% \begin{macrocode} +\cs_new_protected:Npn \regex_compile_quantifier:w #1#2 + { + \token_if_eq_meaning:NNTF #1 \regex_compile_special:N + { + \cs_if_exist_use:cF { regex_compile_quantifier_#2:w } + { \regex_compile_quantifier_none: #1 #2 } + } + { \regex_compile_quantifier_none: #1 #2 } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_compile_quantifier_none:} +% \begin{macro}[aux]{\regex_compile_quantifier_abort:xNN} +% Those functions are called whenever there is no quantifier, or a +% braced construction is invalid (equivalent to no quantifier, and +% whatever characters were grabbed are left raw). +% \begin{macrocode} +\cs_new_protected:Npn \regex_compile_quantifier_none: + { \tl_build_one:n { \if_false: { \fi: } { 1 } { 0 } \c_false_bool } } +\cs_new_protected:Npn \regex_compile_quantifier_abort:xNN #1#2#3 + { + \regex_compile_quantifier_none: + \msg_kernel_warning:nnxx { regex } { invalid-quantifier } {#1} {#3} + \regex_compile_abort_tokens:x {#1} + #2 #3 + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[aux]{\regex_compile_quantifier_lazyness:nnNN} +% Once the \enquote{main} quantifier (\texttt{?}, \texttt{*}, +% \texttt{+} or a braced construction) is found, we check whether it +% is lazy (followed by a question mark). We then add to the compiled +% regex a closing brace (ending \cs{regex_class:NnnnN} and friends), +% the start-point of the range, its end-point, and a boolean, +% \texttt{true} for lazy and \texttt{false} for greedy operators. +% \begin{macrocode} +\cs_new_protected:Npn \regex_compile_quantifier_lazyness:nnNN #1#2#3#4 + { + \str_if_eq:nnTF { #3 #4 } { \regex_compile_special:N ? } + { \tl_build_one:n { \if_false: { \fi: } { #1 } { #2 } \c_true_bool } } + { + \tl_build_one:n { \if_false: { \fi: } { #1 } { #2 } \c_false_bool } + #3 #4 + } } % \end{macrocode} % \end{macro} % -% \begin{macro}{\regex_tests_action_cost:n} -% The argument is the target state if the test succeeds. +% \begin{macro}[aux] +% { +% \regex_compile_quantifier_?:w, +% \regex_compile_quantifier_*:w, +% \regex_compile_quantifier_+:w +% } +% For each \enquote{basic} quantifier, |?|, |*|, |+|, feed the correct +% arguments to \cs{regex_compile_quantifier_lazyness:nnNN}, $-1$ means +% that there is no upper bound on the number of repetitions. +% \begin{macrocode} +\cs_new_protected_nopar:cpn { regex_compile_quantifier_?:w } + { \regex_compile_quantifier_lazyness:nnNN { 0 } { 1 } } +\cs_new_protected_nopar:cpn { regex_compile_quantifier_*:w } + { \regex_compile_quantifier_lazyness:nnNN { 0 } { -1 } } +\cs_new_protected_nopar:cpn { regex_compile_quantifier_+:w } + { \regex_compile_quantifier_lazyness:nnNN { 1 } { -1 } } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]+\regex_compile_quantifier_{:w+ ^^A} +% \begin{macro}[aux] +% { +% \regex_compile_quantifier_braced_i:w, +% \regex_compile_quantifier_braced_ii:w, +% \regex_compile_quantifier_braced_iii:w, +% } +% Three possible syntaxes: \texttt{\{\meta{int}\}}, +% \texttt{\{\meta{int},\}}, or \texttt{\{\meta{int},\meta{int}\}}. Any +% other syntax causes us to abort and put whatever we collected back +% in the input stream, as \texttt{raw} characters, including the +% opening brace. Grab a number into \cs{l_regex_internal_a_int}. If +% the number is followed by a right brace, the range is $[a,a]$. If +% followed by a comma, grab one more number, and call the \texttt{_ii} +% or \texttt{_iii} auxiliary. Those auxiliaries check for a closing +% brace, leading to the range $[a,\infty]$ or $[a,b]$, encoded as +% $\{a\}\{-1\}$ and $\{a\}\{b-a\}$. % \begin{macrocode} -\cs_new:Npn \regex_tests_action_cost:n #1 +\cs_new_protected:cpn { regex_compile_quantifier_ \c_lbrace_str :w } + { + \regex_get_digits:NTFw \l_regex_internal_a_int + { \regex_compile_quantifier_braced_i:w } + { \regex_compile_quantifier_abort:xNN { \c_lbrace_str } } + } +\cs_new_protected:Npn \regex_compile_quantifier_braced_i:w #1#2 + { + \prg_case_str:xxn { #1 #2 } + { + { \regex_compile_special:N \c_rbrace_str } + { + \exp_args:No \regex_compile_quantifier_lazyness:nnNN + { \int_use:N \l_regex_internal_a_int } { 0 } + } + { \regex_compile_special:N , } + { + \regex_get_digits:NTFw \l_regex_internal_b_int + { \regex_compile_quantifier_braced_iii:w } + { \regex_compile_quantifier_braced_ii:w } + } + } + { + \regex_compile_quantifier_abort:xNN + { \c_lbrace_str \int_use:N \l_regex_internal_a_int } + #1 #2 + } + } +\cs_new_protected:Npn \regex_compile_quantifier_braced_ii:w #1#2 + { + \str_if_eq:xxTF + { #1 #2 } { \regex_compile_special:N \c_rbrace_str } + { + \exp_args:No \regex_compile_quantifier_lazyness:nnNN + { \int_use:N \l_regex_internal_a_int } { -1 } + } + { + \regex_compile_quantifier_abort:xNN + { \c_lbrace_str \int_use:N \l_regex_internal_a_int , } + #1 #2 + } + } +\cs_new_protected:Npn \regex_compile_quantifier_braced_iii:w #1#2 { - \exp_not:o \l_regex_tests_tl - \bool_if:NTF \l_regex_tests_bool - { \regex_break_point:TF { \regex_action_cost:n {#1} } { } } - { \regex_break_point:TF { } { \regex_action_cost:n {#1} } } + \str_if_eq:xxTF + { #1 #2 } { \regex_compile_special:N \c_rbrace_str } + { + \if_num:w \l_regex_internal_a_int > \l_regex_internal_b_int + \msg_kernel_error:nnxx { regex } { backwards-quantifier } + { \int_use:N \l_regex_internal_a_int } + { \int_use:N \l_regex_internal_b_int } + \int_zero:N \l_regex_internal_b_int + \else: + \int_sub:Nn \l_regex_internal_b_int \l_regex_internal_a_int + \fi: + \exp_args:Noo \regex_compile_quantifier_lazyness:nnNN + { \int_use:N \l_regex_internal_a_int } + { \int_use:N \l_regex_internal_b_int } + } + { + \regex_compile_quantifier_abort:xNN + { + \c_lbrace_str + \int_use:N \l_regex_internal_a_int , + \int_use:N \l_regex_internal_b_int + } + #1 #2 + } } % \end{macrocode} % \end{macro} +% \end{macro} % % \subsubsection{Raw characters} % -% \begin{macro}[int]{\regex_build_raw_error:N} -% Within character classes, some escaped alphanumeric sequences such -% as |\b| do not have any meaning. They are replaced by a raw -% character, after spitting out an error. +% \begin{macro}[int]{\regex_compile_raw_error:N} +% Within character classes, and following catcode tests, some escaped +% alphanumeric sequences such as |\b| do not have any meaning. They +% are replaced by a raw character, after spitting out an error. % \begin{macrocode} -\cs_new_protected:Npn \regex_build_raw_error:N #1 +\cs_new_protected:Npn \regex_compile_raw_error:N #1 { - \if_int_odd:w \l_regex_build_mode_int - \msg_kernel_error:nnx { regex } { class-bad-escape } {#1} - \else: - \msg_kernel_error:nnx { regex } { catcode-bad-escape } {#1} - \fi: - \regex_build_raw:N #1 + \msg_kernel_error:nnx { regex } { bad-escape } {#1} + \regex_compile_raw:N #1 } % \end{macrocode} % \end{macro} % -% \begin{macro}[int]{\regex_build_raw:N} +% \begin{macro}[int]{\regex_compile_raw:N} % If we are in a character class and the next character is an % unescaped dash, this denotes a range. Otherwise, the current % character |#1| matches itself. % \begin{macrocode} -\cs_new_protected:Npn \regex_build_raw:N #1#2#3 +\cs_new_protected:Npn \regex_compile_raw:N #1#2#3 { - \regex_build_if_in_class:TF - { \str_if_eq:nnTF {#2#3} { \regex_build_special:N - } } - { \use_ii:nn } - { \regex_class_range:Nw #1 } + \regex_if_in_class:TF { - \regex_build_one:x { \regex_item_equal:n { \int_value:w `#1 ~ } } + \str_if_eq:nnTF {#2#3} { \regex_compile_special:N - } + { \regex_compile_range:Nw #1 } + { + \regex_compile_one:x + { \regex_item_equal:n { \int_value:w `#1 ~ } } + #2 #3 + } + } + { + \regex_compile_one:x + { \regex_item_equal:n { \int_value:w `#1 ~ } } #2 #3 } } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_class_range:Nw} +% \begin{macro}[aux]{\regex_compile_range:Nw, \regex_if_end_range:NNTF} % We have just read a raw character followed by a dash; this should be -% followed by an end-point for the range. If the following character -% is an escaped alphanumeric, or if it is an unescaped right bracket, -% then we have an error, so we put the initial character and the dash -% back, as raw characters. Otherwise, build the range, checking that -% it is in the right order, and optimizing for equal end-points. +% followed by an end-point for the range. Valid end-points are: any +% raw character; any special character, except a right bracket. In +% particular, escaped characters are forbidden. % \begin{macrocode} -\cs_new_protected:Npn \regex_class_range:Nw #1#2#3 +\prg_new_protected_conditional:Npnn \regex_if_end_range:NN #1#2 { TF } { - \token_if_eq_meaning:NNTF #2 \regex_build_escaped:N % [ - { \use_i:nn } { \str_if_eq:nnTF { #2#3 } { \regex_build_special:N ] } } - { - \msg_kernel_warning:nnxx { regex } { range-missing-end } % [ - {#1} { \token_if_eq_charcode:NNF #3 ] { \c_backslash_str } #3 } - \tl_put_right:Nx \l_regex_tests_tl - { - \regex_item_equal:n { \int_value:w `#1 ~ } - \regex_item_equal:n { \int_value:w `- ~ } - } - #2#3 - } + \if_meaning:w \regex_compile_raw:N #1 + \prg_return_true: + \else: + \if_meaning:w \regex_compile_special:N #1 + \if_charcode:w ] #2 + \prg_return_false: + \else: + \prg_return_true: + \fi: + \else: + \prg_return_false: + \fi: + \fi: + } +\cs_new_protected:Npn \regex_compile_range:Nw #1#2#3 + { + \regex_if_end_range:NNTF #2 #3 { \if_num:w `#1 > `#3 \exp_stop_f: \msg_kernel_error:nnxx { regex } { range-backwards } {#1} {#3} \else: - \tl_put_right:Nx \l_regex_tests_tl + \tl_build_one:x { \if_num:w `#1 = `#3 \exp_stop_f: \regex_item_equal:n @@ -2315,1796 +2256,2155 @@ } \fi: } + { + \msg_kernel_warning:nnxx { regex } { range-missing-end } + {#1} { \c_backslash_str #3 } + \tl_build_one:x + { + \regex_item_equal:n { \int_value:w `#1 ~ } + \regex_item_equal:n { \int_value:w `- ~ } + } + #2#3 + } } % \end{macrocode} % \end{macro} % % \subsubsection{Character properties} % -% \begin{macro}[aux]{\regex_build_.:} +% \begin{macro}[aux]{\regex_compile_.:, \regex_prop_.:} % In a class, the dot has no special meaning. Outside, insert -% \cs{c_regex_._tl}, which matches any character or control -% sequence, and refuses $-2$, which marks the end of the token list. +% \cs{regex_prop_.:}, which matches any character or control +% sequence, and refuses $-2$ (end-marker). % \begin{macrocode} -\cs_new_protected_nopar:cpx { regex_build_.: } +\cs_new_protected_nopar:cpx { regex_compile_.: } { - \exp_not:N \regex_build_if_in_class:TF - { \regex_build_raw:N . } - { \regex_build_one:n \exp_not:c { c_regex_._tl } } + \exp_not:N \regex_if_in_class:TF + { \regex_compile_raw:N . } + { \regex_compile_one:x \exp_not:c { regex_prop_.: } } + } +\cs_new_protected_nopar:cpn { regex_prop_.: } + { + \if_num:w \l_regex_current_char_int > - \c_two + \exp_after:wN \regex_break_true:w + \fi: } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_/d:,\regex_build_/D:} -% \begin{macro}[aux]{\regex_build_/h:,\regex_build_/H:} -% \begin{macro}[aux]{\regex_build_/s:,\regex_build_/S:} -% \begin{macro}[aux]{\regex_build_/v:,\regex_build_/V:} -% \begin{macro}[aux]{\regex_build_/w:,\regex_build_/W:} -% \begin{macro}[aux]{\regex_build_/N:} -% The constants \cs{c_regex_d_tl}, \emph{etc.} hold +% \begin{macro}[aux] +% { +% \regex_compile_/d:, \regex_compile_/D:, +% \regex_compile_/h:, \regex_compile_/H:, +% \regex_compile_/s:, \regex_compile_/S:, +% \regex_compile_/v:, \regex_compile_/V:, +% \regex_compile_/w:, \regex_compile_/W:, +% \regex_compile_/N:, +% } +% The constants \cs{regex_prop_d:}, \emph{etc.} hold % a list of tests which match the corresponding character % class, and jump to the \cs{regex_break_point:TF} marker. % As for a normal character, we check for quantifiers. % \begin{macrocode} -\tl_map_inline:nn { dDhHsSvVwWN } +\cs_set_protected:Npn \regex_tmp:w #1#2 { - \cs_new_protected_nopar:cpx { regex_build_/#1: } - { \regex_build_one:n \exp_not:c { c_regex_#1_tl } } + \cs_new_protected_nopar:cpx { regex_compile_/#1: } + { \regex_compile_one:x \exp_not:c { regex_prop_#1: } } + \cs_new_protected_nopar:cpx { regex_compile_/#2: } + { + \regex_compile_one:x + { \regex_item_reverse:n \exp_not:c { regex_prop_#1: } } + } } +\regex_tmp:w d D +\regex_tmp:w h H +\regex_tmp:w s S +\regex_tmp:w v V +\regex_tmp:w w W +\cs_new_protected_nopar:cpn { regex_compile_/N: } + { \regex_compile_one:x \regex_prop_N: } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} % % \subsubsection{Anchoring and simple assertions} % -% \begin{macro}[aux]{\regex_build_simple_assertion:nn} -% Assertions are only allowed in modes $0$, $-2$ and $-6$. In -% character classes, or immediately following a category code test, -% escaped letters cause errors, while the meaning of special -% characters is ignored: |#1| is used instead. In modes where the -% assertion is allowed, we insert the test |#2|; if the assertion is -% successful, move from the \texttt{left} state to the \texttt{right} -% state. +% \begin{macro}[aux]{\regex_compile_anchor:NF} +% \begin{macro}[aux]+\regex_compile_^:+ +% \begin{macro}[aux]{\regex_compile_/A:, \regex_compile_/G:} +% \begin{macro}[aux]+\regex_compile_$:+ +% \begin{macro}[aux]{\regex_compile_/Z:, \regex_compile_/z:} +% In modes where assertions are allowed, anchor to the start of the +% query, the start of the match, or the end of the query, depending on +% the integer |#1|. In other modes, |#2| treats the character as raw, +% with an error for escaped letters (|$| is valid in a class, but |\A| +% is definitely a mistake on the user's part). % \begin{macrocode} -\cs_new_protected:Npn \regex_build_simple_assertion:nn #1#2 +\cs_new_protected:Npn \regex_compile_anchor:NF #1#2 { - \regex_build_if_assertions_forbidden:TF {#1} + \regex_if_in_class_or_catcode:TF {#2} { - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int - { - \exp_not:n {#2} - { - \regex_action_free:n - { - \int_eval:n - { \l_regex_right_state_int - \l_regex_left_state_int } - } - } - } - %^^A todo: add \regex_assertion_quantifier: + \tl_build_one:n + { \regex_assertion:Nn \c_true_bool { \regex_anchor:N #1 } } } } -% \end{macrocode} -% \end{macro} -% -% \begin{macro}[aux]+\regex_build_^:+ -% \begin{macro}[aux]{\regex_build_/A:} -% \begin{macro}[aux]{\regex_build_/G:} -% \begin{macro}[aux]+\regex_build_$:+ -% \begin{macro}[aux]{\regex_build_/Z:} -% \begin{macro}[aux]{\regex_build_/z:} -% Anchoring at the start corresponds to checking that the current -% character is the first in the token list. Anchoring to the beginning -% of the match attempt uses \cs{l_regex_start_index_int} instead of -% \cs{c_zero}. End anchors match the end of the token list, marked by -% a character code of $-2$. -% \begin{macrocode} -\cs_new_protected_nopar:cpn { regex_build_^: } - { - \regex_build_simple_assertion:nn { \regex_build_raw:N ^ } - { \int_compare:nNnT \l_regex_min_index_int = \l_regex_current_index_int } - } -\cs_new_protected_nopar:cpn { regex_build_/A: } - { - \regex_build_simple_assertion:nn { \regex_build_raw_error:N A } - { \int_compare:nNnT \l_regex_min_index_int = \l_regex_current_index_int } - } -\cs_new_protected_nopar:cpn { regex_build_/G: } - { - \regex_build_simple_assertion:nn { \regex_build_raw_error:N G } - { \int_compare:nNnT \l_regex_start_index_int = \l_regex_current_index_int } - } -\group_begin: - \char_set_catcode_other:N \$ - \cs_new_protected_nopar:cpn { regex_build_$: } % $ - { - \regex_build_simple_assertion:nn { \regex_build_raw:N $ } % $ - { \int_compare:nNnT \l_regex_current_char_int < \c_minus_one } - } -\group_end: -\cs_new_protected_nopar:cpn { regex_build_/Z: } +\cs_set_protected:Npn \regex_tmp:w #1#2 { - \regex_build_simple_assertion:nn { \regex_build_raw_error:N Z } - { \int_compare:nNnT \l_regex_current_char_int < \c_minus_one } + \cs_new_protected_nopar:cpn { regex_compile_/#1: } + { \regex_compile_anchor:NF #2 { \regex_compile_raw_error:N #1 } } } -\cs_new_protected_nopar:cpn { regex_build_/z: } +\regex_tmp:w A \l_regex_min_pos_int +\regex_tmp:w G \l_regex_start_pos_int +\regex_tmp:w Z \l_regex_max_pos_int +\regex_tmp:w z \l_regex_max_pos_int +\cs_set_protected:Npn \regex_tmp:w #1#2 { - \regex_build_simple_assertion:nn { \regex_build_raw_error:N z } - { \int_compare:nNnT \l_regex_current_char_int < \c_minus_one } + \cs_new_protected_nopar:cpn { regex_compile_#1: } + { \regex_compile_anchor:NF #2 { \regex_compile_raw:N #1 } } } +\exp_args:Nx \regex_tmp:w { \iow_char:N \^ } \l_regex_min_pos_int +\exp_args:Nx \regex_tmp:w { \iow_char:N \$ } \l_regex_max_pos_int % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} -% \end{macro} % -% \begin{macro}[int]{\regex_build_/b:} -% \begin{macro}[int]{\regex_build_/B:} -% \begin{macro}[aux]{\regex_if_word_boundary:TF} -% Contrarily to |^| and |$|, which could be implemented without -% really knowing what precedes in the token list, this requires -% more information, namely, the knowledge of the last character -% code. Case sensitivity does not change word boundaries. +% \begin{macro}[aux]{\regex_compile_/b:, \regex_compile_/B:} +% Contrarily to |^| and |$|, which could be implemented without really +% knowing what precedes in the token list, this requires more +% information, namely, the knowledge of the last character code. % \begin{macrocode} -\cs_new_protected_nopar:cpn { regex_build_/b: } - { - \regex_build_simple_assertion:nn { \regex_build_raw_error:N b } - { \regex_if_word_boundary:T } - } -\cs_new_protected_nopar:cpn { regex_build_/B: } +\cs_new_protected_nopar:cpn { regex_compile_/b: } { - \regex_build_simple_assertion:nn { \regex_build_raw_error:N B } - { \regex_if_word_boundary:TF { } } + \regex_if_in_class_or_catcode:TF + { \regex_compile_raw_error:N b } + { + \tl_build_one:n + { \regex_assertion:Nn \c_true_bool { \regex_b_test: } } + } } -\cs_new_protected_nopar:Npn \regex_if_word_boundary:TF +\cs_new_protected_nopar:cpn { regex_compile_/B: } { - \group_begin: - \int_set_eq:NN \l_regex_current_char_int \l_regex_last_char_int - \c_regex_w_tl - \regex_break_point:TF - { \group_end: \c_regex_W_tl \regex_item_equal:n { -2 } } - { \group_end: \c_regex_w_tl } - \regex_break_point:TF + \regex_if_in_class_or_catcode:TF + { \regex_compile_raw_error:N B } + { + \tl_build_one:n + { \regex_assertion:Nn \c_false_bool { \regex_b_test: } } + } } -\cs_new_protected_nopar:Npn \regex_if_word_boundary:T - { \regex_if_word_boundary:TF \use:n \use_none:n } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} % -% \subsubsection{Entering and exiting character classes} -% -% \begin{macro}[aux]{\regex_build_[:} -% In a class, left brackets mean nothing. Outside a class, this starts -% a class, whose first characters may have a special meaning. -% \begin{macrocode} -\cs_new_protected_nopar:cpn { regex_build_[: } - { - \regex_build_if_in_class:TF - { \regex_build_raw:N [ } - { \regex_class_first:NNNN } - } -% \end{macrocode} -% \end{macro} +% \subsubsection{Character classes} % -% \begin{macro}[aux]{\regex_build_]:} +% \begin{macro}[aux]{\regex_compile_]:} % Outside a class, right brackets have no meaning. In a class, change % the mode ($m\to (m-15)/13$, truncated) to reflect the fact that we -% are leaving the class. If we are still in a class after leaving one, -% then this is the case |[...\cL[...]...]|, and we insert the relevant -% closing material in \cs{l_regex_tests_tl}. Otherwise look for +% are leaving the class. Look for quantifiers, unless we are still in +% a class after leaving one (the case of |[...\cL[...]...]|). % quantifiers. % \begin{macrocode} -\cs_new_protected:cpn { regex_build_]: } +\cs_new_protected:cpn { regex_compile_]: } { - \regex_build_if_in_class:TF + \regex_if_in_class:TF { - \if_num:w \l_regex_build_mode_int > \c_sixteen - \tl_set:Nx \l_regex_tests_tl - { - \exp_not:o \l_regex_tests_saved_tl - \if_num:w \l_regex_catcodes_saved_int < \c_regex_catcodes_all_int - \regex_item_catcode:nT - { \int_use:N \l_regex_catcodes_saved_int } - \else: - \exp_after:wN \use:n - \fi: - { - \exp_not:o \l_regex_tests_tl - \bool_if:NF \l_regex_tests_bool - { \regex_break_point:TF { } \regex_break_true:w } - } - } - \bool_set_eq:NN \l_regex_tests_bool \l_regex_tests_saved_bool + \if_num:w \l_regex_mode_int > \c_sixteen + \tl_build_one:n { \if_false: { \fi: } } \fi: - \tex_advance:D \l_regex_build_mode_int - \c_fifteen - \tex_divide:D \l_regex_build_mode_int \c_thirteen - \if_int_odd:w \l_regex_build_mode_int \else: - \exp_after:wN \regex_build_one_quantifier: + \tex_advance:D \l_regex_mode_int - \c_fifteen + \tex_divide:D \l_regex_mode_int \c_thirteen + \if_int_odd:w \l_regex_mode_int \else: + \exp_after:wN \regex_compile_quantifier:w \fi: } - { \regex_build_raw:N ] } + { \regex_compile_raw:N ] } } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_class_first:NNNN} -% This starts a class. Change the mode by appending $3$ to it, and -% reset the variables \cs{l_regex_tests_tl} and -% \cs{l_regex_bool_tl}. In the special case of mode $63$ -% (|[\c[...]]|), we open a group, to avoid overriding the setting of -% \cs{l_regex_tests_bool} and \cs{l_regex_tests_tl}; the group ends at -% the matching right bracket. If the first character is |^|, then the -% class is inverted. We keep track of this in -% \cs{l_regex_tests_bool}. If the next character is a right bracket, -% then it should be changed to a raw one (dirty hack here; the |F| -% argument of \cs{str_if_eq:nnTF} is the trailing |#3|). +% \begin{macro}[aux]{\regex_compile_[:} +% In a class, left brackets might introduce a \textsc{posix} character +% class, or mean nothing. Immediately following |\c|\meta{category}, +% we must insert the apropriate catcode test, then parse the class; we +% pre-expand the catcode as an optimization. Otherwise (modes $0$, +% $-2$ and $-6$) just parse the class. The mode is updated later. % \begin{macrocode} -\cs_new_protected:Npn \regex_class_first:NNNN #1#2#3#4 +\cs_new_protected_nopar:cpn { regex_compile_[: } { - \l_regex_build_mode_int = \int_value:w \l_regex_build_mode_int 3 ~ % - \if_num:w \l_regex_build_mode_int > \c_sixteen - \tl_set_eq:NN \l_regex_tests_saved_tl \l_regex_tests_tl - \bool_set_eq:NN \l_regex_tests_saved_bool \l_regex_tests_bool - \int_set_eq:NN \l_regex_catcodes_saved_int \l_regex_catcodes_int - \int_set_eq:NN \l_regex_catcodes_int \c_regex_catcodes_all_int - \fi: - \token_if_eq_meaning:NNTF #1 \regex_build_special:N + \regex_if_in_class:TF + { \regex_compile_class_posix_test:w } { - \token_if_eq_charcode:NNTF #2 ^ + \regex_if_within_catcode:TF { - \bool_set_false:N \l_regex_tests_bool % [ - \str_if_eq:nnTF {#3#4} { \regex_build_special:N ] } - { \regex_build_raw:N } - } - { % [ - \token_if_eq_charcode:NNTF #2 ] - { \regex_build_raw:N #2 } - { #1 #2 } + \exp_after:wN \regex_compile_class_catcode:w + \int_use:N \l_regex_catcodes_int ; } + { \regex_compile_class_normal:w } } - { #1 #2 } - #3 #4 } % \end{macrocode} % \end{macro} % -% \subsubsection{Catcodes and csnames} -% -% \begin{macro}[aux]{\regex_build_/c:} -% The |\c| escape sequence is only allowed in modes $0$ and $3$, -% \emph{i.e.}, not within any other |\c| escape sequence. +% \begin{macro}[aux]{\regex_compile_class_normal:w} +% In the \enquote{normal} case, we will insert \cs{regex_class:NnnnN} +% \meta{boolean} in the compiled code. The \meta{boolean} is true for +% positive classes, and false for negative classes, characterized by a +% leading |^|. The auxiliary \cs{regex_compile_class:TFNN} also +% checks for a leading |]| which has a special meaning. % \begin{macrocode} -\cs_new_protected:cpn { regex_build_/c: } +\cs_new_protected_nopar:Npn \regex_compile_class_normal:w { - \if_case:w \l_regex_build_mode_int - \exp_after:wN \regex_build_c_aux:wNN - \or: \or: \or: \exp_after:wN \regex_build_c_aux:wNN - \fi: - \msg_kernel_error:nn { regex } { c-bad-mode } - \s_stop + \regex_compile_class:TFNN + { \regex_class:NnnnN \c_true_bool } + { \regex_class:NnnnN \c_false_bool } } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_c_aux:wNN} -% The |\c| escape sequence can be followed by a capital letter -% representing a character category, by a left bracket which starts a -% list of categories, or by a brace group holding a regular expression -% for a control sequence name. Otherwise, raise an error. +% \begin{macro}[aux]{\regex_compile_class_catcode:w} +% This function is called for a left bracket in modes $2$ or $6$ +% (catcode test, and catcode test within a class). In mode $2$ the +% whole construction needs to be put in a class (like single +% character). Then determine if the class is positive or negative, +% inserting \cs{regex_item_catcode:nT} or the \texttt{reverse} variant +% as appropriate, each with the current catcodes bitmap |#1| as an +% argument, and reset the catcodes. % \begin{macrocode} -\cs_new_protected:Npn \regex_build_c_aux:wNN #1 \s_stop #2#3 +\cs_new_protected:Npn \regex_compile_class_catcode:w #1; { - \token_if_eq_meaning:NNTF #2 \regex_build_raw:N - { - \cs_if_exist:cTF { c_regex_catcode_#3_int } - { - \int_set_eq:Nc \l_regex_catcodes_int { c_regex_catcode_#3_int } - \l_regex_build_mode_int - = \if_case:w \l_regex_build_mode_int \c_two \else: \c_six \fi: - } - } - { \cs_if_exist_use:cF { regex_build_c_#3: } } - { - \msg_kernel_error:nnx { regex } { c-missing-category } {#3} - #2 #3 - } + \if_int_compare:w \l_regex_mode_int = \c_two + \tl_build_one:n + { \regex_class:NnnnN \c_true_bool { \if_false: } \fi: } + \fi: + \int_set_eq:NN \l_regex_catcodes_int \l_regex_default_catcodes_int + \regex_compile_class:TFNN + { \regex_item_catcode:nT {#1} } + { \regex_item_catcode_reverse:nT {#1} } } % \end{macrocode} % \end{macro} % -% \begin{macro}+\regex_build_c_{:+ ^^A } -% The case of a left brace is easy, based on what we have done so far: -% in a group, build the regular expression, after changing the mode to -% forbid nesting |\c|. +% \begin{macro}[aux] +% {\regex_compile_class:TFNN, \regex_compile_class_ii:NN} +% If the first character is |^|, then the class is negative (use +% |#2|), otherwise it is positive (use |#1|). If the next character +% is a right bracket, then it should be changed to a raw one. % \begin{macrocode} -\cs_new_protected:cpn { regex_build_c_ \c_lbrace_str : } +\cs_new_protected:Npn \regex_compile_class:TFNN #1#2#3#4 { - \group_begin: - \l_regex_build_mode_int - = - \if_case:w \l_regex_build_mode_int \c_two \else: \c_six \fi: - \regex_build:w + \l_regex_mode_int = \int_value:w \l_regex_mode_int 3 \exp_stop_f: + \str_if_eq:nnTF { #3 #4 } { \regex_compile_special:N ^ } + { + \tl_build_one:n { #2 { \if_false: } \fi: } + \regex_compile_class_ii:NN + } + { + \tl_build_one:n { #1 { \if_false: } \fi: } + \regex_compile_class_ii:NN #3 #4 + } + } +\cs_new_protected:Npn \regex_compile_class_ii:NN #1#2 + { + \token_if_eq_charcode:NNTF #2 ] + { \regex_compile_raw:N #2 } + { #1 #2 } } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_c_[:} -% \begin{macro}[aux]{\regex_build_c_lbrack_loop:NN} -% \begin{macro}[aux]{\regex_build_c_lbrack_end:} -% \begin{macro}[aux]{\regex_build_add:N} -% When encountering |\c[|, the task is to collect uppercase letters -% representing character categories. +% \begin{macro}[aux] +% { +% \regex_compile_class_posix_test:w, +% \regex_compile_class_posix:NNNNw, +% \regex_compile_class_posix_loop:w, +% \regex_compile_class_posix_end:w +% } +% Here we check for a syntax such as |[:alpha:]|. We also detect |[=| +% and |[.| which have a meaning in \textsc{posix} regular expressions, +% but are not implemented in \pkg{l3regex}. In case we see |[:|, grab +% raw characters until hopefully reaching |:]|. If that's missing, or +% the \textsc{posix} class is unknown, abort. If all is right, add the +% test to the current class, with an extra \cs{regex_item_reverse:n} +% for negative classes. % \begin{macrocode} -\cs_new_protected:cpn { regex_build_c_[: } #1#2 +\cs_new_protected:Npn \regex_compile_class_posix_test:w #1#2 { - \int_zero:N \l_regex_catcodes_int - \str_if_eq:nnTF { #1 #2 } { \regex_build_special:N ^ } - { - \bool_set_false:N \l_regex_catcodes_bool - \regex_build_c_lbrack_loop:NN - } + \token_if_eq_meaning:NNT \regex_compile_special:N #1 { - \bool_set_true:N \l_regex_catcodes_bool - \regex_build_c_lbrack_loop:NN - #1#2 + \prg_case_str:nnn { #2 } + { + : { \regex_compile_class_posix:NNNNw } + = { \msg_kernel_warning:nnx { regex } { posix-unsupported } { = } } + . { \msg_kernel_warning:nnx { regex } { posix-unsupported } { . } } + } + { } } + \regex_compile_raw:N [ #1 #2 } -\cs_new_protected:Npn \regex_build_c_lbrack_loop:NN #1#2 +\cs_new_protected:Npn \regex_compile_class_posix:NNNNw #1#2#3#4#5#6 { - \token_if_eq_meaning:NNTF #1 \regex_build_raw:N + \str_if_eq:nnTF { #5 #6 } { \regex_compile_special:N ^ } { - \cs_if_exist:cTF { c_regex_catcode_#2_int } - { - \exp_args:Nc \regex_build_c_lbrack_add:N - { c_regex_catcode_#2_int } - \regex_build_c_lbrack_loop:NN - } + \bool_set_false:N \l_regex_internal_bool + \tl_set:Nx \l_regex_internal_a_tl { \if_false: } \fi: + \regex_compile_class_posix_loop:w } - { % [ - \token_if_eq_charcode:NNTF #2 ] - { \regex_build_c_lbrack_end: } + { + \bool_set_true:N \l_regex_internal_bool + \tl_set:Nx \l_regex_internal_a_tl { \if_false: } \fi: + \regex_compile_class_posix_loop:w #5 #6 } - { - \msg_kernel_error:nnx { regex } { c-missing-rbrack } {#2} - \regex_build_c_lbrack_end: - } - } -\cs_new_protected_nopar:Npn \regex_build_c_lbrack_end: - { - \l_regex_build_mode_int - = \if_case:w \l_regex_build_mode_int \c_two \else: \c_six \fi: - \if_meaning:w \c_false_bool \l_regex_catcodes_bool - \int_set:Nn \l_regex_catcodes_int - { \c_regex_catcodes_all_int - \l_regex_catcodes_int } - \fi: } -\cs_new_protected:Npn \regex_build_c_lbrack_add:N #1 +\cs_new:Npn \regex_compile_class_posix_loop:w #1#2 { - \if_int_odd:w \int_eval:w \l_regex_catcodes_int / #1 \int_eval_end: - \else: - \tex_advance:D \l_regex_catcodes_int #1 - \fi: + \token_if_eq_meaning:NNTF \regex_compile_raw:N #1 + { #2 \regex_compile_class_posix_loop:w } + { \if_false: { \fi: } \regex_compile_class_posix_end:w #1 #2 } } -% \end{macrocode} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% -% ^^A{ -% \begin{macro}+\regex_build_}:+ -% Non-escaped right braces are only special if they appear when -% building the regular expression for a csname. In that case, make -% sure that any character class and group is closed, anchor the match -% at the end, add the code for making the match successful, and expand -% the contents of the regular expression we just built. Otherwise, -% replace the brace with an escaped brace. -% \begin{macrocode} -\cs_new_protected:cpn { regex_build_ \c_rbrace_str : } +\cs_new_protected:Npn \regex_compile_class_posix_end:w #1#2#3#4 { - \int_compare:nNnTF \l_regex_build_mode_int < \c_zero + \str_if_eq:nnTF { #1 #2 #3 #4 } + { \regex_compile_special:N : \regex_compile_special:N ] } { - \regex_build_exit_class: - \regex_build_exit_groups: - \regex_build_escaped:N Z - \regex_build_end: - \use:x + \cs_if_exist:cTF { regex_posix_ \l_regex_internal_a_tl : } { - \group_end: - \regex_build_one:n - { - \regex_item_cs:n - { \regex_set_aux:N \l_regex_internal_regex } - } + \regex_compile_one:x + { + \bool_if:NF \l_regex_internal_bool \regex_item_reverse:n + \exp_not:c { regex_posix_ \l_regex_internal_a_tl : } + } } - } - { \exp_after:wN \regex_build_raw:N \c_rbrace_str } - } -% \end{macrocode} -% \end{macro} -% -% \subsubsection{Quantifiers} -% -% \begin{macro}[int]{\regex_build_quantifier:w} -% This looks ahead and finds any quantifier (control character -% equal to either of |?+*{|). ^^A} -% When all characters for the quantifier are found, the corresponding -% function is called. -% \begin{macrocode} -\cs_new_protected:Npn \regex_build_quantifier:w #1#2 - { - \token_if_eq_meaning:NNTF #1 \regex_build_special:N - { - \cs_if_exist_use:cF { regex_build_quantifier_#2:w } { - \regex_build_quantifier_end:nn { } { } - #1 #2 + \msg_kernel_warning:nnx { regex } { posix-unknown } + { \l_regex_internal_a_tl } + \regex_compile_abort_tokens:x + { + [: \bool_if:NF \l_regex_internal_bool { ^ } + \l_regex_internal_a_tl :] + } } } { - \regex_build_quantifier_end:nn { } { } - #1 #2 + \msg_kernel_error:nnxx { regex } { posix-missing-close } + { [: \l_regex_internal_a_tl } { #2 #4 } + \regex_compile_abort_tokens:x { [: \l_regex_internal_a_tl } + #1 #2 #3 #4 } } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_quantifier_?:w} -% \begin{macro}[aux]{\regex_build_quantifier_*:w} -% \begin{macro}[aux]{\regex_build_quantifier_+:w} -% For each \enquote{basic} quantifier, |?|, |*|, |+|, feed the correct -% arguments to \cs{regex_build_quantifier_aux:nnNN}. -% \begin{macrocode} -\cs_new_protected_nopar:cpn { regex_build_quantifier_?:w } - { \regex_build_quantifier_aux:nnNN { } { ? } } -\cs_new_protected_nopar:cpn { regex_build_quantifier_*:w } - { \regex_build_quantifier_aux:nnNN { } { * } } -\cs_new_protected_nopar:cpn { regex_build_quantifier_+:w } - { \regex_build_quantifier_aux:nnNN { } { + } } -% \end{macrocode} -% \end{macro} -% \end{macro} -% \end{macro} +% \subsubsection{Groups and alternations} % -% \begin{macro}[aux]{\regex_build_quantifier_aux:nnNN} -% Once the \enquote{main} quantifier (\texttt{?}, \texttt{*}, -% \texttt{+} or a braced construction) is found, we check -% whether it is lazy (followed by a question mark), -% and calls the appropriate function. Here |#1| holds some extra -% arguments that the final function needs in the case of braced -% constructions, and is empty otherwise. +% \begin{macro}[aux]{\regex_compile_group_begin:N, \regex_compile_group_end:} +% The contents of a regex group are turned into compiled code in +% \cs{l_regex_internal_regex}, which ends up with items of the form +% \cs{regex_branch:n} \Arg{concatenation}. This construction is done +% using \pkg{l3tl-build} within a \TeX{} group, which automatically +% makes sure that options (case-sensitivity and default catcode) are +% reset at the end of the group. The argument |#1| is +% \cs{regex_group:nnnN} or a variant thereof. A small subtelty to +% support |\cL(abc)| as a shorthand for |(\cLa\cLb\cLc)|: exit any +% pending catcode test, save the category code at the start of the +% group as the default catcode for that group, and make sure that the +% catcode is restored to the default outside the group. % \begin{macrocode} -\cs_new_protected:Npn \regex_build_quantifier_aux:nnNN #1#2#3#4 +\cs_new_protected:Npn \regex_compile_group_begin:N #1 { - \str_if_eq:nnTF { #3 #4 } { \regex_build_special:N ? } - { \regex_build_quantifier_end:nn { #2 #4 } {#1} } - { - \regex_build_quantifier_end:nn { #2 } {#1} - #3 #4 - } + \tl_build_one:n { #1 { \if_false: } \fi: } + \regex_mode_quit_c: + \tl_set_build:Nw \l_regex_internal_regex + \int_set_eq:NN \l_regex_default_catcodes_int \l_regex_catcodes_int + \int_incr:N \l_regex_group_level_int + \tl_build_one:n { \regex_branch:n { \if_false: } \fi: } + } +\cs_new_protected:Npn \regex_compile_group_end: + { + \if_num:w \l_regex_group_level_int > \c_zero + \tl_build_one:n { \if_false: { \fi: } } + \tl_build_end: + \int_set_eq:NN \l_regex_catcodes_int \l_regex_default_catcodes_int + \tl_build_one:o \l_regex_internal_regex + \exp_after:wN \regex_compile_quantifier:w + \else: + \msg_kernel_warning:nn { regex } { extra-rparen } + \exp_after:wN \regex_compile_raw:N \exp_after:wN ) + \fi: } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]+\regex_build_quantifier_{:w+ ^^A} -% \begin{macro}[aux]{\regex_build_quantifier_lbrace:n} -% \begin{macro}[aux]{\regex_build_quantifier_lbrace:nw} -% \begin{macro}[aux]{\regex_build_quantifier_lbrace:nnw} -% Three possible syntaxes: \texttt{\{\meta{int}\}}, -% \texttt{\{\meta{int},\}}, or \texttt{\{\meta{int},\meta{int}\}}. +% \begin{macro}[aux]{\regex_compile_(:} +% In a class, parentheses are not special. Outside, check for a |?|, +% denoting special groups, and run the code for the corresponding +% special group. % \begin{macrocode} -\cs_new_protected_nopar:cpn { regex_build_quantifier_ \c_lbrace_str :w } - { \regex_get_digits:nw { \regex_build_quantifier_lbrace:n } } -\cs_new_protected:Npn \regex_build_quantifier_lbrace:n #1 +\cs_new_protected_nopar:cpn { regex_compile_(: } { - \tl_if_empty:nTF {#1} - { - \regex_build_quantifier_end:nn { } { } - \exp_after:wN \regex_build_raw:N \c_lbrace_str - } - { \regex_build_quantifier_lbrace:nw {#1} } + \regex_if_in_class:TF { \regex_compile_raw:N ( } + { \regex_compile_lparen:w } } -\cs_new_protected:Npx \regex_build_quantifier_lbrace:nw #1#2#3 +\cs_new_protected:Npn \regex_compile_lparen:w #1#2#3#4 { - \exp_not:N \prg_case_str:nnn { #2 #3 } + \str_if_eq:nnTF { #1 #2 } { \regex_compile_special:N ? } { - { \exp_not:N \regex_build_special:N , } + \cs_if_exist_use:cF + { regex_compile_special_group_\token_to_str:N #4 :w } { - \exp_not:N \regex_get_digits:nw - { \exp_not:N \regex_build_quantifier_lbrace:nnw {#1} } + \msg_kernel_warning:nnx { regex } { special-group-unknown } + { (? \token_to_str:N #4 } + \regex_compile_group_begin:N \regex_group:nnnN + \regex_compile_raw:N ? #3 #4 } - { \exp_not:N \regex_build_special:N \c_rbrace_str } - { \exp_not:N \regex_build_quantifier_end:nn {n} { {#1} } } } { - \exp_not:N \regex_build_quantifier_end:nn { } { } - \exp_not:N \regex_build_raw:N \c_lbrace_str #1#2 - } - } -\cs_new_protected:Npn \regex_build_quantifier_lbrace:nnw #1#2#3#4 - { - \str_if_eq:xxTF - { \exp_not:n {#3#4} } - { \exp_not:N \regex_build_special:N \c_rbrace_str } - { - \tl_if_empty:nTF {#2} - { \regex_build_quantifier_aux:nnNN { {#1} } { n* } } - { \regex_build_quantifier_aux:nnNN { {#1} {#2} } { nn } } - } - { - \regex_build_quantifier_end:nn { } { } - \use:x - { - \exp_args:No \tl_map_function:nN - { \c_lbrace_str #1 , #2 } - \regex_build_raw:N - } - #3 #4 + \regex_compile_group_begin:N \regex_group:nnnN + #1 #2 #3 #4 } } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}[aux]{\regex_build_quantifier_end:nn} -% When all quantifiers are found, we will call the relevant -% \cs{regex_build_one/group_\meta{quantifiers}:} function. +% \begin{macro}[aux]+\regex_compile_|:+ +% In a class, the pipe is not special. Otherwise, end the current +% branch and open another one. % \begin{macrocode} -\cs_new_protected:Npn \regex_build_quantifier_end:nn #1#2 +\cs_new_protected_nopar:cpn { regex_compile_|: } { - \use:c { regex_build_ \l_regex_one_or_group_tl _ #1 : } #2 - \tl_clear:N \l_regex_tests_tl - \bool_set_true:N \l_regex_tests_bool + \regex_if_in_class:TF { \regex_compile_raw:N | } + { + \tl_build_one:n + { \if_false: { \fi: } \regex_branch:n { \if_false: } \fi: } + } } % \end{macrocode} % \end{macro} % -% \subsubsection{Quantifiers for one character or character class} -% -% \begin{macro}[aux]{\regex_build_one_quantifier:} -% Used for one single character, or a character class. -% Contrarily to \cs{regex_build_group_quantifier:}, -% we don't need to keep track of submatches, and no thread -% can be created within one repetition, so things are relatively easy. +% \begin{macro}[aux]{\regex_compile_):} +% Within a class, parentheses are not special. Outside, close a group. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_one_quantifier: +\cs_new_protected_nopar:cpn { regex_compile_): } { - \tl_set:Nx \l_regex_one_or_group_tl { one } - \regex_build_quantifier:w + \regex_if_in_class:TF { \regex_compile_raw:N ) } + { \regex_compile_group_end: } } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_one_n:, \regex_build_one_n?:} -% \begin{macro}[aux]{\regex_build_one_:} -% This function is called in case the syntax is -% \texttt{\{\meta{int}\}}. Greedy and lazy operators are identical, -% since the number of repetitions is fixed. Build one state for each -% repetition, with a transition controlled by the tests that we have -% collected. The case where we find no quantifier is the special case -% $n=1$. +% \begin{macro}[aux]{\regex_compile_special_group_::w} +% \begin{macro}[aux]+\regex_compile_special_group_|:w+ +% Non-capturing, and resetting groups are easy to take care of during +% compilation; for those groups, the harder parts will come when +% building. % \begin{macrocode} -\cs_new_protected:Npn \regex_build_one_n: #1 - { - \prg_replicate:nn {#1} - { - \regex_build_transition:NN - \regex_tests_action_cost:n \l_regex_right_state_int - } - } -\cs_new_eq:cN { regex_build_one_n?: } \regex_build_one_n: -\cs_new_protected_nopar:Npn \regex_build_one_: - { \regex_build_one_n: \c_one } +\cs_new_protected_nopar:cpn { regex_compile_special_group_::w } + { \regex_compile_group_begin:N \regex_group_no_capture:nnnN } +\cs_new_protected_nopar:cpn { regex_compile_special_group_|:w } + { \regex_compile_group_begin:N \regex_group_resetting:nnnN } % \end{macrocode} % \end{macro} % \end{macro} % -% \begin{macro}[aux]{\regex_build_one_n*:, \regex_build_one_n*?:} -% \begin{macro}[aux]{\regex_build_one_*:, \regex_build_one_*?:} -% \begin{macro}[aux]{\regex_build_one_+:, \regex_build_one_+?:} -% Those functions are called in case the syntax is -% \texttt{\{\meta{int},\}}. The |*| and |+| quantifiers (greedy or -% lazy) are special cases for $n=0$ or $n=1$. In the case $n=0$, -% build a costly transition going from the current state to itself, -% and a free transition moving to a new state. In the case $n\geq 1$, -% reuse \cs{regex_build_one_n:} to match $n$ repetition, then add free -% transitions from the last state to the previous one, and from the -% last state to a new one. The greedy and lazy versions only differ in -% the order of transitions. +% \begin{macro}[aux] +% {\regex_compile_special_group_i:w, \regex_compile_special_group_-:w} +% The match can be made case-insensitive by setting the option with +% \texttt{(?i)}; the original behaviour is restored by \texttt{(?-i)}. +% This is the only supported option. % \begin{macrocode} -\cs_new_protected:cpn { regex_build_one_n*: } #1 +\cs_new_protected:Npn \regex_compile_special_group_i:w #1#2 { - \int_compare:nNnTF {#1} = \c_zero + \str_if_eq:nnTF { #1 #2 } { \regex_compile_special:N ) } { - \regex_build_transitions:NNNN - \regex_tests_action_cost:n \l_regex_left_state_int - \regex_action_free:n \l_regex_right_state_int + \cs_set_nopar:Npn \regex_item_equal:n { \regex_item_caseless_equal:n } + \cs_set_nopar:Npn \regex_item_range:nn { \regex_item_caseless_range:nn } } { - \regex_build_one_n: {#1} - \int_set_eq:NN \l_regex_internal_a_int \l_regex_left_state_int - \regex_build_transitions:NNNN - \regex_action_free:n \l_regex_internal_a_int - \regex_action_free:n \l_regex_right_state_int + \msg_kernel_warning:nnx { regex } { unknown-option } { (?i #2 } + \regex_compile_raw:N ( + \regex_compile_raw:N ? + \regex_compile_raw:N i + #1 #2 } } -\cs_new_protected:cpn { regex_build_one_n*?: } #1 +\cs_new_protected_nopar:cpn { regex_compile_special_group_-:w } #1#2#3#4 { - \int_compare:nNnTF {#1} = \c_zero + \str_if_eq:nnTF { #1 #2 #3 #4 } + { \regex_compile_raw:N i \regex_compile_special:N ) } { - \regex_build_transitions:NNNN - \regex_action_free:n \l_regex_right_state_int - \regex_tests_action_cost:n \l_regex_left_state_int + \cs_set_nopar:Npn \regex_item_equal:n { \regex_item_caseful_equal:n } + \cs_set_nopar:Npn \regex_item_range:nn { \regex_item_caseful_range:nn } } { - \regex_build_one_n: {#1} - \int_set_eq:NN \l_regex_internal_a_int \l_regex_left_state_int - \regex_build_transitions:NNNN - \regex_action_free:n \l_regex_right_state_int - \regex_action_free:n \l_regex_internal_a_int + \msg_kernel_warning:nnx { regex } { unknown-option } { (?-#2#4 } + \regex_compile_raw:N ( + \regex_compile_raw:N ? + \regex_compile_raw:N - + #1 #2 #3 #4 } } -\cs_new_protected_nopar:cpx { regex_build_one_*: } - { \exp_not:c { regex_build_one_n*: } \c_zero } -\cs_new_protected_nopar:cpx { regex_build_one_*?: } - { \exp_not:c { regex_build_one_n*?: } \c_zero } -\cs_new_protected_nopar:cpx { regex_build_one_+: } - { \exp_not:c { regex_build_one_n*: } \c_one } -\cs_new_protected_nopar:cpx { regex_build_one_+?: } - { \exp_not:c { regex_build_one_n*?: } \c_one } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}[aux]{\regex_build_one_nn:, \regex_build_one_nn?:} -% \begin{macro}[aux]{\regex_build_one_?:, \regex_build_one_??:} -% \begin{macro}[aux]{\regex_build_one_nn_aux:nnnn} -% Those functions are called when the syntax is -% \texttt{\{\meta{int},\meta{int}\}}, or for the |?| quantifier, with -% $m=0$ and $n=1$. +% \subsubsection{Catcodes and csnames} +% +% \begin{macro}[aux]{\regex_compile_/c:, \regex_compile_c_test:NN} +% The |\c| escape sequence can be followed by a capital letter +% representing a character category, by a left bracket which starts a +% list of categories, or by a brace group holding a regular expression +% for a control sequence name. Otherwise, raise an error. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_one_nn: - { - \regex_build_one_nn_aux:nnnn { } - { - \regex_build_transitions:NNNN - \regex_tests_action_cost:n \l_regex_right_state_int - \regex_action_free:n \l_regex_internal_a_int - } - } -\cs_new_protected_nopar:cpn { regex_build_one_nn?: } +\cs_new_protected:cpn { regex_compile_/c: } + { \regex_chk_c_allowed:T { \regex_compile_c_test:NN } } +\cs_new_protected:Npn \regex_compile_c_test:NN #1#2 { - \regex_build_one_nn_aux:nnnn {?} + \token_if_eq_meaning:NNTF #1 \regex_compile_raw:N { - \regex_build_transitions:NNNN - \regex_action_free:n \l_regex_internal_a_int - \regex_tests_action_cost:n \l_regex_right_state_int + \cs_if_exist:cTF { c_regex_catcode_#2_int } + { + \int_set_eq:Nc \l_regex_catcodes_int { c_regex_catcode_#2_int } + \l_regex_mode_int + = \if_case:w \l_regex_mode_int \c_two \else: \c_six \fi: + } } + { \cs_if_exist_use:cF { regex_compile_c_#2:w } } + { + \msg_kernel_error:nnx { regex } { c-missing-category } {#2} + #1 #2 + } } -\cs_new_protected:Npn \regex_build_one_nn_aux:nnnn #1#2#3#4 +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux] +% { +% \regex_compile_c_[:w, +% \regex_compile_c_lbrack_loop:NN, +% \regex_compile_c_lbrack_add:N, +% \regex_compile_c_lbrack_end:, +% } +% When encountering |\c[|, the task is to collect uppercase letters +% representing character categories. First check for |^| which negates +% the list of category codes. +% \begin{macrocode} +\cs_new_protected:cpn { regex_compile_c_[:w } #1#2 { - \regex_build_one_n: {#3} - \int_compare:nNnTF {#3} > {#4} + \l_regex_mode_int + = \if_case:w \l_regex_mode_int \c_two \else: \c_six \fi: + \int_zero:N \l_regex_catcodes_int + \str_if_eq:nnTF { #1 #2 } { \regex_compile_special:N ^ } { - \msg_kernel_error:nnxxx - { regex } { quantifier-backwards } {#3} {#4} {#1} + \bool_set_false:N \l_regex_catcodes_bool + \regex_compile_c_lbrack_loop:NN } { - \int_set:Nn \l_regex_internal_a_int - { \l_regex_right_state_int + #4 - #3 } - \prg_replicate:nn { #4 - #3 } { #2 } + \bool_set_true:N \l_regex_catcodes_bool + \regex_compile_c_lbrack_loop:NN + #1 #2 } } -\cs_new_protected_nopar:cpn { regex_build_one_?: } - { \regex_build_one_nn: \c_zero \c_one } -\cs_new_protected_nopar:cpx { regex_build_one_??: } - { \exp_not:c { regex_build_one_nn?: } \c_zero \c_one } -% \end{macrocode} -% \end{macro} -% \end{macro} -% \end{macro} -% -% \subsubsection{Groups and alternation} -% -% We support the syntax \texttt{\meta{expr1}|\ldots{}%^^A -% |\meta{expr$\sb{n}$}} for alternations. -% -% \begin{macro}[aux]{\regex_build_(:, \regex_build_):} -% \begin{macro}[aux]{\regex_build_open_aux:} -% \begin{macro}[aux]+\regex_build_|:+ -% \begin{macro}[aux]{\regex_build_begin_alternation:, -% \regex_build_end_alternation:} -% Grouping and alternation go together. -% \begin{itemize} -% \item Allocate the next available number for the end vertex -% of the alternation/group and store it on a stack (so that nested -% alternations work). -% \item Put free transitions to separate all cases of the alternation. -% \item Build each branch separately, and merge them to the common -% end-node. -% \item Test for a quantifier, and if needed, transfer the initial -% vertex to a new vertex. -% \end{itemize} -% \begin{macrocode} -\cs_new_protected:cpn { regex_build_(: } #1#2 +\cs_new_protected:Npn \regex_compile_c_lbrack_loop:NN #1#2 { - \regex_build_if_in_class:TF + \token_if_eq_meaning:NNTF #1 \regex_compile_raw:N { - \regex_build_raw:N ( - #1 #2 + \cs_if_exist:cTF { c_regex_catcode_#2_int } + { + \exp_args:Nc \regex_compile_c_lbrack_add:N + { c_regex_catcode_#2_int } + \regex_compile_c_lbrack_loop:NN + } } { - \str_if_eq:nnTF { #1 #2 } { \regex_build_special:N ? } - { \regex_build_special_group:NN } + \token_if_eq_charcode:NNTF #2 ] + { \regex_compile_c_lbrack_end: } + } { - \int_incr:N \l_regex_capturing_group_int - \regex_seq_push_int:NN - \l_regex_capturing_group_seq \l_regex_capturing_group_int - \regex_build_open_aux: + \msg_kernel_error:nnx { regex } { c-missing-rbrack } {#2} + \regex_compile_c_lbrack_end: #1 #2 } - } } -\cs_new_protected_nopar:Npn \regex_build_open_aux: +\cs_new_protected:Npn \regex_compile_c_lbrack_add:N #1 { - \regex_build_new_state: - \regex_seq_push_int:NN \l_regex_left_state_seq \l_regex_left_state_int - \regex_seq_push_int:NN \l_regex_right_state_seq \l_regex_right_state_int - \bool_if:NTF \l_regex_caseless_bool - { \seq_push:Nn \l_regex_end_group_seq \regex_build_caseless: } - { \seq_push:Nn \l_regex_end_group_seq \regex_build_caseful: } - \seq_push:Nn \l_regex_end_alternation_seq { } - \regex_build_begin_alternation: + \if_int_odd:w \int_eval:w \l_regex_catcodes_int / #1 \int_eval_end: + \else: + \tex_advance:D \l_regex_catcodes_int #1 + \fi: } -\cs_new_protected_nopar:cpn { regex_build_|: } +\cs_new_protected_nopar:Npn \regex_compile_c_lbrack_end: { - \regex_build_if_in_class:TF { \regex_build_raw:N | } - { - \regex_build_end_alternation: - \regex_build_begin_alternation: - } + \if_meaning:w \c_false_bool \l_regex_catcodes_bool + \int_set:Nn \l_regex_catcodes_int + { \c_regex_all_catcodes_int - \l_regex_catcodes_int } + \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}+\regex_compile_c_{:+ +% The case of a left brace is easy, based on what we have done so far: +% in a group, compile the regular expression, after changing the mode +% to forbid nesting |\c|. Additionally, disable submatch tracking +% since groups don't escape the scope of |\c{...}|. +% \begin{macrocode} +\cs_new_protected_nopar:cpn { regex_compile_c_ \c_lbrace_str :w } + { + \regex_compile:w + \regex_disable_submatches: + \l_regex_mode_int + = - \if_case:w \l_regex_mode_int \c_two \else: \c_six \fi: } -\cs_new_protected_nopar:cpn { regex_build_): } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}+\regex_compile_}:+ +% Non-escaped right braces are only special if they appear when +% compiling the regular expression for a csname, but not within a +% class: |\c{[}{]}| matches the control sequences |\}| and +% |\{|\ldots{} Admittedly, that would be better done as +% |\c{[{}]}|. So, end compiling the inner regex (this closes any +% dangling class or group). Then insert the corresponding test in the +% outer regex. +% \begin{macrocode} +\cs_new_protected:cpn { regex_compile_ \c_rbrace_str : } { - \regex_build_if_in_class:TF { \regex_build_raw:N ) } + \regex_if_in_cs:TF { - \seq_if_empty:NTF \l_regex_capturing_group_seq - { \msg_kernel_error:nn { regex } { extra-rparen } } - { - \regex_build_close_aux: - \regex_build_group_quantifier: - } + \regex_compile_end: + \regex_compile_one:x + { \regex_item_cs:n { \exp_not:o \l_regex_internal_regex } } } - } -\cs_new_protected_nopar:Npn \regex_build_close_aux: - { - \regex_build_end_alternation: - \regex_seq_pop_int:NN \l_regex_left_state_seq \l_regex_left_state_int - \regex_seq_pop_int:NN \l_regex_right_state_seq \l_regex_right_state_int - \regex_seq_pop_use:N \l_regex_end_group_seq - \seq_pop:NN \l_regex_end_alternation_seq \l_regex_internal_a_tl + { \exp_after:wN \regex_compile_raw:N \c_rbrace_str } } % \end{macrocode} -% Building each branch. +% \end{macro} +% +% \subsubsection{Raw token lists with \cs{u}} +% +% \begin{macro}[aux]{\regex_compile_/u:} +% \begin{macro}[aux, EXP]{\regex_compile_u_loop:NN} +% The |\u| escape is invalid in classes and directly following a +% catcode test. Otherwise, it must be followed by a left brace. We +% then collect the characters for the argument of |\u| within an +% \texttt{x}-expanding assignment. In principle we could just wait to +% encounter a right brace, but this is unsafe: if the right brace is +% missing, then we will reach the end-markers of the regex, and +% continue, leading to obscure fatal errors. Instead, we only allow +% raw and special characters, and stop when encountering a special +% right brace, any escaped character, or the end-marker. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_begin_alternation: +\cs_new_protected:cpn { regex_compile_/u: } #1#2 { - \regex_build_new_state: - \regex_seq_get_int:NN \l_regex_left_state_seq \l_regex_left_state_int - \regex_toks_put_right:Nx \l_regex_left_state_int + \regex_if_in_class_or_catcode:TF + { \regex_compile_raw_error:N u #1 #2 } { - \regex_action_free:n + \str_if_eq:xxTF {#1#2} { \regex_compile_special:N \c_lbrace_str } { - \int_eval:n - { \l_regex_right_state_int - \l_regex_left_state_int } + \tl_set:Nx \l_regex_internal_a_tl { \if_false: } \fi: + \regex_compile_u_loop:NN + } + { + \msg_kernel_error:nn { regex } { u-missing-lbrace } + \regex_compile_raw:N u #1 #2 } } } -\cs_new_protected_nopar:Npn \regex_build_end_alternation: +\cs_new:Npn \regex_compile_u_loop:NN #1#2 { - \int_set_eq:NN \l_regex_left_state_int \l_regex_right_state_int - \regex_seq_get_int:NN \l_regex_right_state_seq \l_regex_right_state_int - \regex_toks_put_right:Nx \l_regex_left_state_int + \token_if_eq_meaning:NNTF #1 \regex_compile_raw:N + { #2 \regex_compile_u_loop:NN } { - \regex_action_free:n + \token_if_eq_meaning:NNTF #1 \regex_compile_special:N + { + \exp_after:wN \token_if_eq_charcode:NNTF \c_rbrace_str #2 + { \if_false: { \fi: } \regex_compile_u_end: } + { #2 \regex_compile_u_loop:NN } + } { - \int_eval:n - { \l_regex_right_state_int - \l_regex_left_state_int } + \if_false: { \fi: } + \msg_kernel_error:nnx { regex } { u-missing-rbrace } {#2} + \regex_compile_u_end: + #1 #2 } } - \regex_seq_get_use:N \l_regex_end_alternation_seq } % \end{macrocode} % \end{macro} % \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}{\regex_build_special_group:NN} -% Same method as elsewhere: if the combination |(?#1| ^^A ) -% is known, then use that. Otherwise, treat the question mark -% as if it had been escaped. +% \begin{macro}[aux]{\regex_compile_u_end:} +% Once we have extracted the variable's name, we store the contents of +% that variable in \cs{l_regex_internal_a_tl}. The behaviour of |\u| +% then depends on whether we are within a |\c{...}| escape (in this +% case, the variable is turned to a string), or not. % \begin{macrocode} -\cs_new_protected:Npn \regex_build_special_group:NN #1#2 +\cs_new_protected:Npn \regex_compile_u_end: { - \cs_if_exist_use:cF { regex_build_special_group_\token_to_str:N #2 : } - { - \msg_kernel_error:nnx { regex } { special-group-unknown } - { (? \token_to_str:N #2 } %) - \regex_build_special:N ( % ) - \regex_build_raw:N ? - #1 #2 - } + \tl_set:Nv \l_regex_internal_a_tl { \l_regex_internal_a_tl } + \if_num:w \l_regex_mode_int = \c_zero + \regex_compile_u_not_cs: + \else: + \regex_compile_u_in_cs: + \fi: } % \end{macrocode} % \end{macro} % -% \begin{macro}{\regex_build_special_group_::} -% Non-capturing groups are like capturing groups, except that -% we set the group id to \texttt{-1}, which will then inhibit -% submatching in \cs{regex_build_group_submatches:NN}. -% The group number is not increased. +% \begin{macro}[aux]{\regex_compile_u_in_cs:} +% When |\u| appears within a control sequence, we convert the variable +% to a string with escaped spaces. Then for each character insert a +% class matching exactly that character, once. % \begin{macrocode} -\cs_new_protected_nopar:cpn { regex_build_special_group_:: } +\cs_new_protected:Npn \regex_compile_u_in_cs: { - \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_minus_one - \regex_build_open_aux: + \exp_args:NNo \str_gset_other:Nn \g_regex_internal_tl + { \l_regex_internal_a_tl } + \tl_build_one:x + { + \tl_map_function:NN \g_regex_internal_tl + \regex_compile_u_in_cs_aux:n + } + } +\cs_new:Npn \regex_compile_u_in_cs_aux:n #1 + { + \regex_class:NnnnN \c_true_bool + { \regex_item_caseful_equal:n { \int_value:w `#1 } } + { 1 } { 0 } \c_false_bool } % \end{macrocode} % \end{macro} % -% \begin{macro}+\regex_build_special_group_|:+ -% The special group \verb"(?|..|..)" is non-capturing -% (hence we set the |capturing_group| to $-1$), and resets -% the group number in each branch of the alternation. -% We use a variant of \cs{regex_build_open_aux:}, adding -% some code to be performed at every alternation, and at -% the end of the group. Namely, we keep track of -% the maximal value that \cs{l_regex_capturing_group_int} -% takes, and restore that value when the group end, -% and in every branch, we reset the capturing group number. +% \begin{macro}[aux]{\regex_compile_u_not_cs:} +% In mode $0$, the |\u| escape adds one state to the NFA for each +% token in \cs{l_regex_internal_a_tl}. If a given \meta{token} is a +% control sequence, then insert a string comparison test, otherwise, +% \cs{regex_item_exact:nn} which compares catcode and character code. % \begin{macrocode} -\cs_new_protected_nopar:cpn { regex_build_special_group_|: } +\cs_new_protected:Npn \regex_compile_u_not_cs: { - \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_minus_one - \regex_build_new_state: - \regex_seq_push_int:NN \l_regex_left_state_seq \l_regex_left_state_int - \regex_seq_push_int:NN \l_regex_right_state_seq \l_regex_right_state_int - \seq_push:Nx \l_regex_end_alternation_seq + \exp_args:No \tl_analysis_map_inline:nn { \l_regex_internal_a_tl } { - \exp_not:N \int_compare:nNnT - \l_regex_capturing_group_int - > \l_regex_capturing_group_max_int + \tl_build_one:n { - \int_set_eq:NN - \l_regex_capturing_group_max_int - \l_regex_capturing_group_int + \regex_class:NnnnN \c_true_bool + { + \if_num:w "##2 = \c_zero + \regex_item_exact_cs:c { \exp_after:wN \cs_to_str:N ##1 } + \else: + \regex_item_exact:nn { \int_value:w "##2 } { ##3 } + \fi: + } + { 1 } { 0 } \c_false_bool } - \int_set:Nn \l_regex_capturing_group_int - { \int_use:N \l_regex_capturing_group_int } } - \seq_push:Nx \l_regex_end_group_seq - { - \bool_if:NTF \l_regex_caseless_bool - \regex_build_caseless: - \regex_build_caseful: - \int_set_eq:NN - \l_regex_capturing_group_int - \l_regex_capturing_group_max_int - } - \regex_build_begin_alternation: } % \end{macrocode} % \end{macro} % -% \begin{macro}{\regex_build_special_group_i:} -% \begin{macro}{\regex_build_special_group_-:} -% \begin{macro}[aux]{\regex_build_options:NNN} -% \begin{macro}[aux]{\regex_build_option_+i:} -% \begin{macro}[aux]{\regex_build_option_-i:} -% The match can be made case-insensitive by setting the option -% with \texttt{(?i)}. +% \subsubsection{Other} +% +% \begin{macro}[aux]{\regex_compile_/K:} +% The |\K| control sequence is currently the only \enquote{command}, +% which performs some action, rather than matching something. It is +% allowed in the same contexts as |\b|. At the compilation stage, we +% leave it as a single control sequence, defined later. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_special_group_i: +\cs_new_protected_nopar:cpn { regex_compile_/K: } { - \regex_build_options:NNN + - \regex_build_raw:N i + \int_compare:nNnTF \l_regex_mode_int = \c_zero + { \tl_build_one:n { \regex_command_K: } } + { \regex_compile_raw_error:N K } } -\cs_new_protected_nopar:cpn { regex_build_special_group_-: } - { - \regex_build_options:NNN - - } -\cs_new_protected:Npn \regex_build_options:NNN #1#2#3 +% \end{macrocode} +% \end{macro} +% +% \subsubsection{Showing regexes} +% +% \begin{macro}[aux]{\regex_show_aux:Nx} +% Within a \cs{tl_set_build:Nw} \ldots{} \cs{tl_build_end:} group, we +% redefine all the function that can appear in a compiled regex, then +% run the regex. The result is then shown. +% \begin{macrocode} +\cs_new_protected:Npn \regex_show_aux:Nx #1#2 { - \token_if_eq_meaning:NNTF \regex_build_raw:N #2 - { - \cs_if_exist_use:cF { regex_build_option_#1#3: } - { \msg_kernel_error:nnx { regex } { unknown-option } { #3 } } - \regex_build_options:NNN #1 - } - { - \prg_case_str:nnn { #3 } - { % ( - { ) } { } - { - } { \regex_build_options:NNN - } - } - { \msg_kernel_error:nnx { regex } { invalid-in-option } { #3 } } - } + \tl_set_build:Nw \l_regex_internal_a_tl + \cs_set_protected_nopar:Npn \regex_branch:n + { + \seq_pop_right:NN \l_regex_show_prefix_seq \l_regex_internal_a_tl + \regex_show_one:n { +-branch } + \seq_put_right:No \l_regex_show_prefix_seq \l_regex_internal_a_tl + \use:n + } + \cs_set_protected_nopar:Npn \regex_group:nnnN + { \regex_show_group_aux:nnnnN { } } + \cs_set_protected_nopar:Npn \regex_group_no_capture:nnnN + { \regex_show_group_aux:nnnnN { ~(no~capture) } } + \cs_set_protected_nopar:Npn \regex_group_resetting:nnnN + { \regex_show_group_aux:nnnnN { ~(resetting) } } + \cs_set_eq:NN \regex_class:NnnnN \regex_show_class:NnnnN + \cs_set_protected_nopar:Npn \regex_command_K: + { \regex_show_one:n { reset~match~start~(\iow_char:N\\K) } } + \cs_set_protected:Npn \regex_assertion:Nn ##1##2 + { \regex_show_one:n { \bool_if:NF ##1 { negative~ } assertion:~##2 } } + \cs_set_nopar:Npn \regex_b_test: { word~boundary } + \cs_set_eq:NN \regex_anchor:N \regex_show_anchor_to_str:N + \cs_set_protected:Npn \regex_item_caseful_equal:n ##1 + { \regex_show_one:n { char~code~\int_eval:n{##1} } } + \cs_set_protected:Npn \regex_item_caseful_range:nn ##1##2 + { \regex_show_one:n { range~[\int_eval:n{##1}, \int_eval:n{##2}] } } + \cs_set_protected:Npn \regex_item_caseless_equal:n ##1 + { \regex_show_one:n { char~code~\int_eval:n{##1}~(caseless) } } + \cs_set_protected:Npn \regex_item_caseless_range:nn ##1##2 + { + \regex_show_one:n + { Range~[\int_eval:n{##1}, \int_eval:n{##2}]~(caseless) } + } + \cs_set_protected:Npn \regex_item_catcode:nT + { \regex_show_item_catcode:NnT \c_true_bool } + \cs_set_protected:Npn \regex_item_catcode_reverse:nT + { \regex_show_item_catcode:NnT \c_false_bool } + \cs_set_protected:Npn \regex_item_reverse:n + { \regex_show_scope:nn { Reversed~match } } + \cs_set_protected:Npn \regex_item_exact:nn ##1##2 + { \regex_show_one:n { char~##2,~catcode~##1 } } + \cs_set_protected:Npn \regex_item_exact_cs:c ##1 + { \regex_show_one:n { control~sequence~\iow_char:N\\##1 } } + \cs_set_protected:Npn \regex_item_cs:n + { \regex_show_scope:nn { control~sequence } } + \cs_set:cpn { regex_prop_.: } { \regex_show_one:n { any~token } } + \seq_clear:N \l_regex_show_prefix_seq + \regex_show_push:n { ~ } + #1 + \tl_build_end: + \msg_aux_show:x { > Compiled~regex~#2: \l_regex_internal_a_tl } } -\cs_new_protected_nopar:cpn { regex_build_option_+i: } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_show_one:n} +% Every part of the final message go through this function, which adds +% one line to the output, with the apropriate prefix. +% \begin{macrocode} +\cs_new_protected:Npn \regex_show_one:n #1 { - \regex_build_caseless: - \cs_set_eq:NN \regex_match_loop_case_hook: - \regex_match_loop_caseless_hook: + \int_incr:N \l_regex_show_lines_int + \tl_build_one:x + { \iow_newline: \seq_use:N \l_regex_show_prefix_seq #1 } } -\cs_new_protected_nopar:cpn { regex_build_option_-i: } - { \regex_build_caseful: } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} % -% \subsubsection{Quantifiers for groups} -% -% \begin{macro}[aux]{\regex_build_group_quantifier:} -% Used for one group. We need to keep track of submatches, -% threads can be created within one repetition, so things are hard. -% The code for the group that was just built starts -% at \cs{l_regex_left_state_int} and ends at -% \cs{l_regex_right_state_int}. +% \begin{macro}[aux] +% {\regex_show_push:n, \regex_show_pop:, \regex_show_scope:nn} +% Enter and exit levels of nesting. The \texttt{scope} function prints +% its first argument as an \enquote{introduction}, then performs its +% second argument in a deeper level of nesting. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_group_quantifier: +\cs_new_protected:Npn \regex_show_push:n #1 + { \seq_put_right:Nx \l_regex_show_prefix_seq { #1 ~ } } +\cs_new_protected:Npn \regex_show_pop: + { \seq_pop_right:NN \l_regex_show_prefix_seq \l_regex_internal_a_tl } +\cs_new_protected:Npn \regex_show_scope:nn #1#2 { - \tl_set:Nn \l_regex_one_or_group_tl { group } - \regex_build_quantifier:w + \regex_show_one:n {#1} + \regex_show_push:n { ~ } + #2 + \regex_show_pop: } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_group_submatches:NN} -% Once the quantifier is found by \cs{regex_build_quantifier:w}, -% we insert the code for tracking submatches. +% \begin{macro}[aux]{\regex_show_group_aux:nnnnN} +% We display all groups in the same way, simply adding a message, +% \texttt{(no capture)} or \texttt{(resetting)}, to special groups. +% The odd \cs{use_ii:nn} avoids printing a spurious \texttt{+-branch} +% for the first branch. % \begin{macrocode} -\cs_new_protected:Npn \regex_build_group_submatches:NN #1#2 +\cs_new_protected:Npn \regex_show_group_aux:nnnnN #1#2#3#4#5 { - \seq_pop:NN \l_regex_capturing_group_seq \l_regex_internal_a_tl - \int_compare:nNnF { \l_regex_internal_a_tl } < \c_zero - { - \regex_toks_put_left:Nx #1 - { \regex_action_submatch:n { \l_regex_internal_a_tl < } } - \regex_toks_put_left:Nx #2 - { \regex_action_submatch:n { \l_regex_internal_a_tl > } } - } + \regex_show_one:n { ,-group~begin #1 } + \regex_show_push:n { | } + \use_ii:nn #2 + \regex_show_pop: + \regex_show_one:n + { `-group~end \regex_msg_repeated:nnN {#3} {#4} #5 } } % \end{macrocode} % \end{macro} % -% Most quantifiers require to add an extra state before the group. -% This is done by shifting the current contents of the \cs{tex_toks:D} -% \cs{l_regex_internal_a_int} to a new state. -% -% \begin{macro}[aux]{\regex_build_group_qs_aux:NN} -% \begin{macro}[aux]{\regex_build_group_?:} -% \begin{macro}[aux]{\regex_build_group_??:} -% Shift the state at which the group begins using -% \cs{regex_build_group_shift:N}, then add two transitions. -% The first transition is taken once the group has been -% traversed: in the case of \texttt{?} and \texttt{??}, -% we should exit by going to \cs{l_regex_right_state_int}, -% while for \texttt{*} and \texttt{*?} we loop by going to -% \cs{l_regex_internal_a_int}. -% The second transition corresponds to skipping the group; -% it has lower priority (\texttt{put_right}) for greedy -% operators, and higher priority (\texttt{put_left}) for -% lazy operators. -% We shift the left state to avoid interactions with -% \texttt{action_submatch}. +% \begin{macro}[aux]{\regex_show_class:NnnnN} +% I'm entirely unhappy about this function: I couldn't find a way to +% test if a class is a single test. Instead, collect the +% representation of the tests in the class. If that had more than one +% line, write \texttt{Match} or \texttt{Don't match} on its own line, +% with the repeating information if any. Then the various tests on +% lines of their own, and finally a line. Otherwise, we need to +% evaluate the representation of the tests again (since the prefix is +% incorrect). That's clunky, but not too expensive, since it's only +% one test. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_group_qs_aux: +\cs_set:Npn \regex_show_class:NnnnN #1#2#3#4#5 { - \int_set_eq:NN \l_regex_internal_a_int \l_regex_left_state_int - \regex_build_new_state: - \tex_toks:D \l_regex_right_state_int = \tex_toks:D \l_regex_internal_a_int - \regex_toks_put_left:Nx \l_regex_right_state_int + \tl_set_build:Nw \l_regex_internal_a_tl + \int_zero:N \l_regex_show_lines_int + \regex_show_push:n {~} + #2 + \exp_last_unbraced:Nf + \prg_case_int:nnn { \l_regex_show_lines_int } { - \int_sub:Nn \l_regex_current_state_int + {0} { - \int_eval:n % ^^A here we lie! - { \l_regex_right_state_int - \l_regex_internal_a_int } + \tl_build_end: + \regex_show_one:n { \bool_if:NTF #1 { Fail } { Pass } } } - } - \use:x - { - \tex_toks:D \l_regex_internal_a_int + {1} { - \s_stop - \regex_action_free:n + \tl_build_end: + \bool_if:NTF #1 { - \int_eval:n - { \l_regex_right_state_int - \l_regex_internal_a_int } + #2 + \tl_build_one:n { \regex_msg_repeated:nnN {#3} {#4} #5 } + } + { + \regex_show_one:n + { Don't~match~\regex_msg_repeated:nnN {#3} {#4} #5 } + \tl_build_one:o \l_regex_internal_a_tl } } } - \regex_build_group_submatches:NN - \l_regex_right_state_int \l_regex_left_state_int - } -\cs_new_protected:Npn \regex_build_group_qs_aux:NN #1#2 - { - \regex_build_group_qs_aux: - \int_set_eq:NN \l_regex_right_state_int \l_regex_left_state_int - \regex_build_transition:NN \regex_action_free:n #1 - #2 \l_regex_internal_a_int { - \regex_action_free:n - { \int_eval:n { \l_regex_right_state_int - \l_regex_internal_a_int } } + \tl_build_end: + \regex_show_one:n + { + \bool_if:NTF #1 { M } { Don't~m } atch + \regex_msg_repeated:nnN {#3} {#4} #5 + } + \tl_build_one:o \l_regex_internal_a_tl } } -\cs_new_protected_nopar:cpn { regex_build_group_?: } - { - \regex_build_group_qs_aux:NN - \l_regex_right_state_int \regex_toks_put_right:Nx - } -\cs_new_protected_nopar:cpn { regex_build_group_??: } - { - \regex_build_group_qs_aux:NN - \l_regex_right_state_int \regex_toks_put_left:Nx - } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}[aux]{\regex_build_group_n_aux:n} -% \begin{macro}[aux]{\regex_build_group_transition_right_max:} -% The braced quantifiers rely on replicating the states corresponding -% to the group that has just been built, and joining the right state -% of each copy to the left state of the next copy. Once this function -% has been run, \cs{l_regex_internal_a_int} points to the last copy of -% the initial left-most state, \cs{l_regex_left/right_state_int} have -% their initial values. Furthermore, \cs{l_regex_max_state_int} is set -% appropriately to the largest allocated \tn{toks} register plus $1$. +% \begin{macro}[aux, rEXP]{\regex_show_anchor_to_str:N} +% The argument is an integer telling us where the anchor is. We +% convert that to the relevant info. % \begin{macrocode} -\cs_new_protected:Npn \regex_build_group_transition_right_max: +\cs_new:Npn \regex_show_anchor_to_str:N #1 { - \regex_toks_put_right:Nx \l_regex_right_state_int + anchor~at~ + \prg_case_str:nnn { #1 } { - \regex_action_free:n - { \int_eval:n { \l_regex_max_state_int - \l_regex_right_state_int } } + { \l_regex_min_pos_int } { start~(\iow_char:N\\A) } + { \l_regex_start_pos_int } { start~of~match~(\iow_char:N\\G) } + { \l_regex_max_pos_int } { end~(\iow_char:N\\Z) } } + { <error:~'#1'~not~recognized> } } -\cs_new_protected:Npn \regex_build_group_n_aux:n #1 +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_show_item_catcode:NnT} +% Produce a sequence of categories which the catcode bitmap |#2| +% contains, and show it, indenting the tests on which this catcode +% constraint applies. +% \begin{macrocode} +\cs_new_protected:Npn \regex_show_item_catcode:NnT #1#2 { - \int_set_eq:NN \l_regex_internal_a_int \l_regex_left_state_int - \int_set_eq:NN \l_regex_internal_b_int \l_regex_max_state_int - \int_add:Nn \l_regex_right_state_int - { ( #1 - 1 ) * ( \l_regex_max_state_int - \l_regex_left_state_int ) } - \int_set:Nn \l_regex_max_state_int - { - \l_regex_left_state_int - + (#1) * ( \l_regex_max_state_int - \l_regex_left_state_int ) - } - \int_while_do:nNnn \l_regex_internal_b_int < \l_regex_max_state_int + \seq_set_split:Nnn \l_regex_internal_seq { } { CBEMTPUDSLOA } + \seq_set_filter:NNn \l_regex_internal_seq \l_regex_internal_seq + { \int_if_odd_p:n { #2 / \int_use:c { c_regex_catcode_##1_int } } } + \regex_show_scope:nn { - \tex_toks:D \l_regex_internal_b_int - = \tex_toks:D \l_regex_internal_a_int - \int_incr:N \l_regex_internal_a_int - \int_incr:N \l_regex_internal_b_int + categories~\seq_use:N \l_regex_internal_seq, ~ + \bool_if:NF #1 { negative~ } class } } % \end{macrocode} % \end{macro} +% +% \subsection{Building} +% +% \subsubsection{Variables used while building} +% +% \begin{variable}{\l_regex_min_state_int, \l_regex_max_state_int} +% The last state that was allocated is $\cs{l_regex_max_state_int}-1$, +% so that \cs{l_regex_max_state_int} always points to a free state. +% The \texttt{min_state} variable is always $0$, but is included to +% avoid hard-coding this value. +% \begin{macrocode} +\int_new:N \l_regex_min_state_int +\int_new:N \l_regex_max_state_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_regex_left_state_int, \l_regex_right_state_int} +% \begin{variable}{\l_regex_left_state_seq, \l_regex_right_state_seq} +% Alternatives are implemented by branching from a \texttt{left} state +% into the various choices, then merging those into a \texttt{right} +% state. We store information about those states in two sequences. +% Those states are also used to implement group quantifiers. Most +% often, the left and right pointers only differ by~$1$. +% \begin{macrocode} +\int_new:N \l_regex_left_state_int +\int_new:N \l_regex_right_state_int +\seq_new:N \l_regex_left_state_seq +\seq_new:N \l_regex_right_state_seq +% \end{macrocode} +% \end{variable} +% \end{variable} +% +% \begin{variable}{\l_regex_capturing_group_int} +% \cs{l_regex_capturing_group_int} is the \textsc{id} number that will +% be assigned to a capturing group if one was opened now. This starts +% at $0$ for the group enclosing the full regular expression, and +% groups are counted in the order of their left parenthesis, except +% when encountering \texttt{resetting} groups. +% \begin{macrocode} +\int_new:N \l_regex_capturing_group_int +% \end{macrocode} +% \end{variable} +% +% \subsubsection{Framework} +% +% This phase is about going from a compiled regex to an \textsc{nfa}. +% Each state of the \textsc{nfa} is stored in a \tn{toks}. The +% operations which can appear in the \tn{toks} are +% \begin{itemize} +% \item \cs{regex_action_start_wildcard:} inserted at the start +% of the regular expression to make it unanchored. +% \item \cs{regex_action_success:} marks the exit state of the +% \textsc{nfa}. +% \item \cs{regex_action_cost:n} \Arg{shift} is a transition from the +% current \meta{state} to $\meta{state}+\meta{shift}$, which +% consumes the current character: the target state is saved and will +% be considered again when matching at the next position. +% \item \cs{regex_action_free:n} \Arg{shift}, and +% \cs{regex_action_free_group:n} \Arg{shift} are free transitions, +% which immediately perform the actions for the state +% $\meta{state}+\meta{shift}$ of the \textsc{nfa}. They differ in +% how they detect and avoid infinite loops. For now, we just need to +% know that the \texttt{group} variant must be used for transitions +% back to the start of a group. +% \item \cs{regex_action_submatch:n} \Arg{key} where the \meta{key} is +% a group number followed by |<| or |>| for the beginning or end of +% group. This causes the current position in the query to be stored +% as the \meta{key} submatch boundary. +% \end{itemize} +% +% We strive to preserve the following properties while building. +% \begin{itemize} +% \item The current capturing group is +% $\text{\texttt{capturing_group}}-1$, and if a group is opened now, +% it will be labelled \texttt{capturing_group}. +% \item The last allocated state is $\text{\texttt{max_state}}-1$, so +% \texttt{max_state} is a free state. +% \item The \texttt{left_state} points to a state to the left of the +% current group or of the last class. +% \item The \texttt{right_state} points to a newly created, +% empty state, with some transitions leading to it. +% \item The \texttt{left/right} sequences hold a list of the +% corresponding end-points of nested groups. +% \end{itemize} +% +% \begin{macro}[int]{\regex_build:n, \regex_build:N} +% The \texttt{n}-type function first compiles its argument. Reset some +% variables. Allocate two states, and put a wildcard in state $0$ +% (transitions to state $1$ and $0$ state). Then build the regex +% within a (capturing) group, which will be numbered $0$ (current +% value of \texttt{capturing_group}). Finally, if the match reaches the +% last state, it is successful. +% \begin{macrocode} +\cs_new_protected:Npn \regex_build:n #1 + { + \regex_compile:n {#1} + \regex_build:N \l_regex_internal_regex + } +\cs_new_protected:Npn \regex_build:N #1 + { +%<trace> \trace_push:nnn { regex } { 1 } { regex_build } + \int_set:Nn \tex_escapechar:D { 92 } + \int_zero:N \l_regex_capturing_group_int + \int_set_eq:NN \l_regex_max_state_int \l_regex_min_state_int + \regex_build_new_state: + \regex_build_new_state: + \regex_toks_put_right:Nn \l_regex_left_state_int + { \regex_action_start_wildcard: } + \regex_group:nnnN {#1} { 1 } { 0 } \c_false_bool + \regex_toks_put_right:Nn \l_regex_right_state_int + { \regex_action_success: } +%<trace> \regex_trace_states:n { 2 } +%<trace> \trace_pop:nnn { regex } { 1 } { regex_build } + } +% \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_group_n:, \regex_build_group_n?:} -% \begin{macro}[aux]{\regex_build_group_:} -% These functions are called in case the syntax is -% \texttt{\{\meta{int}\}}, or in the absence of quantifier (with -% $n=1$). Greedy and lazy operators are identical, since the number of -% repetitions is fixed. We only record the submatch information at -% the last repetition; and of course, in the case $n=0$, we record -% nothing. +% \begin{macro}[int]{\regex_build_for_cs:n} +% When using a regex to match a cs, we don't insert a wildcard, we +% anchor at the end, and since we ignore submatches, there is no need +% to surround the expression with a group. However, for branches to +% work properly at the outer level, we need to put the appropriate +% \texttt{left} and \texttt{right} states in their sequence. % \begin{macrocode} -\cs_new_protected:Npn \regex_build_group_n: #1 +\cs_new_protected:Npn \regex_build_for_cs:n #1 { - \regex_build_group_transition_right_max: - \regex_build_group_n_aux:n {#1} +%<trace> \trace_push:nnn { regex } { 1 } { regex_build_for_cs } + \int_set_eq:NN \l_regex_max_state_int \l_regex_min_state_int + \regex_build_new_state: \regex_build_new_state: - \int_compare:nNnF {#1} = \c_zero + \regex_push_lr_states: + #1 + \regex_pop_lr_states: + \regex_toks_put_right:Nn \l_regex_right_state_int { - \regex_build_group_submatches:NN - \l_regex_internal_a_int \l_regex_left_state_int + \if_num:w \l_regex_current_pos_int = \l_regex_max_pos_int + \exp_after:wN \regex_action_success: + \fi: } +%<trace> \regex_trace_states:n { 2 } +%<trace> \trace_pop:nnn { regex } { 1 } { regex_build_for_cs } } -\cs_new_eq:cN { regex_build_group_n?: } \regex_build_group_n: -\cs_new_protected_nopar:Npn \regex_build_group_: - { \regex_build_group_n: \c_one } % \end{macrocode} % \end{macro} -% \end{macro} % -% \begin{macro}[aux]{\regex_build_group_n*:, \regex_build_group_n*?:} -% \begin{macro}[aux]{\regex_build_group_*:, \regex_build_group_*?:} -% \begin{macro}[aux]{\regex_build_group_+:, \regex_build_group_+?:} -% \begin{macro}[aux]{\regex_build_group_n_star_aux:NNn} -% These functions are called in case the syntax is -% \texttt{\{\meta{int},\}}. They are somewhat hybrid between -% the \texttt{\{\meta{int}\}} and the \texttt{*} quantifiers. -% Contrarily to the \texttt{*} quantifier, for which we had -% to be careful not to overwrite the submatch information in -% case no iteration was made, here, we know that the submatch -% information is overwritten in any case. +% \subsubsection{Helpers for building an \textsc{nfa}} +% +% \begin{macro}[int]{\regex_push_lr_states:, \regex_pop_lr_states:} +% When building the regular expression, we keep track of pointers to +% the left-end and right-end of each group without help from \TeX{}'s +% grouping. % \begin{macrocode} -\cs_new_protected_nopar:cpn { regex_build_group_n*: } +\cs_new_protected_nopar:Npn \regex_push_lr_states: { - \regex_build_group_n_star_aux:NNn - \regex_toks_put_right:Nx \regex_toks_put_left:Nx + \seq_push:No \l_regex_left_state_seq + { \int_use:N \l_regex_left_state_int } + \seq_push:No \l_regex_right_state_seq + { \int_use:N \l_regex_right_state_int } } -\cs_new_protected_nopar:cpn { regex_build_group_n*?: } +\cs_new_protected_nopar:Npn \regex_pop_lr_states: { - \regex_build_group_n_star_aux:NNn - \regex_toks_put_left:Nx \regex_toks_put_right:Nx + \seq_pop:NN \l_regex_left_state_seq \l_regex_internal_a_tl + \int_set:Nn \l_regex_left_state_int \l_regex_internal_a_tl + \seq_pop:NN \l_regex_right_state_seq \l_regex_internal_a_tl + \int_set:Nn \l_regex_right_state_int \l_regex_internal_a_tl } -\cs_new_protected:Npn \regex_build_group_n_star_aux:NNn #1#2#3 +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int]{\regex_toks_put_left:Nx} +% \begin{macro}[int]{\regex_toks_put_right:Nx, \regex_toks_put_right:Nn} +% During the building phase we wish to add \texttt{x}-expanded +% material to \tn{toks}, either to the left or to the right. The +% expansion is done \enquote{by hand} for optimization (these +% operations are used quite a lot). The \texttt{Nn} version of +% \cs{regex_toks_put_right:Nx} is provided because it is more +% efficient than \texttt{x}-expanding with \cs{exp_not:n}. +% \begin{macrocode} +\cs_new_protected:Npn \regex_toks_put_left:Nx #1#2 { - \int_compare:nNnTF {#3} = \c_zero - { \regex_build_group_qs_aux:NN \l_regex_internal_a_int #1 } - { - \regex_build_group_transition_right_max: - \regex_build_group_n_aux:n {#3} - \regex_build_new_state: - #2 \l_regex_left_state_int - { - \regex_action_free:n - { - \int_eval:n - { \l_regex_internal_a_int - \l_regex_left_state_int } - } - } - \regex_build_group_submatches:NN - \l_regex_internal_a_int \l_regex_left_state_int - } + \cs_set_nopar:Npx \regex_tmp:w { #2 } + \tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN + { \exp_after:wN \regex_tmp:w \tex_the:D \tex_toks:D #1 } } -\cs_new_protected_nopar:cpx { regex_build_group_*: } - { \exp_not:c { regex_build_group_n*: } \c_zero } -\cs_new_protected_nopar:cpx { regex_build_group_*?: } - { \exp_not:c { regex_build_group_n*?: } \c_zero } -\cs_new_protected_nopar:cpx { regex_build_group_+: } - { \exp_not:c { regex_build_group_n*: } \c_one } -\cs_new_protected_nopar:cpx { regex_build_group_+?: } - { \exp_not:c { regex_build_group_n*?: } \c_one } +\cs_new_protected:Npn \regex_toks_put_right:Nx #1#2 + { + \cs_set_nopar:Npx \regex_tmp:w {#2} + \tex_toks:D #1 \exp_after:wN + { \tex_the:D \tex_toks:D \exp_after:wN #1 \regex_tmp:w } + } +\cs_new_protected:Npn \regex_toks_put_right:Nn #1#2 + { \tex_toks:D #1 \exp_after:wN { \tex_the:D \tex_toks:D #1 #2 } } % \end{macrocode} % \end{macro} % \end{macro} -% \end{macro} +% +% \begin{macro}[int] +% { +% \regex_build_transition_left:NNN, +% \regex_build_transition_right:nNn +% } +% Add a transition from |#2| to |#3| using the function |#1|. The +% \texttt{left} function is used for higher priority transitions, and +% the \texttt{right} function for lower priority transitions (which +% should be performed later). The signatures differ to reflect the +% differing usage later on. Both functions could be optimized. +% \begin{macrocode} +\cs_new_protected:Npn \regex_build_transition_left:NNN #1#2#3 + { \regex_toks_put_left:Nx #2 { #1 { \int_eval:n { #3 - #2 } } } } +\cs_new_protected:Npn \regex_build_transition_right:nNn #1#2#3 + { \regex_toks_put_right:Nx #2 { #1 { \int_eval:n { #3 - #2 } } } } +% \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_group_nn:, \regex_build_group_nn?:} -% \begin{macro}[aux]{\regex_build_group_nn_aux:nnnn} -% \begin{macro}[aux, rEXP]{\regex_build_group_nn_aux_ii:w} -% These functions are called when the syntax is either -% \texttt{\{\meta{int},\}} or \texttt{\{\meta{int},\meta{int}\}}. +% \begin{macro}[int]{\regex_build_new_state:} +% Add a new empty state to the \textsc{nfa}. Then update the +% \texttt{left}, \texttt{right}, and \texttt{max} states, so that the +% \texttt{right} state is the new empty state, and the \texttt{left} +% state points to the previously \enquote{current} state. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_group_nn: - { - \regex_build_group_nn_aux:nnnn { } - \regex_build_group_transition_right_max: - } -\cs_new_protected_nopar:cpn { regex_build_group_nn?: } +\cs_new_protected_nopar:Npn \regex_build_new_state: { - \regex_build_group_nn_aux:nnnn { } +%<*trace> + \trace:nnx { regex } { 2 } { - \use:x - { - \tex_toks:D \l_regex_right_state_int - { - \exp_after:wN \regex_build_group_nn_aux_ii:w - \tex_the:D \tex_toks:D \l_regex_right_state_int - } - } + regex~new~state~ + L=\int_use:N \l_regex_left_state_int ~ -> ~ + R=\int_use:N \l_regex_right_state_int ~ -> ~ + M=\int_use:N \l_regex_max_state_int ~ -> ~ + \int_eval:n { \l_regex_max_state_int + \c_one } } +%</trace> + \tex_toks:D \l_regex_max_state_int { } + \int_set_eq:NN \l_regex_left_state_int \l_regex_right_state_int + \int_set_eq:NN \l_regex_right_state_int \l_regex_max_state_int + \int_incr:N \l_regex_max_state_int } -\cs_new_protected:Npn \regex_build_group_nn_aux:nnnn #1#2#3#4 +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int]{\regex_build_transitions_lazyness:NNNNN} +% This function creates a new state, and puts two transitions starting +% from the old current state. The order of the transitions is +% controlled by |#1|, true for lazy quantifiers, and false for greedy +% quantifiers. +% \begin{macrocode} +\cs_new_protected:Npn \regex_build_transitions_lazyness:NNNNN #1#2#3#4#5 { - \int_compare:nNnTF {#3} > {#4} - { - \msg_kernel_error:nnxxx - { regex } { quantifier-backwards } {#3} {#4} {#1} - } + \regex_build_new_state: + \regex_toks_put_right:Nx \l_regex_left_state_int { - \int_compare:nNnTF {#3} = \c_zero - { - \regex_build_group_qs_aux: - \int_set_eq:NN \l_regex_right_state_int \l_regex_left_state_int - \int_set_eq:NN \l_regex_left_state_int \l_regex_internal_a_int - \regex_build_group_transition_right_max: - \regex_build_group_n_aux:n { #4 - #3 } - \int_set_eq:NN \l_regex_internal_c_int \l_regex_right_state_int - \int_set_eq:NN \l_regex_right_state_int \l_regex_left_state_int - \prg_replicate:nn { #4 - #3 } - { - #2 - \int_add:Nn \l_regex_right_state_int - { \l_regex_internal_b_int - \l_regex_internal_a_int } - } - \int_set_eq:NN \l_regex_right_state_int \l_regex_internal_c_int - \int_set_eq:NN \l_regex_left_state_int \l_regex_internal_a_int - \regex_build_new_state: - } - { - \regex_build_group_transition_right_max: - \regex_build_group_n_aux:n {#3} - \int_set_eq:NN \l_regex_left_state_int \l_regex_internal_a_int - \regex_build_group_submatches:NN - \l_regex_left_state_int \l_regex_right_state_int - \regex_build_group_n_aux:n { #4 - #3 + \c_one } - \int_sub:Nn \l_regex_right_state_int - { \l_regex_internal_a_int - \l_regex_left_state_int } - \prg_replicate:nn { #4 - #3 } - { - #2 - \int_add:Nn \l_regex_right_state_int - { \l_regex_internal_b_int - \l_regex_internal_a_int } - } - \int_set_eq:NN \l_regex_left_state_int \l_regex_internal_a_int - \regex_build_new_state: - } + \if_meaning:w \c_true_bool #1 + #2 { \int_eval:n { #3 - \l_regex_left_state_int } } + #4 { \int_eval:n { #5 - \l_regex_left_state_int } } + \else: + #4 { \int_eval:n { #5 - \l_regex_left_state_int } } + #2 { \int_eval:n { #3 - \l_regex_left_state_int } } + \fi: } } -\cs_new_protected:Npn \regex_build_group_nn_aux_ii:w #1 \regex_action_free:n +% \end{macrocode} +% \end{macro} +% +% \subsubsection{Building classes} +% +% \begin{macro}[int]{\regex_class:NnnnN} +% \begin{macro}[int, rEXP]{\regex_tests_action_cost:n} +% The arguments are: \meta{boolean} \Arg{tests} \Arg{min} \Arg{more} +% \meta{lazyness}. First store the tests with a trailing +% \cs{regex_action_cost:n}, in the true branch of +% \cs{regex_break_point:TF} for positive classes, or the false branch +% for negative classes. The integer \meta{more} is $0$ for fixed +% repetitions, $-1$ for unbounded repetitions, and +% $\meta{max}-\meta{min}$ for a range of repetitions. +% \begin{macrocode} +\cs_new_protected:Npn \regex_class:NnnnN #1#2#3#4#5 { - #1 - \regex_action_free:n + \cs_set_nopar:Npx \regex_tests_action_cost:n ##1 { - \int_eval:n - { \l_regex_max_state_int - \l_regex_right_state_int } + \exp_not:n { \exp_not:n {#2} } + \bool_if:NTF #1 + { \regex_break_point:TF { \regex_action_cost:n {##1} } { } } + { \regex_break_point:TF { } { \regex_action_cost:n {##1} } } } - \regex_action_free:n + \if_case:w - #4 \exp_stop_f: + \regex_class_repeat:n {#3} + \or: \regex_class_repeat:nN {#3} #5 + \else: \regex_class_repeat:nnN {#3} {#4} #5 + \fi: } +\cs_new:Npn \regex_tests_action_cost:n { \regex_action_cost:n } % \end{macrocode} % \end{macro} % \end{macro} -% \end{macro} -% -% \subsubsection{Matching raw token lists with \cs{u}} -% -%^^A todo: document interactions between caselessness and |\u{...}| -%^^A todo: clarify when l_regex_catcodes_int is reset. % -% \begin{macro}[aux]{\regex_build_/u:} -% \begin{macro}[aux]{\regex_build_u_loop:NN} -% The |\u| escape is invalid in classes nad directly following a -% catcode test. Otherwise, it must be followed by a left brace. We -% then collect the characters for the argument of |\u| within an -% \texttt{x}-expanding assignment. In principle we could just wait to -% encounter a right brace, but this is unsafe: if the right brace is -% missing, then we will reach the end-markers of the regex, and -% continue, leading to obscure fatal errors. Instead, we only allow -% raw and special characters, and stop when encountering a special -% right brace. +% \begin{macro}[aux]{\regex_class_repeat:n} +% This is used for a fixed number of repetitions. Build one state for +% each repetition, with a transition controlled by the tests that we +% have collected. That works just fine for |#1|${}=0$ repetitions: +% nothing is built. % \begin{macrocode} -\cs_new_protected:cpn { regex_build_/u: } #1#2 +\cs_new_protected:Npn \regex_class_repeat:n #1 { - \regex_build_if_assertions_forbidden:TF - { \regex_build_raw_error:N u #1#2 } + \prg_replicate:nn {#1} { - \str_if_eq:xxTF {#1#2} { \regex_build_special:N \c_lbrace_str } - { - \tl_set:Nx \l_regex_internal_a_tl { \if_false: } \fi: - \regex_build_u_loop:NN - } - { - \msg_kernel_error:nn { regex } { u-missing-lbrace } - #1#2 - } + \regex_build_new_state: + \regex_build_transition_right:nNn \regex_tests_action_cost:n + \l_regex_left_state_int \l_regex_right_state_int } } -\cs_new:Npn \regex_build_u_loop:NN #1#2 +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_class_repeat:nN} +% This implements unbounded repetitions of a single class (\emph{e.g.} +% the |*| and |+| quantifiers). If the minimum number |#1| of +% repetitions is $0$, then build a transition from the current state +% to itself governed by the tests, and a free transition to a new +% state (hence skipping the tests). Otherwise, call +% \cs{regex_class_repeat:n} for the code to match |#1| repetitions, +% and add free transitions from the last state to the previous one, +% and to a new one. In both cases, the order of transitions is +% controlled by the lazyness boolean |#2|. +% \begin{macrocode} +\cs_new_protected:Npn \regex_class_repeat:nN #1#2 + { + \if_num:w #1 = \c_zero + \regex_build_transitions_lazyness:NNNNN #2 + \regex_action_free:n \l_regex_right_state_int + \regex_tests_action_cost:n \l_regex_left_state_int + \else: + \regex_class_repeat:n {#1} + \int_set_eq:NN \l_regex_internal_a_int \l_regex_left_state_int + \regex_build_transitions_lazyness:NNNNN #2 + \regex_action_free:n \l_regex_right_state_int + \regex_action_free:n \l_regex_internal_a_int + \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_class_repeat:nnN} +% We want to build the code to match from |#1| to $|#1|+|#2|$ +% repetitions. Match |#1| repetitions (can be $0$). Compute the final +% state of the next construction as \texttt{a}. Build $|#2|>0$ states, +% each with a transition to the next state governed by the tests, and +% a transition to the final state \texttt{a}. The computation of +% \texttt{a} is safe because states are allocated in order, starting +% from \texttt{max_state}. +% \begin{macrocode} +\cs_new_protected:Npn \regex_class_repeat:nnN #1#2#3 { - \token_if_eq_meaning:NNTF #1 \regex_build_raw:N - { #2 \regex_build_u_loop:NN } + \regex_class_repeat:n {#1} + \int_set:Nn \l_regex_internal_a_int + { \l_regex_max_state_int + #2 - \c_one } + \prg_replicate:nn { #2 } { - \token_if_eq_meaning:NNTF #1 \regex_build_special:N - { - \exp_after:wN \token_if_eq_charcode:NNTF \c_rbrace_str #2 - { \if_false: { \fi: } \regex_build_u_end: } - { #2 \regex_build_u_loop:NN } - } - { - \if_false: { \fi: } - \msg_kernel_error:nnx { regex } { u-missing-rbrace } {#2} - \regex_build_u_end: - #1#2 - } + \regex_build_transitions_lazyness:NNNNN #3 + \regex_action_free:n \l_regex_internal_a_int + \regex_tests_action_cost:n \l_regex_right_state_int } } % \end{macrocode} % \end{macro} -% \end{macro} % -% \begin{macro}[aux]{\regex_build_u_end:} -% Once we have extracted the variable's name, we store the contents of -% that variable in \cs{l_regex_internal_a_tl}. The behaviour of |\u| -% then depends on whether we are within a |\c{...}| escape (in this -% case, the variable is turned to a string), or not. Afterwards, clean -% up the value of \cs{l_regex_tests_tl}, but not -% \cs{l_regex_tests_bool}, because this is guaranteed to be -% \texttt{true} at the start, and is not affected by anything we do -% here. +% \subsubsection{Building groups} +% +% \begin{macro}[aux]{\regex_group_aux:nnnnN} +% Arguments: \Arg{label} \Arg{contents} \Arg{min} \Arg{more} +% \meta{lazyness}. If \meta{min} is $0$, we need to add a state before +% building the group, so that the thread which skips the group does +% not also set the start-point of the submatch. After adding one more +% state, the \texttt{left_state} is the left end of the group, from +% which all branches will stem, and the \texttt{right_state} is the +% right end of the group, and all branches end their course in that +% state. We store those two integers to be queried for each branch, we +% build the \textsc{nfa} states for the contents |#2| of the group, +% and we forget about the two integers. Once this is done, perform the +% repetition: either exactly |#3| times, or |#3| or more times, or +% between |#3| and $|#3|+|#4|$ times, with lazyness |#5|. The +% \meta{label} |#1| is used for submatch tracking. Each of the three +% auxiliaries expects \texttt{left_state} and \texttt{right_state} to +% be set properly. % \begin{macrocode} -\cs_new_protected:Npn \regex_build_u_end: +\cs_new_protected:Npn \regex_group_aux:nnnnN #1#2#3#4#5 { - \tl_set:Nv \l_regex_internal_a_tl \l_regex_internal_a_tl - \if_num:w \l_regex_build_mode_int = \c_zero - \regex_build_u_not_cs: - \else: - \regex_build_u_in_cs: - \fi: - \tl_clear:N \l_regex_tests_tl +%<trace> \trace_push:nnn { regex } { 1 } { regex_group } + \if_num:w #3 = \c_zero + \regex_build_new_state: +%<assert>\assert_int:n { \l_regex_max_state_int = \l_regex_right_state_int + 1 } + \regex_build_transition_right:nNn \regex_action_free_group:n + \l_regex_left_state_int \l_regex_right_state_int + \fi: + \regex_build_new_state: + \regex_push_lr_states: + #2 + \regex_pop_lr_states: + \if_case:w - #4 \exp_stop_f: + \regex_group_repeat:nn {#1} {#3} + \or: \regex_group_repeat:nnN {#1} {#3} #5 + \else: \regex_group_repeat:nnnN {#1} {#3} {#4} #5 + \fi: +%<trace> \trace_pop:nnn { regex } { 1 } { regex_group } } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_u_in_cs:} -% When |\u| appears within a control sequence, the variable is -% converted to a string, and we add to the NFA one state per -% character. We use \cs{regex_build_one_:} directly, as this avoids -% having to search for a quantifier. +% \begin{macro}[int]{\regex_group:nnnN, \regex_group_no_capture:nnnN} +% Hand to \cs{regex_group_aux:nnnnnN} the label of that group +% (expanded), and the group itself, with some extra commands to +% perform. % \begin{macrocode} -\cs_new_protected:Npn \regex_build_u_in_cs: +\cs_new_protected:Npn \regex_group:nnnN #1 { - \exp_args:NNo \str_gset_other:Nn \g_regex_internal_tl - { \l_regex_internal_a_tl } - \tl_map_inline:Nn \g_regex_internal_tl + \exp_args:No \regex_group_aux:nnnnN + { \int_use:N \l_regex_capturing_group_int } { - \tl_set:Nx \l_regex_tests_tl { \regex_item_equal:n { `##1 } } - \regex_build_one_: + \int_incr:N \l_regex_capturing_group_int + #1 } } +\cs_new_protected_nopar:Npn \regex_group_no_capture:nnnN + { \regex_group_aux:nnnnN { -1 } } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_u_not_cs:} -% In mode $0$, the |\u| escape adds one state to the NFA for each -% token in \cs{l_regex_internal_a_tl}. The tests corresponding to a -% given \meta{token} are: first test the category, then, if the -% \meta{token} is a control sequence, compare the string -% representations, otherwise compare the character codes. We could -% compare the string representations in all cases, but that doesn't -% interact well with caseless matching. +% \begin{macro}[int]{\regex_group_resetting:nnnN} +% \begin{macro}[aux]{\regex_group_resetting_loop:nnNn} +% Again, hand the label $-1$ to \cs{regex_group_aux:nnnnN}, but this +% time we work a little bit harder to keep track of the maximum group +% label at the end of any branch, and to reset the group number at +% each branch. This relies on the fact that a compiled regex always is +% a sequence of items of the form \cs{regex_branch:n} \Arg{branch}. % \begin{macrocode} -\cs_new_protected:Npn \regex_build_u_not_cs: +\cs_new_protected:Npn \regex_group_resetting:nnnN #1 { - \exp_args:No \tl_analysis_map_inline:nn { \l_regex_internal_a_tl } + \regex_group_aux:nnnnN { -1 } { - \tl_set:Nx \l_regex_tests_tl - { - \exp_not:N \int_compare:nNnT - \l_regex_current_catcode_int = { \int_value:w "##2 } - { - \int_compare:nNnTF { "##2 } = \c_zero - { - \exp_not:n - { \str_if_eq:xxT { \l_regex_current_token_tl } {##1} } - { \regex_break_true:w } - } - { \regex_item_equal:n {##3} } - } - } - \regex_build_one_: + \exp_args:Noo \regex_group_resetting_loop:nnNn + { \int_use:N \l_regex_capturing_group_int } + { \int_use:N \l_regex_capturing_group_int } + #1 + { ?? \prg_map_break:n } { } + \prg_break_point:n { } } } +\cs_new_protected:Npn \regex_group_resetting_loop:nnNn #1#2#3#4 + { + \use_none:nn #3 { \int_set:Nn \l_regex_capturing_group_int {#1} } + \int_set:Nn \l_regex_capturing_group_int {#2} + #3 {#4} + \exp_args:Nf \regex_group_resetting_loop:nnNn + { \int_max:nn {#1} { \l_regex_capturing_group_int } } + {#2} + } % \end{macrocode} % \end{macro} +% \end{macro} % -% \subsection{Matching} -% -% At every index, we unpack that array of active states and -% empty it. Then loop over all active states, and perform the -% instruction at that state of the NFA. This can involve -% \enquote{free} transitions to other states, or transitions -% which \enquote{consume} the current character. For free -% transitions, the instruction at the new state of the NFA is -% performed. When a transition consumes a character, the new -% state is put in the array of \tn{skip} registers: it will -% be active again when the next character is read. -% -% If two paths through the NFA \enquote{collide} in the sense -% that they reach the same state when reading a given -% character, then any future execution will be identical for -% both. Hence, it is indeed enough to keep track of which -% states are active. [In the presence of back-references, the -% future execution is affected by how the previous match took -% place; this is why we cannot support those non-regular -% features.] -% -% Many of the functions require extracting the submatches for -% the \enquote{best} match. Execution paths through the NFA -% are ordered by precedence: for instance, the regular -% expression \texttt{a?} creates two paths, matching either -% an empty token list or a single \texttt{a}; the path matching -% an \texttt{a} has higher precedence. When two paths collide, -% the path with the highest precedence is kept, and the other -% one is discarded. The submatch information for a given path -% is stored at the start of the \tn{toks} register which -% holds the state at which that path currently is. -% -% Deciding to store the submatch information in \tn{toks} -% registers alongside with states of the NFA unfortunately -% implies some shuffling around. The two other options are to -% store the submatch information in one control sequence per -% path, which wastes csnames, or to store all of the submatch -% information in one property list, which turns out to be too -% slow. A tricky aspect of submatch tracking is to know when -% to get rid of submatch information. This naturally happens -% when submatch information is stored in \tn{toks} registers: -% if the information is not moved, it will be overwritten -% later. -% -% The presence of $\epsilon$-transitions (transitions which -% consume no character) leads to potential infinite loops; -% for instance the regular expression |(a??)*| could lead to -% an infinite recursion, where |a??| matches no character, |*| -% loops back to the start of the group, and |a??| matches no -% character again. Therefore, we need to keep track of the -% states of the NFA visited at the current step. More -% precisely, a state is marked as \enquote{visited} if the -% instructions for that state have been inserted in the input -% stream, by setting the corresponding \tn{dimen} register to -% a value which uniquely identifies at which step it was last -% inserted. -% -% \subsubsection{Helpers for running the NFA} -% -% \begin{macro}[aux]{\regex_store_state:n} -% Put the given state in the array of \tn{skip} registers. -% This is done by increasing the pointer -% \cs{l_regex_max_active_int}, and converting the integer -% to a dimension (suitable for a \tn{skip} assignment) in -% scaled points. +% \begin{macro}[int]{\regex_branch:n} +% Add a free transition from the left state of the current group to a +% brand new state, starting point of this branch. Once the branch is +% built, add a transition from its last state to the right state of +% the group. The left and right states of the group are extracted from +% the relevant sequences. % \begin{macrocode} -\cs_new_protected:Npn \regex_store_state:n #1 +\cs_new_protected:Npn \regex_branch:n #1 { - \int_incr:N \l_regex_max_active_int - \tex_skip:D \l_regex_max_active_int #1 sp \scan_stop: - \regex_store_submatches:n {#1} +%<trace> \trace_push:nnn { regex } { 1 } { regex_branch } + \regex_build_new_state: + \seq_get:NN \l_regex_left_state_seq \l_regex_internal_a_tl + \int_set:Nn \l_regex_left_state_int \l_regex_internal_a_tl + \regex_build_transition_right:nNn \regex_action_free:n + \l_regex_left_state_int \l_regex_right_state_int + #1 + \seq_get:NN \l_regex_right_state_seq \l_regex_internal_a_tl + \regex_build_transition_right:nNn \regex_action_free:n + \l_regex_right_state_int \l_regex_internal_a_tl +%<trace> \trace_pop:nnn { regex } { 1 } { regex_branch } } % \end{macrocode} % \end{macro} % -% \begin{macro}[int]{\regex_state_use:} -% \begin{macro}[int]{\regex_state_use_with_submatches:} -% \begin{macro}[aux]{\regex_state_use_aux_ii:w} -% \begin{macro}[aux]{\regex_state_use_aux:n} -% Use a given program instruction, unless it has already been -% executed at this step. The \tn{toks} registers begin with -% some submatch information, ignored by \cs{regex_state_use:}, -% but not by \cs{regex_state_use_with_submatches:}. -% A state is free if it is not marker as taken, namely -% if the corresponding \tn{dimen} register is not -% \cs{l_regex_step_int} in \texttt{sp}. -% The primitive conditional is ended before unpacking -% the \tn{toks} register. +% \begin{macro}[int]{\regex_group_repeat:nn} +% This function is called to repeat a group a fixed number of times +% |#2|; if this is $0$ we remove the group altogether (but don't reset +% the \texttt{capturing_group} label). Otherwise, the auxiliary +% \cs{regex_group_repeat_aux:n} copies |#2| times the \tn{toks} for +% the group, and leaves \texttt{internal_a} pointing to the left end +% of the last repetition. We only record the submatch information at +% the last repetition. Finally, add a state at the end (the transition +% to it has been taken care of by the replicating auxiliary. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_state_use_with_submatches: - { \regex_state_use_aux:n { } } -\cs_new_protected_nopar:Npn \regex_state_use: - { \regex_state_use_aux:n { \exp_after:wN \use_none_delimit_by_s_stop:w } } -\cs_new_protected:Npn \regex_state_use_aux:n #1 +\cs_new_protected:Npn \regex_group_repeat:nn #1#2 { - \if_num:w \tex_dimen:D \l_regex_current_state_int < \l_regex_step_int - \tex_dimen:D \l_regex_current_state_int - = \l_regex_step_int sp \scan_stop: - #1 \tex_the:D \tex_toks:D \exp_after:wN \l_regex_current_state_int + \if_num:w #2 = \c_zero + \int_set:Nn \l_regex_max_state_int + { \l_regex_left_state_int - \c_one } + \regex_build_new_state: + \else: + \regex_group_repeat_aux:n {#2} + \regex_group_submatches:nNN {#1} + \l_regex_internal_a_int \l_regex_right_state_int + \regex_build_new_state: \fi: - \scan_stop: } % \end{macrocode} % \end{macro} +% +% \begin{macro}[aux]{\regex_group_submatches:nNN} +% This inserts in states |#2| and |#3| the code for tracking +% submatches of the group |#1|, unless inhibited by a label of $-1$. +% \begin{macrocode} +\cs_new_protected:Npn \regex_group_submatches:nNN #1#2#3 + { + \if_num:w #1 > \c_minus_one + \regex_toks_put_left:Nx #2 { \regex_action_submatch:n { #1 < } } + \regex_toks_put_left:Nx #3 { \regex_action_submatch:n { #1 > } } + \fi: + } +% \end{macrocode} % \end{macro} -% \end{macro} +% +% \begin{macro}[aux]{\regex_group_repeat_aux:n} +% Here we repeat \tn{toks} ranging from \texttt{left_state} to +% \texttt{max_state}, $|#1|>0$ times. First add a transition so that +% the copies will \enquote{chain} properly. Compute the shift +% \texttt{c} between the original copy and the last copy we +% want. Shift the \texttt{right_state} and \texttt{max_state} to their +% final values. We then want to perform \texttt{c} copy operations. At +% the end, \texttt{b} is equal to the \texttt{max_state}, and +% \texttt{a} points to the left of the last copy of the group. +% \begin{macrocode} +\cs_new_protected:Npn \regex_group_repeat_aux:n #1 + { + \regex_build_transition_right:nNn \regex_action_free:n + \l_regex_right_state_int \l_regex_max_state_int + \int_set_eq:NN \l_regex_internal_a_int \l_regex_left_state_int + \int_set_eq:NN \l_regex_internal_b_int \l_regex_max_state_int + \if_num:w \int_eval:w #1 > \c_one + \int_set:Nn \l_regex_internal_c_int + { + ( #1 - \c_one ) + * ( \l_regex_internal_b_int - \l_regex_internal_a_int ) + } + \tex_advance:D \l_regex_right_state_int \l_regex_internal_c_int + \tex_advance:D \l_regex_max_state_int \l_regex_internal_c_int + \prg_replicate:nn \l_regex_internal_c_int + { + \tex_toks:D \l_regex_internal_b_int + = \tex_toks:D \l_regex_internal_a_int + \tex_advance:D \l_regex_internal_a_int \c_one + \tex_advance:D \l_regex_internal_b_int \c_one + } + \fi: + } +% \end{macrocode} % \end{macro} % -% \subsubsection{Submatch tracking when running the NFA} +% \begin{macro}[aux]{\regex_group_repeat:nnN} +% This function is called to repeat a group at least $n$ times; the +% case $n=0$ is very different from $n>0$. Assume first that $n=0$. +% Insert submatch tracking information at the start and end of the +% group, add a free transition from the right end to the +% \enquote{true} left state \texttt{a} (remember: in this case we had +% added an extra state before the left state). This forms the loop, +% which we break away from by adding a free transition from \texttt{a} +% to a new state. % -% \begin{macro}[int]{\regex_disable_submatches:} -% Some user functions don't require tracking submatches. -% We get a performance improvement by simply defining the -% relevant functions to remove their argument and do nothing -% with it. +% Now consider the case $n>0$. Repeat the group $n$ times, chaining +% various copies with a free transition. Add submatch tracking only to +% the last copy, then add a free transition from the right end back to +% the left end of the last copy, either before or after the transition +% to move on towards the rest of the \textsc{nfa}. This transition can +% end up before submatch tracking, but that is irrelevant since it +% only does so when going again through the group, recording new +% matches. Finally, add a state; we already have a transition pointing +% to it from \cs{regex_group_repeat_aux:n}. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_disable_submatches: +\cs_new_protected:Npn \regex_group_repeat:nnN #1#2#3 { - \cs_set_eq:NN \regex_state_use_with_submatches: \regex_state_use: - \cs_set_eq:NN \regex_store_submatches:n - \regex_protected_use_none:n - \cs_set_eq:NN \regex_action_submatches:n - \regex_protected_use_none:n + \if_num:w #2 = \c_zero + \regex_group_submatches:nNN {#1} + \l_regex_left_state_int \l_regex_right_state_int + \int_set:Nn \l_regex_internal_a_int + { \l_regex_left_state_int - \c_one } + \regex_build_transition_right:nNn \regex_action_free:n + \l_regex_right_state_int \l_regex_internal_a_int + \regex_build_new_state: + \if_meaning:w \c_true_bool #3 + \regex_build_transition_left:NNN \regex_action_free:n + \l_regex_internal_a_int \l_regex_right_state_int + \else: + \regex_build_transition_right:nNn \regex_action_free:n + \l_regex_internal_a_int \l_regex_right_state_int + \fi: + \else: + \regex_group_repeat_aux:n {#2} + \regex_group_submatches:nNN {#1} + \l_regex_internal_a_int \l_regex_right_state_int + \if_meaning:w \c_true_bool #3 + \regex_build_transition_right:nNn \regex_action_free_group:n + \l_regex_right_state_int \l_regex_internal_a_int + \else: + \regex_build_transition_left:NNN \regex_action_free_group:n + \l_regex_right_state_int \l_regex_internal_a_int + \fi: + \regex_build_new_state: + \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_group_repeat:nnnN} +% We wish to repeat the group between |#2| and $|#2|+|#3|$ times, with +% a lazyness controlled by |#4|. We insert submatch tracking up front: +% in principle, we could avoid recording submatches for the first |#2| +% copies of the group, but that forces us to treat specially the case +% $|#2|=0$. Repeat that group with submatch tracking $|#2|+|#3|$ times +% (the maximum number of repetitions). Then our goal is to add |#3| +% transitions from the end of the |#2|-th group, and each subsequent +% groups, to the end. For a lazy quantifier, we add those transitions +% to the left states, before submatch tracking. For the greedy case, +% we add the transitions to the right states, after submatch tracking +% and the transitions which go on with more repetitions. In the greedy +% case with $|#2|=0$, the transition which skips over all copies of +% the group must be added separately, because its starting state does +% not follow the normal pattern: we had to add it \enquote{by hand} +% earlier. +% \begin{macrocode} +\cs_new_protected:Npn \regex_group_repeat:nnnN #1#2#3#4 + { + \regex_group_submatches:nNN {#1} + \l_regex_left_state_int \l_regex_right_state_int + \regex_group_repeat_aux:n { #2 + #3 } + \if_meaning:w \c_true_bool #4 + \int_set_eq:NN \l_regex_left_state_int \l_regex_max_state_int + \prg_replicate:nn { #3 } + { + \int_sub:Nn \l_regex_left_state_int + { \l_regex_internal_b_int - \l_regex_internal_a_int } + \regex_build_transition_left:NNN \regex_action_free:n + \l_regex_left_state_int \l_regex_max_state_int + } + \else: + \prg_replicate:nn { #3 - \c_one } + { + \int_sub:Nn \l_regex_right_state_int + { \l_regex_internal_b_int - \l_regex_internal_a_int } + \regex_build_transition_right:nNn \regex_action_free:n + \l_regex_right_state_int \l_regex_max_state_int + } + \if_num:w #2 = \c_zero + \int_set:Nn \l_regex_right_state_int + { \l_regex_left_state_int - \c_one } + \else: + \int_sub:Nn \l_regex_right_state_int + { \l_regex_internal_b_int - \l_regex_internal_a_int } + \fi: + \regex_build_transition_right:nNn \regex_action_free:n + \l_regex_right_state_int \l_regex_max_state_int + \fi: + \regex_build_new_state: } -\cs_new_protected:Npn \regex_protected_use_none:n #1 { } % \end{macrocode} % \end{macro} % -% \begin{macro}[int]{\regex_store_submatches:n} -% \begin{macro}[aux]{\regex_store_submatches_aux:w} -% \begin{macro}[aux]{\regex_store_submatches_aux_ii:Nnnw} -% The submatch information pertaining to one given thread is moved -% from state to state as we execute the NFA. -% We make sure that most of the \tn{toks} register is not read -% before being assigned again to that same register. +% \subsubsection{Others} +% +% \begin{macro}[int]{\regex_assertion:Nn, \regex_b_test:, \regex_anchor:N} +% Usage: \cs{regex_assertion:Nn} \meta{boolean} \Arg{test}, where the +% \meta{test} is either of the two other functions. Add a free +% transition to a new state, conditionally to the assertion test. The +% \cs{regex_b_test:} test is used by the |\b| and |\B| escape: check +% if the last character was a word character or not, and do the same +% to the current character. The boundary-markers of the string are +% non-word characters for this purpose. Anchors at the start or end +% of match use \cs{regex_anchor:N}, with a position controlled by the +% integer |#1|. % \begin{macrocode} -\cs_new_protected:Npn \regex_store_submatches:n #1 +\cs_new_protected:Npn \regex_assertion:Nn #1#2 { - \tex_toks:D #1 \exp_after:wN + \regex_build_new_state: + \regex_toks_put_right:Nx \l_regex_left_state_int { - \tex_romannumeral:D - \exp_after:wN \regex_store_submatches_aux:w - \tex_the:D \tex_toks:D #1 + \exp_not:n {#2} + \regex_break_point:TF + \bool_if:NF #1 { { } } + { + \regex_action_free:n + { + \int_eval:n + { \l_regex_right_state_int - \l_regex_left_state_int } + } + } + \bool_if:NT #1 { { } } } } -\cs_new_protected:Npn \regex_store_submatches_aux:w #1 \s_stop +\cs_new_protected:Npn \regex_anchor:N #1 { - \regex_store_submatches_aux_ii:Nnnw - #1 - \regex_state_submatches:nn \c_minus_one \q_prop - \s_stop + \if_num:w #1 = \l_regex_current_pos_int + \exp_after:wN \regex_break_true:w + \fi: } -\cs_new_protected:Npn \regex_store_submatches_aux_ii:Nnnw - \regex_state_submatches:nn #1 #2 #3 \s_stop +\cs_new_protected_nopar:Npn \regex_b_test: { - \exp_after:wN \c_zero - \exp_after:wN \regex_state_submatches:nn \exp_after:wN - { \int_value:w \int_eval:w \l_regex_step_int + \c_one \exp_after:wN } - \exp_after:wN { \l_regex_current_submatches_prop } - \regex_state_submatches:nn {#1} {#2} - \s_stop + \group_begin: + \int_set_eq:NN \l_regex_current_char_int \l_regex_last_char_int + \regex_prop_w: + \regex_break_point:TF + { \group_end: \regex_item_reverse:n \regex_prop_w: } + { \group_end: \regex_prop_w: } } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}[aux]{\regex_state_submatches:nn} -% This function is inserted by \cs{regex_store_submatches:n} -% in the \tn{toks} register holding a given state, and it is -% performed when the state is used. +% \begin{macro}[int]{\regex_command_K:} +% Change the starting point of the $0$-th submatch (full match), and +% transition to a new state, pretending that this is a fresh thread. % \begin{macrocode} -\cs_new_protected:Npn \regex_state_submatches:nn #1#2 +\cs_new_protected_nopar:Npn \regex_command_K: { - \if_num:w #1 = \l_regex_step_int - \tl_set:Nn \l_regex_current_submatches_prop { #2 } - \fi: + \regex_build_new_state: + \regex_toks_put_right:Nx \l_regex_left_state_int + { + \regex_action_submatch:n { 0< } + \bool_set_true:N \l_regex_fresh_thread_bool + \regex_action_free:n + { \int_eval:n { \l_regex_right_state_int - \l_regex_left_state_int } } + \bool_set_false:N \l_regex_fresh_thread_bool + } } % \end{macrocode} % \end{macro} % +% \subsection{Matching} +% +% We search for matches by running all the execution threads through the +% \textsc{nfa} in parallel, reading one token of the query at each step. +% The \textsc{nfa} contains \enquote{free} transitions to other states, +% and transitions which \enquote{consume} the current token. For free +% transitions, the instruction at the new state of the \textsc{nfa} is +% performed immediately. When a transition consumes a character, the +% new state is appended to a list of \enquote{active states}, stored in +% \tn{skip} registers: this thread will be active again when the next +% token is read from the query. At every step (for each token in the +% query), we unpack that list of active states and the corresponding +% submatch props, and empty those. +% +% If two paths through the \textsc{nfa} \enquote{collide} in the sense +% that they reach the same state after reading a given token, then they +% only differ in how they previously matched, and the future execution +% will be identical for both. (Note that this would be wrong in the +% presence of back-references.) Hence, we only need to keep one of the +% two threads: the thread with the highest priority. Our \textsc{nfa} is +% built in such a way that higher priority actions always come before +% lower priority actions, which makes things work. +% +% The explanation in the previous paragraph may make us think that we +% simply need to keep track of which states were visited at a given +% step: after all, the loop generated when matching |(a?)*| against |a| +% is broken, isn't it? No. The group first matches |a|, as it should, +% then repeats; it attempts to match |a| again but fails; it skips |a|, +% and finds out that this state has already been seen at this position +% in the query: the match stops. The capturing group is (wrongly) |a|. +% What went wrong is that a thread collided with itself, and the later +% version, which has gone through the group one more times with an empty +% match, should have a higher priority than not going through the group. +% +% We solve this by distinguishing \enquote{normal} free transitions +% \cs{regex_action_free:n} from transitions +% \cs{regex_action_free_group:n} which go back to the start of the +% group. The former will keep threads unless they have been visited by a +% \enquote{completed} thread, while the latter kind of transition also +% prevents going back to a state visited by the current thread. +% +% \subsubsection{Variables used when matching} +% +% \begin{variable} +% { +% \l_regex_min_pos_int, +% \l_regex_max_pos_int, +% \l_regex_current_pos_int, +% \l_regex_start_pos_int, +% \l_regex_success_pos_int, +% } +% The tokens in the query are indexed from \texttt{min_pos} for the +% first to $\texttt{max_pos}-1$ for the last, and their information is +% stored in \tn{muskip} and \tn{toks} registers with those numbers. We +% don't start from $0$ because the \tn{toks} registers with low +% numbers are used to hold the states of the \textsc{nfa}. We match +% without backtracking, keeping all threads in lockstep at the +% \texttt{current_pos} in the query. The starting point of the current +% match attempt is \texttt{start_pos}, and \texttt{success_pos}, +% updated whenever a thread succeeds, is used as the next starting +% position. +% \begin{macrocode} +\int_new:N \l_regex_min_pos_int +\int_new:N \l_regex_max_pos_int +\int_new:N \l_regex_current_pos_int +\int_new:N \l_regex_start_pos_int +\int_new:N \l_regex_success_pos_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable} +% { +% \l_regex_current_char_int, +% \l_regex_current_catcode_int, +% \l_regex_last_char_int, +% \l_regex_case_changed_char_int +% } +% The character and category codes of the token at the current +% position; the character code of the token at the previous position; +% and the character code of the result of changing the case of the +% current token (|A-Z|$\leftrightarrow$|a-z|). This last integer is +% only computed when necessary, and is otherwise \cs{c_max_int}. The +% \texttt{current_char} variable is also used in various other phases +% to hold a character code. +% \begin{macrocode} +\int_new:N \l_regex_current_char_int +\int_new:N \l_regex_current_catcode_int +\int_new:N \l_regex_last_char_int +\int_new:N \l_regex_case_changed_char_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_regex_current_state_int} +% For every character in the token list, each of the active states is +% considered in turn. The variable \cs{l_regex_current_state_int} +% holds the state of the \textsc{nfa} which is currently considered: +% transitions are then given as shifts relative to the current state. +% \begin{macrocode} +\int_new:N \l_regex_current_state_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable} +% {\l_regex_current_submatches_prop, \l_regex_success_submatches_prop} +% The submatches for the thread which is currently active are stored +% in the \texttt{current_submatches} property list variable. This +% property list is stored by \cs{regex_action_cost:n} into the +% \tn{toks} register for the target state of the transition, to be +% retrieved when matching at the next position. When a thread +% succeeds, this property list is copied to +% \cs{l_regex_success_submatches_prop}: only the last sucessful thread +% will remain there. +% \begin{macrocode} +\prop_new:N \l_regex_current_submatches_prop +\prop_new:N \l_regex_success_submatches_prop +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_regex_step_int} +% This integer, always even, is increased every time a character in +% the query is read, and not reset when doing multiple matches. For +% each \meta{state} in the \textsc{nfa} we store in +% \tn{dimen}\meta{state} the last step in which this state was +% encountered. This lets us break infinite loops by not visiting the +% same state twice in the same step. In fact, \tn{dimen}\meta{state} +% is equal \texttt{step} when we have started performing the +% operations of \tn{toks}\meta{state}, but not finished yet. However, +% once we finish, we set \tn{dimen}\meta{state} to +% $\text{\texttt{step}}+1$. This is needed to track submatches +% properly (see building phase). The \texttt{step} is also used to +% attach each set of submatch information to a given iteration (and +% automatically discard it when it corresponds to a past step). +% \begin{macrocode} +\int_new:N \l_regex_step_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_regex_min_active_int, \l_regex_max_active_int} +% All the currently active states are kept in order of precedence in +% the \tn{skip} registers, and the corresponding submatches in the +% \tn{toks}. For our purposes, those serve as an array, indexed from +% \texttt{min_active} (inclusive) to \texttt{max_active} (excluded). +% At the start of every step, the whole array is unpacked, so that the +% space can immediately be reused, and \texttt{max_active} is reset to +% \texttt{min_active}, effectively clearing the array. +% \begin{macrocode} +\int_new:N \l_regex_min_active_int +\int_new:N \l_regex_max_active_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_regex_every_match_tl} +% Every time a match is found, this token list is used. For single +% matching, the token list is empty. For multiple matching, the token +% list is set to repeat the matching, after performing some operation +% which depends on the user function. See \cs{regex_single_match:} and +% \cs{regex_multi_match:n}. +% \begin{macrocode} +\tl_new:N \l_regex_every_match_tl +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_regex_fresh_thread_bool, \l_regex_empty_success_bool} +% \begin{macro}[aux]{\regex_if_two_empty_matches:F} +% When doing multiple matches, we need to avoid infinite loops where +% each iteration matches the same empty token list. When an empty +% token list is matched, the next successful match of the same empty +% token list is suppressed. We detect empty matches by setting +% \cs{l_regex_fresh_thread_bool} to \texttt{true} for threads which +% directly come from the start of the regex or from the |\K| command, +% and testing that boolean whenever a thread succeeds. The function +% \cs{regex_if_two_empty_matches:F} is redefined at every match +% attempt, depending on whether the previous match was empty or not: +% if it was, then the function must cancel a purported success if it +% is empty and at the same spot as the previous match; otherwise, we +% definitely don't have two identical empty matches, so the function +% is \cs{use:n}. +% \begin{macrocode} +\bool_new:N \l_regex_fresh_thread_bool +\bool_new:N \l_regex_empty_success_bool +\cs_new_eq:NN \regex_if_two_empty_matches:F \use:n +% \end{macrocode} +% \end{macro} +% \end{variable} +% +% \begin{variable} +% { +% \g_regex_success_bool, +% \l_regex_saved_success_bool, +% \l_regex_match_success_bool +% } +% The boolean \cs{l_regex_match_success_bool} is true if the current +% match attempt was successful, and \cs{g_regex_success_bool} is true +% if there was at least one successful match. This is the only global +% variable in this whole module, but we would need it to be local when +% matching a control sequence with |\c{...}|. This is done by saving +% the global variable into \cs{l_regex_saved_success_bool}, which is +% local, hence not affected by the changes due to inner regex +% functions. +% \begin{macrocode} +\bool_new:N \g_regex_success_bool +\bool_new:N \l_regex_saved_success_bool +\bool_new:N \l_regex_match_success_bool +% \end{macrocode} +% \end{variable} +% % \subsubsection{Matching: framework} % % \begin{macro}[int]{\regex_match:n} -%^^A todo: update doc. -% Then reset a few variables which should be set only once, -% before the first match, even in the case of multiple matches. -% Then run the NFA (\cs{regex_match_once:} matches multiple times -% when appropriate). +% First store the query into \tn{toks} and \tn{muskip} registers (see +% \cs{regex_query_set:nnn}). Then initialize the variables that should +% be set once for each user function (even for multiple +% matches). Namely, the overall matching is not yet sucessful; none of +% the states should be marked as visited (\tn{dimen} registers), and +% we start at step $0$; we pretend that there was a previous match +% ending at the start of the query, which was not empty (to avoid +% smothering an empty match at the start). Once all this is set up, we +% are ready for the ride. Find the first match. % \begin{macrocode} \cs_new_protected:Npn \regex_match:n #1 { - \int_zero:N \l_regex_nesting_int - \int_set_eq:NN \l_regex_current_index_int \l_regex_max_state_int +%<trace> \trace_push:nnx { regex } { 1 } { regex_match } +%<trace> \trace:nnx { regex } { 1 } { analyzing~query~token~list } + \int_zero:N \l_regex_balance_int + \int_set:Nn \l_regex_current_pos_int { \c_two * \l_regex_max_state_int } \regex_query_set:nnn { } { -1 } { -2 } - \int_set_eq:NN \l_regex_min_index_int \l_regex_current_index_int + \int_set_eq:NN \l_regex_min_pos_int \l_regex_current_pos_int \tl_analysis_map_inline:nn {#1} - { - \regex_query_set:nnn {##1} {"##2} {##3} - \if_case:w "##2 \exp_stop_f: - \or: \int_incr:N \l_regex_nesting_int - \or: \int_decr:N \l_regex_nesting_int - \fi: - } - \int_set_eq:NN \l_regex_max_index_int \l_regex_current_index_int + { \regex_query_set:nnn {##1} {"##2} {##3} } + \int_set_eq:NN \l_regex_max_pos_int \l_regex_current_pos_int \regex_query_set:nnn { } { -1 } { -2 } - \regex_match_initial_setup: +%<trace> \trace:nnx { regex } { 1 } { initializing } + \bool_gset_false:N \g_regex_success_bool + \prg_stepwise_inline:nnnn + \l_regex_min_state_int \c_one { \l_regex_max_state_int - \c_one } + { \tex_dimen:D ##1 \c_one sp \scan_stop: } + \int_set_eq:NN \l_regex_min_active_int \l_regex_max_state_int + \int_set_eq:NN \l_regex_step_int \c_zero + \int_set_eq:NN \l_regex_success_pos_int \l_regex_min_pos_int + \int_set:Nn \l_regex_submatch_int + { \c_two * \l_regex_max_state_int } + \bool_set_false:N \l_regex_empty_success_bool \regex_match_once: - } -\cs_new_protected:Npn \regex_query_set:nnn #1#2#3 - { - \tex_muskip:D \l_regex_current_index_int - = \etex_gluetomu:D - #3 sp - plus #2 sp - minus \l_regex_nesting_int sp - \scan_stop: - \tex_toks:D \l_regex_current_index_int {#1} - \int_incr:N \l_regex_current_index_int - } -\cs_new_protected_nopar:Npn \regex_query_get: - { - \tl_set:Nx \l_regex_current_token_tl - { \tex_the:D \tex_toks:D \l_regex_current_index_int } - \l_regex_current_char_int - = \etex_mutoglue:D \tex_muskip:D \l_regex_current_index_int - \l_regex_current_catcode_int = \etex_gluestretch:D - \etex_mutoglue:D \tex_muskip:D \l_regex_current_index_int +%<trace> \trace_pop:nnx { regex } { 1 } { regex_match } } % \end{macrocode} % \end{macro} % % \begin{macro}[int]{\regex_match_once:} -% Set up more variables in \cs{regex_match_setup:}. -% If there was a match, use the token list \cs{l_regex_every_match_tl}, -% which may call \cs{regex_match_once:} again to achieve multiple matches. +% This function finds one match, then does some action defined by the +% \texttt{every_match} token list, which may recursively call +% \cs{regex_match_once:}. First initialize some variables: set the +% conditional which detects identical empty matches; this match +% attempt starts at the previous \texttt{success_pos}, is not yet +% sucessful, and has no submatches yet; clear the array of active +% threads, and put the starting state $0$ in it. We are then almost +% ready to read our first token in the query, but we actually start +% one position earlier than the start, and \texttt{get} that token, so +% that the \texttt{last_char} will be set properly for word +% boundaries. Then call \cs{regex_match_loop:}, which runs through the +% query until the end or until a successful match breaks early. % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_match_once: { - \regex_match_setup: + \if_meaning:w \c_true_bool \l_regex_empty_success_bool + \cs_set_nopar:Npn \regex_if_two_empty_matches:F + { \int_compare:nNnF \l_regex_start_pos_int = \l_regex_current_pos_int } + \else: + \cs_set_eq:NN \regex_if_two_empty_matches:F \use:n + \fi: + \int_set_eq:NN \l_regex_start_pos_int \l_regex_success_pos_int + \bool_set_false:N \l_regex_match_success_bool + \prop_clear:N \l_regex_current_submatches_prop + \int_set_eq:NN \l_regex_max_active_int \l_regex_min_active_int + \regex_store_state:n { \l_regex_min_state_int } + \int_set:Nn \l_regex_current_pos_int + { \l_regex_start_pos_int - \c_one } \regex_query_get: \regex_match_loop: - \prg_break_point:n { } - \bool_if:NT \l_regex_success_match_bool - { - \bool_gset_true:N \g_regex_success_bool - \l_regex_every_match_tl - } + \l_regex_every_match_tl } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_match_initial_setup:} -% This function holds the setup that should be done -% only once for one given pattern matching. -% It is called only once for the whole token list. -% On the other hand, \cs{regex_match_setup:} -% is called for every match in the token list in case of -% repeated matches. +% \begin{macro}[int]{\regex_single_match:, \regex_multi_match:n} +% For a single match, the overall success is determined by whether the +% only match attempt is a success. When doing multiple matches, the +% overall matching is successful as soon as any match +% succeeds. Perform the action |#1|, then find the next match. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_match_initial_setup: +\cs_new_protected_nopar:Npn \regex_single_match: { - \int_set_eq:NN \l_regex_step_int \c_minus_one - \prg_stepwise_inline:nnnn - \c_zero \c_one { \l_regex_max_state_int - \c_one } - { \tex_dimen:D ##1 \l_regex_step_int sp } - \int_set:Nn \l_regex_start_index_int - { \l_regex_min_index_int - \c_one } - \int_set_eq:NN \l_regex_current_index_int \l_regex_min_index_int - \int_set_eq:NN \l_regex_success_index_int \l_regex_min_index_int - \int_set_eq:NN \l_regex_submatch_int \l_regex_max_state_int - \bool_set_false:N \l_regex_success_empty_bool - \bool_gset_false:N \g_regex_success_bool + \tl_set:Nn \l_regex_every_match_tl + { \bool_gset_eq:NN \g_regex_success_bool \l_regex_match_success_bool } } -% \end{macrocode} -% \end{macro} -% -% \begin{macro}[aux]{\regex_match_setup:} -% Every time a match starts, \cs{regex_match_setup:} resets -% a few variables. -% \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_match_setup: +\cs_new_protected:Npn \regex_multi_match:n #1 { - \prop_clear:N \l_regex_current_submatches_prop - \bool_if:NTF \l_regex_success_empty_bool - { \cs_set_eq:NN \regex_if_two_empty_matches:F \regex_if_match_empty:F } - { \cs_set_eq:NN \regex_if_two_empty_matches:F \use:n } - \int_set_eq:NN \l_regex_start_index_int \l_regex_success_index_int - \int_set:Nn \l_regex_current_index_int - { \l_regex_start_index_int - \c_one } - \bool_set_false:N \l_regex_success_match_bool - \int_zero:N \l_regex_max_active_int - \regex_store_state:n {0} %^^A _state_int! + \tl_set:Nn \l_regex_every_match_tl + { + \if_meaning:w \c_true_bool \l_regex_match_success_bool + \bool_gset_true:N \g_regex_success_bool + #1 + \exp_after:wN \regex_match_once: + \fi: + } } % \end{macrocode} % \end{macro} % % \begin{macro}[aux]{\regex_match_loop:} -% \begin{macro}[aux]{\regex_match_one_active:n} -% \begin{macro}[aux]{\regex_match_one_active_aux:n} -% Setup what needs to be reset at every character, -% then set \cs{l_regex_current_char_int} to the -% character code of the token that is read -% (and $-1$ for the end of the token list), and loop -% over the elements of the \tn{skip} array. Then repeat. -% There are a couple of tests to stop reading the token list -% when no active state is left, or when the end is reached. -% At every step in reading the token list, we store the character -% code of the current character in \cs{l_regex_current_char_int}, -% unless the end was reached: then we store $-1$. +% \begin{macro}[aux, rEXP]{\regex_match_one_active:w} +% At each new position, set some variables and get the new character +% and category from the query. Then unpack the array of active +% threads, and clear it by resetting its length +% (\texttt{max_active}). This results in a sequence of +% \cs{regex_use_state_and_submatches:nn} \Arg{state} \Arg{prop}, and +% we consider those states one by one in order. As soon as a thread +% succeeds, exit the step, and, if there are threads to consider at the +% next position, and we have not reached the end of the string, +% repeat the loop. Otherwise, the last thread that succeeded is what +% \cs{regex_match_once:} matches. We explain the \texttt{fresh_thread} +% business when describing \cs{regex_action_wildcard:}. % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_match_loop: { - \int_incr:N \l_regex_current_index_int - \int_incr:N \l_regex_step_int - \bool_set_false:N \l_regex_fresh_thread_bool + \tex_advance:D \l_regex_step_int \c_two + \int_incr:N \l_regex_current_pos_int \int_set_eq:NN \l_regex_last_char_int \l_regex_current_char_int + \int_set_eq:NN \l_regex_case_changed_char_int \c_max_int \regex_query_get: - \regex_match_loop_case_hook: \use:x { - \int_zero:N \l_regex_max_active_int - \regex_match_one_active:w 1 ; \prg_break_point:n { } - \exp_not:N \prg_break_point:n { } + \int_set_eq:NN \l_regex_max_active_int \l_regex_min_active_int + \exp_after:wN \regex_match_one_active:w + \int_use:N \l_regex_min_active_int ; } - \if_num:w \l_regex_current_char_int < \c_minus_one - \exp_after:wN \prg_map_break: - \fi: - \if_num:w \l_regex_max_active_int = \c_zero - \exp_after:wN \prg_map_break: + \prg_break_point:n { \bool_set_false:N \l_regex_fresh_thread_bool } + \if_num:w \l_regex_max_active_int > \l_regex_min_active_int + \if_num:w \l_regex_current_pos_int < \l_regex_max_pos_int + \exp_after:wN \exp_after:wN \exp_after:wN \regex_match_loop: + \fi: \fi: - \regex_match_loop: } \cs_new:Npn \regex_match_one_active:w #1; { - \if_num:w #1 > \l_regex_max_active_int - \exp_after:wN \prg_map_break: + \if_num:w #1 < \l_regex_max_active_int + \regex_use_state_and_submatches:nn + { \int_value:w \tex_skip:D #1 } + { \tex_the:D \tex_toks:D #1 } + \exp_after:wN \regex_match_one_active:w + \int_use:N \int_eval:w #1 + \c_one \exp_after:wN ; \fi: - \regex_match_one_active_aux:n - { \int_value:w \tex_skip:D #1 } - \exp_after:wN \regex_match_one_active:w - \int_use:N \int_eval:w #1 + \c_one ; } -\cs_new_protected:Npn \regex_match_one_active_aux:n #1 +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[aux]{\regex_query_set:nnn} +% The arguments are: tokens that \texttt{o} and \texttt{x} expand to +% one token of the query, the catcode, and the character code. Store +% those, and the current brace balance (used later to check for +% overall brace balance) in a \tn{muskip} register and a \tn{toks}, +% then update the \texttt{balance}. +% \begin{macrocode} +\cs_new_protected:Npn \regex_query_set:nnn #1#2#3 { - \int_set:Nn \l_regex_current_state_int {#1} - \prop_clear:N \l_regex_current_submatches_prop - \regex_state_use_with_submatches: + \tex_muskip:D \l_regex_current_pos_int + = \etex_gluetomu:D + #3 sp + plus #2 sp + minus \l_regex_balance_int sp + \scan_stop: + \tex_toks:D \l_regex_current_pos_int {#1} + \int_incr:N \l_regex_current_pos_int + \if_case:w #2 \exp_stop_f: + \or: \int_incr:N \l_regex_balance_int + \or: \int_decr:N \l_regex_balance_int + \fi: } % \end{macrocode} % \end{macro} +% +% \begin{macro}[aux]{\regex_query_get:} +% Extract the current character and category codes from the +% \tn{muskip} register of the current position: those are the main and +% the stretch components, and we need a conversion to avoid \TeX{}'s +% \enquote{incompatible glue units} error. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \regex_query_get: + { + \l_regex_current_char_int + = \etex_mutoglue:D \tex_muskip:D \l_regex_current_pos_int + \l_regex_current_catcode_int = \etex_gluestretch:D + \etex_mutoglue:D \tex_muskip:D \l_regex_current_pos_int + } +% \end{macrocode} % \end{macro} +% +% \subsubsection{Using states of the \textsc{nfa}} +% +% \begin{macro}[int]{\regex_use_state:} +% Use the current \textsc{nfa} instruction. The state is initially +% marked as belonging to the current \texttt{step}: this allows normal +% free transition to repeat, but group-repeating transitions +% won't. Once we are done exploring all the branches it spawned, the +% state is marked as $\texttt{step}+1$: any thread hitting it at that +% point will be terminated. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \regex_use_state: + { +%<*trace> + \trace:nnx { regex } { 2 } { state~\int_use:N \l_regex_current_state_int } +%</trace> + \tex_dimen:D \l_regex_current_state_int + = \l_regex_step_int sp \scan_stop: + \tex_the:D \tex_toks:D \l_regex_current_state_int + \tex_dimen:D \l_regex_current_state_int + = \int_eval:w \l_regex_step_int + \c_one \int_eval_end: sp \scan_stop: + } +% \end{macrocode} % \end{macro} % -% \begin{macro}{\regex_match_loop_case_hook:} -% \begin{macro}{\regex_match_loop_caseless_hook:} -% In the case where the regular expression contains caseless matching, -% the \cs{regex_match_loop_case_hook:} (normally empty) is redefined -% to set \cs{l_regex_case_changed_char_int} properly. +% \begin{macro}[int]{\regex_use_state_and_submatches:nn} +% This function is called as one item in the array of active threads +% after that array has been unpacked for a new step. Update the +% \texttt{current_state} and \texttt{current_submatches} and use the +% state if it has not yet been encountered at this step. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_match_loop_case_hook: { } -\cs_new_protected_nopar:Npn \regex_match_loop_caseless_hook: +\cs_new_protected:Npn \regex_use_state_and_submatches:nn #1 #2 { - \int_set_eq:NN \l_regex_case_changed_char_int \l_regex_current_char_int - \if_num:w \l_regex_current_char_int < \c_ninety_one - \if_num:w \l_regex_current_char_int < \c_sixty_five - \else: - \int_add:Nn \l_regex_case_changed_char_int { \c_thirty_two } - \fi: - \else: - \if_num:w \l_regex_current_char_int < \c_one_hundred_twenty_three - \if_num:w \l_regex_current_char_int < \c_ninety_seven - \else: - \int_sub:Nn \l_regex_case_changed_char_int { \c_thirty_two } - \fi: - \fi: + \int_set:Nn \l_regex_current_state_int {#1} + \if_num:w \tex_dimen:D \l_regex_current_state_int < \l_regex_step_int + \tl_set:Nn \l_regex_current_submatches_prop {#2} + \exp_after:wN \regex_use_state: \fi: + \scan_stop: } % \end{macrocode} % \end{macro} -% \end{macro} % % \subsubsection{Actions when matching} % % \begin{macro}[int]{\regex_action_start_wildcard:} -%: the first state has a free transition to the second -% state, where the regular expression really begins, and a costly -% transition to itself, to try again at the next character. ^^A ?? -% The search is made unanchored at the start by putting -% a free transition to the real start of the NFA, and a -% costly transition to the same state, waiting for the -% next token in the query. This combination -% could be reused (with some changes). We sometimes need -% to know that the match for a given thread starts at -% this character. For that, we use the boolean -% \cs{l_regex_fresh_thread_bool}. +% For an unanchored match, state $0$ has a free transition to the next +% and a costly one to itself, to repeat at the next position. To catch +% repeated identical empty matches, we need to know if a successful +% thread corresponds to an empty match. The instruction resetting +% \cs{l_regex_fresh_thread_bool} may be skipped by a successful +% thread, hence we had to add it to \cs{regex_match_loop:} too. % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_action_start_wildcard: { @@ -4116,176 +4416,348 @@ % \end{macrocode} % \end{macro} % +% \begin{macro}[int]{\regex_action_free:n, \regex_action_free_group:n} +% \begin{macro}[aux]{\regex_action_free_aux:nn} +% These functions copy a thread after checking that the \textsc{nfa} +% state has not already been used at this position. If not, store +% submatches in the new state, and insert the instructions for that +% state in the input stream. Then restore the old value of +% \cs{l_regex_current_state_int} and of the current submatches. The +% two types of free transitions differ by how they test that the state +% has not been encountered yet: the \texttt{group} version is +% stricter, and will not use a state if it was used earlier in the +% current thread, hence forcefully breaking the loop, while the +% \enquote{normal} version will revisit a state when within the thread +% itself. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \regex_action_free:n + { \regex_action_free_aux:nn { > \l_regex_step_int \else: } } +\cs_new_protected_nopar:Npn \regex_action_free_group:n + { \regex_action_free_aux:nn { < \l_regex_step_int } } +\cs_new_protected:Npn \regex_action_free_aux:nn #1#2 + { + \use:x + { + \int_add:Nn \l_regex_current_state_int {#2} + \exp_not:n + { + \if_num:w \tex_dimen:D \l_regex_current_state_int #1 + \exp_after:wN \regex_use_state: + \fi: + } + \int_set:Nn \l_regex_current_state_int + { \int_use:N \l_regex_current_state_int } + \tl_set:Nn \exp_not:N \l_regex_current_submatches_prop + { \exp_not:o \l_regex_current_submatches_prop } + } + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% % \begin{macro}[int]{\regex_action_cost:n} -% A transition which consumes the current character and moves -% to state |#1|. +% A transition which consumes the current character and shifts the +% state by |#1|. The resulting state is stored in the \tn{skip} array +% for use at the next position, and we also store the current +% submatches. % \begin{macrocode} \cs_new_protected:Npn \regex_action_cost:n #1 { - \exp_args:Nf \regex_store_state:n - { \int_eval:n { \l_regex_current_state_int + #1 } } + \exp_args:No \regex_store_state:n + { \int_use:N \int_eval:w \l_regex_current_state_int + #1 } } % \end{macrocode} % \end{macro} % -% \begin{macro}[int]{\regex_action_success:} -% There is a successful match when an execution path reaches -% the end of the regular expression. Then store the current -% step and submatches. The current step is then interrupted -% with \cs{prg_map_break:}, -% and only paths with higher precedence are pursued further. -% The values stored here may be overwritten by a later success -% of a path with higher precedence. +% \begin{macro}[int]{\regex_store_state:n} +% \begin{macro}[aux]{\regex_store_submatches:} +% Put the given state in the array of \tn{skip} registers (converted +% to a dimension in scaled points), and increment the length of the +% array. Then store the current submatch in the This is done by +% increasing the pointer \cs{l_regex_max_active_int}, and converting +% the integer to a dimension (suitable for a \tn{skip} assignment) in +% scaled points. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_action_success: +\cs_new_protected:Npn \regex_store_state:n #1 { - \regex_if_two_empty_matches:F - { - \bool_set_true:N \l_regex_success_match_bool - \bool_set_eq:NN \l_regex_success_empty_bool - \l_regex_fresh_thread_bool - \int_set_eq:NN \l_regex_success_index_int \l_regex_current_index_int - \prop_set_eq:NN \l_regex_success_submatches_prop - \l_regex_current_submatches_prop - \prg_map_break: - } + \regex_store_submatches: + \tex_skip:D \l_regex_max_active_int = #1 sp \scan_stop: + \int_incr:N \l_regex_max_active_int + } +\cs_new_protected_nopar:Npn \regex_store_submatches: + { + \tex_toks:D \l_regex_max_active_int \exp_after:wN + { \l_regex_current_submatches_prop } } % \end{macrocode} % \end{macro} +% \end{macro} % -% \begin{macro}[int]{\regex_action_free:n} -% To copy a thread, check whether the program state has already -% been used at this character. If not, store submatches in the -% new state, and insert the instructions for that state in the -% input stream. -% Then restore the old value of \cs{l_regex_current_state_int} -% and of the current submatches. +% \begin{macro}[int]{\regex_disable_submatches:} +% Some user functions don't require tracking submatches. +% We get a performance improvement by simply defining the +% relevant functions to remove their argument and do nothing +% with it. % \begin{macrocode} -\cs_new_protected:Npn \regex_action_free:n #1 +\cs_new_protected_nopar:Npn \regex_disable_submatches: { - \use:x - { - \int_add:Nn \l_regex_current_state_int {#1} - \regex_state_use: - \int_set:Nn \l_regex_current_state_int - { \int_use:N \l_regex_current_state_int } - \tl_set:Nn \exp_not:N \l_regex_current_submatches_prop - { \exp_not:o \l_regex_current_submatches_prop } - } + \cs_set_protected_nopar:Npn \regex_store_submatches: { } + \cs_set_protected:Npn \regex_action_submatch:n ##1 { } } % \end{macrocode} % \end{macro} % % \begin{macro}[int]{\regex_action_submatch:n} -% Update the current submatches with the information -% from the current index. +% Update the current submatches with the information from the current +% position. Maybe a bottleneck. % \begin{macrocode} \cs_new_protected:Npn \regex_action_submatch:n #1 { \prop_put:Nno \l_regex_current_submatches_prop {#1} - { \int_use:N \l_regex_current_index_int } + { \int_use:N \l_regex_current_pos_int } } % \end{macrocode} % \end{macro} % -% \subsubsection{??}%^^A todo: title -% -% \begin{macro}{\regex_if_match_empty:T, \regex_if_match_empty:F} -% +% \begin{macro}[int]{\regex_action_success:} +% There is a successful match when an execution path reaches the last +% state in the \textsc{nfa}, unless this marks a second identical +% empty match. Then mark that there was a successful match; it is +% empty if it is \enquote{fresh}; and we store the current position +% and submatches. The current step is then interrupted with +% \cs{prg_map_break:}, and only paths with higher precedence are +% pursued further. The values stored here may be overwritten by a +% later success of a path with higher precedence. % \begin{macrocode} -\cs_new:Npn \regex_if_match_empty:T - { \int_compare:nNnT \l_regex_start_index_int = \l_regex_current_index_int } -\cs_new:Npn \regex_if_match_empty:F - { \int_compare:nNnF \l_regex_start_index_int = \l_regex_current_index_int } +\cs_new_protected_nopar:Npn \regex_action_success: + { + \regex_if_two_empty_matches:F + { + \bool_set_true:N \l_regex_match_success_bool + \bool_set_eq:NN \l_regex_empty_success_bool + \l_regex_fresh_thread_bool + \int_set_eq:NN \l_regex_success_pos_int \l_regex_current_pos_int + \prop_set_eq:NN \l_regex_success_submatches_prop + \l_regex_current_submatches_prop + \prg_map_break: + } + } % \end{macrocode} % \end{macro} % % \subsection{Replacement} % -% \begin{macro}[rEXP]{\regex_submatch_nesting_aux:n} +% \subsubsection{Variables and helpers used in replacement} +% +% \begin{variable}{\l_regex_replacement_csnames_int} +% The behaviour of closing braces inside a replacement text depends on +% whether a sequences |\c{| or |\u{| has been encountered. The number +% of \enquote{open} such sequences that should be closed by |}| is +% stored in \cs{l_regex_replacement_csnames_int}, and decreased by +% $1$ by each |}|. % \begin{macrocode} -\cs_new_protected:Npn \regex_submatch_nesting_aux:n #1 - { - + \etex_glueshrink:D \etex_mutoglue:D \etex_muexpr:D - \tex_muskip:D \etex_gluestretch:D \tex_skip:D #1 - - \tex_muskip:D \tex_skip:D #1 - \scan_stop: +\int_new:N \l_regex_replacement_csnames_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_regex_balance_tl} +% This token list holds the replacement text for +% \cs{regex_replacement_balance_one_match:n} while it is being built +% incrementally. +% \begin{macrocode} +\tl_new:N \l_regex_balance_tl +% \end{macrocode} +% \end{variable} +% +% \begin{macro}[aux, rEXP]{\regex_replacement_balance_one_match:n} +% This expects as an argument the first index of a range of \tn{skip} +% registers which hold the submatch information for a given match. It +% can be used within an integer expression to obtain the brace balance +% incurred by performing the replacement on that match. This combines +% the braces lost by removing the match, braces added by all the +% submatches appearing in the replacement, and braces appearing +% explicitly in the replacement. Even though it is always redefined +% before use, we initialize it as for an empty replacement. An +% important property is that concatenating several calls to that +% function must result in a valid integer expression (hence a leading +% |+| in the actual definition). +% \begin{macrocode} +\cs_new:Npn \regex_replacement_balance_one_match:n #1 + { - \regex_submatch_balance:n {#1} } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux, rEXP]{\regex_replacement_do_one_match:n} +% The input is the same as \cs{regex_replacement_balance_one_match:n}. +% This function is redefined to expand to the part of the token list +% from the end of the previous match to a given match, followed by the +% replacement text. Hence concatenating the result of this function +% with all possible arguments (one call for each match), as well as +% the range from the end of the last match to the end of the string, +% will produce the fully replaced token list. The initialization does +% not matter, but we set it as for an empty replacement. +% \begin{macrocode} +\cs_new:Npn \regex_replacement_do_one_match:n #1 + { + \regex_query_range:nn + { \etex_glueshrink:D \tex_skip:D #1 } + { \tex_skip:D #1 } } % \end{macrocode} % \end{macro} % -% \begin{macro}[int]{\regex_replacement:n} -% Our goal here is to analyse the replacement text. First take care -% of detecting escaped and non-escaped characters using -% \cs{regex_escape_use:nnnn} with three protected arguments. -% This inserts in the input stream a token list of the form -% \meta{fn 1} \meta{char 1} \ldots{} \meta{fn $N$} \meta{char $N$}, -% where \meta{fn $i$} is one of the three functions, and \meta{char $i$} -% a character in the string |#1|. -% \begin{macrocode} -\cs_new:Npn \regex_nesting:n #1 { } %^^A move. Rename? -\tl_new:N \l_regex_nesting_tl -\cs_new_protected:Npn \regex_replacement:n #1 +% \begin{macro}[aux]{\regex_replacement_exp_not:N} +% This function lets us navigate around the fact that the primitive +% \cs{exp_not:n} requires a braced argument. As far as I can tell, it +% is only needed if the user tries to include in the replacement text +% a control sequence set equal to a macro parameter character, such as +% \cs{c_parameter_token}. Indeed, within an \texttt{x}-expanding +% assignment, \cs{exp_not:N}~|#| behaves as a single |#|, whereas +% \cs{exp_not:n}~|{#}| behaves as a doubled |##|. +% \begin{macrocode} +\cs_new:Npn \regex_replacement_exp_not:N #1 { \exp_not:n {#1} } +% \end{macrocode} +% \end{macro} +% +% \subsubsection{Query and brace balance} +% +% \begin{macro}[int, rEXP]{\regex_query_range:nn} +% \begin{macro}[aux, rEXP]{\regex_query_range_loop:ww} +% When it is time to extract submatches from the token list, the +% various tokens are stored in \tn{toks} registers numbered from +% \cs{l_regex_min_pos_int} inclusive to \cs{l_regex_max_pos_int} +% exclusive. The function \cs{regex_query_range:nn} \Arg{min} +% \Arg{max} unpacks registers from the position \meta{min} to the +% position $\meta{max}-1$ included. Once this is expanded, a second +% \texttt{x}-expansion will result in the actual tokens from the +% query. That second expansion is only done by user functions at the +% very end of their operation, after checking (and correcting) the +% brace balance first. +% \begin{macrocode} +\cs_new:Npn \regex_query_range:nn #1#2 { - \int_zero:N \l_regex_replacement_int - \int_zero:N \l_regex_nesting_int - \tl_clear:N \l_regex_nesting_tl - \regex_escape_use:nnnn - { \regex_replacement_unescaped:N ##1 } - { \regex_replacement_escaped:N ##1 } - { \regex_replacement_raw:N ##1 } - {#1} - \prg_do_nothing: \prg_do_nothing: - \cs_set:Npx \regex_nesting:n ##1 - { - + \int_use:N \l_regex_nesting_int - \l_regex_nesting_tl - - \regex_submatch_nesting_aux:n {##1} - } - \use:x - { - \exp_not:n { \cs_set:Npn \regex_replacement_tl:n ##1 } - { \regex_toks_range:nn \c_zero \l_regex_replacement_int } - } - % ^^A rename! + \exp_after:wN \regex_query_range_loop:ww + \int_use:N \int_eval:w #1 \exp_after:wN ; + \int_use:N \int_eval:w #2 ; + \prg_break_point:n { } + } +\cs_new:Npn \regex_query_range_loop:ww #1 ; #2 ; + { + \if_num:w #1 < #2 \exp_stop_f: + \else: + \exp_after:wN \prg_map_break: + \fi: + \tex_the:D \tex_toks:D #1 \exp_stop_f: + \exp_after:wN \regex_query_range_loop:ww + \int_use:N \int_eval:w #1 + \c_one ; #2 ; } % \end{macrocode} % \end{macro} +% \end{macro} % -% \begin{macro}[aux]{\regex_replacement_raw:N} +% \begin{macro}[int]{\regex_query_submatch:n} +% When this function is called, \tn{skip}$i$ holds the start and end +% positions for the $i$-th overall submatch as its main and stretch +% components. In the case of repeated matches, submatches from all the +% matches are put one after the other in blocks of +% \cs{l_regex_capturing_group_int} \tn{skip} registers. % \begin{macrocode} -\cs_new_protected:Npn \regex_replacement_raw:N #1 - { \regex_replacement_put:n {#1} } +\cs_new:Npn \regex_query_submatch:n #1 + { + \regex_query_range:nn + { \tex_skip:D \int_eval:w #1 } + { \etex_gluestretch:D \tex_skip:D \int_eval:w #1 } + } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_replacement_put:n} -% Raw characters are stored in a toks register. +% \begin{macro}[rEXP]{\regex_submatch_balance:n} +% Every user function must result in a balanced token list (unbalanced +% token lists cannot be stored by TeX). When we unpacked the query, we +% kept track of the brace balance as the shrink component of +% \tn{muskip} registers, hence the contribution from a given range is +% the difference between the shrink components of +% \tn{muskip}\meta{max~pos} and \tn{muskip}\meta{min~pos}. For the +% $i$-th submatch, the end-points of the range are the main and +% stretch components of \tn{skip}$i$. The trailing \cs{scan_stop:} is +% gobbled by \cs{etex_muexpr:D}, and the whole expression can be cast +% safely to an integer (no trailing expansion). % \begin{macrocode} -\cs_new_protected:Npn \regex_replacement_put:n #1 +\cs_new_protected:Npn \regex_submatch_balance:n #1 { - \tex_toks:D \l_regex_replacement_int {#1} - \int_incr:N \l_regex_replacement_int + \etex_glueshrink:D \etex_mutoglue:D \etex_muexpr:D + \tex_muskip:D \etex_gluestretch:D \tex_skip:D #1 + - \tex_muskip:D \tex_skip:D #1 + \scan_stop: } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_replacement_escaped:N} -% \begin{macro}[aux]{\regex_replacement_submatch:w} -% \begin{macro}[aux]{\regex_replacement_submatch_aux:nN} +% \subsubsection{Framework} +% +% \begin{macro}[int]{\regex_replacement:n} +% \begin{macro}[aux]{\regex_replacement_aux:n} +% The replacement text is built incrementally by abusing \tn{toks} +% within a group (see \pkg{l3tl-build}). We keep track in +% \cs{l_regex_balance_int} of the balance of explicit begin- and +% end-group tokens and \cs{l_regex_balance_tl} will consist of some +% code to compute the brace balance from submatches (see its +% description). Detect unescaped right braces, and escaped characters, +% with trailing \cs{prg_do_nothing:} because some of the later +% function look-ahead. Once the whole replacement text has been +% parsed, make sure that there is no open csname. Finally, define the +% \texttt{balance_one_match} and \texttt{do_one_match} functions. % \begin{macrocode} -\cs_new_protected:Npn \regex_replacement_unescaped:N #1 +\cs_new_protected:Npn \regex_replacement:n #1 { - \if_charcode:w \c_rbrace_str #1 - \if_num:w \l_regex_replacement_csnames_int > \c_zero - \regex_replacement_put:n \cs_end: - \else: - \regex_replacement_put:n #1 +%<trace> \trace_push:nnn { regex } { 1 } { regex_replacement:n } + \tl_set_build:Nw \l_regex_internal_a_tl + \int_zero:N \l_regex_balance_int + \tl_clear:N \l_regex_balance_tl + \regex_escape_use:nnnn + { + \if_charcode:w \c_rbrace_str ##1 + \regex_replacement_rbrace:N \else: \tl_build_one:n \fi: ##1 + } + { \regex_replacement_escaped:N ##1 } + { \tl_build_one:n ##1 } + {#1} + \prg_do_nothing: \prg_do_nothing: + \if_int_compare:w \l_regex_replacement_csnames_int > \c_zero + \msg_kernel_error:nnx { regex } { replacement-missing-rbrace } + { \int_use:N \l_regex_replacement_csnames_int } + \tl_build_one:x + { \prg_replicate:nn \l_regex_replacement_csnames_int \cs_end: } \fi: - \else: - \regex_replacement_put:n #1 - \fi: + \cs_gset:Npx \regex_replacement_balance_one_match:n ##1 + { + + \int_use:N \l_regex_balance_int + \l_regex_balance_tl + - \regex_submatch_balance:n {##1} + } + \tl_build_end: + \exp_args:No \regex_replacement_aux:n \l_regex_internal_a_tl +%<trace> \trace_pop:nnn { regex } { 1 } { regex_replacement:n } } +\cs_new_protected:Npn \regex_replacement_aux:n #1 + { + \cs_set:Npn \regex_replacement_do_one_match:n ##1 + { + \regex_query_range:nn + { \etex_glueshrink:D \tex_skip:D ##1 } + { \tex_skip:D ##1 } + #1 + } + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[aux]{\regex_replacement_escaped:N} +% As in parsing a regular expression, we use an auxiliary built from +% |#1| if defined. Otherwise, check for escaped digits (standing from +% submatches from $0$ to $9$): anything else is a raw character. +% \begin{macrocode} \cs_new_protected:Npn \regex_replacement_escaped:N #1 { \cs_if_exist_use:cF { regex_replacement_#1:w } @@ -4293,34 +4765,45 @@ \if_num:w \c_one < 1#1 \exp_stop_f: \regex_replacement_put_submatch:n {#1} \else: - \regex_replacement_put:n #1 + \tl_build_one:n #1 \fi: } } +% \end{macrocode} +% \end{macro} +% +% \subsubsection{Submatches} +% +% \begin{macro}[aux]{\regex_replacement_put_submatch:n} +% Insert a submatch in the replacement text. This is dropped if the +% submatch number is larger than the number of capturing groups. +% Unless the submatch appears inside a |\c{...}| or |\u{...}| +% construction, it must be taken into account in the brace balance. +% Here, |##1| will receive a pointer to the $0$-th submatch for a +% given match. We cannot use \cs{int_eval:n} because it is +% expandable, and would be expanded too early (short of adding +% \cs{exp_not:N}, making the code messy again). +% \begin{macrocode} \cs_new_protected:Npn \regex_replacement_put_submatch:n #1 { - \regex_replacement_put:n - { \regex_query_submatch:nn {#1} {##1} } - \if_num:w \l_regex_replacement_csnames_int = \c_zero - \tl_put_right:Nn \l_regex_nesting_tl - { - \exp_not:N \if_num:w #1 < \l_regex_capturing_group_int - \regex_submatch_nesting_aux:n { \int_eval:w #1+##1 \int_eval_end: } - \exp_not:N \fi: - } + \if_num:w #1 < \l_regex_capturing_group_int + \tl_build_one:n { \regex_query_submatch:n { #1 + ##1 } } + \if_num:w \l_regex_replacement_csnames_int = \c_zero + \tl_put_right:Nn \l_regex_balance_tl + { + \regex_submatch_balance:n { \int_eval:w #1+##1 \int_eval_end: } } + \fi: \fi: } -\cs_new_protected:Npn \regex_replacement_error:NNN #1#2#3 - { - \msg_kernel_error:nnxx { regex } { #1-command } - { replacement~text } {#3} - #2 #3 - } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_replacement_g:w, \regex_replacement_g_digits:NN} +% An ugly method to grab digits for the |\g| escape sequence. At the +% end of the run of digits, check that it ends with a right brace. +% \begin{macrocode} \cs_new_protected:Npn \regex_replacement_g:w #1#2 { - \str_if_eq:xxTF - { \exp_not:n { #1#2 } } - { \regex_replacement_unescaped:N \c_lbrace_str } + \str_if_eq:xxTF { #1#2 } { \tl_build_one:n \c_lbrace_str } { \int_zero:N \l_regex_internal_a_int \regex_replacement_g_digits:NN @@ -4329,40 +4812,48 @@ } \cs_new_protected:Npn \regex_replacement_g_digits:NN #1#2 { - \token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N + \token_if_eq_meaning:NNTF #1 \tl_build_one:n { \if_num:w \c_one < 1#2 \exp_stop_f: - \int_set:Nn \l_regex_internal_a_int { \c_ten * \l_regex_internal_a_int + #2 } + \int_set:Nn \l_regex_internal_a_int + { \c_ten * \l_regex_internal_a_int + #2 } \exp_after:wN \use_i:nnn \exp_after:wN \regex_replacement_g_digits:NN \else: - \if_charcode:w \c_rbrace_str #2 - \exp_args:No \regex_replacement_put_submatch:n - { \int_use:N \l_regex_internal_a_int } - \exp_after:wN \exp_after:wN \exp_after:wN \use_none:nn - \else: - \exp_after:wN \exp_after:wN - \exp_after:wN \regex_replacement_error:NNN - \exp_after:wN \exp_after:wN \exp_after:wN g - \fi: + \exp_after:wN \regex_replacement_error:NNN + \exp_after:wN g \fi: } - { \regex_replacement_error:NNN g } - #1 #2 - } -\cs_new_protected:Npn \regex_replacement_u:w #1#2 - { - \str_if_eq:xxTF { #1#2 } { \regex_replacement_unescaped:N \c_lbrace_str } { - \int_incr:N \l_regex_replacement_csnames_int - \regex_replacement_put:n - { \exp_not:n { \exp_after:wN \exp_not:V \cs:w } } + \if_meaning:w \regex_replacement_rbrace:N #1 + \exp_args:No \regex_replacement_put_submatch:n + { \int_use:N \l_regex_internal_a_int } + \exp_after:wN \use_none:nn + \else: + \exp_after:wN \regex_replacement_error:NNN + \exp_after:wN g + \fi: } - { \regex_replacement_error:NNN u #1#2 } + #1 #2 } +% \end{macrocode} +% \end{macro} +% +% \subsubsection{Csnames in replacement} +% +% \begin{macro}[aux]{\regex_replacement_c:w} +% \begin{macro}[aux]+\regex_replacement_c_{:w+ +% |\c| can be followed by a left brace, or by a letter for which we +% have defined a way to produce that category of characters. The +% appropriate definitions for catcodes are introduced later. For +% control sequences, if we are within a control sequence, convert +% the token list to a string, otherwise simply prevent expansion, +% with a weird cross-over between \cs{exp_not:n} and \cs{exp_not:N} +% (see this helper's description for an explanation). +% \begin{macrocode} \cs_new_protected:Npn \regex_replacement_c:w #1#2 { - \token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N + \token_if_eq_meaning:NNTF #1 \tl_build_one:n { \cs_if_exist_use:cF { regex_replacement_c_#2:w } { \regex_replacement_error:NNN c #1#2 } @@ -4371,245 +4862,341 @@ } \cs_new_protected_nopar:cpn { regex_replacement_c_ \c_lbrace_str :w } { + \if_case:w \l_regex_replacement_csnames_int + \tl_build_one:n + { \exp_not:n { \exp_after:wN \regex_replacement_exp_not:N \cs:w } } + \else: + \tl_build_one:n { \exp_not:n { \exp_after:wN \tl_to_str:N \cs:w } } + \fi: \int_incr:N \l_regex_replacement_csnames_int - \regex_replacement_put:n - { \exp_not:n { \exp_after:wN \regex_replacement_exp_not:N \cs:w } } } -\cs_new:Npn \regex_replacement_exp_not:N #1 { \exp_not:n {#1} } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[aux]{\regex_replacement_u:w} +% Check that |\u| is followed by a left brace. If so, start a control +% sequence with \cs{cs:w}, which is then unpacked either with +% \cs{exp_not:V} or \cs{tl_to_str:V} depending on the current context. +% \begin{macrocode} +\cs_new_protected:Npn \regex_replacement_u:w #1#2 + { + \str_if_eq:xxTF { #1#2 } { \tl_build_one:n \c_lbrace_str } + { + \if_case:w \l_regex_replacement_csnames_int + \tl_build_one:n { \exp_not:n { \exp_after:wN \exp_not:V \cs:w } } + \else: + \tl_build_one:n { \exp_not:n { \exp_after:wN \tl_to_str:V \cs:w } } + \fi: + \int_incr:N \l_regex_replacement_csnames_int + } + { \regex_replacement_error:NNN u #1#2 } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_replacement_rbrace:N} +% Within a |\c{...}| or |\u{...}| construction, end the control +% sequence, and decrease the brace count. Otherwise, this is a raw +% right brace. +% \begin{macrocode} +\cs_new_protected:Npn \regex_replacement_rbrace:N #1 + { + \if_int_compare:w \l_regex_replacement_csnames_int > \c_zero + \tl_build_one:n \cs_end: + \int_decr:N \l_regex_replacement_csnames_int + \else: + \tl_build_one:n #1 + \fi: + } +% \end{macrocode} +% \end{macro} +% +% \subsubsection{Characters in replacement} +% +% We will need to change the category code of the null character many +% times, hence work in a group. The catcode-specific macros below are +% defined in alphabetical order; if you are trying to understand the +% code, start from the end of the alphabet as those categories are +% simpler than active or begin-group. +% \begin{macrocode} \group_begin: - - \char_set_catcode_math_superscript:N \^^@ - \cs_new_protected_nopar:Npn \regex_replacement_c_U:w - { \regex_replacement_char:nNN { ^^@ } } - - \char_set_catcode_alignment:N \^^@ - \cs_new_protected_nopar:Npn \regex_replacement_c_T:w - { \regex_replacement_char:nNN { ^^@ } } - - \cs_new_protected:Npn \regex_replacement_c_S:w #1#2 +% \end{macrocode} +% +% \begin{macro}[aux]{\regex_replacement_char:nNN} +% The only way to produce an arbitrary character--catcode pair is to +% use the \tn{lowercase} or \tn{uppercase} primitives. This is a +% wrapper for our purposes. The first argument is the null character +% with various catcodes. The second and third arguments are grabbed +% from the input stream: |#3| is the character whose character code to +% reproduce. +% \begin{macrocode} + \cs_new_protected:Npn \regex_replacement_char:nNN #1#2#3 { - \int_compare:nNnTF { `#2 } = \c_zero - { \regex_replacement_error:NNN c #1#2 } - { - \char_set_lccode:nn {32} { `#2 } - \tl_to_lowercase:n { \regex_replacement_put:n {~} } - } + \if_meaning:w \prg_do_nothing: #3 + \msg_kernel_error:nn { regex } { replacement-catcode-end } + \else: + \tex_lccode:D \c_zero = `#3 \scan_stop: + \tl_to_lowercase:n { \tl_build_one:n {#1} } + \fi: } - - \char_set_catcode_parameter:N \^^@ - \cs_new_protected_nopar:Npn \regex_replacement_c_P:w - { \regex_replacement_char:nNN { ^^@^^@^^@^^@^^@^^@^^@^^@ } } - - \char_set_catcode_other:N \^^@ - \cs_new_protected_nopar:Npn \regex_replacement_c_O:w - { \regex_replacement_char:nNN { ^^@ } } - - \char_set_catcode_math_toggle:N \^^@ - \cs_new_protected_nopar:Npn \regex_replacement_c_M:w - { \regex_replacement_char:nNN { ^^@ } } - - \char_set_catcode_letter:N \^^@ - \cs_new_protected_nopar:Npn \regex_replacement_c_L:w - { \regex_replacement_char:nNN { ^^@ } } - - \char_set_catcode_group_end:N \^^@ - \cs_new_protected_nopar:Npn \regex_replacement_c_E:w +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_replacement_c_A:w} +% For an active character, expansion must be avoided, twice because we +% later do two \texttt{x}-expansions, to unpack \tn{toks} for the +% query, and to expand their contents to tokens of the query. +% \begin{macrocode} + \char_set_catcode_active:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_A:w + { \regex_replacement_char:nNN { \exp_not:n { \exp_not:N ^^@ } } } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_replacement_c_B:w} +% An explicit begin-group token increases the balance, unless within a +% |\c{...}| or |\u{...}| construction. Add the desired begin-group +% character, using the standard \cs{if_false:} trick. We eventually +% \texttt{x}-expand twice. The first time must yield a balanced token +% list, and the second one gives the bare begin-group token. The +% \cs{exp_after:wN} is not strictly needed, but is more consistent +% with \pkg{l3tl-analysis}. +% \begin{macrocode} + \char_set_catcode_group_begin:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_B:w { - \int_decr:N \l_regex_nesting_int - \regex_replacement_char:nNN { \if_false: { \fi: ^^@ } + \if_num:w \l_regex_replacement_csnames_int = \c_zero + \int_incr:N \l_regex_balance_int + \fi: + \regex_replacement_char:nNN + { \exp_not:n { \exp_after:wN ^^@ \if_false: } \fi: } } } - +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_replacement_c_C:w} +% This is not quite catcode-related: when the user requests a +% character with category \enquote{control sequence}, the +% one-character control symbol is returned. As for the active +% character, we prepare for two \texttt{x}-expansions. +% \begin{macrocode} + \cs_new_protected:Npn \regex_replacement_c_C:w #1#2 + { \tl_build_one:n { \exp_not:N \exp_not:N \exp_not:c {#2} } } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_replacement_c_D:w} +% Subscripts fit the mould: \tn{lowercase} the null byte with the +% correct category. +% \begin{macrocode} \char_set_catcode_math_subscript:N \^^@ \cs_new_protected_nopar:Npn \regex_replacement_c_D:w { \regex_replacement_char:nNN { ^^@ } } - - \char_set_catcode_group_begin:N \^^@ - \cs_new_protected_nopar:Npn \regex_replacement_c_B:w +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_replacement_c_E:w} +% Similar to the begin-group case, the second \texttt{x}-expansion +% produces the bare end-group token. +% \begin{macrocode} + \char_set_catcode_group_end:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_E:w { - \int_incr:N \l_regex_nesting_int - \regex_replacement_char:nNN { \exp_after:wN ^^@ \if_false: } \fi: } + \if_num:w \l_regex_replacement_csnames_int = \c_zero + \int_decr:N \l_regex_balance_int + \fi: + \regex_replacement_char:nNN + { \exp_not:n { \if_false: { \fi: ^^@ } } } - - \char_set_catcode_active:N \^^@ - \cs_new_protected_nopar:Npn \regex_replacement_c_A:w - { \regex_replacement_char:nNN { \exp_not:N ^^@ } } - -\group_end: - -\cs_new_protected:Npn \regex_replacement_char:nNN #1#2#3 - { - \char_set_lccode:nn \c_zero { `#3 } - \tl_to_lowercase:n - { \regex_replacement_put:n { \exp_not:n {#1} } } - } -% \end{macrocode} -% \end{macro} -% \end{macro} -% \end{macro} -% -% \subsection{User commands} -% -% \subsubsection{Precompiled pattern} -% -% A given pattern is often reused to match many different query token lists. -% We thus give a means of storing the NFA corresponding to a given -% pattern in a token list variable of the form -% \begin{quote} -% \cs{regex_nfa:Nw} \meta{variable~name} \\ -% \meta{assignments} \\ -% \cs{tex_toks:D} 0 \{ \meta{instruction0} \} \\ -% \ldots{} \\ -% \cs{tex_toks:D} $n$ \{ \meta{instruction$\sb{n}$} \} \\ -% \cs{s_stop} -% \end{quote} -% where $n$ is the number of states in the NFA, -% and the various \meta{instruction$\sb{i}$} control -% how the NFA behaves in state $i$. The \cs{regex_nfa:Nw} -% function removes the whole NFA from the input stream -% and produces an error: the \meta{nfa var} should only be -% accessed through dedicated functions. This rather drastic -% approach is taken because assignments triggered by the -% contents of \meta{nfa var} may overwrite data which is used -% elsewhere, unless everything is done carefully in a group. -% -% \begin{macro}{\regex_gset:Nn} -% \begin{macro}{\regex_const:Nn} -% \begin{macro}{\regex_set:Nn} -% \begin{macro}[aux]{\regex_set_aux:NNn} -% The three user functions only differ with which function is used -% to assign the pre-compiled regular expression to the user's variable. -% Internally, they all first build the NFA corresponding to the regex, -% then store the contents of all the necessary \tn{toks} registers -% in the user's variable. -% \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_set:Nn - { \regex_set_aux:NNn \tl_set:Nn } -\cs_new_protected_nopar:Npn \regex_gset:Nn - { \regex_set_aux:NNn \tl_gset:Nn } -\cs_new_protected_nopar:Npn \regex_const:Nn - { \regex_set_aux:NNn \tl_const:Nn } -\cs_new_protected:Npn \regex_set_aux:NNn #1#2#3 - { - \group_begin: - \regex_build:n {#3} - \use:x - { - \group_end: - #1 \exp_not:N #2 { \regex_set_aux:N #2 } - } - } % \end{macrocode} % \end{macro} +% +% \begin{macro}[aux]{\regex_replacement_c_L:w} +% Simply \tn{lowercase} a letter null byte to produce an arbitrary letter. +% \begin{macrocode} + \char_set_catcode_letter:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_L:w + { \regex_replacement_char:nNN { ^^@ } } +% \end{macrocode} % \end{macro} +% +% \begin{macro}[aux]{\regex_replacement_c_M:w} +% No surprise here, we lowercase the null math toggle. +% \begin{macrocode} + \char_set_catcode_math_toggle:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_M:w + { \regex_replacement_char:nNN { ^^@ } } +% \end{macrocode} % \end{macro} +% +% \begin{macro}[aux]{\regex_replacement_c_O:w} +% Lowercase an other null byte. +% \begin{macrocode} + \char_set_catcode_other:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_O:w + { \regex_replacement_char:nNN { ^^@ } } +% \end{macrocode} % \end{macro} % -% \begin{macro}[int, rEXP]{\regex_set_aux:N} -% \begin{macro}[aux, rEXP]{\regex_set_aux:n} -% \begin{macro}[aux, EXP]{\regex_nfa:Nw} -% Within a group, build the NFA corresponding to the given regular -% expression, with submatch tracking. Then save the contents of all -% relevant \tn{toks} registers into the variable outside the group. -% The auxiliary \cs{regex_nfa:Nw} is not protected: this ensures that -% the NFA will properly be replaced by an error message in expansion -% contexts. +% \begin{macro}[aux]{\regex_replacement_c_P:w} +% For macro parameters, expansion is a tricky issue. We need to +% prepare for two \texttt{x}-expansions and passing through various +% macro definitions. Note that we cannot replace one \cs{exp_not:n} by +% doubling the macro parameter characters because this would misbehave +% if a mischievous user asks for |\c{\cP\#}|, since that macro +% parameter character would be doubled. % \begin{macrocode} -\cs_new:Npn \regex_set_aux:N #1 - { - \exp_not:n { \regex_nfa:Nw #1 } - \l_regex_max_state_int - = \int_use:N \l_regex_max_state_int - \l_regex_capturing_group_int - = \int_use:N \l_regex_capturing_group_int - \token_if_eq_meaning:NNT - \regex_match_loop_case_hook: - \regex_match_loop_caseless_hook: - { - \cs_set_eq:NN \regex_match_loop_case_hook: - \regex_match_loop_caseless_hook: - } - \prg_stepwise_function:nnnN - \c_zero \c_one {\l_regex_max_state_int - \c_one } - \regex_set_aux:n - \s_stop - } -\cs_new:Npn \regex_set_aux:n #1 - { \tex_toks:D #1 { \tex_the:D \tex_toks:D #1 } } -\cs_new:Npn \regex_nfa:Nw #1 - { - \msg_expandable_kernel_error:nnn { regex } { nfa-misused } {#1} - \use_none_delimit_by_s_stop:w - } + \char_set_catcode_parameter:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_P:w + { + \regex_replacement_char:nNN + { \exp_not:n { \exp_not:n { ^^@^^@^^@^^@ } } } + } % \end{macrocode} % \end{macro} +% +% \begin{macro}[aux]{\regex_replacement_c_S:w} +% Spaces are normalized on input by \TeX{} to have character code +% $32$. It is in fact impossible to get a token with character code +% $0$ and category code $10$. Hence we use $32$ instead of $0$ as our +% base character. +% \begin{macrocode} + \cs_new_protected:Npn \regex_replacement_c_S:w #1#2 + { + \if_meaning:w \prg_do_nothing: #2 + \msg_kernel_error:nn { regex } { replacement-catcode-end } + \else: + \if_num:w `#2 = \c_zero + \msg_kernel_error:nn { regex } { replacement-null-space } + \fi: + \tex_lccode:D 32 = `#2 \scan_stop: + \tl_to_lowercase:n { \tl_build_one:n {~} } + \fi: + } +% \end{macrocode} % \end{macro} +% +% \begin{macro}[aux]{\regex_replacement_c_T:w} +% No surprise for alignment tabs here. Those are surrounded by the +% appropriate braces whenever necessary, hence they don't cause +% trouble in alignment settings. +% \begin{macrocode} + \char_set_catcode_alignment:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_T:w + { \regex_replacement_char:nNN { ^^@ } } +% \end{macrocode} % \end{macro} % -% \begin{macro}[int]{\regex_use:N} -% No error-checking. +% \begin{macro}[aux]{\regex_replacement_c_U:w} +% Simple call to \cs{regex_replacement_char:nNN} which lowercases the +% math superscript |^^@|. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_use:N - { \exp_last_unbraced:No \use_none:nn } + \char_set_catcode_math_superscript:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_U:w + { \regex_replacement_char:nNN { ^^@ } } % \end{macrocode} % \end{macro} % -% \subsubsection{Generic auxiliary functions} +% Restore the catcode of the null byte. +% \begin{macrocode} +\group_end: +% \end{macrocode} % -% Most of \pkg{l3regex}'s work is done within a group. +% \subsubsection{An error} % -% \begin{macro}[aux]{\regex_aux_return:} -% This function triggers either \cs{prg_return_false:} -% or \cs{prg_return_true:} as appropriate to whether a -% match was found or not. +% \begin{macro}[aux]{\regex_replacement_error:NNN} +% Simple error reporting by calling one of the messages +% \texttt{replacement-c}, \texttt{replacement-g}, or +% \texttt{replacement-u}. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_aux_return: +\cs_new_protected:Npn \regex_replacement_error:NNN #1#2#3 { - \if_meaning:w \c_true_bool \g_regex_success_bool - \prg_return_true: - \else: - \prg_return_false: - \fi: + \msg_kernel_error:nnx { regex } { replacement-#1 } {#3} + #2 #3 + } +% \end{macrocode} +% \end{macro} +% +% \subsection{User functions} +% +% \begin{macro}{\regex_new:N} +% Before being assigned a sensible value, a regex variable matches +% nothing. +% \begin{macrocode} +\cs_new_protected:Npn \regex_new:N #1 + { \cs_new_eq:NN #1 \c_regex_no_match_regex } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\regex_set:Nn, \regex_gset:Nn, \regex_const:Nn} +% Compile, then store the result in the user variable with the +% appropriate assignment function. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \regex_set:Nn #1#2 + { + \regex_compile:n {#2} + \tl_set_eq:NN #1 \l_regex_internal_regex + } +\cs_new_protected_nopar:Npn \regex_gset:Nn #1#2 + { + \regex_compile:n {#2} + \tl_gset_eq:NN #1 \l_regex_internal_regex + } +\cs_new_protected_nopar:Npn \regex_const:Nn #1#2 + { + \regex_compile:n {#2} + \tl_const:Nx #1 { \exp_not:o \l_regex_internal_regex } } % \end{macrocode} % \end{macro} % -% \begin{macro}{\regex_aux_build_match:nn} -% This auxiliary is used by user functions whose \meta{regex} argument -% is given as an explicit regular expression within braces. In that -% case, we need to build the automaton corresponding to that regular -% expression, then perform the matching on the given token list |#2|. +% \begin{macro}{\regex_show:N, \regex_show:n} +% User functions: the \texttt{n} variant requires compilation first. +% Then show the variable with some appropriate text. The auxiliary +% \cs{regex_show_aux:Nx} is defined in a different section. % \begin{macrocode} -\cs_new_protected:Npn \regex_aux_build_match:nn #1#2 +\cs_new_protected:Npn \regex_show:n #1 { - \regex_build:n {#1} - \regex_match:n {#2} + \regex_compile:n {#1} + \regex_show_aux:Nx \l_regex_internal_regex + { { \tl_to_str:n {#1} } } } +\cs_new_protected:Npn \regex_show:N #1 + { \regex_show_aux:Nx #1 { variable~\token_to_str:N #1 } } % \end{macrocode} % \end{macro} % -% \begin{macro}{\regex_aux_use_match:Nn} -% This auxiliary is used by user functions whose \meta{regex} argument -% is given as a pre-compiled regex variable. We make sure that -% the token list variable indeed is an automaton (by testing the -% first token). If not, the match is deemed unsuccessful, after -% raising an error. If we have an automaton, \enquote{use} it, -% then perform the matching on the given token list |#2|. +% \begin{macro}[TF]{\regex_match:nn, \regex_match:Nn} +% Those conditionals are based on a common auxiliary defined +% later. Its first argument builds the \textsc{nfa} corresponding to +% the regex, and the second argument is the query token list. Once we +% have performed the match, convert the resulting boolean to +% \cs{prg_return_true:} or \texttt{false}. % \begin{macrocode} -\cs_new_protected:Npn \regex_aux_use_match:Nn #1#2 +\prg_new_protected_conditional:Npnn \regex_match:nn #1#2 { T , F , TF } { - \exp_args:No \tl_if_head_eq_meaning:nNTF {#1} \regex_nfa:Nw - { - \regex_use:N #1 - \regex_match:n {#2} - } - { - \msg_kernel_error:nnx { regex } { not-nfa } { \token_to_str:N #1 } - \bool_gset_false:N \g_regex_success_bool - } + \regex_match_aux:nn { \regex_build:n {#1} } {#2} + \regex_aux_return: + } +\prg_new_protected_conditional:Npnn \regex_match:Nn #1#2 { T , F , TF } + { + \regex_match_aux:nn { \regex_build:N #1 } {#2} + \regex_aux_return: } % \end{macrocode} % \end{macro} % +% \begin{macro}{\regex_count:nnN, \regex_count:NnN} +% Again, use an auxiliary whose first argument builds the \textsc{nfa}. +% \begin{macrocode} +\cs_new_protected:Npn \regex_count:nnN #1 + { \regex_count_aux:nnN { \regex_build:n {#1} } } +\cs_new_protected:Npn \regex_count:NnN #1 + { \regex_count_aux:nnN { \regex_build:N #1 } } +% \end{macrocode} +% \end{macro} +% % \begin{macro} % { % \regex_extract_once:nnN, \regex_extract_once:NnN, @@ -4626,92 +5213,206 @@ % \regex_replace_all:nnN, \regex_replace_all:NnN, % \regex_split:nnN, \regex_split:NnN % } -% We define here $40$ user functions, following a common pattern -% in terms of an auxiliary such as \cs{regex_extract_once_aux:NnnN} -% (those auxiliaries are defined in the coming sections). -% The arguments handed to the auxiliary are -% \cs{regex_aux_build_match:nn} or \cs{regex_aux_use_match:Nn}, -% followed by the three arguments of the user function. -% The conditionals call \cs{regex_aux_return:} to return -% either \texttt{true} or \texttt{false} once matching -% has been performed. +% We define here $40$ user functions, following a common pattern in +% terms of \texttt{_aux:nnN} auxiliaries, defined in the coming +% subsections. The auxiliary is handed \cs{regex_build:n} or +% \cs{regex_build:N} with the appropriate regex argument, then all +% other necessary arguments (replacement text, token list, \emph{etc.} +% The conditionals call \cs{regex_aux_return:} to return either +% \texttt{true} or \texttt{false} once matching has been performed. % \begin{macrocode} \cs_set_protected:Npn \regex_tmp:w #1#2#3 { - \cs_new_protected_nopar:Npn #1 - { #3 \regex_aux_build_match:nn } - \cs_new_protected_nopar:Npn #2 - { #3 \regex_aux_use_match:Nn } - \prg_new_protected_conditional:Npnn #1 ##1##2##3 { T , F , TF } - { - #3 \regex_aux_build_match:nn {##1} {##2} ##3 - \regex_aux_return: - } + \cs_new_protected:Npn #2 ##1 { #1 { \regex_build:n {##1} } } + \cs_new_protected:Npn #3 ##1 { #1 { \regex_build:N ##1 } } \prg_new_protected_conditional:Npnn #2 ##1##2##3 { T , F , TF } - { - #3 \regex_aux_use_match:Nn {##1} {##2} ##3 - \regex_aux_return: - } - } -\tl_map_inline:nn - { - { extract_once } { extract_all } - { replace_once } { replace_all } - { split } - } - { - \exp_args:Nccc \regex_tmp:w - { regex_#1:nnN } { regex_#1:NnN } { regex_#1_aux:NnnN } + { #1 { \regex_build:n {##1} } {##2} ##3 \regex_aux_return: } + \prg_new_protected_conditional:Npnn #3 ##1##2##3 { T , F , TF } + { #1 { \regex_build:N ##1 } {##2} ##3 \regex_aux_return: } } +\regex_tmp:w \regex_extract_once_aux:nnN + \regex_extract_once:nnN \regex_extract_once:NnN +\regex_tmp:w \regex_extract_all_aux:nnN + \regex_extract_all:nnN \regex_extract_all:NnN +\regex_tmp:w \regex_replace_once_aux:nnN + \regex_replace_once:nnN \regex_replace_once:NnN +\regex_tmp:w \regex_replace_all_aux:nnN + \regex_replace_all:nnN \regex_replace_all:NnN +\regex_tmp:w \regex_split_aux:nnN \regex_split:nnN \regex_split:NnN % \end{macrocode} % \end{macro} % \end{macro} % -% \subsubsection{Submatches, once the correct match is found} +% \subsubsection{Variables and helpers for user functions} % -% \begin{macro}[int]{\regex_extract:} -% \begin{macro}[aux]{\regex_extract_aux_b:wn} -% \begin{macro}[aux]{\regex_extract_aux_e:wn} +% \begin{variable}{\l_regex_match_count_int} +% The number of matches found so far is stored +% in \cs{l_regex_match_count_int}. This is only used +% in the \cs{regex_count:nnN} functions. +% \begin{macrocode} +\int_new:N \l_regex_match_count_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{regex_begin, regex_end} +% Those flags are raised to indicate extra begin-group +% or end-group tokens when extracting submatches. +% \begin{macrocode} +\flag_new:n { regex_begin } +\flag_new:n { regex_end } +% \end{macrocode} +% \end{variable} % +% \begin{variable}{\l_regex_submatch_int, \l_regex_zeroth_submatch_int} +% The end-points of each submatch are stored as main and stretch +% components of \tn{skip}\meta{submatch}, where \meta{submatch} ranges +% from \cs{l_regex_max_state_int} (inclusive) to +% \cs{l_regex_submatch_int} (exclusive). Each successful match comes +% with a $0$-th submatch (the full match), and one match for each +% capturing group: submatches corresponding to the last successful +% match are labelled starting at +% \texttt{zeroth_submatch}. Additionally, the shrink component of this +% $0$-th submatch is the position at which that match attempt started: +% this is used for splitting and replacements. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_extract: +\int_new:N \l_regex_submatch_int +\int_new:N \l_regex_zeroth_submatch_int +% \end{macrocode} +% \end{variable} +% +% \begin{macro}[aux]{\regex_aux_return:} +% This function triggers either \cs{prg_return_false:} or +% \cs{prg_return_true:} as appropriate to whether a match was found or +% not. It is used by all user conditionals. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \regex_aux_return: { - \int_set_eq:NN \l_regex_submatch_start_int \l_regex_submatch_int \if_meaning:w \c_true_bool \g_regex_success_bool - \prg_replicate:nn \l_regex_capturing_group_int - { - \tex_skip:D \l_regex_submatch_int \c_zero sp \scan_stop: - \int_incr:N \l_regex_submatch_int - } - \prop_map_inline:Nn \l_regex_success_submatches_prop - { - \if_num:w ##1 \c_max_int - \exp_after:wN \regex_extract_aux_b:wn \int_use:N - \else: - \exp_after:wN \regex_extract_aux_e:wn \int_use:N - \fi: - \int_eval:w \l_regex_submatch_start_int + ##1 {##2} - } + \prg_return_true: + \else: + \prg_return_false: \fi: } -\cs_new_protected:Npn \regex_extract_aux_b:wn #1 < #2 +% \end{macrocode} +% \end{macro} +% +% \subsubsection{Matching} +% +% \begin{macro}[aux]{\regex_match_aux:nn} +% We don't track submatches, and stop after a single match. Build the +% \textsc{nfa} with |#1|, and perform the match on the query |#2|. +% \begin{macrocode} +\cs_new_protected:Npn \regex_match_aux:nn #1#2 { - \tex_skip:D #1 = #2 sp - plus \etex_gluestretch:D \tex_skip:D #1 \scan_stop: + \group_begin: + \regex_disable_submatches: + \regex_single_match: + #1 + \regex_match:n {#2} + \group_end: } -\cs_new_protected:Npn \regex_extract_aux_e:wn #1 > #2 +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_count_aux:nnN} +% Again, we don't care about submatches. Instead of aborting after the +% first \enquote{longest match} is found, we search for multiple +% matches, incrementing \cs{l_regex_match_count_int} every time to +% record the number of matches. Build the \textsc{nfa} and match. At +% the end, store the result in the user's variable. +% \begin{macrocode} +\cs_new_protected:Npn \regex_count_aux:nnN #1#2#3 { - \tex_skip:D #1 - = 1 \tex_skip:D #1 plus #2 sp \scan_stop: + \group_begin: + \regex_disable_submatches: + \int_zero:N \l_regex_match_count_int + \regex_multi_match:n { \int_incr:N \l_regex_match_count_int } + #1 + \regex_match:n {#2} + \exp_args:NNNo + \group_end: + \int_set:Nn #3 { \int_use:N \l_regex_match_count_int } } % \end{macrocode} % \end{macro} +% +% \subsubsection{Extracting submatches} +% +% \begin{macro}[aux]{\regex_extract_once_aux:nnN, \regex_extract_all_aux:nnN} +% Match once or multiple times. After each match (or after the only +% match), extract the submatches using \cs{regex_extract:}. At the +% end, store the sequence containing all the submatches into the user +% variable |#3| after closing the group. +% \begin{macrocode} +\cs_new_protected:Npn \regex_extract_once_aux:nnN #1#2#3 + { + \group_begin: + \regex_single_match: + #1 + \regex_match:n {#2} + \regex_extract: + \regex_group_end_extract_seq:N #3 + } +\cs_new_protected:Npn \regex_extract_all_aux:nnN #1#2#3 + { + \group_begin: + \regex_multi_match:n { \regex_extract: } + #1 + \regex_match:n {#2} + \regex_group_end_extract_seq:N #3 + } +% \end{macrocode} % \end{macro} +% +% \begin{macro}[aux]{\regex_split_aux:nnN} +% Splitting at submatches is a bit more tricky. For each match, +% extract all submatches, and replace the zeroth submatch by the part +% of the query between the start of the match attempt and the start of +% the zeroth submatch. This is inhibited if the delimiter matched an +% empty token list at the start of this match attempt. After the last +% match, store the last part of the token list, which ranges from the +% start of the match attempt to the end of the query. This step is +% inhibited if the last match was empty and at the very end: decrement +% \cs{l_regex_submatch_int}, which controls which \tn{skip} registers +% will be used. +% \begin{macrocode} +\cs_new_protected:Npn \regex_split_aux:nnN #1#2#3 + { + \group_begin: + \regex_multi_match:n + { + \if_num:w \l_regex_start_pos_int < \l_regex_success_pos_int + \regex_extract: + \tex_skip:D \l_regex_zeroth_submatch_int + = \l_regex_start_pos_int sp + plus \tex_skip:D \l_regex_zeroth_submatch_int \scan_stop: + \fi: + } + #1 + \regex_match:n {#2} +%<assert>\assert_int:n { \l_regex_current_pos_int = \l_regex_max_pos_int } + \tex_skip:D \l_regex_submatch_int + = \l_regex_start_pos_int sp plus \l_regex_max_pos_int sp \scan_stop: + \int_incr:N \l_regex_submatch_int + \if_meaning:w \c_true_bool \l_regex_empty_success_bool + \if_num:w \l_regex_start_pos_int = \l_regex_max_pos_int + \int_decr:N \l_regex_submatch_int + \fi: + \fi: + \regex_group_end_extract_seq:N #3 + } +% \end{macrocode} % \end{macro} % -% \begin{macro}{\regex_group_end_extract_seq:N} -% \begin{macro}[aux]{\regex_extract_seq_aux:n} -% \begin{macro}[aux]{\regex_extract_seq_aux:ww} +% \begin{macro}[aux]{\regex_group_end_extract_seq:N} +% The end-points of submatches are stored as the main and stretch +% components of \tn{skip} registers from \cs{l_regex_max_state_int} to +% \cs{l_regex_submatch_int} (exclusive). Extract the relevant ranges +% into \cs{l_regex_internal_a_tl}. We detect unbalanced results using +% the two flags \texttt{regex_begin} and \texttt{regex_end}, raised +% whenever we see too many begin-group or end-group tokens in a +% submatch. We disable \cs{seq_item:n} to prevent two +% \texttt{x}-expansions. % \begin{macrocode} \cs_new_protected:Npn \regex_group_end_extract_seq:N #1 { @@ -4721,7 +5422,7 @@ \tl_set:Nx \l_regex_internal_a_tl { \prg_stepwise_function:nnnN - \l_regex_max_state_int + { \c_two * \l_regex_max_state_int } \c_one { \l_regex_submatch_int - \c_one } \regex_extract_seq_aux:n @@ -4729,17 +5430,33 @@ \int_compare:nNnF { \flag_height:n { regex_begin } + \flag_height:n { regex_end } } = \c_zero - { \msg_kernel_error:nn { regex } { sequence-unbalanced } } - \tl_set:Nx \l_regex_internal_a_tl { \l_regex_internal_a_tl } - \exp_args:NNNo \group_end: - \tl_set:Nn #1 \l_regex_internal_a_tl + { + \msg_kernel_error:nnxxx { regex } { result-unbalanced } + { splitting~or~extracting~submatches } + { \flag_height:n { regex_end } } + { \flag_height:n { regex_begin } } + } + \use:x + { + \group_end: + \tl_set:Nn \exp_not:N #1 { \l_regex_internal_a_tl } + } } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux, EXP]{\regex_extract_seq_aux:n, \regex_extract_seq_aux:ww} +% The \texttt{:n} auxiliary builds one item of the sequence of +% submatches. First compute the brace balance of the submatch, then +% extract the submatch from the query, adding the appropriate braces +% and raising a flag if the submatch is not balanced. +% \begin{macrocode} \cs_new:Npn \regex_extract_seq_aux:n #1 { \seq_item:n { \exp_after:wN \regex_extract_seq_aux:ww - \int_value:w \regex_submatch_nesting_aux:n {#1} ; #1; + \int_value:w \regex_submatch_balance:n {#1} ; #1; } } \cs_new:Npn \regex_extract_seq_aux:ww #1; #2; @@ -4748,7 +5465,7 @@ \flag_raise:n { regex_end } \prg_replicate:nn {-#1} { \exp_not:n { { \if_false: } \fi: } } \fi: - \regex_query_submatch:w #2; + \regex_query_submatch:n {#2} \if_num:w #1 > \c_zero \flag_raise:n { regex_begin } \prg_replicate:nn {#1} { \exp_not:n { \if_false: { \fi: } } } @@ -4756,278 +5473,182 @@ } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} -% -% \subsubsection{Matching} -% -% \begin{macro}[TF]{\regex_match:nn} -% \begin{macro}[TF]{\regex_match:Nn} -% \begin{macro}{\regex_match_aux:n} -% We don't track submatches. Then either build the NFA corresponding -% to the regular expression, or use a precompiled pattern. Then match, -% using the internal \cs{regex_match:n}. Finally return the result -% after closing the group. -% \begin{macrocode} -\prg_new_protected_conditional:Npnn \regex_match:nn #1#2 { T , F , TF } - { - \regex_match_aux:n - { \regex_aux_build_match:nn {#1} {#2} } - } -\prg_new_protected_conditional:Npnn \regex_match:Nn #1#2 { T , F , TF } - { - \regex_match_aux:n - { \regex_aux_use_match:Nn #1 {#2} } - } -\cs_new_protected:Npn \regex_match_aux:n #1 - { - \group_begin: - \tl_clear:N \l_regex_every_match_tl - \regex_disable_submatches: - #1 - \group_end: - \regex_aux_return: - } -% \end{macrocode} -% \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}{\regex_count:nnN} -% \begin{macro}{\regex_count:NnN} -% \begin{macro}{\regex_count_aux:n} -% Instead of aborting once the first \enquote{longest match} is found, -% we repeat the search. The code is such that the search will not -% start on the same character, hence avoiding infinite loops. +% \begin{macro}[aux] +% {\regex_extract:, \regex_extract_aux_b:wn, \regex_extract_aux_e:wn} +% Our task here is to extract from the property list +% \cs{l_regex_success_submatches_prop} the list of end-points of +% submatches, and store them in \tn{skip} registers, from +% \cs{l_regex_zeroth_submatch_int} upwards. We begin by emptying those +% \tn{skip} registers. Then for each \meta{key}--\meta{value} pair in +% the property list update the appropriate \tn{skip} component. This +% is somewhat a hack: the \meta{key} is a non-negative integer +% followed by |<| or |>|, which we use in a comparison to $-1$. At the +% end, store the information about the position at which the match +% attempt started, as a shrink component. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_count:nnN - { \regex_count_aux:NnnN \regex_aux_build_match:nn } -\cs_new_protected_nopar:Npn \regex_count:NnN - { \regex_count_aux:NnnN \regex_aux_use_match:Nn } -\cs_new_protected:Npn \regex_count_aux:NnnN #1#2#3#4 +\cs_new_protected_nopar:Npn \regex_extract: { - \group_begin: - \regex_disable_submatches: - \int_zero:N \l_regex_match_count_int - \tl_set:Nn \l_regex_every_match_tl + \if_meaning:w \c_true_bool \g_regex_success_bool + \int_set_eq:NN \l_regex_zeroth_submatch_int \l_regex_submatch_int + \prg_replicate:nn \l_regex_capturing_group_int { - \int_incr:N \l_regex_match_count_int - \regex_match_once: + \tex_skip:D \l_regex_submatch_int \c_zero sp \scan_stop: + \int_incr:N \l_regex_submatch_int } - #1 {#2} {#3} - \exp_args:NNNo - \group_end: - \int_set:Nn #4 { \int_use:N \l_regex_match_count_int } - } -% \end{macrocode} -% \end{macro} -% \end{macro} -% \end{macro} -% -% \subsubsection{Submatch extraction} -% -% \begin{macro}[aux]{\regex_extract_once_aux:NnnN} -% As announced, here comes the auxiliary for extracting one match. -% Since we only want one match, \cs{l_regex_every_match_tl} -% is empty, and does not trigger the matching code again. -% After matching, \cs{regex_extract:} extracts submatches -% into various \tn{skip} registers, and those are then concatenated -% into a sequence by \cs{regex_group_end_extract_seq:N}. That function -% is also responsible for closing the group. -% \begin{macrocode} -\cs_new_protected:Npn \regex_extract_once_aux:NnnN #1#2#3#4 - { - \group_begin: - \tl_set:Nn \l_regex_every_match_tl { \regex_extract: } - #1 {#2} {#3} - \regex_group_end_extract_seq:N #4 + \prop_map_inline:Nn \l_regex_success_submatches_prop + { + \if_num:w ##1 \c_minus_one + \exp_after:wN \regex_extract_aux_e:wn \int_value:w + \else: + \exp_after:wN \regex_extract_aux_b:wn \int_value:w + \fi: + \int_eval:w \l_regex_zeroth_submatch_int + ##1 {##2} + } + \tex_skip:D \l_regex_zeroth_submatch_int + = \tex_the:D \tex_skip:D \l_regex_zeroth_submatch_int + minus \l_regex_start_pos_int sp \scan_stop: + \fi: } -% \end{macrocode} -% \end{macro} -% -% \begin{macro}[aux]{\regex_extract_all_aux:NnnN} -%^^A todo: update doc -% The set of submatches will be built progressively -% in \cs{l_regex_result_seq}. For each match, extract -% the submatches, and concatenate that to the right of -% the result sequence, then start matching again. -% Finally, copy the result in the user's sequence variable. -% \begin{macrocode} -\cs_new_protected:Npn \regex_extract_all_aux:NnnN #1#2#3#4 +\cs_new_protected:Npn \regex_extract_aux_b:wn #1 < #2 { - \group_begin: - \tl_set:Nn \l_regex_every_match_tl - { \regex_extract: \regex_match_once: } - #1 {#2} {#3} - \regex_group_end_extract_seq:N #4 + \tex_skip:D #1 = #2 sp + plus \etex_gluestretch:D \tex_skip:D #1 \scan_stop: } -% \end{macrocode} -% \end{macro} -% -% \subsubsection{Splitting a token list by matches of a regex} -% -% \begin{macro}[aux]{\regex_split_aux:NnnN} -% Recurse through the matches, and for each, do the following. -% Extract the submatches into various \tn{skip} registers, -% then replace the match |\0|, which should not be kept in the -% final result, and replace it by the part of the token list -% before the match. -% This process must be inhibited to avoid creating empty items -% if the regex matched an empty token list at the place where -% the match attempt started. -% After the last successful match, we need to add to the result -% the part of the token list after the last match, unless the -% last match was empty and at the very end. -% Finally, \cs{regex_group_end_extract_seq:N} builds a sequence -% from all the \tn{skip} registers, and assigns it to |#4| -% after closing the group. -% \begin{macrocode} -\cs_new_protected:Npn \regex_split_aux:NnnN #1#2#3#4 +\cs_new_protected:Npn \regex_extract_aux_e:wn #1 > #2 { - \group_begin: - \tl_set:Nn \l_regex_every_match_tl - { - \if_num:w \l_regex_start_index_int < \l_regex_success_index_int - \regex_extract: - \tex_skip:D \l_regex_submatch_start_int - = \l_regex_start_index_int sp - plus \tex_skip:D \l_regex_submatch_start_int \scan_stop: - \fi: - \regex_match_once: - } - #1 {#2} {#3} - \tex_skip:D \l_regex_submatch_int - = \l_regex_start_index_int sp - plus \l_regex_current_index_int sp \scan_stop: - \int_incr:N \l_regex_submatch_int - \if_num:w \l_regex_start_index_int = \l_regex_current_index_int - \if_meaning:w \c_true_bool \l_regex_success_empty_bool - \int_decr:N \l_regex_submatch_int - \fi: - \fi: - \regex_group_end_extract_seq:N #4 + \tex_skip:D #1 + = 1 \tex_skip:D #1 plus #2 sp \scan_stop: } % \end{macrocode} % \end{macro} % % \subsubsection{Replacement} % -% \begin{macro}[aux]{\regex_replace_once_aux:NnnN} -% The replacement text is analysed by \cs{regex_replacement:n}, -% which defines \cs{regex_replacement_tl:n} to expand to the -% replaced token list, assuming that submatches are stored in -% various \tn{skip} registers, as done by \cs{regex_extract:}. -% If there is a match, we grab -% the parts before and after it, and get the result -% by \texttt{x}-expanding twice. -% \begin{macrocode} -\cs_new_protected:Npn \regex_replace_once_aux:NnnN #1#2#3#4 +% \begin{macro}[aux]{\regex_replace_once_aux:nnN} +% Build the \textsc{nfa} and the replacement functions, then find a +% single match. If the match failed, simply exit the +% group. Otherwise, we do the replacement. Extract submatches. Compute +% the brace balance corresponding to replacing this match by the +% replacement (this depends on submatches). Prepare the replaced token +% list: the replacement function produces the tokens from the start of +% the query to the start of the match and the replacement text for +% this match; we need to add the tokens from the end of the match to +% the end of the query. Finally, store the result in the user's +% variable after closing the group: this step involves an additional +% \texttt{x}-expansion, and checks that braces are balanced in the +% final result. +% \begin{macrocode} +\cs_new_protected:Npn \regex_replace_once_aux:nnN #1#2#3 { \group_begin: - \tl_clear:N \l_regex_every_match_tl - \regex_replacement:n {#3} - \exp_args:Nno #1 {#2} #4 - \if_meaning:w \c_true_bool \g_regex_success_bool + \regex_single_match: + #1 + \regex_replacement:n {#2} + \exp_args:No \regex_match:n { #3 } + \if_meaning:w \c_false_bool \g_regex_success_bool + \group_end: + \else: \regex_extract: - \int_set:Nn \l_regex_internal_a_int - { \regex_nesting:n { \l_regex_submatch_start_int } } - \if_num:w \l_regex_internal_a_int = \c_zero - \else: - \msg_kernel_error:nnx { regex } { replace-unbalanced } - { \l_regex_internal_a_int } - \fi: + \int_set:Nn \l_regex_balance_int + { + \regex_replacement_balance_one_match:n + { \l_regex_zeroth_submatch_int } + } \tl_set:Nx \l_regex_internal_a_tl { - \if_num:w \l_regex_internal_a_int < \c_zero - \prg_replicate:nn { - \l_regex_internal_a_int } - { \exp_not:n { { \if_false: } \fi: } } - \fi: - \regex_toks_range:nn - { \l_regex_min_index_int } - { \tex_skip:D \l_regex_submatch_start_int } - \regex_replacement_tl:n { \l_regex_submatch_start_int } - \regex_toks_range:nn - { \etex_gluestretch:D \tex_skip:D \l_regex_submatch_start_int } - { \l_regex_max_index_int } - \if_num:w \l_regex_internal_a_int > \c_zero - \prg_replicate:nn { \l_regex_internal_a_int } - { \exp_not:n { \if_false: { \fi: } } } - \fi: + \regex_replacement_do_one_match:n { \l_regex_zeroth_submatch_int } + \regex_query_range:nn + { \etex_gluestretch:D \tex_skip:D \l_regex_zeroth_submatch_int } + { \l_regex_max_pos_int } } - \tl_set:Nx \l_regex_internal_a_tl { \l_regex_internal_a_tl } - \exp_args:NNNo \group_end: - \tl_set:Nn #4 \l_regex_internal_a_tl - \else: - \group_end: + \regex_group_end_replace:N #3 \fi: } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_replace_all_aux:NnnN} -% For every match, extract submatches, and add the part before -% the beginning of the match, as well as the replacement, -% to the result. After the last match, extract the end -% of the token list, and add it to the replaced token list. +% \begin{macro}[aux]{\regex_replace_all_aux:nnN} +% Match multiple times, and for every match, extract submatches and +% additionally store the position at which the match attempt started +% (as the shrink component of a \tn{skip} register). The \tn{skip} +% registers from \cs{l_regex_max_state_int} to +% \cs{l_regex_submatch_int} hold information about submatches of every +% match in order; each match corresponds to +% \cs{l_regex_capturing_group_int} consecutive \tn{skip} registers. +% Compute the brace balance corresponding to doing all the +% replacements: this is the sum of brace balances for replacing each +% match. Join together the replacement texts for each match (including +% the part of the query before the match), and the end of the query. % \begin{macrocode} -\cs_new_protected:Npn \regex_replace_all_aux:NnnN #1#2#3#4 +\cs_new_protected:Npn \regex_replace_all_aux:nnN #1#2#3 { \group_begin: - \tl_set:Nn \l_regex_every_match_tl - { - \regex_extract: - \tex_skip:D \l_regex_submatch_start_int - = \tex_the:D \tex_skip:D \l_regex_submatch_start_int - minus \l_regex_start_index_int sp \scan_stop: - \regex_match_once: - } - \regex_replacement:n {#3} - \exp_args:Nno #1 {#2} #4 - \int_set:Nn \l_regex_internal_a_int + \regex_multi_match:n { \regex_extract: } + #1 + \regex_replacement:n {#2} + \exp_args:No \regex_match:n {#3} + \int_set:Nn \l_regex_balance_int { 0 \prg_stepwise_function:nnnN - \l_regex_max_state_int + { \c_two * \l_regex_max_state_int } \l_regex_capturing_group_int { \l_regex_submatch_int - \c_one } - \regex_nesting:n + \regex_replacement_balance_one_match:n } - \if_num:w \l_regex_internal_a_int = \c_zero - \else: - \msg_kernel_error:nnx { regex } { replace-unbalanced } - { \l_regex_internal_a_int } - \fi: \tl_set:Nx \l_regex_internal_a_tl { - \if_num:w \l_regex_internal_a_int < \c_zero - \prg_replicate:nn { - \l_regex_internal_a_int } - { \exp_not:n { { \if_false: } \fi: } } - \fi: \prg_stepwise_function:nnnN - \l_regex_max_state_int + { \c_two * \l_regex_max_state_int } \l_regex_capturing_group_int { \l_regex_submatch_int - \c_one } - \regex_replace_all_aux:n - \regex_toks_range:nn - \l_regex_start_index_int \l_regex_max_index_int - \if_num:w \l_regex_internal_a_int > \c_zero - \prg_replicate:nn { \l_regex_internal_a_int } - { \exp_not:n { \if_false: { \fi: } } } - \fi: + \regex_replacement_do_one_match:n + \regex_query_range:nn + \l_regex_start_pos_int \l_regex_max_pos_int } - \tl_set:Nx \l_regex_internal_a_tl { \l_regex_internal_a_tl } - \exp_args:NNNo \group_end: - \tl_set:Nn #4 \l_regex_internal_a_tl + \regex_group_end_replace:N #3 } -\cs_new:Npn \regex_replace_all_aux:n #1 +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_group_end_replace:N} +% If the brace balance is not $0$, raise an error. Then set the user's +% variable |#1| to the \texttt{x}-expansion of +% \cs{l_regex_internal_a_tl}, adding the appropriate braces to produce +% a balanced result. And end the group. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \regex_group_end_replace:N #1 { - \regex_toks_range:nn - { \etex_glueshrink:D \tex_skip:D #1 } { \tex_skip:D #1 } - \regex_replacement_tl:n {#1} + \if_num:w \l_regex_balance_int = \c_zero + \else: + \msg_kernel_error:nnxxx { regex } { result-unbalanced } + { replacing } + { \int_max:nn { - \l_regex_balance_int } { \c_zero } } + { \int_max:nn { \l_regex_balance_int } { \c_zero } } + \fi: + \use:x + { + \group_end: + \tl_set:Nn \exp_not:N #1 + { + \if_int_compare:w \l_regex_balance_int < \c_zero + \prg_replicate:nn { - \l_regex_balance_int } + { { \if_false: } \fi: } + \fi: + \l_regex_internal_a_tl + \if_int_compare:w \l_regex_balance_int > \c_zero + \prg_replicate:nn { \l_regex_balance_int } + { \if_false: { \fi: } } + \fi: + } + } } % \end{macrocode} % \end{macro} % +% \subsubsection{Storing and showing compiled patterns} +% % \subsection{Messages} % % Messages for the preparsing phase. @@ -5054,6 +5675,17 @@ } % \end{macrocode} % +% Invalid quantifier. +% \begin{macrocode} +\msg_kernel_new:nnnn { regex } { invalid-quantifier } + { Braced~quantifier~'#1'~may~not~be~followed~by~'#2'. } + { + The~character~'#2'~is~invalid~in~the~braced~quantifier~'#1'.~ + The~only~valid~quantifiers~are~'*',~'?',~'+',~'{<int>}',~ + '{<min>,}'~and~'{<min>,<max>}',~followed~or~not~by~'?'. + } +% \end{macrocode} +% % Messages for missing or extra closing brackets and parentheses, with % some fancy singular/plural handling for the case of parentheses. % \begin{macrocode} @@ -5081,21 +5713,31 @@ } % \end{macrocode} % -% Sometimes escaped alphanumerics are not allowed everywhere. +% Some escaped alphanumerics are not allowed everywhere. % \begin{macrocode} -\msg_kernel_new:nnnn { regex } { class-bad-escape } - { Invalid~escape~\c_backslash_str #1~in~character~class. } +\msg_kernel_new:nnnn { regex } { bad-escape } { - The~escape~sequence~\iow_char:N\\#1~may~not~appear~within~ - a~character~class.~For~instance,~assertions~(which~match~zero~ - characters)~would~not~make~sense~there. + Invalid~escape~\c_backslash_str #1~ + \regex_if_in_cs:TF { within~a~control~sequence. } + { + \regex_if_in_class:TF + { in~a~character~class. } + { following~a~category~test. } + } } -\msg_kernel_new:nnnn { regex } { catcode-bad-escape } - { Invalid~escape~\c_backslash_str #1~following~category~test. } { - The~escape~sequence~\iow_char:N\\#1~may~not~appear~following~ - a~category~test~such~as~\iow_char:N\\cL~or~ - \iow_char:N\\c[\iow_char:N\^BE]. + The~escape~sequence~\iow_char:N\\#1~may~not~appear~ + \regex_if_in_cs:TF + { + within~a~control~sequence~test~introduced~by~ + \iow_char:N\\c\iow_char:N\{. + } + { + \regex_if_in_class:TF + { within~a~character~class~ } + { following~a~category~test~such~as~\iow_char:N\\cL~ } + because~it~does~not~match~exactly~one~character. + } } % \end{macrocode} % @@ -5141,7 +5783,7 @@ regular~expression,~but~the~closing~']'~is~not~present. } \msg_kernel_new:nnnn { regex } { c-missing-category } - { Invalid~character~'#1'~following~\iow_char\\c~escape. } + { Invalid~character~'#1'~following~\iow_char:N\\c~escape. } { In~regular~expressions,~the~\iow_char:N\\c~escape~sequence~ may~only~be~followed~by~a~left~brace,~a~left~bracket,~or~a~ @@ -5165,91 +5807,155 @@ } % \end{macrocode} % -% In various cases, the result of a \pkg{l3regex} operation can leave us -% with an unbalanced token list, which we must re-balance by adding -% begin-group or end-group character tokens. +% Errors when encountering the \textsc{posix} syntax |[:...:]|. % \begin{macrocode} -\msg_kernel_new:nnnn { regex } { sequence-unbalanced } +\msg_kernel_new:nnnn { regex } { posix-unsupported } + { POSIX~collating~element~'[#1 ~ #1]'~not~supported. } { - Missing~ - \flag_if_raised:nTF { regex_end } - { - left~ - \flag_if_raised:nTF { regex_begin } - { and~right~braces } { brace } - } - { right~brace } - \ inserted~in~extracted~match. + The~[.foo.]~and~[=bar=]~syntaxes~have~a~special~meaning~in~POSIX~ + regular~expressions.~This~is~not~supported~by~LaTeX.~Maybe~you~ + forgot~to~escape~a~left~bracket~in~a~character~class? } +\msg_kernel_new:nnnn { regex } { posix-unknown } + { POSIX~class~[:#1:]~unknown. } { - LaTeX~was~asked~to~extract~submatches~or~split~a~token~list~ - according~to~a~given~regular~expression,~but~some~of~the~resulting~ - items~were~not~balanced. + [:#1:]~is~not~among~the~known~POSIX~classes~ + [:alnum:],~[:alpha:],~[:ascii:],~[:blank:],~ + [:cntrl:],~[:digit:],~[:graph:],~[:lower:],~ + [:print:],~[:punct:],~[:space:],~[:upper:],~ + [:word:],~and~[:xdigit:]. } -\msg_kernel_new:nnnn { regex } { replace-unbalanced } - { The~result~of~a~replacement~does~not~have~balanced~braces. } +\msg_kernel_new:nnnn { regex } { posix-missing-close } + { Missing~closing~':]'~for~POSIX~class. } + { The~POSIX~syntax~'#1'~must~be~followed~by~':]',~not~'#2'. } +% \end{macrocode} +% +% In various cases, the result of a \pkg{l3regex} operation can leave us +% with an unbalanced token list, which we must re-balance by adding +% begin-group or end-group character tokens. +% \begin{macrocode} +\msg_kernel_new:nnnn { regex } { result-unbalanced } + { Missing~brace~inserted~when~#1. } { - LaTeX~was~asked~to~do~some~regular~expression~replacement,~ + LaTeX~was~asked~to~do~some~regular~expression~operation,~ and~the~resulting~token~list~would~not~have~the~same~number~ - of~begin-group~and~end-group~tokens. \\ \\ - \ \ \ \ - \prg_case_int:nnn {#1} - { - { -1 } { A~left~brace~was } - { 1 } { A~right~brace~was } - } - { - \int_abs:n {#1} ~ - \int_compare:nNnTF {#1} < \c_zero { left } { right } ~ - braces ~ were - } - \ inserted. + of~begin-group~and~end-group~tokens.~Braces~were~inserted:~ + #2~left,~#3~right. } % \end{macrocode} % -% Messages related to NFA variables. +% Error message for unknown options. % \begin{macrocode} -\msg_kernel_new:nnnn { regex } { not-nfa } - { This~is~not~a~regular~expression~variable. } +\msg_kernel_new:nnnn { regex } { unknown-option } + { Unknown~option~`#1'~for~regular~expressions. } { - LaTeX~was~expecting~`#1'~to~be~a~regular~expression~variable.\\ - This~control~sequence~is~not~a~regex~variable.~It's~current~meaning~ - is~\\\\ - \ \ \ \ \token_to_str:N #1 = \token_to_meaning:N #1 . + The~only~available~option~is~'case-insensitive',~toggled~by~ + '(?i)'~and~'(?-i)'. } -\msg_kernel_new:nnn { regex } { nfa-misused } - { Automaton~#1 used~incorrectly. } % \end{macrocode} % +% Errors in the replacement text. % \begin{macrocode} -\msg_kernel_new:nnnn { regex } { c-command } - { Misused~\iow_char:N\\c~or\iow_char:N\\C~command~in~a~#1. } +\msg_kernel_new:nnnn { regex } { replacement-c } + { Misused~\iow_char:N\\c~command~in~a~replacement~text. } { - In~a~#1,~the~\iow_char:N\\C~escape~sequence~ + In~a~replacement~text,~the~\iow_char:N\\c~escape~sequence~ can~be~followed~by~one~of~the~letters~ABCDELMOPSTU~ - or~a~brace~group,~not~by~'#2'. - } -\msg_kernel_new:nnnn { regex } { unknown-option } - { Unknown~option~`#1'~for~regular~expressions. } - { - LaTeX~came~across~something~like~`(?#1)'~in~a~regular~expression,~ - but~the~option~`#1'~is~not~known.~It~will~be~ignored. + or~a~brace~group,~not~by~'#1'. } -\msg_kernel_new:nnnn { regex } { invalid-in-option } - { Invalid~character~in~option~of~a~regular~expression. } +\msg_kernel_new:nnnn { regex } { replacement-u } + { Misused~\iow_char:N\\u~command~in~a~replacement~text. } { - The~character~or~escape~sequence~`#1'~is~not~defined~ - as~an~option~within~regular~expressions. + In~a~replacement~text,~the~\iow_char:N\\u~escape~sequence~ + must~be~~followed~by~a~brace~group~holding~the~name~of~the~ + variable~to~use. } -\msg_kernel_new:nnnn { regex } { g-command } +\msg_kernel_new:nnnn { regex } { replacement-g } { Missing~brace~for~the~\iow_char:N\\g~construction~in~a~replacement~text. } { In~the~replacement~text~for~a~regular~expression~search,~ submatches~are~represented~either~as~\iow_char:N \\g{dd..d},~ or~\\d,~where~`d'~are~single~digits.~Here,~a~brace~is~missing. } +\msg_kernel_new:nnnn { regex } { replacement-catcode-end } + { + Missing~character~for~the~\iow_char:N\\c<category><character>~ + construction~in~a~replacement~text. + } + { + In~a~replacement~text,~the~\iow_char:N\\c~escape~sequence~ + can~be~followed~by~one~of~the~letters~ABCDELMOPSTU~representing~ + the~character~category.~Then,~a~character~must~follow.~LaTeX~ + reached~the~end~of~the~replacement~when~looking~for~that. + } +\msg_kernel_new:nnnn { regex } { replacement-null-space } + { TeX~cannot~build~a~space~token~with~character~code~0. } + { + You~asked~for~a~character~token~with~category~'space',~ + and~character~code~0,~for~instance~through~ + '\iow_char:N\\cS\iow_char:N\\x00'.~ + This~specific~case~is~impossible~and~will~be~replaced~ + by~a~normal~space. + } +\msg_kernel_new:nnnn { regex } { replacement-missing-rbrace } + { Missing~right~brace~inserted~in~replacement~text. } + { + There~were~\int_use:N \l_regex_replacement_csnames_int \ + missing~right~braces. + } % \end{macrocode} % +% \begin{macro}[aux]{\regex_msg_repeated:nnN} +% This is not technically a message, but seems related enough to go +% there. The arguments are: |#1| is the minimum number of repetitions; +% |#2| is the number of allowed extra repetitions ($-1$ for infinite +% number), and |#3| tells us aboud lazyness. +% \begin{macrocode} +\cs_new:Npn \regex_msg_repeated:nnN #1#2#3 + { + \str_if_eq:xxF { #1 #2 } { 1 0 } + { + , ~ repeated ~ + \prg_case_int:nnn {#2} + { + { -1 } { #1~or~more~times,~\bool_if:NTF #3 { lazy } { greedy } } + { 0 } { #1~times } + } + { + between~#1~and~\int_eval:n {#1+#2}~times,~ + \bool_if:NTF #3 { lazy } { greedy } + } + } + } +% \end{macrocode} +% \end{macro} +% +% \subsection{Code for tracing} +% +% The tracing code is still very experimental, and is meant to be used +% with the \pkg{l3trace} package, currently in \texttt{l3trial}. +% +% \begin{macro}[int]{\regex_trace_states:n} +% This function lists the contents of all states of the \textsc{nfa}, +% stored in \tn{toks} from $0$ to \cs{l_regex_max_state_int} +% (excluded). +% \begin{macrocode} +%<*trace> +\cs_new_protected:Npn \regex_trace_states:n #1 + { + \prg_stepwise_inline:nnnn + \l_regex_min_state_int + \c_one + { \l_regex_max_state_int - 1 } + { + \trace:nnx { regex } { #1 } + { \iow_char:N \\toks ##1 = { \tex_the:D \tex_toks:D ##1 } } + } + } +%</trace> +% \end{macrocode} +% \end{macro} +% % \begin{macrocode} %</package> % \end{macrocode} @@ -5257,84 +5963,27 @@ % \end{implementation} % % \endinput -%^^A -%^^A \msg_new:nnn { regex } { 4 } -%^^A { Numbers~out~of~order~in~\iow_char:N\{\iow_char\}~quantifier. } -%^^A \msg_new:nnn { regex } { 44 } { Invalid~UTF-8~string } -%^^A \msg_new:nnn { regex } { 46 } -%^^A { Malformed~\iow_char:N\\P~or\iow_char:N\\p~sequence } -%^^A \msg_new:nnn { regex } { 47 } -%^^A { Unknown~property~after~\iow_char:N\\P~or\iow_char:N\\p } -%^^A %^^A NOT IMPLEMENTED -%^^A \cx "control-x", where x is any ASCII character -%^^A \C one byte, even in UTF-8 mode (best avoided) %^^A \p{xx} a character with the xx property %^^A \P{xx} a character without the xx property +%^^A [[:xxx:]] positive POSIX named set +%^^A [[:^xxx:]] negative POSIX named set +%^^A (?=...) positive look ahead +%^^A (?!...) negative look ahead +%^^A (?<=...) positive look behind +%^^A (?<!...) negative look behind +%^^A (?<name>...) or (?'name'...) or (?P<name>...) +%^^A named capturing group %^^A \R a newline sequence %^^A \X an extended Unicode sequence -%^^A [[:xxx:]] positive POSIX named set -%^^A [[:^xxx:]] negative POSIX named set -%^^A ?+ 0 or 1, possessive -%^^A *+ 0 or more, possessive -%^^A ++ 1 or more, possessive -%^^A {n,m}+ at least n, no more than m, possessive -%^^A {n,}+ n or more, possessive -%^^A \K reset start of match -%^^A (?<name>...) named capturing group (Perl) -%^^A (?'name'...) named capturing group (Perl) -%^^A (?P<name>...) named capturing group (Python) -%^^A (?:...) non-capturing group -%^^A (?|...) non-capturing group; reset group numbers for -%^^A capturing groups in each alternative -%^^A (?>...) atomic, non-capturing group -%^^A (?#....) comment (not nestable) -%^^A (?i) caseless -%^^A (?J) allow duplicate names -%^^A (?m) multiline -%^^A (?s) single line (dotall) -%^^A (?U) default ungreedy (lazy) -%^^A (?x) extended (ignore white space) -%^^A (?-...) unset option(s) -%^^A (*NO_START_OPT) no start-match optimization (PCRE_NO_START_OPTIMIZE) -%^^A (*UTF8) set UTF-8 mode (PCRE_UTF8) -%^^A (*UCP) set PCRE_UCP (use Unicode properties for \d etc) -%^^A (?=...) positive look ahead -%^^A (?!...) negative look ahead -%^^A (?<=...) positive look behind -%^^A (?<!...) negative look behind -%^^A \n reference by number (can be ambiguous) -%^^A \gn reference by number -%^^A \g{n} reference by number -%^^A \g{-n} relative reference by number -%^^A \k<name> reference by name (Perl) -%^^A \k'name' reference by name (Perl) -%^^A \g{name} reference by name (Perl) -%^^A \k{name} reference by name (.NET) -%^^A (?P=name) reference by name (Python) +%^^A (?C) or (?Cn) callout with data n %^^A (?R) recurse whole pattern -%^^A (?n) call subpattern by absolute number -%^^A (?+n) call subpattern by relative number -%^^A (?-n) call subpattern by relative number -%^^A (?&name) call subpattern by name (Perl) -%^^A (?P>name) call subpattern by name (Python) -%^^A \g<name> call subpattern by name (Oniguruma) -%^^A \g'name' call subpattern by name (Oniguruma) -%^^A \g<n> call subpattern by absolute number (Oniguruma) -%^^A \g'n' call subpattern by absolute number (Oniguruma) -%^^A \g<+n> call subpattern by relative number (PCRE extension) -%^^A \g'+n' call subpattern by relative number (PCRE extension) -%^^A \g<-n> call subpattern by relative number (PCRE extension) -%^^A \g'-n' call subpattern by relative number (PCRE extension) -%^^A (?(n)... absolute reference condition -%^^A (?(+n)... relative reference condition -%^^A (?(-n)... relative reference condition -%^^A (?(<name>)... named reference condition (Perl) -%^^A (?('name')... named reference condition (Perl) -%^^A (?(name)... named reference condition (PCRE) -%^^A (?(R)... overall recursion condition -%^^A (?(Rn)... specific group recursion condition -%^^A (?(R&name)... specific recursion condition +%^^A (?[+-]n) or \g<[+-]n> or (?&name) or (?P>name) or \g<name> +%^^A call subpattern +%^^A (?([+-]n)... or (?(<name>)... +%^^A reference condition +%^^A (?(R)... or (?(Rn)... or (?(R&name)... +%^^A recursion condition %^^A (?(DEFINE)... define subpattern for reference %^^A (?(assert)... assertion condition %^^A (*ACCEPT) force successful match @@ -5343,12 +5992,21 @@ %^^A (*PRUNE) advance to next starting character %^^A (*SKIP) advance start to current matching position %^^A (*THEN) local failure, backtrack to next alternation -%^^A (*CR) carriage return only -%^^A (*LF) linefeed only -%^^A (*CRLF) carriage return followed by linefeed -%^^A (*ANYCRLF) all three of the above -%^^A (*ANY) any Unicode newline sequence -%^^A (*BSR_ANYCRLF) CR, LF, or CRLF -%^^A (*BSR_UNICODE) any Unicode newline sequence -%^^A (?C) callout -%^^A (?Cn) callout with data n
\ No newline at end of file +%^^A (*CR) or (*LF) or (*CRLF) or (*ANYCRLF) or (*ANY) +%^^A newline convention +%^^A (*BSR_ANYCRLF) or (*BSR_UNICODE) +%^^A change what \R matches. +%^^A +%^^A \cx "control-x", where x is any ASCII character +%^^A \C one byte, even in UTF-8 mode (best avoided) +%^^A + possessive quantifiers +%^^A (?>...) atomic, non-capturing group +%^^A (?#....) comment (not nestable) +%^^A (?JmsUx) options (duplicate names; multiline; single line; +%^^A ungreedy; extended) +%^^A (*NO_START_OPT) no start-match optimization (PCRE_NO_START_OPTIMIZE) +%^^A (*UTF8) set UTF-8 mode (PCRE_UTF8) +%^^A (*UCP) set PCRE_UCP (use Unicode properties for \d etc) +%^^A \n or \gn or \g{[-]n} or \g{name} or (?P=name) +%^^A or \k<name> or \k'name' or \k{name} +%^^A back-references diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx index 3d73ed10b8f..ef2bc9b5460 100644 --- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx @@ -35,7 +35,7 @@ % %<*driver|package> \RequirePackage{expl3} -\GetIdInfo$Id: l3str.dtx 3257 2012-01-29 13:37:03Z bruno $ +\GetIdInfo$Id: l3str.dtx 3460 2012-02-26 12:16:41Z will $ {L3 Experimental Strings} %</driver|package> %<*driver> @@ -189,7 +189,7 @@ % var} are not automatically converted to a string. % \end{function} % -% \section{Characters given by their position} +% \section{Accessing the contents of a string} % % \begin{function}[EXP] % {\str_length:N, \str_length:n, \str_length_ignore_spaces:n} @@ -291,8 +291,8 @@ % \str_if_eq:nV, \str_if_eq:VV, \str_if_eq:xx % } % \begin{syntax} -% \cs{str_if_eq_p:nn} \Arg{tl1} \Arg{tl2} -% \cs{str_if_eq:nnTF} \Arg{tl1} \Arg{tl2} \Arg{true code} \Arg{false code} +% \cs{str_if_eq_p:nn} \Arg{tl_1} \Arg{tl_2} +% \cs{str_if_eq:nnTF} \Arg{tl_1} \Arg{tl_2} \Arg{true code} \Arg{false code} % \end{syntax} % Compares the two \meta{token lists} on a character by character % basis, and is \texttt{true} if the two lists contain the same @@ -305,6 +305,15 @@ % expansion). % \end{function} % +% \section{Viewing strings} +% +% \begin{function}{\str_show:N, \str_show:c, \str_show:n} +% \begin{syntax} +% \cs{str_show:N} \meta{tl~var} +% \end{syntax} +% Displays the content of the \meta{str~var} on the terminal. +% \end{function} +% % \section{Encoding functions} % % Traditionally, string encodings only specify how strings of characters @@ -571,7 +580,7 @@ % \begin{macrocode} \ProvidesExplPackage {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription} -\RequirePackage{l3tl-analysis,l3flag} +\RequirePackage{l3tl-analysis,l3tl-build,l3flag} % \end{macrocode} % % The following string-related functions are currently defined in @@ -601,6 +610,14 @@ % % \subsubsection{Assigning strings} % +% \begin{macro}{\str_new:N, \str_new:c} +% A string is simply a token list. +% \begin{macrocode} +\cs_new_eq:NN \str_new:N \tl_new:N +\cs_generate_variant:Nn \str_new:N { c } +% \end{macrocode} +% \end{macro} +% % \begin{macro} % { % \str_set:Nn, \str_set:Nx, @@ -1337,31 +1354,38 @@ % \end{macrocode} % \end{macro} % +% \subsection{Viewing strings} +% +% \begin{macro}{\str_show:n, \str_show:N, \str_show:c} +% Displays a string on the terminal. +% \begin{macrocode} +\cs_new_eq:NN \str_show:n \tl_show:n +\cs_new_eq:NN \str_show:N \tl_show:N +\cs_generate_variant:Nn \str_show:N { c } +% \end{macrocode} +% \end{macro} +% % \subsection{Conversions} % % \subsubsection{Producing one byte or character} % % \begin{variable}{\c_str_byte_0_tl, \c_str_byte_1_tl, \c_str_byte_255_tl} % \begin{variable}{\c_str_byte_-1_tl} -% \begin{variable}{\c_str_positive_bytes_tl} % For each integer $N$ in the range $[0,255]$, we create a constant % token list which holds three character tokens with category code % other: the character with character code $N$, followed by the % representation of $N$ as two hexadecimal digits. The value $-1$ is % given a default token list which ensures that later functions give -% an empty result for the input $-1$. Simultaneously, we build a list -% of all bytes (from which we remove the null byte) in -% \cs{c_str_positive_bytes_tl}. +% an empty result for the input $-1$. % \begin{macrocode} \group_begin: - \tl_clear:N \l_str_internal_tl \char_set_catcode_other:n { \c_zero } - \tl_gset:Nx \g_str_result_tl { \tl_to_str:n { 0123456789ABCDEF } } - \exp_args:No \tl_map_inline:nn { \g_str_result_tl " } + \tl_set:Nx \l_str_internal_tl { \tl_to_str:n { 0123456789ABCDEF } } + \exp_args:No \tl_map_inline:nn { \l_str_internal_tl " } { \char_set_lccode:nn {`#1} { \c_zero } } - \tl_map_inline:Nn \g_str_result_tl + \tl_map_inline:Nn \l_str_internal_tl { - \tl_map_inline:Nn \g_str_result_tl + \tl_map_inline:Nn \l_str_internal_tl { \char_set_lccode:nn { \c_zero } {"#1##1} \tl_to_lowercase:n @@ -1369,18 +1393,14 @@ \tl_const:cx { c_str_byte_ \int_eval:n {"#1##1} _tl } { ^^@ #1 ##1 } - \tl_put_right:Nn \l_str_internal_tl { ^^@ } } } } - \tl_const:Nx \c_str_positive_bytes_tl - { \exp_after:wN \use_none:n \l_str_internal_tl } \group_end: \tl_const:cn { c_str_byte_-1_tl } { { } \use_none:n { } } % \end{macrocode} % \end{variable} % \end{variable} -% \end{variable} % % \begin{macro}[int, EXP]{\str_output_byte:n} % \begin{macro}[int, EXP]{\str_output_byte:w} @@ -1943,68 +1963,34 @@ % \begin{macro}[aux, rEXP]{\str_encode_native_filter:N} % In Unicode-aware engines, since building particular characters % cannot be done expandably in \TeX{}, we cannot hope to get a -% linear-time function. However, we get quite close by building the -% result one block of $255$ characters at a time. Set the lowercase -% code of the \textsc{ascii} character $1$ to the first character code -% in the block, \emph{etc.}, then lowercase the contents of -% \cs{c_str_positive_bytes_tl}. Unfortunately, this is complicated by -% the fact that \TeX{} won't lowercase a character to the null -% character, thus the need for filtering: whenever the lowercase code -% of a given character is zero, it is replaced \enquote{by hand} by -% the null character. Unicode-aware engines will never incur an -% overflow because the internal string is guaranteed to only contain -% code points in $[0,\hexnum{10FFFF}]$. +% linear-time function. However, we get quite close using the +% \pkg{l3tl-build} module, which abuses \tn{toks} to reach an almost +% linear time. Use the standard lowercase trick to produce an +% arbitrary character from the null character, and add that character +% to the end of the token list being built. At the end of the loop, +% put the token list together with \cs{tl_build_end:}. Note that we +% use an \texttt{x}-expanding assignment because it is slightly +% faster. Unicode-aware engines will never incur an overflow because +% the internal string is guaranteed to only contain code points in +% $[0,\hexnum{10FFFF}]$. % \begin{macrocode} { \cs_new_protected_nopar:Npn \str_convert_encode_: { - \tex_lccode:D \c_zero \c_zero - \int_zero:N \l_str_internal_int - \tl_set_eq:NN \l_str_internal_tl \g_str_result_tl - \tl_gclear:N \g_str_result_tl - \exp_after:wN \str_encode_native_loop:w - \l_str_internal_tl \s_tl { \q_stop \prg_map_break: } \s_tl - \prg_break_point:n { \str_encode_native_flush: } + \int_zero:N \l_tl_build_offset_int + \tl_gset_build_x:Nw \g_str_result_tl + \exp_after:wN \str_encode_native_loop:w + \g_str_result_tl \s_tl { \q_stop \prg_map_break: } \s_tl + \prg_break_point:n { } + \tl_build_end: } \cs_new_protected:Npn \str_encode_native_loop:w #1 \s_tl #2 \s_tl { \use_none_delimit_by_q_stop:w #2 \q_stop - \if_num:w \l_str_internal_int = \c_two_hundred_fifty_five - \str_encode_native_flush: - \l_str_internal_int \c_zero - \fi: - \tex_advance:D \l_str_internal_int \c_one \tex_lccode:D \l_str_internal_int \int_eval:w #2 \int_eval_end: + \tl_to_lowercase:n { \tl_build_one:n { ^^@ } } \str_encode_native_loop:w } - \cs_new_protected_nopar:Npn \str_encode_native_flush: - { - \use:x - { - \tl_to_lowercase:n - { - \tl_gput_right:Nx \exp_not:N \g_str_result_tl - { - \exp_after:wN \str_encode_native_filter:N - \c_str_positive_bytes_tl - { ? = \c_zero \fi: \prg_map_break: } - \prg_break_point:n { } - } - } - } - } - \cs_new:Npn \str_encode_native_filter:N #1 - { - \if_num:w `#1 > \l_str_internal_int - \exp_after:wN \prg_map_break: - \fi: - \if_num:w \tex_lccode:D `#1 = \c_zero - ^^@ - \else: - #1 - \fi: - \str_encode_native_filter:N - } } % \end{macrocode} % \end{macro} @@ -3044,7 +3030,8 @@ % character; % \item $[\hexnum{E000}, \hexnum{FFFF}]$: converted to two bytes; % \item $[\hexnum{10000}, \hexnum{10FFFF}]$: converted to a pair of -% surrogates, each two bytes. +% surrogates, each two bytes. The magic \hexnum{D7C0} is +% $\hexnum{D800}-\hexnum{10000}/\hexnum{400}$. % \end{itemize} % For the duration of this operation, \cs{str_tmp:w} is defined as a % function to convert a number in the range $[0, \hexnum{FFFF}]$ to a @@ -3083,7 +3070,7 @@ \str_tmp:w {#1} \fi: \else: - \exp_args:Nf \str_tmp:w { \int_div_truncate:nn {#1} {"400} + "D800 } + \exp_args:Nf \str_tmp:w { \int_div_truncate:nn {#1} {"400} + "D7C0 } \exp_args:Nf \str_tmp:w { \int_mod:nn {#1} {"400} + "DC00 } \fi: \fi: @@ -3257,6 +3244,10 @@ % looping with \cs{str_decode_utf_xvi_pair:NN}. Otherwise, of course, % complain about the missing surrogate. % +% The magic number \hexnum{D7F7} is such that +% $\hexnum{D7F7}*\hexnum{400} = \hexnum{D800}*\hexnum{400} + +% \hexnum{DC00} - \hexnum{10000}$. +% % Every time we read a pair of bytes, we test for the end-marker % \cs{q_nil}. When reaching the end, we additionally check that the % string had an even length. Also, if the end is reached when @@ -3292,8 +3283,8 @@ #1 #2 #4 #5 \s_tl \int_eval:n { - ( "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 - "D800 ) * "400 - + "100 * \str_tmp:w #4#5 + \str_tmp:w #5#4 - "DC00 + ( "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 - "D7F7 ) * "400 + + "100 * \str_tmp:w #4#5 + \str_tmp:w #5#4 } \s_tl \exp_after:wN \use_i:nnn diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins index 676a39bea7b..7dc0c00357d 100644 --- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins @@ -39,7 +39,8 @@ Do not distribute a modified version of this file. \generate{\file{l3str.sty} {\from{l3str.dtx} {package}}} \generate{\file{l3regex.sty} {\from{l3regex.dtx} {package}}} \generate{\file{l3tl-analysis.sty} {\from{l3tl-analysis.dtx} {package}}} -%\generate{\file{l3regex-trace.sty} {\from{l3regex.dtx} {package,trace}}} +\generate{\file{l3tl-build.sty} {\from{l3tl-build.dtx} {package}}} +\generate{\file{l3regex-trace.sty} {\from{l3regex.dtx} {package,trace}}} % Escapings. \generate{% diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx index f9294dbb87c..89ce9c22f3d 100644 --- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx @@ -82,8 +82,8 @@ % \end{syntax} % Displays to the terminal the detailed decomposition of the % \meta{token list} into tokens, showing the category code of each -% character token, and the meaning of control sequences and active -% characters. +% character token, the meaning of control sequences and active +% characters, and the value of registers. % \end{function} % % \subsection{Internal functions} @@ -203,6 +203,14 @@ % \end{macrocode} % \end{variable} % +% \begin{variable}{\l_tl_analysis_internal_tl} +% This token list variable is used to hand the argument of +% \cs{tl_show_analysis:n} to \cs{tl_show_analysis:N}. +% \begin{macrocode} +\tl_new:N \l_tl_analysis_internal_tl +% \end{macrocode} +% \end{variable} +% % \begin{variable}{\l_tl_analysis_token} % \begin{variable}{\l_tl_analysis_char_token} % The tokens in the token list are probed with the \TeX{} primitive @@ -953,8 +961,8 @@ } \cs_new_protected:Npn \tl_show_analysis:n #1 { - \tl_set:Nn \l_tl_tmpa_tl {#1} - \tl_show_analysis:N \l_tl_tmpa_tl + \tl_set:Nn \l_tl_analysis_internal_tl {#1} + \tl_show_analysis:N \l_tl_analysis_internal_tl } % \end{macrocode} % \end{macro} @@ -1004,10 +1012,31 @@ % \end{macrocode} % \end{macro} % +% \begin{macro}[EXP]{\tl_show_analysis_value:N} +% This expands to the value of |#1| if it has any. +% \begin{macrocode} +\cs_new:Npn \tl_show_analysis_value:N #1 + { + \token_if_expandable:NF #1 + { + \token_if_chardef:NTF #1 \prg_map_break: { } + \token_if_mathchardef:NTF #1 \prg_map_break: { } + \token_if_dim_register:NTF #1 \prg_map_break: { } + \token_if_int_register:NTF #1 \prg_map_break: { } + \token_if_skip_register:NTF #1 \prg_map_break: { } + \token_if_toks_register:NTF #1 \prg_map_break: { } + \use_none:nnnn + \prg_break_point:n { } + \use:n { = \tex_the:D #1 } + } + } +% \end{macrocode} +% \end{macro} +% % \begin{macro}[aux, rEXP]{\tl_show_analysis_cs:n} % \begin{macro}[aux, rEXP]{\tl_show_analysis_active:n} % \begin{macro}[aux, rEXP]{\tl_show_analysis_long:nn} -% \begin{macro}[aux, rEXP]{\tl_show_analysis_long_aux:nnn} +% \begin{macro}[aux, rEXP]{\tl_show_analysis_long_aux:nnnn} % Control sequences and active characters are printed in the same way, % making sure not to go beyond the \cs{l_iow_line_length_int}. In case % of an overflow, we replace the last characters by @@ -1019,25 +1048,27 @@ { \exp_args:No \tl_show_analysis_long:nn {#1} { active~character= } } \cs_new:Npn \tl_show_analysis_long:nn #1 { - \exp_args:Noo \tl_show_analysis_long_aux:nnn + \tl_show_analysis_long_aux:oofn { \token_to_str:N #1 } { \token_to_meaning:N #1 } + { \tl_show_analysis_value:N #1 } } -\cs_new:Npn \tl_show_analysis_long_aux:nnn #1#2#3 +\cs_new:Npn \tl_show_analysis_long_aux:nnnn #1#2#3#4 { \int_compare:nNnTF - { \str_length:n { #1 ~ ( #3 #2 ) } } + { \str_length:n { #1 ~ ( #4 #2 #3 ) } } > { \l_iow_line_length_int - \c_three } { - \str_substr:nnn { #1 ~ ( #3 #2 ) } \c_zero + \str_substr:nnn { #1 ~ ( #4 #2 #3 ) } \c_zero { \l_iow_line_length_int - \c_three - \str_length:N \c_tl_show_analysis_etc_str } \c_tl_show_analysis_etc_str } - { #1 ~ ( #3 #2 ) } + { #1 ~ ( #4 #2 #3 ) } } +\cs_generate_variant:Nn \tl_show_analysis_long_aux:nnnn { oof } % \end{macrocode} % \end{macro} % \end{macro} @@ -1060,7 +1091,8 @@ \msg_kernel_new:nnn { tl-analysis } { show } { The~token~list~ - \str_if_eq:nnF {#1} { \l_tl_tmpa_tl } { \token_to_str:N #1 ~ } + \str_if_eq:nnF {#1} { \l_tl_analysis_internal_tl } + { \token_to_str:N #1 ~ } \tl_if_empty:NTF #1 { is~empty } { contains~the~tokens: } diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-build.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-build.dtx new file mode 100644 index 00000000000..d3fa2d4e366 --- /dev/null +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-build.dtx @@ -0,0 +1,301 @@ +% \iffalse meta-comment +% +%% File: l3tl-build.dtx Copyright (C) 2011-2012 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +% +%<*driver|package> +\RequirePackage{expl3} +\GetIdInfo$Id: l3tl-build.dtx 3039 2011-12-08 09:22:35Z bruno $ + {L3 Experimental Build of Token lists} +%</driver|package> +%<*driver> +\documentclass[full]{l3doc} +\usepackage{amsmath} +\begin{document} + \DocInput{\jobname.dtx} +\end{document} +%</driver> +% \fi +% +% +% \title{^^A +% The \textsf{l3tl-build} package: building token lists^^A +% \thanks{This file describes v\ExplFileVersion, +% last revised \ExplFileDate.}^^A +% } +% +% \author{^^A +% The \LaTeX3 Project\thanks +% {^^A +% E-mail: +% \href{mailto:latex-team@latex-project.org} +% {latex-team@latex-project.org}^^A +% }^^A +% } +% +% \date{Released \ExplFileDate} +% +% \maketitle +% +% \begin{documentation} +% +% \section{\pkg{l3tl-build} documentation} +% +% This module provides no user function. +% +% There are two main ways of building token lists from individual +% tokens. Either in one go within an \texttt{x}-expanding assignment, or +% by repeatedly using \cs{tl_put_right:Nn}. The first method takes a +% linear time, but only allows expandable operations. The second method +% takes a time quadratic in the length of the token list, but allows +% expandable and non-expandable operations. +% +% The goal of this module is to provide functions to build a token list +% piece by piece in linear time, while allowing non-expandable +% operations. This is achieved by abusing \tn{toks}: adding some tokens +% to the token list is done by storing them in a free token register +% (time $O(1)$ for each such operation). Those token registers are only +% put together at the end, within an \texttt{x}-expanding assignment, +% which takes a linear time.\footnote{If we run out of token registers, +% then the currently filled-up \tn{toks} are put together in a +% temporary token list, and cleared, and we ultimately use +% \cs{tl_put_right:Nx} to put those chunks together. Hence the true +% asymptotic is quadratic, with a very small constant.} Of course, +% all this must be done in a group: we can't go and clobber the values +% of legitimate \tn{toks} used by \LaTeXe{}. +% +% Since none of the current applications need the ability to insert +% material on the left of the token list, I have not implemented +% that. This could be done for instance by using odd-numbered \tn{toks} +% for the left part, and even-numbered \tn{toks} for the right part. +% +% \subsection{Internal functions} +% +% \begin{function} +% { +% \tl_set_build:Nw, \tl_gset_build:Nw, +% \tl_set_build_x:Nw, \tl_gset_build_x:Nw +% } +% \begin{syntax} +% \cs{tl_set_build:Nw} \meta{tl~var} \texttt{\ldots{}} +% \cs{tl_build_one:n} \Arg{tokens_1} \texttt{\ldots{}} +% \cs{tl_build_one:n} \Arg{tokens_2} \texttt{\ldots{}} +% \ldots{} +% \cs{tl_build_end:} +% \end{syntax} +% Defines the \meta{tl~var} to contain the contens of \meta{tokens1} +% followed by \meta{tokens2}, \emph{etc.} This is built in such a way +% to be more efficient than repeatedly using \cs{tl_put_right:Nn}. The +% code in \enquote{\texttt{\ldots{}}} does not need to be +% expandable. The commands \cs{tl_set_build:Nw} and \cs{tl_build_end:} +% start and end a group. The assignment to the \meta{tl~var} occurs +% just after the end of that group, using \cs{tl_set:Nn}, +% \cs{tl_gset:Nn}, \cs{tl_set:Nx}, or \cs{tl_gset:Nx}. +% \end{function} +% +% \begin{function}{\tl_build_one:n, \tl_build_one:o, \tl_build_one:x} +% \begin{syntax} +% \cs{tl_build_one:n} \Arg{tokens} +% \end{syntax} +% This function may only be used within the scope of a +% \cs{tl_set_build:Nw} function. It adds the \meta{tokens} on the +% right of the current token list. +% \end{function} +% +% \begin{function}{\tl_build_end:} +% Ends the scope started by \cs{tl_set_build:Nw}, and performs the +% relevant assignment. +% \end{function} +% +% \end{documentation} +% +% \begin{implementation} +% +% \section{\pkg{l3tl-build} implementation} +% +% \begin{macrocode} +%<*initex|package> +% \end{macrocode} +% +% \begin{macrocode} +\ProvidesExplPackage + {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription} +% \end{macrocode} +% +% \subsection{Variables and helper functions} +% +% \begin{variable}{\l_tl_build_start_index_int, \l_tl_build_index_int} +% Integers pointing to the starting index (currently always starts at +% zero), and the current index. The corresponding \tn{toks} are +% accessed directly by number. +% \begin{macrocode} +\int_new:N \l_tl_build_start_index_int +\int_new:N \l_tl_build_index_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_tl_build_result_tl} +% The resulting token list is normally built in one go by unpacking +% all \tn{toks} in some range. In the rare cases where there are too +% many \cs{tl_build_one:n} commands, leading to the depletion of +% registers, the contents of the current set of \tn{toks} is unpacked +% into \cs{l_tl_build_result_tl}. This prevents overflow from +% affecting the end-user (beyond an obvious performance hit). +% \begin{macrocode} +\tl_new:N \l_tl_build_result_tl +% \end{macrocode} +% \end{variable} +% +% \begin{macro}{\tl_build_unpack:} +% \begin{macro}[aux, EXP]{\tl_build_unpack_loop:w} +% The various pieces of the token list are built in \tn{toks} from the +% \texttt{start_index} (inclusive) to the (current) \texttt{index} +% (excluded). Those \tn{toks} are unpacked and stored in order in the +% \texttt{result} token list. Optimizations would be possible here, +% for instance, unpacking $10$ \tn{toks} at a time with a macro +% expanding to |\the\toks#10...\the\toks#19|, but this should be kept +% for much later. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \tl_build_unpack: + { + \tl_put_right:Nx \l_tl_build_result_tl + { + \exp_after:wN \tl_build_unpack_loop:w + \int_use:N \l_tl_build_start_index_int ; + \prg_break_point:n { } + } + } +\cs_new:Npn \tl_build_unpack_loop:w #1 ; + { + \if_num:w #1 = \l_tl_build_index_int + \exp_after:wN \prg_map_break: + \fi: + \tex_the:D \tex_toks:D #1 \exp_stop_f: + \exp_after:wN \tl_build_unpack_loop:w + \int_use:N \int_eval:w #1 + \c_one ; + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \subsection{Building the token list} +% +% \begin{macro} +% { +% \tl_set_build:Nw , \tl_set_build_x:Nw , +% \tl_gset_build:Nw , \tl_gset_build_x:Nw +% } +% \begin{macro}[aux]{\tl_set_build_aux:NNw} +% Similar to what is done for coffins: redefine some command, here +% \cs{tl_build_end_aux:n} to hold the relevant assignment (see +% \cs{tl_build_end:} for details). Then initialize the start index and +% the current index at zero, and empty the \texttt{result} token list. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \tl_set_build:Nw + { \tl_set_build_aux:NNw \tl_set:Nn } +\cs_new_protected_nopar:Npn \tl_set_build_x:Nw + { \tl_set_build_aux:NNw \tl_set:Nx } +\cs_new_protected_nopar:Npn \tl_gset_build:Nw + { \tl_set_build_aux:NNw \tl_gset:Nn } +\cs_new_protected_nopar:Npn \tl_gset_build_x:Nw + { \tl_set_build_aux:NNw \tl_gset:Nx } +\cs_new_protected:Npn \tl_set_build_aux:NNw #1#2 + { + \group_begin: + \cs_set_nopar:Npn \tl_build_end_assignment:n + { \group_end: #1 #2 } + \int_zero:N \l_tl_build_start_index_int + \int_zero:N \l_tl_build_index_int + \tl_clear:N \l_tl_build_result_tl + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}{\tl_build_end:} +% \begin{macro}[aux]{\tl_build_end_assignment:n} +% When we are done building a token list, unpack all \tn{toks} into +% the \texttt{result} token list, and expand this list before closing +% the group. The \cs{tl_build_end_assignment:n} function is defined by +% \cs{tl_set_build_aux:NNw} to end the group and hold the relevant +% assignment. Its value outside is irrelevant, but just in case, we +% set it to a function which would clean up the contents of +% \cs{l_tl_build_result_tl}. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \tl_build_end: + { + \tl_build_unpack: + \exp_args:No + \tl_build_end_assignment:n \l_tl_build_result_tl + } +\cs_new_eq:NN \tl_build_end_assignment:n \use_none:n +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}{\tl_build_one:n, \tl_build_one:o, \tl_build_one:x} +% Store the tokens in a free \tn{toks}, then move the pointer to the +% next one. If we overflow, unpack the current \tn{toks}, and reset +% the current index, preparing to fill more \tn{toks}. This could be +% optimized by avoiding to read |#1|, using \tn{afterassignment}. +% \begin{macrocode} +\cs_new_protected:Npn \tl_build_one:n #1 + { + \tex_toks:D \l_tl_build_index_int {#1} + \tex_advance:D \l_tl_build_index_int \c_one + \if_num:w \l_tl_build_index_int > \c_max_register_int + \tl_build_unpack: + \l_tl_build_index_int \l_tl_build_start_index_int + \fi: + } +\cs_new_protected:Npn \tl_build_one:o #1 + { + \tex_toks:D \l_tl_build_index_int \exp_after:wN {#1} + \tex_advance:D \l_tl_build_index_int \c_one + \if_num:w \l_tl_build_index_int > \c_max_register_int + \tl_build_unpack: + \l_tl_build_index_int \l_tl_build_start_index_int + \fi: + } +\cs_new_protected:Npn \tl_build_one:x #1 + { \use:x { \tl_build_one:n {#1} } } +% \end{macrocode} +% \end{macro} +% +% \begin{macrocode} +%</initex|package> +% \end{macrocode} +% +% \end{implementation} +% +% \PrintIndex diff --git a/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins.dtx b/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins.dtx index 30a58c18817..be9e4e24f75 100644 --- a/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins.dtx +++ b/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins.dtx @@ -1,5 +1,5 @@ % \iffalse meta-comment -% +% %% File: xcoffins.dtx Copyright(C) 2010,2011 The LaTeX3 Project %% %% It may be distributed and/or modified under the conditions of the @@ -36,7 +36,7 @@ % %<*driver|package> \RequirePackage{xparse} -\GetIdInfo$Id: xcoffins.dtx 2963 2011-11-15 22:06:41Z joseph $ +\GetIdInfo$Id: xcoffins.dtx 3385 2012-02-12 20:37:09Z bruno $ {L3 Experimental design-level coffins} %</driver|package> %<*driver> @@ -56,13 +56,13 @@ \end{document} %</driver> % \fi -% +% % \title{^^A % The \textsf{xcoffins} package\\ Design-level coffins^^A % \thanks{This file describes v\ExplFileVersion, % last revised \ExplFileDate.}^^A % } -% +% % \author{^^A % The \LaTeX3 Project\thanks % {^^A @@ -89,7 +89,7 @@ % positions, for example the top or horizontal centre. The points % where these poles intersect are called \enquote{handles}. Two % coffins can then be aligned by describing the relationship between -% a handle on one coffin with a handle on the second. In words, an +% a handle on one coffin with a handle on the second. In words, an % example might then read % \begin{quote} % Align the top-left handle of coffin A with the bottom-right @@ -97,20 +97,20 @@ % \end{quote} % % The locations of coffin handles are much easier to understand -% visually. Figure~\ref{fgr:handles} shows the standard handle -% positions for a coffin typeset in horizontal mode (left) and in +% visually. Figure~\ref{fgr:handles} shows the standard handle +% positions for a coffin typeset in horizontal mode (left) and in % vertical mode (right). Notice that the later case results in a greater % number of handles being available. As illustrated, each handle % results from the intersection of two poles. For example, the centre % of the coffin is marked |(hc,vc)|, \emph{i.e.}~it is the -% point of intersection of the horizontal centre pole with the +% point of intersection of the horizontal centre pole with the % vertical centre pole. New handles are generated automatically when % poles are added to a coffin: handles are \enquote{dynamic} entities. % % \begin{figure} % \hfil % \begin{minipage}{0.4\textwidth} -% \SetHorizontalCoffin\ExampleCoffin +% \SetHorizontalCoffin\ExampleCoffin % {\color{black!10!white}\rule{1 in}{1 in}} % \DisplayCoffinHandles\ExampleCoffin{blue} % \end{minipage} @@ -123,9 +123,9 @@ % \hfil % \caption{Standard coffin handles: left, horizontal coffin; right, % vertical coffin} -% \label{fgr:handles} +% \label{fgr:handles} % \end{figure} -% +% %\section{Creating and setting coffins} % % Before any alignment can take place, coffins must be created and @@ -139,7 +139,7 @@ % \begin{syntax} % \cs{NewCoffin} \meta{coffin} % \end{syntax} -% Before a \meta{coffin} can be used, it must be allocated using +% Before a \meta{coffin} can be used, it must be allocated using % \cs{NewCoffin}. The name of the \meta{coffin} should be a % control sequence (starting with the escape character, usually % |\|), for example @@ -175,7 +175,7 @@ % \end{syntax} % Typesets the \meta{material} in vertical mode constrained to the % given \meta{width} and stores the result in the \meta{coffin}. The -% standard poles for the \meta{coffin} are then set up based on the +% standard poles for the \meta{coffin} are then set up based on the % size of the typeset material. % \end{function} % @@ -194,7 +194,7 @@ % box of the coffin; % \item[b] a pole running along the bottom edge of the bounding % box of the coffin; -% \item[vc] a pole running horizontally through the centre of the +% \item[vc] a pole running horizontally through the centre of the % coffin half-way between the bottom and top edges of the bounding % box (\emph{i.e.}~the \enquote{vertical centre}); % \item[t] a pole running along the top edge of the bounding @@ -202,7 +202,7 @@ % \item[H] a pole running along the baseline of the typeset material % contained in the coffin. % \end{itemize} -% In addition, coffins containing vertical-mode material also +% In addition, coffins containing vertical-mode material also % feature poles which reflect the richer nature of these systems: % \begin{itemize} % \item[B] a pole running along the baseline of the material at the @@ -219,10 +219,10 @@ % The \meta{pole} will be located at the \meta{offset} from the % bottom edge of the bounding box of the \meta{coffin}. The % \meta{offset} should be given as a dimension expression; this may -% include the terms \cs{TotalHeight}, \cs{Height}, \cs{Depth} and -% \cs{Width}, which will evaluate to the appropriate dimensions of -% the \meta{coffin}. For example, to create a pole running -% horizontally through the coffin at one third of the distance from +% include the terms \cs{TotalHeight}, \cs{Height}, \cs{Depth} and +% \cs{Width}, which will evaluate to the appropriate dimensions of +% the \meta{coffin}. For example, to create a pole running +% horizontally through the coffin at one third of the distance from % the base of the coffin to the top, the appropriate instruction would % be % \begin{verbatim} @@ -230,7 +230,7 @@ % \end{verbatim} % Note that poles which run \emph{horizontally} are described in terms % of their \emph{vertical} location in the coffin. Also notice that -% the total height of the coffin is described by the sum of +% the total height of the coffin is described by the sum of % \cs{Height} and \cs{Depth}: these are both measured from the % horizontal baseline of the material in the coffin. % \end{function} @@ -243,10 +243,10 @@ % The \meta{pole} will be located at the \meta{offset} from the % left-hand edge of the bounding box of the \meta{coffin}. The % \meta{offset} should be given as a dimension expression; this may -% include the terms \cs{TotalHeight}, \cs{Height}, \cs{Depth} and -% \cs{Width}, which will evaluate to the appropriate dimensions of +% include the terms \cs{TotalHeight}, \cs{Height}, \cs{Depth} and +% \cs{Width}, which will evaluate to the appropriate dimensions of % the \meta{coffin}. For example, to create a pole running vertically -% through the coffin at one third of the distance from the left-hand +% through the coffin at one third of the distance from the left-hand % edge, the appropriate instruction would be % \begin{verbatim} % \SetVerticalPole \MyCoffin {width/3} {\Width/3} @@ -260,7 +260,7 @@ % \cs{TotalHeight} % \end{syntax} % Within the \meta{offset} argument of \cs{SetHorizontalPole} and -% \cs{SetVerticalPole}, \cs{TotalHeight} will give the distance from +% \cs{SetVerticalPole}, \cs{TotalHeight} will give the distance from % the base to the top of the bounding box of the relevant coffin. % \end{function} % @@ -300,10 +300,10 @@ % \end{syntax} % Rotates the \meta{coffin} by the given \meta{angle} about its reference % point (given in degrees counter-clockwise) . This process will rotate both -% the coffin content and poles. Multiple rotations will not result in +% the coffin content and poles. Multiple rotations will not result in % the bounding box of the coffin growing unnecessarily. -% -% The effect of rotation on a coffin is illustrated in +% +% The effect of rotation on a coffin is illustrated in % Figure~\ref{fgr:rotation}. As is shown, the coffin handles will % remain correctly positioned relative to the content of the coffin. % The \enquote{top} of a rotated coffin may of course no longer be the @@ -312,7 +312,7 @@ % % \begin{figure} % \hfil -% \SetHorizontalCoffin\ExampleCoffin +% \SetHorizontalCoffin\ExampleCoffin % {^^A % \color{black!10!white}\rule{0.5 in}{1 in}^^A % \color{black!20!white}\rule{0.5 in}{1 in}^^A @@ -339,8 +339,8 @@ % \end{syntax} % Resized the \meta{coffin} to \meta{width} and \meta{total-height}, % both of which should be given as dimension expressions. These may -% include the terms \cs{TotalHeight}, \cs{Height}, \cs{Depth} and -% \cs{Width}, which will evaluate to the appropriate dimensions of +% include the terms \cs{TotalHeight}, \cs{Height}, \cs{Depth} and +% \cs{Width}, which will evaluate to the appropriate dimensions of % the \meta{coffin}. % \end{function} % @@ -348,8 +348,8 @@ % \begin{syntax} % \cs{ScaleCoffin} \meta{coffin} \Arg{x-scale} \Arg{y-scale} % \end{syntax} -% Scales the \meta{coffin} by a factors \meta{x-scale} and -% \meta{y-scale} in the horizontal and vertical directions, +% Scales the \meta{coffin} by a factors \meta{x-scale} and +% \meta{y-scale} in the horizontal and vertical directions, % respectively. The two scale factors should be given as real numbers. % \end{function} % @@ -359,7 +359,7 @@ % % \begin{figure} % \hfil -% \SetHorizontalCoffin\ExampleCoffin +% \SetHorizontalCoffin\ExampleCoffin % {^^A % \color{black!10!white}\rule{0.5 in}{1 in}^^A % \color{black!20!white}\rule{0.5 in}{1 in}^^A @@ -376,26 +376,26 @@ % \hfil % \caption{Coffin resizing: left, resized to exactly $4$\,cm by % $6$\,cm; right, scaled a factors of $2$ and $0.5$ in -% $x$ and $y$, respectively (example coffin as in +% $x$ and $y$, respectively (example coffin as in % Figure~\ref{fgr:rotation}).} -% \label{fgr:resizing} +% \label{fgr:resizing} % \end{figure} % % \section{Joining coffins} % % The key operation for coffins is joining coffins to each other. This -% is always carried out such that the first coffin is the +% is always carried out such that the first coffin is the % \enquote{parent}, and is updated by the alignment. The second % \enquote{child} coffin is not altered by the alignment process. -% +% % \begin{function}{\JoinCoffins} % \begin{syntax} % \cs{JoinCoffins} * -% ~~\meta{coffin1} [ \meta{coffin1-pole1} , \meta{coffin1-pole2} ] +% ~~\meta{coffin1} [ \meta{coffin1-pole1} , \meta{coffin1-pole2} ] % ~~\meta{coffin2} [ \meta{coffin2-pole1} , \meta{coffin2-pole2} ] -% ~~( \meta{x-offset} , \meta{y-offset} ) +% ~~( \meta{x-offset} , \meta{y-offset} ) % \end{syntax} -% Joining of two coffins is carried out by the \cs{JoinCoffins} +% Joining of two coffins is carried out by the \cs{JoinCoffins} % function, which takes two mandatory arguments: the \enquote{parent} % \meta{coffin1} and the \enquote{child} \meta{coffin2}. All of the % other arguments shown are optional. @@ -403,15 +403,15 @@ % % The standard \cs{JoinCoffins} functions joins \meta{coffin2} to % \meta{coffin1} such that the bounding box of \meta{coffin1} after the -% process will expand. The new bounding box will be the smallest -% rectangle covering the bounding boxes of the two input coffins. +% process will expand. The new bounding box will be the smallest +% rectangle covering the bounding boxes of the two input coffins. % When the starred variant of \cs{JoinCoffins} is used, the bounding % box of \meta{coffin1} is not altered, \emph{i.e.}~\meta{coffin2} may % protrude outside of the bounding box of the updated \meta{coffin1}. % The difference between the two forms of alignment is best illustrated % using a visual example. In Figure~\ref{fgr:alignment}, the two % processes are contrasted. In both cases, the small red coffin has been -% aligned with the large grey coffin. In the left-hand illustration, +% aligned with the large grey coffin. In the left-hand illustration, % the \cs{JoinCoffins} function was used, resulting in an expanded % bounding box. In contrast, on the right \cs{AttachCoffin} was used, % meaning that the bounding box does not include the area of the @@ -419,9 +419,9 @@ % % \begin{figure} % \fboxsep 0 pt\relax -% \SetHorizontalCoffin\ExampleCoffin +% \SetHorizontalCoffin\ExampleCoffin % {\color{black!20!white}\rule{1 in}{1 in}} -% \SetHorizontalCoffin \SmallCoffin +% \SetHorizontalCoffin \SmallCoffin % {\color{red!20!white}\rule{0.1 in}{0.1 in}} % \hfil % \begin{minipage}{0.4\textwidth} @@ -436,14 +436,14 @@ % \fbox{\TypesetCoffin\ExampleCoffin}% % \end{minipage} % \hfil -% \caption{Contrast between \cs{JoinCoffins} (left) and +% \caption{Contrast between \cs{JoinCoffins} (left) and % \cs{JoinCoffins*} (right); the bounding box of the coffin is show % in black.} -% \label{fgr:alignment} +% \label{fgr:alignment} % \end{figure} -% +% % The alignment is carried out by first calculating \meta{handle1}, the -% point of intersection of \meta{coffin1-pole1} and +% point of intersection of \meta{coffin1-pole1} and % \meta{coffin1-pole2}, and \meta{handle2}, the point of intersection % of \meta{coffin2-pole1} and \meta{coffin2-pole2}. If the two % \meta{poles} are not specified, \cs{JoinCoffins} will use the @@ -456,19 +456,19 @@ % |(0 pt, 0 pt)| is used. % % Notice that when \cs{JoinCoffins} is used the new bounding box is -% the smallest rectangle containing the bounding boxes of the two input +% the smallest rectangle containing the bounding boxes of the two input % coffins. As a result, it will include additional white space unless -% one coffin entirely overlaps the other (Figure~\ref{fgr:bounding}, +% one coffin entirely overlaps the other (Figure~\ref{fgr:bounding}, % left). Rotation of coffins will take account of the extent of the % material after rotation when re-calculating the bounding box. This -% means that no \emph{unnecessary} white space will be added on +% means that no \emph{unnecessary} white space will be added on % rotation (Figure~\ref{fgr:bounding}, right). % % \begin{figure} % \fboxsep 0 pt\relax -% \SetHorizontalCoffin\ExampleCoffin +% \SetHorizontalCoffin\ExampleCoffin % {\color{black!20!white}\rule{1 in}{1 in}} -% \SetHorizontalCoffin\SmallCoffin +% \SetHorizontalCoffin\SmallCoffin % {\color{red!20!white}\rule{0.1 in}{0.1 in}} % \JoinCoffins\ExampleCoffin[vc,r]\SmallCoffin[vc,l] % \hfil @@ -485,7 +485,7 @@ % \hfil % \caption{The effect of rotation of a joined coffin: the black line % shows the coffin bounding box.} -% \label{fgr:bounding} +% \label{fgr:bounding} % \end{figure} % % As part of the joining procedure, the poles of the two input coffins @@ -495,35 +495,35 @@ % \begin{enumerate} % \item The \enquote{native} poles of the updated coffin, such as % \texttt{l}, \texttt{r}, \texttt{hc}, \emph{etc}. -% \item Poles derived from \meta{coffin1}, such as +% \item Poles derived from \meta{coffin1}, such as % \texttt{\meta{coffin1}-l}, \texttt{\meta{coffin1}-r}, % \texttt{\meta{coffin1}-hc}, \emph{etc.} -% \item Poles derived from \meta{coffin2}, such as +% \item Poles derived from \meta{coffin2}, such as % \texttt{\meta{coffin2}-l}, \texttt{\meta{coffin2}-r}, % \texttt{\meta{coffin2}-hc}, \emph{etc.} % \end{enumerate} -% +% % Applying this ability allows a series of joining operations to % take place, as illustrated in Figure~\ref{fgr:nested}. In this % example, the scheme used for alignment was as follows: % \begin{verbatim} % \SetHorizontalCoffin\OutputCoffin{} -% \SetHorizontalCoffin\RedCoffin +% \SetHorizontalCoffin\RedCoffin % {\color{red!20!white}\rule{0.2 in}{0.2 in}} % \JoinCoffins\OutputCoffin[vc,hc]\RedCoffin[vc,hc] -% \SetHorizontalCoffin\BlueCoffin +% \SetHorizontalCoffin\BlueCoffin % {\color{blue!20!white}\rule{0.2 in}{0.2 in}} % \JoinCoffins\OutputCoffin[\RedCoffin-vc,\RedCoffin-hc] % \BlueCoffin[b,l] -% \SetHorizontalCoffin\GreenCoffin +% \SetHorizontalCoffin\GreenCoffin % {\color{green!20!white}\rule{0.2 in}{0.2 in}} % \JoinCoffins\OutputCoffin[\BlueCoffin-vc,\BlueCoffin-hc] % \GreenCoffin[b,l] -% \SetHorizontalCoffin\YellowCoffin +% \SetHorizontalCoffin\YellowCoffin % {\color{yellow!20!white}\rule{0.2 in}{0.2 in}} % \JoinCoffins\OutputCoffin[\GreenCoffin-vc,\GreenCoffin-hc] % \YellowCoffin[b,l] -% \SetHorizontalCoffin \OrangeCoffin +% \SetHorizontalCoffin \OrangeCoffin % {\color{orange!20!white}\rule{0.2 in}{0.2 in}} % \JoinCoffins\OutputCoffin[\BlueCoffin-t,\BlueCoffin-l] % \OrangeCoffin[b,r] @@ -533,62 +533,62 @@ % output. Each join then takes place placing the new addition relative % to the previous one. As each coffin joined has a unique name it is % possible to align relative to each one of the component parts of the -% assembly. This is illustrated by the addition of the final +% assembly. This is illustrated by the addition of the final % \cs{OrangeCoffin} based on the earlier placement of the % \cs{BlueCoffin}. % % \begin{figure} % \centering % \SetHorizontalCoffin\OutputCoffin{} -% \SetHorizontalCoffin\RedCoffin +% \SetHorizontalCoffin\RedCoffin % {\color{red!20!white}\rule{0.2 in}{0.2 in}} % \JoinCoffins\OutputCoffin[vc,hc]\RedCoffin[vc,hc] -% \SetHorizontalCoffin\BlueCoffin +% \SetHorizontalCoffin\BlueCoffin % {\color{blue!20!white}\rule{0.2 in}{0.2 in}} % \JoinCoffins\OutputCoffin[\RedCoffin-vc,\RedCoffin-hc] % \BlueCoffin[b,l] -% \SetHorizontalCoffin\GreenCoffin +% \SetHorizontalCoffin\GreenCoffin % {\color{green!20!white}\rule{0.2 in}{0.2 in}} % \JoinCoffins\OutputCoffin[\BlueCoffin-vc,\BlueCoffin-hc] % \GreenCoffin[b,l] -% \SetHorizontalCoffin\YellowCoffin +% \SetHorizontalCoffin\YellowCoffin % {\color{yellow!20!white}\rule{0.2 in}{0.2 in}} % \JoinCoffins\OutputCoffin[\GreenCoffin-vc,\GreenCoffin-hc] % \YellowCoffin[b,l] -% \SetHorizontalCoffin \OrangeCoffin +% \SetHorizontalCoffin \OrangeCoffin % {\color{orange!20!white}\rule{0.2 in}{0.2 in}} % \JoinCoffins\OutputCoffin[\BlueCoffin-t,\BlueCoffin-l] % \OrangeCoffin[b,r] % \TypesetCoffin\OutputCoffin % \caption{Aligning coffins using poles from previous operations.} -% \label{fgr:nested} +% \label{fgr:nested} % \end{figure} % % \section{Typesetting coffins} % % \begin{function}{\TypesetCoffin} % \begin{syntax} -% \cs{TypesetCoffin} +% \cs{TypesetCoffin} % ~~\meta{coffin} [ \meta{pole1} , \meta{pole2} ] -% ~~( \meta{x-offset} , \meta{y-offset} ) +% ~~( \meta{x-offset} , \meta{y-offset} ) % \end{syntax} % Typesetting is carried out by first calculating \meta{handle}, the % point of intersection of \meta{pole1} and \meta{pole2}. This is an % optional argument, and if not given then |(H,l)|, the \TeX{} % reference point of the underlying box, is used. The coffin -% is then typeset such that the relationship between the current +% is then typeset such that the relationship between the current % reference point in the document and the \meta{handle} is described % by the \meta{x-offset} and \meta{y-offset}. This \meta{offset} is % optional, and if not given |(0 pt, 0 pt)| is used. Typesetting a % coffin is therefore analogous to carrying out an alignment where the % \enquote{parent} coffin is the current insertion point. % \end{function} -% +% % \section{Measuring coffins} % % There are places in the design process where it is useful to be able to % measure coffins outside of pole-setting procedures. -% +% % \begin{function}{\CoffinDepth} % \begin{syntax} % \cs{CoffinDepth} \meta{coffin} @@ -606,7 +606,7 @@ % in a form suitable for use in a \meta{dimension expression}, for example % |\setlength{\mylength}{\CoffinHeight\ExampleCoffin}|. % \end{function} -% +% % \begin{function}{\CoffinTotalHeight} % \begin{syntax} % \cs{CoffinTotalHeight} \meta{coffin} @@ -627,7 +627,7 @@ % % \section{Diagnostic functions} % -% Diagnostic data for following the coffin-building process is +% Diagnostic data for following the coffin-building process is % available both graphically and at the terminal. This reflects the % fact that coffins are visual constructs. % @@ -636,22 +636,22 @@ % \cs{DisplayCoffinHandles} \meta{coffin} \Arg{colour} % \end{syntax} % This function first calculates the intersections between all of -% the \meta{poles} of the \meta{coffin} to give a set of -% \meta{handles}. It then prints the \meta{coffin} at the current -% location in the source, with the position of the \meta{handles} -% marked on the coffin. The \meta{handles} will be labelled as part -% of this process: the locations of the \meta{handles} and the labels +% the \meta{poles} of the \meta{coffin} to give a set of +% \meta{handles}. It then prints the \meta{coffin} at the current +% location in the source, with the position of the \meta{handles} +% marked on the coffin. The \meta{handles} will be labelled as part +% of this process: the locations of the \meta{handles} and the labels % are both printed in the \meta{colour} specified. % \end{function} % % \begin{function}{\MarkCoffinHandle} % \begin{syntax} -% \cs{MarkCoffinHandle} \meta{coffin} +% \cs{MarkCoffinHandle} \meta{coffin} % ~~[ \meta{pole1} , \meta{pole2} ] \Arg{colour} % \end{syntax} -% This function first calculates the \meta{handle} for the +% This function first calculates the \meta{handle} for the % \meta{coffin} as defined by the intersection of \meta{pole1} and -% \meta{pole2}. It then marks the position of the \meta{handle} +% \meta{pole2}. It then marks the position of the \meta{handle} % on the \meta{coffin}. The \meta{handle} will be labelled as part of % this process: the location of the \meta{handle} and the label are % both printed in the \meta{colour} specified. If no \meta{poles} are @@ -663,9 +663,9 @@ % \cs{ShowCoffinStructure} \meta{coffin} % \end{syntax} % This function shows the structural information about the -% \meta{coffin} in the terminal. The width, height and depth of the -% typeset material are given, along with the location of all of the -% poles of the coffin. For example, for the rotated coffin in +% \meta{coffin} in the terminal. The width, height and depth of the +% typeset material are given, along with the location of all of the +% poles of the coffin. For example, for the rotated coffin in % Figure~\ref{fgr:rotation}, the output of \cs{ShowCoffinStructure} % is: % \begin{verbatim} @@ -673,7 +673,6 @@ % > ht = 72.26999pt % > dp = 0.0pt % > wd = 72.26999pt -% % Poles of coffin \ExampleCoffin: % > l => {0pt}{0pt}{0pt}{1000pt} % > B => {0pt}{0pt}{1000pt}{0pt} @@ -684,11 +683,11 @@ % > vc => {0pt}{36.135pt}{1000pt}{0pt} % > t => {0pt}{72.26999pt}{1000pt}{0pt} % > b => {0pt}{0.0pt}{1000pt}{0pt}. -% <argument> ...ffin_show_aux:w \l_coffin_tmp_tl } +% <recently read> } % \end{verbatim} % Notice that the poles of a coffin are defined by four values: -% the $x$ and $y$ co-ordinates of a point that the pole -% passes through and the $x$- and $y$-components of a +% the $x$ and $y$ co-ordinates of a point that the pole +% passes through and the $x$- and $y$-components of a % vector denoting the direction of the pole. It is the ratio between % the later, rather than the absolute values, which determines the % direction of the pole. @@ -699,17 +698,17 @@ % \begin{implementation} % % \section{Implementation} -% +% % \begin{macrocode} %<*package> % \end{macrocode} -% +% % \begin{macrocode} \ProvidesExplPackage {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription} % \end{macrocode} -% -% +% +% % \begin{variable} % { % \l_coffin_A_hpole_tl , @@ -722,7 +721,7 @@ % } % Key--value definitions for the alignment system. With the exception % of \texttt{grow-bounding-box}, all of these have to be given with a -% value. +% value. % \begin{macrocode} \keys_define:nn { coffin } { @@ -755,7 +754,7 @@ % \end{variable} % % A lot of this is more-or-less just passing data straight through. -% +% % \begin{macro}{\NewCoffin} % This is a very easy conversion. % \begin{macrocode} @@ -763,7 +762,7 @@ { \coffin_new:N #1 } % \end{macrocode} % \end{macro} -% +% % \begin{macro}{\BoxToCoffin} % For allowing a user-level \LaTeXe{} box to be used as a coffin. % \begin{macrocode} @@ -855,12 +854,12 @@ % which need to be split. This is a simpler case of the code needed for % \cs{JoinCoffins}. % \begin{macrocode} -\NewDocumentCommand \TypesetCoffin +\NewDocumentCommand \TypesetCoffin { - m - > { \SplitArgument { 1 } { , } } O { H , l } + m + > { \SplitArgument { 1 } { , } } O { H , l } > { \SplitArgument { 1 } { , } } D ( ) { 0 pt , 0 pt } - } + } { \coffin_typeset:Nnnnn #1 #2 #3 } % \end{macrocode} % \end{macro} @@ -880,7 +879,7 @@ % \end{macro} % \end{macro} % \end{macro} -% +% % \begin{macro}{\CoffinDepth, \CoffinHeight, \CoffinTotalHeigth, \CoffinWidth} % Nothing too complex, except that the total height is set up as an % expression so that it will act correctly if prefixed with a negative @@ -911,10 +910,10 @@ % \end{macro} % % \begin{macro}{\MarkCoffinHandle} -% Marking a handle requires a bit of work with the input, so that +% Marking a handle requires a bit of work with the input, so that % the design-level interface is \enquote{nice}. % \begin{macrocode} -\NewDocumentCommand \MarkCoffinHandle +\NewDocumentCommand \MarkCoffinHandle { m > { \SplitArgument { 1 } { , } } O { H , l } m } { \cs_if_exist:cTF { l_coffin_poles_ \int_value:w #1 _prop } |