diff options
Diffstat (limited to 'Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx')
| -rw-r--r-- | Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx | 2487 |
1 files changed, 1556 insertions, 931 deletions
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx index 94bd054f0ce..9c276b6922d 100644 --- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx @@ -35,7 +35,7 @@ % %<*driver|package> \RequirePackage{expl3} -\GetIdInfo$Id: l3regex.dtx 3017 2011-11-28 00:10:02Z bruno $ +\GetIdInfo$Id: l3regex.dtx 3073 2011-12-21 01:53:06Z bruno $ {L3 Experimental Regular Expressions} %</driver|package> %<*driver> @@ -74,36 +74,53 @@ % \section{\pkg{l3regex} documentation} % % The \pkg{l3regex} package provides regular expression testing, -% extraction of submatches, splitting, and replacement, all acting on strings -% of characters. The syntax of regular expressions is mostly a subset -% of the PCRE syntax (and very close to POSIX). For performance -% reasons, only a limited set of features are implemented. Notably, -% back-references are not supported. +% extraction of submatches, splitting, and replacement, all acting +% on token lists. The syntax of regular expressions is mostly a subset +% of the PCRE syntax (and very close to POSIX), with some additions +% due to the fact that we act on lists of tokens rather than characters. +% For performance reasons, only a limited set of features are implemented. +% Notably, back-references are not supported. % % Let us give a few examples. After % \begin{verbatim} -% \str_set:Nn \l_my_str { That~cat. } -% \regex_replace_once:nnN { at } { is } \l_my_str +% \tl_set:Nn \l_my_tl { That~cat. } +% \regex_replace_once:nnN { at } { is } \l_my_tl % \end{verbatim} -% the string variable \cs{l_my_str} holds the text +% the token list variable \cs{l_my_tl} holds the text % \enquote{\texttt{This cat.}}, where the first % occurrence of \enquote{\texttt{at}} was replaced % by \enquote{\texttt{is}}. A more complicated example is % a pattern to add a comma at the end of each word: % \begin{verbatim} -% \regex_replace_all:nnN { \w+ } { \0 , } \l_my_str +% \regex_replace_all:nnN { \w+ } { \0 , } \l_my_tl % \end{verbatim} % The |\w| sequence represents any \enquote{word} character, % and |+| indicates that the |\w| sequence should be repeated % as many times as possible (at least once), hence matching a word in the -% input string. In the replacement text, |\0| denotes the full match +% input token list. In the replacement text, |\0| denotes the full match % (here, a word). % +% If a regular expression is to be used several times, +% it can be compiled once, and stored in a token list +% variable using \cs{regex_const:Nn}. For example, +% \begin{verbatim} +% \regex_const:Nn \l_foo_regex_tl { \c{begin} \cB (\CB*?) \cE } +% \end{verbatim} +% stores in \cs{l_foo_regex_tl} a regular expression which matches +% the starting marker for an environment: \cs{begin}, followed +% by a begin-group token (|\cB|), then tokens which are not +% begin-group characters (|\CB|), as few as possible (|*?|), +% followed by an end-group token (|\cE|). As explained in the +% next section, the parentheses \enquote{capture} the result of +% |\CB*?|, giving us access to the name of the environment +% when doing replacements. +% % \subsection{Syntax of regular expressions} % -% Most characters match exactly themselves. Some characters are +% Most characters match exactly themselves, +% with an arbitrary category code. Some characters are % special and must be escaped with a backslash (\emph{e.g.}, |\*| -% matches an explicit star character). Some escape sequences of +% matches a star character). Some escape sequences of % the form backslash--letter also have a special meaning % (for instance |\d| matches any digit). As a rule, % \begin{itemize} @@ -113,7 +130,7 @@ % |\A|, |\B|, \ldots{} have special meanings; % \item non-alphanumeric printable ascii characters can (and should) % always be escaped: many of them have special meanings (\emph{e.g.}, -% |(|, |)|, |?|, |.|); +% use |\(|, |\)|, |\?|, |\.|); % \item spaces should always be escaped (even in character % classes); % \item any other character may be escaped or not, without any @@ -121,12 +138,11 @@ % \end{itemize} % Note that these rules play nicely with the fact that many % non-alphanumeric characters are difficult to input into \TeX{} -% under normal category codes. For instance, |\$\%\^\\abc\#| -% matches the literal string |$%^\abc#|. -% \begin{texnote} -% When converting the regular expression to a string, -% the value of the escape character is set to be a backslash. -% \end{texnote} +% under normal category codes. For instance, |\\abc\%| +% matches the characters |\abc%| (with arbitrary category codes), +% but does not match the control sequence |\abc| followed by a +% percent character. Matching control sequences can be done +% using the |\c|\Arg{regex} syntax (see below). % % Any special character which appears at a place where its special % behaviour cannot apply matches itself instead (for instance, @@ -146,7 +162,7 @@ % % Character types. % \begin{l3regex-syntax} -% \item[.] A single period matches any character, including newlines. +% \item[.] A single period matches any token. % \item[\\d] Any decimal digit. % \item[\\h] Any horizontal space character, % equivalent to |[\ \^^I]|: space and tab. @@ -157,22 +173,27 @@ % but not a space, for compatibility with Perl. % \item[\\w] Any word character, \emph{i.e.}, % alpha-numerics and underscore, equivalent to |[A-Za-z0-9\_]|. -% \item[\\D] Any character not matched by |\d|. -% \item[\\H] Any character not matched by |\h|. -% \item[\\N] Any character other than the |\n| character (hex 0A). -% \item[\\S] Any character not matched by |\s|. -% \item[\\V] Any character not matched by |\v|. -% \item[\\W] Any character not matched by |\w|. +% \item[\\D] Any token not matched by |\d|. +% \item[\\H] Any token not matched by |\h|. +% \item[\\N] Any token other than the |\n| character (hex 0A). +% \item[\\S] Any token not matched by |\s|. +% \item[\\V] Any token not matched by |\v|. +% \item[\\W] Any token not matched by |\w|. % \end{l3regex-syntax} +% Of those, |.|, |\D|, |\H|, |\N|, |\S|, |\V|, and |\W| will match arbitrary +% control sequences. % % Character classes match exactly one character in the subject string. % \begin{l3regex-syntax} % \item[{[\ldots{}]}] Positive character class. -% Matches any of the specified characters. +% Matches any of the specified tokens. % \item[{[\char`\^\ldots{}]}] Negative character class. -% Matches any character other than the specified characters. +% Matches any token other than the specified characters. % \item[{[x-y]}] Range (can be used with escaped characters). % \end{l3regex-syntax} +% For instance, |[a-oq-z\cC]| matches any lowercase latin letter +% except |p|, as well as control sequences (see below for a description +% of |\c|). % % Quantifiers (repetition). % \begin{l3regex-syntax} @@ -191,12 +212,15 @@ % % Anchors and simple assertions. % \begin{l3regex-syntax} -% \item[\\b] Word boundary. -% \item[\\B] Not a word boundary. +% \item[\\b] Word boundary: either the previous token is matched by +% |\w| and the next by |\W|, or the opposite. For this purpose, +% the ends of the token list are considered as |\W|. +% \item[\\B] Not a word boundary: between two |\w| tokens +% or two |\W| tokens (including the boundary). % \item[\char`^ \textrm{or} \\A] -% Start of the subject string. +% Start of the subject token list. % \item[\char`$\textrm{,} \\Z \textrm{or} \\z] -% End of the subject string. +% End of the subject token list. % \item[\\G] Start of the current match. This is only different from |^| % in the case of multiple matches: for instance % |\regex_count:nnN { \G a } { aaba } \l_tmpa_int| yields $2$, but @@ -216,6 +240,30 @@ % group number. % \end{l3regex-syntax} % +% Each catcode is represented by a single uppercase letter: +% |C| for control sequences; +% |B| for begin-group tokens; +% |E| for end-group tokens; +% |M| for math shift; +% |T| for alignment tab tokens; +% |P| for macro parameter tokens; +% |U| for superscript tokens (up); +% |D| for subscript tokens (down); +% |S| for spaces; +% |L| for letters; +% |O| for others; and +% |A| for active characters. +% \begin{l3regex-syntax} +% \item[\\cX] Any character token with catcode |X|, +% where |X| can be any of |CBEMTPUDSLOA|. +% \item[\\CX] Any token with catcode different from |X|, +% where |X| can be any of |CBEMTPUDSLOA|. +% \item[\\c\Arg{regex}] A control sequence whose csname matches +% the \meta{regex}, anchored at the beginning and end, +% so that |\c{begin}| matches exactly \cs{begin}, and nothing +% else. +% \end{l3regex-syntax} +% % Options can be set with |(?|\meta{option}|)| and % unset with |(?-|\meta{option}|)|. Options are local % to the group in which they are set, and revert to their @@ -234,7 +282,7 @@ % In character classes, only |^|, |-|, |]|, |\| and spaces are special, % and should be escaped. Other non-alphanumeric characters can % still be escaped without harm. The escape sequences |\d|, -% |\D|, |\w|, |\W| are also supported in character classes. +% |\D|, etc. are also supported in character classes. % If the first character is |^|, then the meaning of the character % class is inverted. Ranges of characters can be expressed using % |-|, for instance, |[\D 0-5]| is equivalent to |[^6-9]|. @@ -256,10 +304,24 @@ % % For instance, % \begin{verbatim} -% \str_set:Nn \l_my_str { Hello,~world! } -% \regex_replace_all:nnN { ([er]?l|o) . } { \(\0\-\-\1\) } \l_my_str +% \tl_set:Nn \l_my_tl { Hello,~world! } +% \regex_replace_all:nnN { ([er]?l|o) . } { \(\0\-\-\1\) } \l_my_tl % \end{verbatim} -% results in \cs{l_my_str} holding |H(ell--el)(o,--o) w(or--o)(ld--l)!| +% results in \cs{l_my_tl} holding |H(ell--el)(o,--o) w(or--o)(ld--l)!| +% +% The characters inserted by the replacement have category code $12$ +% (other) by default. The escape sequence |\c| allows to insert characters +% with arbitrary category codes, as well as control sequences. +% \begin{l3regex-syntax} +% \item[\\cXY] Produces the character |Y| (which can be given as +% an escape sequence such as |\t| for tab) with category code |X|, +% which must be one of |CBEMTPUDSLOA|. +% \item[\\c\Arg{text}] Produces the control sequence with csname +% \meta{text}. The \meta{text} may contain references to the submatches +% |\0|, |\1| \emph{etc.} As an experimental feature, |\c{...}| can be +% nested, but the behaviour is not yet fully specified. Use at own risks. +%^^A todo: decide what the right behaviour should be. +% \end{l3regex-syntax} % % \subsection{Precompiling regular expressions} % @@ -285,14 +347,32 @@ % The assignment is local for \cs{regex_set:Nn} and global for % \cs{regex_gset:Nn}. Use \cs{regex_const:Nn} for precompiled expressions % which will never change. -% \begin{texnote} -% Precompiled regular expressions can be safely written to a file -% and read when the \LaTeX3 syntax is active (as triggered by -% \cs{ExplSyntaxOn}). -% \end{texnote} % \end{function} % -% \subsection{String matching} +% \begin{function}{\regex_to_str:N} +% \begin{syntax} +% \cs{regex_to_str:N} \meta{regex var} +% \end{syntax} +% Converts the \meta{regex var}, previously defined using +% \cs{regex_(g)set:Nn}, to a string, which can be safely +% read back when the \LaTeX3 syntax is active +% (as triggered by \cs{ExplSyntaxOn}). +% Line breaks are inserted at various places, to \emph{try} +% and keep the line length short. +% For instance, if \cs{l_my_stream} is an open stream for writing, +% and \cs{l_regex_tl} is a regex variable, you can save its definition +% using +% \begin{verbatim} +% \iow_now:Nx \l_my_stream +% { +% \tl_to_str:n { \tl_set:Nn \l_my_regex_tl } { \iow_newline: +% \regex_to_str:N \l_my_regex_tl +% } +% } +% \end{verbatim} +% \end{function} +% +% \subsection{Matching} % % All regular expression functions are available in both |:n| and |:N| % variants. The former require a \enquote{standard} regular expression, @@ -301,10 +381,10 @@ % % \begin{function}[TF]{\regex_match:nn, \regex_match:Nn} % \begin{syntax} -% \cs{regex_match:nnTF} \Arg{regex} \Arg{string} \Arg{true code} \Arg{false code} +% \cs{regex_match:nnTF} \Arg{regex} \Arg{token list} \Arg{true code} \Arg{false code} % \end{syntax} -% Tests whether the \meta{regular expression} matches any substring -% of \meta{string}. For instance, +% Tests whether the \meta{regular expression} matches any part +% of the \meta{token list}. For instance, % \begin{verbatim} % \regex_match:nnTF { b [cde]* } { abecdcx } { TRUE } { FALSE } % \regex_match:nnTF { [b-dq-w] } { example } { TRUE } { FALSE } @@ -314,15 +394,18 @@ % % \begin{function}{\regex_count:nnN, \regex_count:NnN} % \begin{syntax} -% \cs{regex_count:nnN} \Arg{regex} \Arg{string} \meta{int var} +% \cs{regex_count:nnN} \Arg{regex} \Arg{token list} \meta{int var} % \end{syntax} % Sets \meta{int var} within the current \TeX{} group level % equal to the number of times -% \meta{regular expression} appears in \meta{string}. +% \meta{regular expression} appears in \meta{token list}. % The search starts by finding the left-most longest match, % respecting greedy and ungreedy operators. Then the search % starts again from the character following the last character -% of the previous match, until reaching the end of the string. +% of the previous match, until reaching the end of the token list. +% Infinite loops are prevented in the case where the regular expression +% can match an empty string: then we count one match between each +% pair of characters. % For instance, % \begin{verbatim} % \int_new:N \l_foo_int @@ -335,25 +418,25 @@ % % \begin{function}[TF]{\regex_extract_once:nnN, \regex_extract_once:NnN} % \begin{syntax} -% \cs{regex_extract_once:nnN} \Arg{regex} \Arg{string} \meta{seq~var} -% \cs{regex_extract_once:nnNTF} \Arg{regex} \Arg{string} \meta{seq~var} \Arg{true code} \Arg{false code} +% \cs{regex_extract_once:nnN} \Arg{regex} \Arg{token list} \meta{seq~var} +% \cs{regex_extract_once:nnNTF} \Arg{regex} \Arg{token list} \meta{seq~var} \Arg{true code} \Arg{false code} % \end{syntax} % Finds the first match of the \meta{regular expression} -% in the \meta{string}. If it exists, the match is stored +% in the \meta{token list}. If it exists, the match is stored % as the zeroeth item of the \meta{seq~var}, and further % items are the contents of capturing groups, in the order % of their opening parenthesis. The \meta{seq~var} % is assigned locally. If there is no match, -% the \meta{seq~var} is not altered. +% the \meta{seq~var} is cleared. % The testing versions insert the \meta{true code} into the input % stream if a match was found, and the \meta{false code} otherwise. % For instance, assume that you type % \begin{verbatim} -% \regex_extract_once:nnNTF { ^(La)?TeX(!*)$ } { LaTeX!!! } -% \l_foo_seq { true } { false } +% \regex_extract_once:nnNTF { \A(La)?TeX(!*)\Z } { LaTeX!!! } \l_foo_seq +% { true } { false } % \end{verbatim} -% Then the regular expression (anchored at the start with |^| and -% at the end with |$|) will match the whole string. The first +% Then the regular expression (anchored at the start with |\A| and +% at the end with |\Z|) will match the whole token list. The first % capturing group, |(La)?|, matches |La|, and the second capturing % group, |(!*)|, matches |!!!|. Thus, |\l_foo_seq| will contain % the items |{LaTeX!!!}|, |{La}|, and |{!!!}|, and the \texttt{true} @@ -362,64 +445,79 @@ % % \begin{function}[TF]{\regex_extract_all:nnN, \regex_extract_all:NnN} % \begin{syntax} -% \cs{regex_extract_all:nnNTF} \Arg{regex} \Arg{string} \meta{seq~var} \Arg{true code} \Arg{false code} +% \cs{regex_extract_all:nnN} \Arg{regex} \Arg{token list} \meta{seq~var} +% \cs{regex_extract_all:nnNTF} \Arg{regex} \Arg{token list} \meta{seq~var} \Arg{true code} \Arg{false code} % \end{syntax} % Finds all matches of the \meta{regular expression} -% in the \meta{string}, and stores all the submatch information +% in the \meta{token list}, and stores all the submatch information % in a single sequence (concatenating the results of % multiple \cs{regex_extract_once:nnN} calls). +% The \meta{seq~var} is assigned locally. If there is no match, +% the \meta{seq~var} is cleared. +% The testing versions insert the \meta{true code} into the input +% stream if a match was found, and the \meta{false code} otherwise. % For instance, assume that you type % \begin{verbatim} -% \regex_extract_all:nnN { \w+ } { Hello,~world! } \l_foo_seq +% \regex_extract_all:nnNTF { \w+ } { Hello,~world! } \l_foo_seq +% { true } { false } % \end{verbatim} % Then the regular expression will match twice, and the resulting -% sequence contains the two items \texttt{Hello} and \texttt{world}. +% sequence contains the two items |{Hello}| and |{world}|, +% and the \texttt{true} branch is left in the input stream. % \end{function} % -% \begin{function}{\regex_split:nnN, \regex_split:NnN} +% \begin{function}[TF]{\regex_split:nnN, \regex_split:NnN} % \begin{syntax} -% \cs{regex_split:nnN} \Arg{regular expression} \meta{string} \meta{seq~var} +% \cs{regex_split:nnN} \Arg{regular expression} \Arg{token list} \meta{seq~var} +% \cs{regex_split:nnNTF} \Arg{regular expression} \Arg{token list} \meta{seq~var} \Arg{true code} \Arg{false code} % \end{syntax} -% Searches the \meta{string} into a sequence of substrings, delimited by +% Splits the \meta{token list} into a sequence of parts, delimited by % matches of the \meta{regular expression}. If the \meta{regular expression} -% has capturing groups, then the substrings that they match are stored as +% has capturing groups, then the token lists that they match are stored as % items of the sequence as well. The assignment to \meta{seq~var} is local. -% If no match is found the resulting \meta{seq~var} has the \meta{string} as -% its sole item. If the \meta{regular expression} matches the empty string, -% then the \meta{string} is split into single character substrings. +% If no match is found the resulting \meta{seq~var} has the +% \meta{token list} as its sole item. If the \meta{regular expression} +% matches the empty token list, then the \meta{token list} is split +% into single tokens. +% The testing versions insert the \meta{true code} into the input +% stream if a match was found, and the \meta{false code} otherwise. % For example, after % \begin{verbatim} % \seq_new:N \l_path_seq -% \regex_split:nnN { / } { the/path/for/this/file.tex } \l_path_seq +% \regex_split:nnNTF { / } { the/path/for/this/file.tex } \l_path_seq +% { true } { false } % \end{verbatim} % the sequence |\l_path_seq| contains the items |{the}|, |{path}|, -% |{for}|, |{this}|, and |{file.tex}|. +% |{for}|, |{this}|, and |{file.tex}|, and the \texttt{true} branch +% is left in the input stream. % \end{function} % -% \subsection{String replacement} +% \subsection{Replacement} % % \begin{function}[TF]{\regex_replace_once:nnN,\regex_replace_once:NnN} % \begin{syntax} -% \cs{regex_replace_once:nnN} \Arg{regular expression} \Arg{replacement} \meta{str~var} +% \cs{regex_replace_once:nnN} \Arg{regular expression} \Arg{replacement} \meta{tl~var} +% \cs{regex_replace_once:nnNTF} \Arg{regular expression} \Arg{replacement} \meta{tl~var} \Arg{true code} \Arg{false code} % \end{syntax} -% Searches for the \meta{regular expression} in the \meta{string} -% and replaces the matching part with the \meta{replacement}. The result -% is assigned locally to \meta{str~var}. In the \meta{replacement}, +% Searches for the \meta{regular expression} in the \meta{token list} +% and replaces the first match with the \meta{replacement}. The result +% is assigned locally to \meta{tl~var}. In the \meta{replacement}, % |\0| represents the full match, |\1| represent the contents % of the first capturing group, |\2| of the second, \emph{etc.} % \end{function} % -% \begin{function}{\regex_replace_all:nnN, \regex_replace_all:NnN} +% \begin{function}[TF]{\regex_replace_all:nnN, \regex_replace_all:NnN} % \begin{syntax} -% \cs{regex_replace_all:nnN} \Arg{regular expression} \Arg{replacement} \meta{str~var} +% \cs{regex_replace_all:nnN} \Arg{regular expression} \Arg{replacement} \meta{tl~var} +% \cs{regex_replace_all:nnNTF} \Arg{regular expression} \Arg{replacement} \meta{tl~var} \Arg{true code} \Arg{false code} % \end{syntax} % Replaces all occurrences of the \cs{regular expression} -% in the \meta{string} by the \meta{replacement}, where +% in the \meta{token list} by the \meta{replacement}, where % |\0| represents the full match, |\1| % represent the contents of the first capturing group, % |\2| of the second, \emph{etc.} Every match % is treated independently, and matches cannot overlap. -% The result is assigned locally to \meta{str~var}. +% The result is assigned locally to \meta{tl~var}. % \end{function} % % \subsection{Bugs, misfeatures, future work, and other possibilities} @@ -427,11 +525,27 @@ % The following need to be done now. % \begin{itemize} % \item The \texttt{\{\}} quantifiers are only partially implemented. -% \item Check the catcode of spaces everywhere. -% \item Newline conventions are not done. -% In particular, we should have an option for |.| not to match newlines. -% Also, |\A| should differ from |^|, and |\Z|, |\z| and |$| should -% differ. +% \item Clean up the use of messages. +% \item Add tests for every regex and replacement feature. +% \item Make sure the documentation is correct, and the code comments +% are as well. +% \end{itemize} +% +% Code improvements to come. +% \begin{itemize} +% \item Test for the maximum register \cs{c_max_register_int}. +% \item Use \tn{dimen} registers rather than \cs{l_regex_nesting_tl} +% to build \cs{regex_nesting:n}. +% \item Reduce the number of epsilon-transitions in alternatives. +% \item Merge the \texttt{build} and \texttt{class} functions where possible. +% \item Improve nesting of |\c| in the replacement text. +% \item Move the \enquote{reconstruction} part of \pkg{l3regex} +% to \pkg{l3tl-analysis}. +% \item Optimize regexes for csnames when the regex is a simple string. +% \item Optimize simple strings: use less states +% (|abcade| should give two states, for |abc| and |ade|). +% \item Optimize groups with no alternative. +% \item Optimize the use of \cs{prg_stepwise_...} functions. % \end{itemize} % % The following features are likely to be implemented at some point @@ -441,30 +555,33 @@ % \item Regex matching on external files. % \item Conditional subpatterns with look ahead/behind: \enquote{if % what follows is [\ldots{}], then [\ldots{}]}. -% \item |(*..)| and |(?..)| sequences to set some options. -% \item Reduce the number of epsilon-transitions in alternatives. -% \item Optimize simple strings: use less states -% (|abcade| should give two states, for |abc| and |ade|). -% \item Optimize groups with no alternative. -% \item Optimize the use of \cs{prg_stepwise_...} functions. +% \item |(*..)| and |(?..)| sequences to set some options +% (partially implemented). % \item |\K| for resetting the beginning of the match. % \item UTF-8 mode for pdf\TeX{}. +% \item Newline conventions are not done. +% In particular, we should have an option for |.| not to match newlines. +% Also, |\A| should differ from |^|, and |\Z|, |\z| and |$| should +% differ. +% \item Unicode properties: |\p{..}| and |\P{..}|; +% |\X| which should match any \enquote{extended} Unicode sequence. +% This requires to manipulate a lot of data, hence a lot of optimization +% ahead. % \end{itemize} % % The following features of PCRE or Perl will probably not be implemented. % \begin{itemize} -% \item |\cx|, similar to \TeX{}'s own |\^^x|; % \item |\ddd|, matching the character with code \texttt{ddd} in octal; -% \item POSIX character classes |[:alpha:]| \emph{etc.}; -% \item Unicode properties: |\p{..}| and |\P{..}|; -% \item |\X| which should match any \enquote{extended} Unicode sequence; -% \item Callout with |(?C...)|; +% \item POSIX character classes |[:alpha:]| \emph{etc.}, this is redundant; +% \item Callout with |(?C...)|, we cannot run arbitrary user code during +% the matching, because the regex code uses registers in an unsafe way; % \item Conditional subpatterns (other than with a look-ahead % or look-behind condition): this is non-regular, isn't it? % \end{itemize} % % The following features of PCRE or perl will definitely not be implemented. % \begin{itemize} +% \item |\cx|, similar to \TeX{}'s own |\^^x|; % \item Comments: \TeX{} already has its own system for comments. % \item Named subpatterns: \TeX{} programmers have lived so far without % any need for named macro parameters. @@ -495,7 +612,7 @@ % \begin{macrocode} \ProvidesExplPackage {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription} -\RequirePackage{l3str} +\RequirePackage{l3str, l3tl-analysis, l3flag} % \end{macrocode} % % Most regex engines use backtracking. This allows to provide very @@ -506,8 +623,8 @@ % % Given a regular expression of $n$ characters, we build a % non-deterministic finite automaton (NFA) with roughly $n$ states, -% which accepts precisely those strings matching that regular expression. -% We then run the string through the NFA, and check the return value. +% which accepts precisely those token lists matching that regular expression. +% We then run the token list through the NFA, and check the return value. % % The code is structured as follows. Various helper functions are % introduced in the next subsection, to limit the clutter in later @@ -520,14 +637,14 @@ % \subsection{Constants and variables} % % \begin{macro}{\regex_tmp:w} -% \begin{variable}{\g_regex_tmpa_tl, \l_regex_tmpa_tl, \l_regex_tmpb_tl} +% \begin{variable}{\l_regex_tmpa_tl, \l_regex_tmpb_tl, \l_regex_tmpc_tl} % \begin{variable}{\l_regex_tmpa_int, \l_regex_tmpb_int} % Temporary variables. % \begin{macrocode} \cs_new:Npn \regex_tmp:w { } -\tl_new:N \g_regex_tmpa_tl \tl_new:N \l_regex_tmpa_tl \tl_new:N \l_regex_tmpb_tl +\tl_new:N \l_regex_tmpc_tl \int_new:N \l_regex_tmpa_int \int_new:N \l_regex_tmpb_int % \end{macrocode} @@ -535,14 +652,27 @@ % \end{variable} % \end{macro} % +% \begin{variable}{\l_regex_group_begin_flag} +% \begin{variable}{\l_regex_group_end_flag} +% Those flags are raised to indicate extra begin-group +% or end-group tokens when extracting submatches. +% \begin{macrocode} +\flag_new:N \l_regex_group_begin_flag +\flag_new:N \l_regex_group_end_flag +% \end{macrocode} +% \end{variable} +% \end{variable} +% % \subsubsection{Variables used while building} % % \begin{variable}{\l_regex_max_state_int} % \begin{variable}{\l_regex_left_state_int, \l_regex_right_state_int} -% The last state that was allocated is stored in \cs{l_regex_max_state_int}, -% and \cs{l_regex_left/right_state_int} point to both end-points of the -% last group (which any quantifier would repeat). For simple strings of -% characters, the left and right pointers only differ by one. +% The end-point states of the last group (which any quantifier +% would repeat) are stored as \cs{l_regex_left/right_state_int}. +% For simple strings of characters, the left and right pointers +% only differ by one. +% The last state that was allocated is $\cs{l_regex_max_state_int}-1$, +% so that \cs{l_regex_max_state_int} always points to a free state. % \begin{macrocode} \int_new:N \l_regex_max_state_int \int_new:N \l_regex_left_state_int @@ -593,6 +723,16 @@ % \end{variable} % \end{variable} % +% \begin{variable}{\l_regex_build_or_class_tl} +% The actions of various escape sequences and special characters +% are different within a class and outside a class. Rather than +% using a boolean, we store either \texttt{build} or \texttt{class} +% in this token list. +% \begin{macrocode} +\tl_new:N \l_regex_build_or_class_tl +% \end{macrocode} +% \end{variable} +% % \begin{variable}{\l_regex_one_or_group_tl} % When looking for quantifiers, this variable holds either % \enquote{one} or \enquote{group} depending on whether the @@ -603,6 +743,22 @@ % \end{macrocode} % \end{variable} % +% \begin{variable}{\l_regex_in_csname_bool} +% This boolean is true when building the regular expression +% for a given csname. +% \begin{macrocode} +\bool_new:N \l_regex_in_csname_bool +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_regex_tmpa_regex_tl} +% This holds a temporary pre-compiled regular expression +% when matching a control sequence name. +% \begin{macrocode} +\tl_new:N \l_regex_tmpa_regex_tl +% \end{macrocode} +% \end{variable} +% % \subsubsection{Character classes} % % \begin{macro}{\regex_build_tmp_class:n} @@ -710,19 +866,37 @@ % % \subsubsection{Variables used when matching} % -% \begin{variable}{\l_regex_query_other_str} -% The string that is being matched. -% In that string, spaces have category code \enquote{other}. +% \begin{variable}{\l_regex_nesting_int} +% This integer is used to keep track of begin and end-group tokens. +% \begin{macrocode} +\int_new:N \l_regex_nesting_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_regex_min_step_int} +% \begin{variable}{\l_regex_max_step_int} +% This integer holds the value of the step corresponding to +% the left end of the token list, \emph{i.e.}, when the token +% list is stored in \tn{toks} registers, the \tn{toks} register +% \cs{l_regex_min_step_int} holds the first token in the token +% list. In fact, this number is always one more than +% \cs{l_regex_max_state_int}, but it is more practical to give +% that a name. % \begin{macrocode} -\tl_new:N \l_regex_query_other_str +\int_new:N \l_regex_min_step_int +\int_new:N \l_regex_max_step_int % \end{macrocode} % \end{variable} +% \end{variable} % % \begin{variable}{\l_regex_current_step_int} % \begin{variable}{\l_regex_start_step_int} % \begin{variable}{\l_regex_success_step_int} -% While reading through the query string, \cs{l_regex_current_step_int} -% is the position in the string. Each match begins at the position +% While reading through the query token list, +% \cs{l_regex_current_step_int} is the position in the +% token list, starting at \cs{l_regex_min_step_int} for +% the left-most token. +% Each match begins at the position % given by \cs{l_regex_start_step_int}. Whenever an execution thread % succeeds, the corresponding step is stored into % \cs{l_regex_success_step_int}, which will be the next starting step @@ -736,24 +910,26 @@ % \end{variable} % \end{variable} % -% \begin{variable}{\l_regex_unique_step_int} +% \begin{variable}{\l_regex_unique_id_int} % In the case of repeated matches, \cs{l_regex_current_step_int} % is reset to the end-position of the previous match. In contrast, -% \cs{l_regex_unique_step_int} is simply incremented to provide +% \cs{l_regex_unique_id_int} is simply incremented to provide % a unique number for each iteration of the matching loop. This % is handy to attach each set of submatch information to a given % iteration (and automatically discard it when it corresponds to % a past iteration). % \begin{macrocode} -\int_new:N \l_regex_unique_step_int +\int_new:N \l_regex_unique_id_int % \end{macrocode} % \end{variable} % % \begin{variable}{\l_regex_current_char_int} +% \begin{variable}{\l_regex_current_catcode_int} +% \begin{variable}{\l_regex_current_token_tl} % \begin{variable}{\l_regex_last_char_int} % \begin{variable}{\l_regex_case_changed_char_int} % The character codes of the character at the current position -% in the string, and at the previous position, and the current +% in the token list, and at the previous position, and the current % character with its case changed (|A-Z|$\leftrightarrow$|a-z|). % The \cs{l_regex_last_char_int} is used to test for word boundaries % (|\b| and |\B|). The \cs{l_regex_case_changed_char_int} is @@ -761,12 +937,16 @@ % is used in the regex. % \begin{macrocode} \int_new:N \l_regex_current_char_int +\int_new:N \l_regex_current_catcode_int +\tl_new:N \l_regex_current_token_tl \int_new:N \l_regex_last_char_int \int_new:N \l_regex_case_changed_char_int % \end{macrocode} % \end{variable} % \end{variable} % \end{variable} +% \end{variable} +% \end{variable} % % \begin{variable}{\l_regex_caseless_bool} % True if caseless matching is used within the regular expression. @@ -777,7 +957,7 @@ % \end{variable} % % \begin{variable}{\l_regex_current_state_int} -% For every character in the string, each of the active states is +% For every character in the token list, each of the active states is % considered in turn. % The variable \cs{l_regex_current_state_int} holds the state % of the NFA which is currently considered: transitions are then @@ -823,26 +1003,38 @@ % \begin{macro}[int]{\l_regex_every_match_tl} % Every time a match is found, this token list is used. % For single matching, the token list is set to removing -% the remainder of the query string. For multiple matching, +% the remainder of the query token list. For multiple matching, % the token list is set to repeat the matching. % \begin{macrocode} \tl_new:N \l_regex_every_match_tl % \end{macrocode} % \end{macro} % -% \begin{variable}{\l_regex_success_bool} -% The boolean \cs{l_regex_success_bool} is true if the current match -% attempt was successful. +% \begin{variable}{\g_regex_success_bool} +% \begin{variable}{\l_regex_saved_success_bool} +% \begin{variable}{\l_regex_success_match_bool} +% The boolean \cs{g_regex_success_bool} is true if there was +% at least one match, and \cs{l_regex_success_match_bool} is +% true if the current match attempt was successful. +% The variable \cs{g_regex_success_bool} is the only global +% variable in this whole module. When nesting \cs{regex} +% functions internally, the value of \cs{g_regex_success_bool} +% is saved into \cs{l_regex_saved_success_bool}, which is local, +% hence not affected by the changes due to inner regex functions. % \begin{macrocode} -\bool_new:N \l_regex_success_bool +\bool_new:N \g_regex_success_bool +\bool_new:N \l_regex_saved_success_bool +\bool_new:N \l_regex_success_match_bool % \end{macrocode} % \end{variable} +% \end{variable} +% \end{variable} % % \begin{macro}{\regex_last_match_empty:F} % \begin{macro}[aux]{\regex_last_match_empty_no:F} % \begin{macro}[aux]{\regex_last_match_empty_yes:F} % When doing multiple matches, we need to avoid infinite loops where -% each iteration matches the same empty string. When we detect such +% each iteration matches the same empty token list. When we detect such % a situation, the next match attempt is shifted by one character. % Namely, an empty match is discarded if it follows an empty match % at the same position. If the previous match was non-empty, @@ -876,86 +1068,61 @@ % \end{variable} % \end{variable} % -% \subsubsection{Variables used for user functions} +% \subsubsection{Variables used when building the replacement} % -% \begin{variable}{\g_regex_submatches_seq} -% This holds temporarily a sequence of submatches, -% global so that it exits the group. +% \begin{variable}{\l_regex_replacement_int} % \begin{macrocode} -\seq_new:N \g_regex_submatches_seq +\int_new:N \l_regex_replacement_int % \end{macrocode} % \end{variable} % -% \begin{variable}{\g_regex_match_count_int} -% The number of matches found so far is stored -% in \cs{g_regex_match_count_int}. This is only used -% in the \cs{regex_count:nnN} functions. +% \begin{variable}{\l_regex_replacement_csnames_int} % \begin{macrocode} -\int_new:N \g_regex_match_count_int +\int_new:N \l_regex_replacement_csnames_int % \end{macrocode} % \end{variable} % -% \begin{variable}{\g_regex_extract_all_seq} -% The \cs{regex_extract_all:nnN} function stores its result -% in that sequence variable before assigning it to the -% variable provided by the user. +% \begin{variable}{\l_regex_submatch_int} +% \begin{variable}{\l_regex_submatch_start_int} % \begin{macrocode} -\seq_new:N \g_regex_extract_all_seq +\int_new:N \l_regex_submatch_int +\int_new:N \l_regex_submatch_start_int % \end{macrocode} % \end{variable} -% -% \begin{variable}{\g_regex_split_seq} -% The \cs{regex_split:nnN} function stores its result -% in that sequence variable before assigning it to the -% variable provided by the user. -% \begin{macrocode} -\seq_new:N \g_regex_split_seq -% \end{macrocode} % \end{variable} % -% \begin{variable}{\l_regex_replacement_tl} -% \begin{variable}{\g_regex_replaced_str} -% The replacement code stores a processed version -% of the user's argument in \cs{l_regex_replacement_tl}. -% The result of the replacement is stored in \cs{g_regex_replaced_str}, -% global to exit the group. +% \subsubsection{Variables used for user functions} +% +% \begin{variable}{\l_regex_match_count_int} +% The number of matches found so far is stored +% in \cs{l_regex_match_count_int}. This is only used +% in the \cs{regex_count:nnN} functions. % \begin{macrocode} -\tl_new:N \l_regex_replacement_tl -\tl_new:N \g_regex_replaced_str +\int_new:N \l_regex_match_count_int % \end{macrocode} % \end{variable} -% \end{variable} % % \subsection{Helpers} % -% \subsubsection{Scan markers} -% -% \begin{macro}[int]{\s_regex_stop} -% Unexpandable markers, which does nothing if it reaches \TeX{}'s stomach. -% When performing the matching, the \tn{toks} registers hold submatch -% information, followed by the instruction for a given state of the NFA. -% The two parts are separated by \cs{s_regex_stop}. -% See \cs{regex_nfa:Nw} for another use. -% \begin{macrocode} -\cs_new_eq:NN \s_regex_stop \scan_stop: -% \end{macrocode} -% \end{macro} -% % \subsubsection{Toks} % +% When performing the matching, the \tn{toks} registers hold submatch +% information, followed by the instruction for a given state of the NFA. +% The two parts are separated by \cs{s_stop}. +% % \begin{macro}[int]{\regex_toks_put_left:Nx} % \begin{macro}[int]{\regex_toks_put_right:Nx} % During the building phase, every \tn{toks} register starts with -% \cs{s_regex_stop}, and we wish to add \texttt{x}-expanded material +% \cs{s_stop}, and we wish to add \texttt{x}-expanded material % to those registers. The expansion is done \enquote{by hand} for % optimization (these operations are used quite a lot). When adding % material to the left, we define \cs{regex_tmp:w} to remove the -% \cs{s_regex_stop} marker and put it back to the left of the new +% \cs{s_stop} marker and put it back to the left of the new % material. % \begin{macrocode} \cs_new_protected:Npn \regex_toks_put_left:Nx #1#2 { - \cs_set_nopar:Npx \regex_tmp:w \s_regex_stop { \s_regex_stop #2 } + \cs_set_nopar:Npx \regex_tmp:w \s_stop { \s_stop #2 } \tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN { \exp_after:wN \regex_tmp:w \tex_the:D \tex_toks:D #1 } } @@ -969,30 +1136,37 @@ % \end{macro} % \end{macro} % -% \subsubsection{Unsafe substring extraction} +% \subsubsection{Extracting parts of the query token list} % -% \begin{macro}[int]{\regex_query_substr:NN} -% \begin{macro}[aux]{\regex_query_substr_aux:nnn} -% Extracting submatches from the query string is done -% in a way similar to \cs{str_substr:Nnn}, but omitting -% most checks, hence faster. +% \begin{macro}[int]{\regex_query_substr:nn} +% \begin{macro}[int]{\regex_query_submatch:nn} +% \begin{macro}[int]{\regex_query_submatch:w} +% The token list is stored in \tn{toks} registers: +% the first is \cs{l_regex_min_step_int}, the last is (we don't care yet). % \begin{macrocode} -\cs_new_nopar:Npn \regex_query_substr:NN #1#2 +\cs_new:Npn \regex_query_substr:nn #1#2 { - \exp_after:wN \regex_query_substr_aux:NN - \exp_after:wN #1 - \exp_after:wN #2 - \l_regex_query_other_str - \q_stop + \if_num:w \int_eval:w #1 < \int_eval:w #2 \int_eval_end: + \str_aux_toks_range:nn {#1} {#2} + \fi: + } +\cs_new:Npn \regex_query_submatch:nn #1#2 + { + \if_num:w #1 < \l_regex_capturing_group_int + \exp_after:wN \regex_query_submatch:w + \int_use:N \int_eval:w #1 + #2 ; + \fi: } -\cs_new:Npn \regex_query_substr_aux:NN #1#2 +\cs_new:Npn \regex_query_submatch:w #1 ; { - \str_skip_do:nn {#1} - { \str_collect_do:nn { #2 - #1 } { \use_i_delimit_by_q_stop:nw } } + \regex_query_substr:nn + { \tex_skip:D #1 } + { \etex_gluestretch:D \tex_skip:D #1 } } % \end{macrocode} % \end{macro} % \end{macro} +% \end{macro} % % \subsubsection{Sequences} % @@ -1042,20 +1216,46 @@ % \end{macro} % \end{macro} % +% \begin{macro}[int, EXP]{\regex_seq_item:Nn} +% \begin{macro}[aux, EXP]{\regex_seq_item_aux:wNn} +% \begin{macrocode} +\cs_new:Npn \regex_seq_item:Nn #1#2 + { + \exp_after:wN \regex_seq_item_aux:wNn + \int_use:N \int_eval:w #2 \exp_after:wN ; + #1 { ? \prg_map_break: } { } + \prg_break_point:n { } + } +\cs_new:Npn \regex_seq_item_aux:wNn #1; #2#3 + { + \use_none:n #2 + \if_num:w #1 = \c_zero + \exp_after:wN \prg_map_break:n + \else: + \exp_after:wN \use_none:n + \fi: + { \exp_not:n {#3} } + \exp_after:wN \regex_seq_item_aux:wNn + \int_use:N \int_eval:w #1 - \c_one ; + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% % \subsubsection{Testing characters} % % \begin{macro}{\regex_item_dot:T} % The dot meta-character matches any character, -% except the end-of-string marker. +% except the end marker. % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_item_dot:T - { \int_compare:nNnF \l_regex_current_char_int = \c_minus_one } + { \int_compare:nNnF \l_regex_current_char_int < \c_minus_one } % \end{macrocode} % \end{macro} % % \begin{macro}[int]{\regex_break_point:TF} % \begin{macro}[int]{\regex_break_true:w,\regex_break_false:w} -% When testing whether a character of the query string matches +% When testing whether a character of the query token list matches % a given character class in the regular expression, we often % have to test it against several ranges of characters, checking % if any one of those matches. This is done with a structure like @@ -1187,6 +1387,56 @@ % \end{macrocode} % \end{macro} % +% \begin{macro}[int]{\regex_item_catcode:n} +% \begin{macro}[int]{\regex_item_catcode_neq:n} +% Checking the category code is quite straightforward. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \regex_item_catcode:n #1 + { + \if_num:w #1 = \l_regex_current_catcode_int + \exp_after:wN \regex_break_true:w + \fi: + } +\cs_new_protected_nopar:Npn \regex_item_catcode_neq:n #1 + { + \if_num:w #1 = \l_regex_current_catcode_int + \else: + \exp_after:wN \regex_break_true:w + \fi: + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int]{\regex_item_cs:n} +% Match a control sequence (the argument is a pre-compiled regex). +% First test the catcode of the current token to be zero. +% Then perform the matching test, and break if the csname +% indeed matches. The three \cs{exp_after:wN} expand the contents +% of \cs{l_regex_current_token_tl} (of the form \cs{exp_not:n} +% \Arg{control sequence}) to \meta{control sequence}. +% \begin{macrocode} +\cs_new_protected:Npn \regex_item_cs:n #1 + { + \int_compare:nNnT \l_regex_current_catcode_int = \c_zero + { + \tl_set:Nn \l_regex_tmpa_regex_tl {#1} + \bool_set_eq:NN \l_regex_saved_success_bool \g_regex_success_bool + \exp_args:NNx \regex_match:NnTF \l_regex_tmpa_regex_tl + { + \exp_after:wN \exp_after:wN + \exp_after:wN \cs_to_str:N \l_regex_current_token_tl + } + { + \bool_gset_eq:NN \g_regex_success_bool \l_regex_saved_success_bool + \regex_break_true:w + } + { \bool_gset_eq:NN \g_regex_success_bool \l_regex_saved_success_bool } + } + } +% \end{macrocode} +% \end{macro} +% % \subsubsection{Grabbing digits} % % \begin{macro}[int]{\regex_get_digits:nw} @@ -1323,19 +1573,15 @@ % \begin{macro}[int]{\regex_build_new_state:} % Here, we add a new state to the NFA. At the end of the building % phase, we want every \tn{toks} register to start with -% \cs{s_regex_stop}, hence initialize the new register appropriately. +% \cs{s_stop}, hence initialize the new register appropriately. % Then set \cs{l_regex_left/right_state_int} to their new values. % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_build_new_state: { - \int_compare:nNnTF \l_regex_max_state_int > { 32766 } - { \msg_error:nn { regex } { -997 } } - { - \int_incr:N \l_regex_max_state_int - \tex_toks:D \l_regex_max_state_int { \s_regex_stop } - } + \tex_toks:D \l_regex_max_state_int { \s_stop } \int_set_eq:NN \l_regex_left_state_int \l_regex_right_state_int \int_set_eq:NN \l_regex_right_state_int \l_regex_max_state_int + \int_incr:N \l_regex_max_state_int } % \end{macrocode} % \end{macro} @@ -1377,7 +1623,9 @@ % First, reset a few variables. Then use the generic framework defined % in \pkg{l3str} to parse the regular expression once, recognizing % which characters are raw characters, and which have special meanings. -% The result is stored in \cs{g_str_result_tl}, and can be run directly. +% The search is not anchored: to achieve that, we insert state(s) +% responsible for repeating the match attempt on every token +% in the token list. % The trailing \cs{prg_do_nothing:} ensure that the look-ahead done by % some of the operations is harmless. % Finally, \cs{regex_build_end:} adds the finishing code @@ -1385,16 +1633,18 @@ % \begin{macrocode} \cs_new_protected:Npn \regex_build:n #1 { - \regex_build_setup: - \str_aux_escape:NNNn - \regex_build_i_unescaped:N - \regex_build_i_escaped:N - \regex_build_i_raw:N - { #1 } + \bool_set_false:N \l_regex_in_csname_bool + \regex_build:w + \regex_build_new_state: + \regex_toks_put_right:Nx \l_regex_left_state_int + { \regex_action_start_wildcard:nn {0} {1} } \regex_build_open_aux: - \g_str_result_tl \prg_do_nothing: \prg_do_nothing: - \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_zero - \regex_build_close_aux: \regex_build_group_: + \str_escape_use:NNNn + \regex_build_i_unescaped:N + \regex_build_i_escaped:N + \regex_build_i_raw:N + { #1 } + \prg_do_nothing: \prg_do_nothing: \regex_build_end: } % \end{macrocode} @@ -1403,7 +1653,7 @@ % \begin{macro}[aux]{\regex_build_i_unescaped:N} % \begin{macro}[aux]{\regex_build_i_escaped:N} % \begin{macro}[aux]{\regex_build_i_raw:N} -% The \pkg{l3str} function \cs{str_aux_escape:NNNn} goes through +% The \pkg{l3str} function \cs{str_escape_use:NNNn} goes through % the regular expression and finds the |\a|, |\e|, |\f|, |\n|, |\r|, % |\t|, and |\x| escape sequences, then distinguishes three cases: % non-escaped characters, escaped characters, and \enquote{raw} @@ -1432,35 +1682,36 @@ % \end{macro} % \end{macro} % -% \begin{macro}[aux]{\regex_build_default_control:N} -% If the control character has a particular meaning in regular expressions, -% the corresponding function is used. Otherwise, it is interpreted as a raw -% character. The \cs{regex_build_default_raw:N} function is defined later. +% \begin{macro}[aux]{\regex_build_control:N} +% If the control character has a particular meaning in regexes, +% the corresponding function is used. Otherwise, it is interpreted +% as a raw character. The \cs{regex_build/class_default_raw:N} +% functions are defined later. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_default_control:N #1 +\cs_new_protected_nopar:Npn \regex_build_control:N #1 { - \cs_if_exist_use:cF { regex_build_#1: } - { \regex_build_default_raw:N #1 } + \cs_if_exist_use:cF { regex_ \l_regex_build_or_class_tl _#1: } + { \use:c { regex_ \l_regex_build_or_class_tl _default_raw:N } #1 } } % \end{macrocode} % \end{macro} % -% \begin{macro}[int]{\regex_build_setup:} +%^^A todo: update +% This function is similar to \cs{regex_build_control:N}. If the control +% character has a meaning in character classes, call the corresponding +% function, otherwise, treat it as a raw character, with the +% \cs{regex_class_default_raw:N} function, defined later. +% +% \begin{macro}[int]{\regex_build:w} % Hopefully, we didn't forget to initialize anything here. -% The search is not anchored: to achieve that, we insert state(s) -% responsible for repeating the match attempt on every character -% of the string. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_setup: +\cs_new_protected_nopar:Npn \regex_build:w { - \cs_set_eq:NN \regex_build_control:N \regex_build_default_control:N \cs_set_eq:NN \regex_build_raw:N \regex_build_default_raw:N + \tl_set:Nn \l_regex_build_or_class_tl { build } \int_set_eq:NN \l_regex_capturing_group_int \c_zero \int_zero:N \l_regex_max_state_int \regex_build_new_state: - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int - { \regex_action_start_wildcard:nn {0} {1} } } % \end{macrocode} % \end{macro} @@ -1472,15 +1723,19 @@ % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_build_end: { + \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_zero + \regex_build_close_aux: \regex_build_group_: \seq_if_empty:NF \l_regex_capturing_group_seq { - \msg_error:nn { regex } {22} + \msg_kernel_error:nnx { regex } { missing-rparen } + { \seq_length:N \l_regex_capturing_group_seq } \prg_replicate:nn { \seq_length:N \l_regex_capturing_group_seq } { \regex_build_close_aux: \regex_build_group_: } } \regex_toks_put_right:Nx \l_regex_right_state_int { \regex_action_success: } + \int_incr:N \l_regex_capturing_group_int } % \end{macrocode} % \end{macro} @@ -1492,14 +1747,14 @@ % \begin{macro}[int]{\regex_build_G:} % \begin{macro}[aux]{\regex_build_anchor_start:N} % Anchoring at the start corresponds to checking that the current -% character is the first in the string. Anchoring to the beginning +% character is the first in the token list. Anchoring to the beginning % of the match attempt uses \cs{l_regex_start_step_int} instead of % \cs{c_zero}. % \begin{macrocode} \cs_new_protected_nopar:cpn { regex_build_^: } - { \regex_build_anchor_start:N \c_zero } + { \regex_build_anchor_start:N \l_regex_min_step_int } \cs_new_protected_nopar:Npn \regex_build_A: - { \regex_build_anchor_start:N \c_zero } + { \regex_build_anchor_start:N \l_regex_min_step_int } \cs_new_protected_nopar:Npn \regex_build_G: { \regex_build_anchor_start:N \l_regex_start_step_int } \cs_new_protected_nopar:Npn \regex_build_anchor_start:N #1 @@ -1507,7 +1762,7 @@ \regex_build_new_state: \regex_toks_put_right:Nx \l_regex_left_state_int { - \exp_not:N \int_compare:nNnT {#1} = \l_regex_current_step_int + \exp_not:N \int_compare:nNnT #1 = \l_regex_current_step_int { \regex_action_free:n { @@ -1526,7 +1781,7 @@ % \begin{macro}[aux]{\regex_build_Z:} % \begin{macro}[aux]{\regex_build_z:} % \begin{macro}[aux]+\regex_build_$:+ -% This matches the end of the string, marked by a character code of $-1$. +% This matches the end of the token list, marked by a character code of $-2$. % \begin{macrocode} \cs_new_protected_nopar:cpn { regex_build_$: } % $ { @@ -1534,7 +1789,7 @@ \regex_toks_put_right:Nx \l_regex_left_state_int { \exp_not:N \int_compare:nNnT - \c_minus_one = \l_regex_current_char_int + \l_regex_current_char_int < \c_minus_one { \regex_action_free:n { @@ -1555,7 +1810,7 @@ % \begin{macro}[int]{\regex_build_B:} % \begin{macro}[aux]{\regex_if_word_boundary:TF} % Contrarily to |^| and |$|, which could be implemented without -% really knowing what precedes in the string, this requires +% really knowing what precedes in the token list, this requires % more information, namely, the knowledge of the last character % code. Case sensitivity does not change word boundaries. % \begin{macrocode} @@ -1597,7 +1852,7 @@ \int_set_eq:NN \l_regex_current_char_int \l_regex_last_char_int \c_regex_w_tl \regex_break_point:TF - { \group_end: \c_regex_W_tl \regex_item_equal:n { -1 } } + { \group_end: \c_regex_W_tl \regex_item_equal:n { -2 } } { \group_end: \c_regex_w_tl } \regex_break_point:TF } @@ -1608,6 +1863,23 @@ % % \subsubsection{Normal character, and simple character classes} % +% \begin{macro}[int]{\regex_build_one:n, \regex_build_one:x} +% +% \begin{macrocode} +\cs_new_protected:Npn \regex_build_one:n #1 + { \regex_build_one:x { \exp_not:n {#1} } } +\cs_new_protected:Npn \regex_build_one:x #1 + { + \cs_set_nopar:Npx \regex_build_tmp_class:n ##1 + { + \exp_not:N \exp_not:n {#1} + \regex_break_point:TF { \regex_action_cost:n {##1} } { } + } + \regex_build_one_quantifier: + } +% \end{macrocode} +% \end{macro} +% % \begin{macro}[aux]{\regex_build_default_raw:N} % A normal alphanumeric or an escaped non-alphanumeric % (actually, any unknown combination) will match itself @@ -1618,20 +1890,13 @@ % yet what this will be before checking for quantifiers. % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_build_default_raw:N #1 - { - \cs_set:Npx \regex_build_tmp_class:n ##1 - { - \regex_item_equal:n { \int_value:w `#1 ~ } - \regex_break_point:TF { \regex_action_cost:n { ##1 } } { } - } - \regex_build_one_quantifier: - } + { \regex_build_one:x { \regex_item_equal:n { \int_value:w `#1 ~ } } } % \end{macrocode} % \end{macro} % % \begin{macro}[aux]{\regex_build_.:} % Similar to \cs{regex_build_default_raw:N} but accepts any character, -% and refuses $-1$, which marks the end of the string. +% and refuses $-1$, which marks the end of the token list. % \begin{macrocode} \cs_new_protected_nopar:cpn { regex_build_.: } { @@ -1642,34 +1907,21 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_d:,\regex_build_D} -% \begin{macro}[aux]{\regex_build_h:,\regex_build_H} -% \begin{macro}[aux]{\regex_build_s:,\regex_build_S} -% \begin{macro}[aux]{\regex_build_v:,\regex_build_V} -% \begin{macro}[aux]{\regex_build_w:,\regex_build_W} +% \begin{macro}[aux]{\regex_build_d:,\regex_build_D:} +% \begin{macro}[aux]{\regex_build_h:,\regex_build_H:} +% \begin{macro}[aux]{\regex_build_s:,\regex_build_S:} +% \begin{macro}[aux]{\regex_build_v:,\regex_build_V:} +% \begin{macro}[aux]{\regex_build_w:,\regex_build_W:} % \begin{macro}[aux]{\regex_build_N:} -% \begin{macro}[aux]{\regex_build_char_type:N} % The constants \cs{c_regex_d_tl}, \emph{etc.} hold % a list of tests which match the corresponding character % class, and jump to the \cs{regex_break_point:TF} marker. % As for a normal character, we check for quantifiers. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_char_type:N #1 - { - \cs_set:Npn \regex_build_tmp_class:n ##1 - { - \exp_not:N #1 - \regex_break_point:TF { \regex_action_cost:n {##1} } { } - } - \regex_build_one_quantifier: - } \tl_map_inline:nn { dDhHsSvVwWN } { \cs_new_protected_nopar:cpx { regex_build_#1: } - { - \exp_not:N \regex_build_char_type:N - \exp_not:c { c_regex_#1_tl } - } + { \regex_build_one:n \exp_not:c { c_regex_#1_tl } } } % \end{macrocode} % \end{macro} @@ -1678,10 +1930,20 @@ % \end{macro} % \end{macro} % \end{macro} -% \end{macro} % % \subsubsection{Character classes} % +% \begin{macro}[aux]{\regex_class_one:n, \regex_class_one:x} +% This is the function used to add material +% to the class. +% \begin{macrocode} +\cs_new_protected:Npn \regex_class_one:n + { \tl_put_right:Nn \l_regex_class_tl } +\cs_new_protected:Npn \regex_class_one:x + { \tl_put_right:Nx \l_regex_class_tl } +% \end{macrocode} +% \end{macro} +% % \begin{macro}[aux]{\regex_build_[:} % This starts a class. The code for the class is collected % in \cs{l_regex_class_tl}. The first character is special. @@ -1689,41 +1951,26 @@ \cs_new_protected_nopar:cpn { regex_build_[: } { \tl_clear:N \l_regex_class_tl - \cs_set_eq:NN \regex_build_control:N \regex_class_control:N - \cs_set_eq:NN \regex_build_raw:N \regex_class_raw:N + \cs_set_eq:NN \regex_build_raw:N \regex_class_default_raw:N + \tl_set:Nn \l_regex_build_or_class_tl { class } \regex_class_first:NN } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_class_control:N} -% This function is similar to \cs{regex_build_control:N}. If the control -% character has a meaning in character classes, call the corresponding -% function, otherwise, treat it as a raw character, with the -% \cs{regex_class_raw:N} function, defined later. -% \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_class_control:N #1 - { - \cs_if_exist_use:cF { regex_class_#1: } - { \regex_class_raw:N #1 } - } -% \end{macrocode} -% \end{macro} -% % \begin{macro}[aux]{\regex_class_]:} % If \texttt{]} appears as the first item of a class, then % it doesn't end the class. Otherwise, it's the end, % act just as for a single character, but with a more -% complicated test. And restore \cs{regex_build_control:N} -% and \cs{regex_build_raw:N}. +% complicated test. And restore \cs{regex_build_raw:N}. % \begin{macrocode} \cs_new_protected_nopar:cpn { regex_class_]: } { \tl_if_empty:NTF \l_regex_class_tl %[ - { \regex_class_raw:N ] } + { \regex_class_default_raw:N ] } { - \cs_set_eq:NN \regex_build_control:N \regex_build_default_control:N \cs_set_eq:NN \regex_build_raw:N \regex_build_default_raw:N + \tl_set:Nn \l_regex_build_or_class_tl { build } \cs_set:Npn \regex_build_tmp_class:n ##1 { \exp_not:o \l_regex_class_tl @@ -1753,12 +2000,12 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_class_raw:N} +% \begin{macro}[aux]{\regex_class_default_raw:N} % \begin{macro}[aux]{\regex_class_single:N} % Most characters are treated here. We look ahead for an unescaped dash. % If there is none, then the character matches itself. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_class_raw:N #1#2#3 +\cs_new_protected_nopar:Npn \regex_class_default_raw:N #1#2#3 { \str_if_eq:nnTF {#2#3} { \regex_build_control:N - } { \regex_class_range:Nw #1 } @@ -1768,10 +2015,7 @@ } } \cs_new_protected_nopar:Npn \regex_class_single:N #1 - { - \tl_put_right:Nx \l_regex_class_tl - { \regex_item_equal:n { \int_value:w `#1 } } - } + { \regex_class_one:x { \regex_item_equal:n { \int_value:w `#1 } } } % \end{macrocode} % \end{macro} % \end{macro} @@ -1804,9 +2048,9 @@ \cs_new_protected_nopar:Npn \regex_class_range_put:NN #1#2 { \if_num:w `#1 > `#2 \exp_stop_f: - \msg_error:nn { regex } {8} + \msg_kernel_error:nnxx { regex } { backwards-range } {#1} {#2} \else: - \tl_put_right:Nx \l_regex_class_tl + \regex_class_one:x { \if_num:w `#1 = `#2 \exp_stop_f: \regex_item_equal:n @@ -1834,9 +2078,59 @@ \tl_map_inline:nn { dDhHsSvVwWN } { \cs_new_protected_nopar:cpx { regex_class_#1: } + { \regex_class_one:n \exp_not:c { c_regex_#1_tl } } + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \subsubsection{Catcodes and csnames} +% +% \begin{macro}[aux]{\regex_build_c:} +% \begin{macro}[aux]{\regex_build_C:} +% \begin{macro}[aux]{\regex_class_c:} +% \begin{macro}[aux]{\regex_class_C:} +% \begin{macro}[aux]{\regex_aux_c:nnNN} +% +% \begin{macrocode} +\cs_new_protected_nopar:Npn \regex_build_c: + { \regex_aux_c:NNNN \regex_item_catcode:n \use_i:nn } +\cs_new_protected_nopar:Npn \regex_build_C: + { \regex_aux_c:NNNN \regex_item_catcode_neq:n \use_ii:nn } +\cs_new_eq:NN \regex_class_c: \regex_build_c: +\cs_new_eq:NN \regex_class_C: \regex_build_C: +\cs_new_protected:Npn \regex_aux_c:NNNN #1#2#3#4 + { + \token_if_eq_meaning:NNTF #3 \regex_build_raw:N + { + \regex_aux_c_test:NTF #4 + { + \use:c { regex_ \l_regex_build_or_class_tl _one:x } + { #1 { \int_use:N \l_regex_current_catcode_int } } + } + { + \msg_kernel_error:nnx { regex } { build-c } { #4 } + #3 #4 + } + } { - \tl_put_right:Nn \exp_not:N \l_regex_class_tl - { \exp_not:c { c_regex_#1_tl } } + \if_charcode:w \c_lbrace_str \exp_not:N #4 + \exp_after:wN #2 + \else: + \exp_after:wN \use_ii:nn + \fi: + { + \group_begin: + \bool_set_true:N \l_regex_in_csname_bool + \regex_build:w + } + { + \msg_kernel_error:nnx { regex } { build-c } { #4 } + #3 #4 + } } } % \end{macrocode} @@ -1846,6 +2140,81 @@ % \end{macro} % \end{macro} % +% The |\c| and |\C| escape sequences are used both in the +% regex and the replacement texts. They can be followed either +% by a capital letter representing a category code, or by a left +% brace (in the case of |\c| only). +% +% \begin{macro}{\regex_aux_c_test:NTF} +% If the first argument is a known catcode identifier +% (capital letters listed below), then +% \cs{l_regex_current_catcode_int} is set to the relevant +% catcode value (between $0$ for cs and $13$ for active chars), +% and the \texttt{true} branch is taken. +% Otherwise, the \texttt{false} branch is taken, +% and the current catcode is set to $-1$. +% \begin{macrocode} +\cs_new_protected:Npn \regex_aux_c_test:NTF #1 + { + \int_set:Nn \l_regex_current_catcode_int + { + \if_case:w \int_eval:w `#1 - `A \int_eval_end: + \c_thirteen % Active + \or: \c_one % Begin-group + \or: \c_zero % Control sequence + \or: \c_eight % Down (subscript) + \or: \c_two % End-group + \else: + \if_case:w \int_eval:w `#1 - `L \int_eval_end: + \c_eleven % Letter + \or: \c_three % Math toggle + \or: \c_minus_one + \or: \c_twelve % Other + \or: \c_six % Parameter + \or: \c_minus_one + \or: \c_minus_one + \or: \c_ten % Space + \or: \c_four % Tab (alignment) + \or: \c_seven % Up (superscript) + \else: \c_minus_one + \fi: + \fi: + } + \if_num:w \l_regex_current_catcode_int = \c_minus_one + \exp_after:wN \use_ii:nn + \else: + \exp_after:wN \use_i:nn + \fi: + } +% \end{macrocode} +% \end{macro} +% +% ^^A{ +% \begin{macro}+\regex_build}:+ +% Non-escaped right braces are only special if they appear +% when building the regular expression for a csname. Otherwise, +% replace the brace with an escaped brace. +% \begin{macrocode} +\cs_new_protected:cpn { regex_build_ \c_rbrace_str : } + { + \bool_if:NTF \l_regex_in_csname_bool + { + \regex_build_Z: + \regex_toks_put_right:Nx \l_regex_right_state_int + { \regex_action_success: } + \int_incr:N \l_regex_capturing_group_int + \use:x + { + \group_end: + \exp_not:n { \use:c { regex_ \l_regex_build_or_class_tl _one:n } } + { \regex_item_cs:n { \regex_set_aux:N ? } } + } + } + { \exp_after:wN \regex_build_raw:N \c_rbrace_str } + } +% \end{macrocode} +% \end{macro} +% % \subsubsection{Quantifiers} % % \begin{macro}[int]{\regex_build_quantifier:w} @@ -1952,12 +2321,8 @@ { \tl_if_empty:nTF {#2} { \regex_build_quantifier_aux:nnNN { {#1} } { n* } } - { - \int_compare:nNnT {#1} > {#2} - { \msg_error:nn { regex } {4} } - \regex_build_quantifier_aux:nnNN { {#1} {#2} } { nn } - } - } + { \regex_build_quantifier_aux:nnNN { {#1} {#2} } { nn } } + } %^^A todo: catch the case m>n { \regex_build_quantifier_end:n { } \use:x @@ -2190,7 +2555,7 @@ \cs_new_protected_nopar:cpn { regex_build_): } { \seq_if_empty:NTF \l_regex_capturing_group_seq - { \msg_error:nn { regex } { 22 } } + { \msg_kernel_error:nn { regex } { extra-rparen } } { \regex_build_close_aux: \regex_build_group_quantifier: @@ -2249,7 +2614,8 @@ { \cs_if_exist_use:cF { regex_build_special_group_\token_to_str:N #2 : } { - \msg_error:nn { regex } { -998 } + \msg_kernel_error:nnx { regex } { unsupported } + { (? \token_to_str:N #2 } %) \regex_build_control:N ( % ) \regex_build_raw:N ? #1 #2 @@ -2433,7 +2799,7 @@ { \tex_toks:D \l_regex_tmpa_int { - \s_regex_stop + \s_stop #1 { \int_eval:n { \l_regex_right_state_int - \l_regex_tmpa_int } } } } @@ -2544,16 +2910,19 @@ \regex_toks_put_right:Nx \l_regex_right_state_int { \regex_action_free:n - { \int_eval:n { \l_regex_max_state_int - \l_regex_left_state_int } } + { + \int_eval:n %^^A todo: document why that value. + { \l_regex_max_state_int - \c_one - \l_regex_left_state_int } + } } \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int - \int_set:Nn \l_regex_tmpb_int { \l_regex_max_state_int + \c_one } + \int_set_eq:NN \l_regex_tmpb_int \l_regex_max_state_int \int_set:Nn \l_regex_max_state_int { - \l_regex_left_state_int - \c_one - + #1 * ( \l_regex_max_state_int - \l_regex_left_state_int + \c_one ) + \l_regex_left_state_int + + #1 * ( \l_regex_max_state_int - \l_regex_left_state_int ) } - \int_until_do:nNnn \l_regex_tmpb_int > \l_regex_max_state_int + \int_while_do:nNnn \l_regex_tmpb_int < \l_regex_max_state_int { \tex_toks:D \l_regex_tmpb_int = \tex_toks:D \l_regex_tmpa_int \int_incr:N \l_regex_tmpa_int @@ -2640,12 +3009,12 @@ % % The first step in matching a regular expression is to build % the corresponding NFA and store its states in the \tn{toks} -% registers. Then loop through the query string one character +% registers. Then loop through the query token list one character % (one \enquote{step}) at a time, exploring in parallel every % possible path through the NFA. We keep track of an array of % the states currently \enquote{active}. More precisely, % \tn{skip} registers hold the state numbers to be considered -% when the next character of the string is read. +% when the next token is read. % % At every step, we unpack that array of active states and % empty it. Then loop over all active states, and perform the @@ -2670,7 +3039,7 @@ % the \enquote{best} match. Execution paths through the NFA % are ordered by precedence: for instance, the regular % expression \texttt{a?} creates two paths, matching either -% an empty string or a single \texttt{a}; the path matching +% an empty token list or a single \texttt{a}; the path matching % an \texttt{a} has higher precedence. When two paths collide, % the path with the highest precedence is kept, and the other % one is discarded. The submatch information for a given path @@ -2704,6 +3073,7 @@ % % The current approach means that stretch and shrink components % of \tn{skip} registers, +%^^A todo: update: submatches' end-points are muskips. % as well as all \tn{muskip} registers are unused. It could seem that % \tn{count} registers are also free for use, but we still want to be % able to safely use integers, which are implemented as \tn{count} @@ -2737,21 +3107,20 @@ % but not by \cs{regex_state_use_with_submatches:}. % A state is free if it is not marker as taken, namely % if the corresponding \tn{dimen} register is not -% \cs{l_regex_unique_step_int} in \texttt{sp}. +% \cs{l_regex_unique_id_int} in \texttt{sp}. % The primitive conditional is ended before unpacking % the \tn{toks} register. % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_state_use_with_submatches: { \regex_state_use_aux:n { } } \cs_new_protected_nopar:Npn \regex_state_use: - { \regex_state_use_aux:n { \exp_after:wN \regex_state_use_aux_ii:w } } -\cs_new_nopar:Npn \regex_state_use_aux_ii:w #1 \s_regex_stop { } + { \regex_state_use_aux:n { \exp_after:wN \use_none_delimit_by_s_stop:w } } \cs_new_protected_nopar:Npn \regex_state_use_aux:n #1 { \if_num:w \tex_dimen:D \l_regex_current_state_int - < \l_regex_unique_step_int + < \l_regex_unique_id_int \tex_dimen:D \l_regex_current_state_int - = \l_regex_unique_step_int sp \scan_stop: + = \l_regex_unique_id_int sp \scan_stop: #1 \tex_the:D \tex_toks:D \exp_after:wN \l_regex_current_state_int \fi: \scan_stop: @@ -2799,26 +3168,26 @@ \tex_the:D \tex_toks:D #1 } } -\cs_new_protected:Npn \regex_store_submatches_aux:w #1 \s_regex_stop +\cs_new_protected:Npn \regex_store_submatches_aux:w #1 \s_stop { \regex_store_submatches_aux_ii:Nnnw #1 \regex_state_submatches:nn \c_minus_one \q_prop - \s_regex_stop + \s_stop } \cs_new_protected:Npn \regex_store_submatches_aux_ii:Nnnw - \regex_state_submatches:nn #1 #2 #3 \s_regex_stop + \regex_state_submatches:nn #1 #2 #3 \s_stop { \exp_after:wN \c_zero \exp_after:wN \regex_state_submatches:nn \exp_after:wN { \int_value:w \int_eval:w - \l_regex_unique_step_int + \c_one + \l_regex_unique_id_int + \c_one \exp_after:wN } \exp_after:wN { \l_regex_current_submatches_prop } \regex_state_submatches:nn {#1} {#2} - \s_regex_stop + \s_stop } % \end{macrocode} % \end{macro} @@ -2832,7 +3201,7 @@ % \begin{macrocode} \cs_new_protected:Npn \regex_state_submatches:nn #1#2 { - \if_num:w #1 = \l_regex_unique_step_int + \if_num:w #1 = \l_regex_unique_id_int \tl_set:Nn \l_regex_current_submatches_prop { #2 } \fi: } @@ -2842,7 +3211,7 @@ % \subsubsection{Matching: framework} % % \begin{macro}[int]{\regex_match:n} -% Store the query string in \cs{l_regex_query_other_str}. +%^^A todo: update doc. % Then reset a few variables which should be set only once, % before the first match, even in the case of multiple matches. % Then run the NFA (\cs{regex_match_once:} matches multiple times @@ -2850,70 +3219,95 @@ % \begin{macrocode} \cs_new_protected:Npn \regex_match:n #1 { - \tl_set:Nx \l_regex_query_other_str { \tl_to_other_str:n {#1} } + \tl_set_analysis:Nn \l_regex_tmpa_tl {#1} + + \int_zero:N \l_regex_nesting_int + \int_set_eq:NN \l_regex_current_step_int \l_regex_max_state_int + \regex_query_set:nnn { } { -1 } { -2 } + \int_set_eq:NN \l_regex_min_step_int \l_regex_current_step_int + \exp_after:wN \regex_query_set_loop:ww + \l_regex_tmpa_tl + \q_mark { ? \prg_map_break: } , + \prg_break_point:n { } + \int_set_eq:NN \l_regex_max_step_int \l_regex_current_step_int + \regex_query_set:nnn { } { -1 } { -2 } + \regex_match_initial_setup: \regex_match_once: } +\cs_new:Npn \regex_query_set_loop:ww #1 \q_mark #2#3 , + { + \use_none:n #2 + \regex_query_set:nnn {#1} {"#2} {#3} + \if_case:w "#2 \exp_stop_f: + \or: \int_incr:N \l_regex_nesting_int + \or: \int_decr:N \l_regex_nesting_int + \fi: + \regex_query_set_loop:ww + } +\cs_new_protected:Npn \regex_query_set:nnn #1#2#3 + { + \tex_muskip:D \l_regex_current_step_int + = \etex_gluetomu:D + #3 sp + plus #2 sp + minus \l_regex_nesting_int sp + \scan_stop: + \tex_toks:D \l_regex_current_step_int {#1} + \int_incr:N \l_regex_current_step_int + } +\cs_new_protected_nopar:Npn \regex_query_get: + { + \tl_set:Nx \l_regex_current_token_tl + { \tex_the:D \tex_toks:D \l_regex_current_step_int } + \l_regex_current_char_int + = \etex_mutoglue:D \tex_muskip:D \l_regex_current_step_int + \l_regex_current_catcode_int = \etex_gluestretch:D + \etex_mutoglue:D \tex_muskip:D \l_regex_current_step_int + } % \end{macrocode} % \end{macro} % % \begin{macro}[int]{\regex_match_once:} -% \begin{macro}[aux]{\regex_match_once_aux:} -% \begin{macro}[aux]{\regex_match_once_aux_ii:N} -% After setting up more variables in \cs{regex_match_setup:}, -% skip the \cs{l_regex_start_step_int} first characters of the -% query string, and loop over it. +% Set up more variables in \cs{regex_match_setup:}. % If there was a match, use the token list \cs{l_regex_every_match_tl}, -% which may call \cs{regex_match_once:} to achieve multiple matches. +% which may call \cs{regex_match_once:} again to achieve multiple matches. % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_match_once: { \regex_match_setup: - \exp_after:wN \regex_match_once_aux: \l_regex_query_other_str - \q_recursion_tail \q_recursion_stop - \bool_if:NT \l_regex_success_bool { \l_regex_every_match_tl } - } -\cs_new_protected_nopar:Npn \regex_match_once_aux: - { - \int_compare:nNnTF \l_regex_start_step_int = \c_zero - { - \int_set_eq:NN \l_regex_current_char_int \c_minus_one - \regex_match_loop:N - } + \regex_query_get: + \regex_match_loop: + \prg_break_point:n { } + \bool_if:NT \l_regex_success_match_bool { - \str_skip_do:nn - { \l_regex_start_step_int - \c_one } - { \regex_match_once_aux_ii:N } + \bool_gset_true:N \g_regex_success_bool + \l_regex_every_match_tl } } -\cs_new_protected_nopar:Npn \regex_match_once_aux_ii:N #1 - { - \int_set:Nn \l_regex_current_char_int { `#1 } - \regex_match_loop:N - } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} % % \begin{macro}[aux]{\regex_match_initial_setup:} % This function holds the setup that should be done -% only once for one given pattern matching on a given -% string. It is called only once for the whole string. +% only once for one given pattern matching. +% It is called only once for the whole token list. % On the other hand, \cs{regex_match_setup:} -% is called for every match in the string in case of -% repeated matches, and \cs{regex_match_loop_setup:N} -% is called at every step. +% is called for every match in the token list in case of +% repeated matches. % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_match_initial_setup: { - \prg_stepwise_inline:nnnn {1} {1} { \l_regex_max_state_int } + \prg_stepwise_inline:nnnn {0} {1} { \l_regex_max_state_int - \c_one } { \tex_dimen:D ##1 \c_minus_one sp \scan_stop: } - \int_set_eq:NN \l_regex_unique_step_int \c_minus_one - \int_set_eq:NN \l_regex_start_step_int \c_minus_one - \int_set_eq:NN \l_regex_current_step_int \c_zero - \int_set_eq:NN \l_regex_success_step_int \c_zero + \int_set_eq:NN \l_regex_unique_id_int \c_minus_one + \int_set:Nn \l_regex_start_step_int + { \l_regex_min_step_int - \c_one } + \int_set_eq:NN \l_regex_current_step_int \l_regex_min_step_int + \int_set_eq:NN \l_regex_success_step_int \l_regex_min_step_int + \int_set_eq:NN \l_regex_submatch_int \l_regex_max_state_int \bool_set_false:N \l_regex_success_empty_bool + \bool_gset_false:N \g_regex_success_bool } % \end{macrocode} % \end{macro} @@ -2929,47 +3323,61 @@ { \cs_set_eq:NN \regex_last_match_empty:F \regex_last_match_empty_yes:F } { \cs_set_eq:NN \regex_last_match_empty:F \regex_last_match_empty_no:F } \int_set_eq:NN \l_regex_start_step_int \l_regex_success_step_int - \int_set_eq:NN \l_regex_current_step_int \l_regex_start_step_int - \int_decr:N \l_regex_current_step_int - \bool_set_false:N \l_regex_success_bool + \int_set:Nn \l_regex_current_step_int + { \l_regex_start_step_int - \c_one } + \bool_set_false:N \l_regex_success_match_bool \int_zero:N \l_regex_max_index_int - \regex_store_state:n {1} + \regex_store_state:n {0} %^^A _state_int! } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_match_loop:N} +% \begin{macro}[aux]{\regex_match_loop:} % \begin{macro}[aux]{\regex_match_one_index:n} % \begin{macro}[aux]{\regex_match_one_index_aux:n} % Setup what needs to be reset at every character, % then set \cs{l_regex_current_char_int} to the -% character code of the character that is read -% (and $-1$ for the end of the string), and loop +% character code of the token that is read +% (and $-1$ for the end of the token list), and loop % over the elements of the \tn{skip} array. Then repeat. -% There are a couple of tests to stop reading the string +% There are a couple of tests to stop reading the token list % when no active state is left, or when the end is reached. +% At every step in reading the token list, we store the character +% code of the current character in \cs{l_regex_current_char_int}, +% unless the end was reached: then we store $-1$. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_match_loop:N #1 +\cs_new_protected_nopar:Npn \regex_match_loop: { - \regex_match_loop_setup:N #1 + \int_incr:N \l_regex_current_step_int + \int_incr:N \l_regex_unique_id_int + \bool_set_false:N \l_regex_fresh_thread_bool + \int_set_eq:NN \l_regex_last_char_int \l_regex_current_char_int + \regex_query_get: + \regex_match_loop_case_hook: \cs_set_nopar:Npx \regex_tmp:w { \int_zero:N \l_regex_max_index_int - \prg_stepwise_function:nnnN - {1} {1} { \l_regex_max_index_int } - \regex_match_one_index:n + \regex_match_one_index:w 1 ; \prg_break_point:n { } + \exp_not:N \prg_break_point:n { } } - \regex_tmp:w \prg_break_point:n { } + \regex_tmp:w + \if_num:w \l_regex_current_char_int < \c_minus_one + \exp_after:wN \prg_map_break: + \fi: \if_num:w \l_regex_max_index_int = \c_zero - \exp_after:wN \use_none_delimit_by_q_recursion_stop:w + \exp_after:wN \prg_map_break: \fi: - \quark_if_recursion_tail_stop:N #1 - \regex_match_loop:N + \regex_match_loop: } -\cs_new_nopar:Npn \regex_match_one_index:n #1 +\cs_new_nopar:Npn \regex_match_one_index:w #1; { + \if_num:w #1 > \l_regex_max_index_int + \exp_after:wN \prg_map_break: + \fi: \regex_match_one_index_aux:n { \int_value:w \tex_skip:D #1 } + \exp_after:wN \regex_match_one_index:w + \int_use:N \int_eval:w #1 + \c_one ; } \cs_new_protected_nopar:Npn \regex_match_one_index_aux:n #1 { @@ -2982,28 +3390,6 @@ % \end{macro} % \end{macro} % -% \begin{macro}[aux]{\regex_match_loop_setup:N} -% This is called for every character in the string. -% At every step in reading the query string, we store the character -% code of the current character in \cs{l_regex_current_char_int}, -% unless the end was reached: then we store $-1$. -% \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_match_loop_setup:N #1 - { - \int_incr:N \l_regex_current_step_int - \int_incr:N \l_regex_unique_step_int - \bool_set_false:N \l_regex_fresh_thread_bool - \int_set_eq:NN \l_regex_last_char_int \l_regex_current_char_int - \if_meaning:w #1 \q_recursion_tail - \int_set_eq:NN \l_regex_current_char_int \c_minus_one - \else: - \int_set:Nn \l_regex_current_char_int {`#1} - \fi: - \regex_match_loop_case_hook: - } -% \end{macrocode} -% \end{macro} -% % \begin{macro}{\regex_match_loop_case_hook:} % \begin{macro}{\regex_match_loop_caseless_hook:} % In the case where the regular expression contains caseless matching, @@ -3041,7 +3427,7 @@ % The search is made unanchored at the start by putting % a free transition to the real start of the NFA, and a % costly transition to the same state, waiting for the -% next character in the query string. This combination +% next token in the query. This combination % could be reused (with some changes). We sometimes need % to know that the match for a given thread starts at % this character. For that, we use the boolean @@ -3063,7 +3449,7 @@ % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_action_cost:n #1 { - \exp_args:Nf \regex_store_state:n %^^A optimize! + \exp_args:Nf \regex_store_state:n { \int_eval:n { \l_regex_current_state_int + #1 } } } % \end{macrocode} @@ -3082,11 +3468,10 @@ { \regex_last_match_empty:F { - \bool_set_true:N \l_regex_success_bool + \bool_set_true:N \l_regex_success_match_bool \bool_set_eq:NN \l_regex_success_empty_bool \l_regex_fresh_thread_bool - \int_set_eq:NN \l_regex_success_step_int - \l_regex_current_step_int + \int_set_eq:NN \l_regex_success_step_int \l_regex_current_step_int \prop_set_eq:NN \l_regex_success_submatches_prop \l_regex_current_submatches_prop \prg_map_break: @@ -3131,46 +3516,252 @@ % \end{macrocode} % \end{macro} % -% \subsection{Submatches, once the correct match is found} +% \subsection{Replacement} % -% \begin{macro}[int]{\regex_extract:} -% \begin{macro}[aux]{\regex_extract_aux:nTF} +% \begin{macro}[rEXP]{\regex_submatch_nesting_aux:n} % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_extract: +\cs_new_protected:Npn \regex_submatch_nesting_aux:n #1 + { + + \etex_glueshrink:D \etex_mutoglue:D \etex_muexpr:D + \tex_muskip:D \etex_gluestretch:D \tex_skip:D #1 + - \tex_muskip:D \tex_skip:D #1 + \scan_stop: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int]{\regex_replacement:n} +% Our goal here is to analyse the replacement text. First take care +% of detecting escaped and non-escaped characters using +% \cs{str_escape_use:NNNn} with three protected arguments. +% This inserts in the input stream a token list of the form +% \meta{fn 1} \meta{char 1} \ldots{} \meta{fn $N$} \meta{char $N$}, +% where \meta{fn $i$} is one of the three functions, and \meta{char $i$} +% a character in the string |#1|. +% \begin{macrocode} +\cs_new:Npn \regex_nesting:n #1 { } %^^A move. Rename? +\tl_new:N \l_regex_nesting_tl +\cs_new_protected:Npn \regex_replacement:n #1 { - \seq_gclear:N \g_regex_submatches_seq - \prg_stepwise_inline:nnnn - {0} {1} { \l_regex_capturing_group_int } + \int_zero:N \l_regex_replacement_int + \int_zero:N \l_regex_nesting_int + \tl_clear:N \l_regex_nesting_tl + \str_escape_use:NNNn + \regex_replacement_unescaped:N + \regex_replacement_escaped:N + \regex_replacement_raw:N + {#1} + \prg_do_nothing: \prg_do_nothing: + \cs_set_nopar:Npx \regex_nesting:n ##1 { - \regex_extract_aux:nTF { ##1 } - { - \seq_gput_right:Nx \g_regex_submatches_seq - { \regex_query_substr:NN \l_regex_tmpa_tl \l_regex_tmpb_tl } - } - { \seq_gput_right:Nn \g_regex_submatches_seq { } } + + \int_use:N \l_regex_nesting_int + \l_regex_nesting_tl + - \regex_submatch_nesting_aux:n {##1} + } + \use:x + { + \exp_not:n { \cs_set_nopar:Npn \regex_replacement_tl:n ##1 } + { \str_aux_toks_range:nn \c_zero \l_regex_replacement_int } + } + % ^^A rename! Careful with \cP\#. + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_replacement_raw:N} +% \begin{macrocode} +\cs_new_protected:Npn \regex_replacement_raw:N #1 + { \regex_replacement_put:n {#1} } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_replacement_put:n} +% Raw characters are stored in a toks register. +% \begin{macrocode} +\cs_new_protected:Npn \regex_replacement_put:n #1 + { + \tex_toks:D \l_regex_replacement_int {#1} + \int_incr:N \l_regex_replacement_int + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_replacement_escaped:N} +% \begin{macro}[aux]{\regex_replacement_submatch:w} +% \begin{macro}[aux]{\regex_replacement_submatch_aux:nN} +% \begin{macrocode} +\cs_new_protected:Npn \regex_replacement_unescaped:N #1 + { + \if_charcode:w \c_rbrace_str #1 + \if_num:w \l_regex_replacement_csnames_int > \c_zero + \regex_replacement_put:n \cs_end: + \else: + \regex_replacement_put:n #1 + \fi: + \else: + \regex_replacement_put:n #1 + \fi: + } +\cs_new_protected:Npn \regex_replacement_escaped:N #1 + { + \if_charcode:w c #1 + \exp_after:wN \regex_replacement_c:w + \else: + \if_charcode:w g #1 + \exp_after:wN \exp_after:wN \exp_after:wN \regex_replacement_g:w + \else: + \if_num:w \c_one < 1#1 \exp_stop_f: + \regex_replacement_put_submatch:n {#1} + \else: + \regex_replacement_put:n #1 + \fi: + \fi: + \fi: + } +\cs_new_protected:Npn \regex_replacement_put_submatch:n #1 + { + \regex_replacement_put:n + { \regex_query_submatch:nn {#1} {##1} } + \if_num:w \l_regex_replacement_csnames_int = \c_zero + \tl_put_right:Nn \l_regex_nesting_tl + { + \exp_not:N \if_num:w #1 < \l_regex_capturing_group_int + \regex_submatch_nesting_aux:n { \int_eval:w #1+##1 \int_eval_end: } + \exp_not:N \fi: + } + \fi: + } +\cs_new_protected:Npn \regex_replacement_error:NNN #1#2#3 + { + \msg_kernel_error:nn { regex } { replacement-#1 } + #2 #3 + } +\cs_new_protected:Npn \regex_replacement_g:w #1#2 + { + \str_if_eq:xxTF + { \exp_not:n { #1#2 } } + { \regex_replacement_unescaped:N \c_lbrace_str } + { + \int_zero:N \l_regex_tmpa_int + \regex_replacement_g_digits:NN } + { \regex_replacement_error:NNN g #1 #2 } } -\cs_new_protected_nopar:Npn \regex_extract_aux:nTF #1#2#3 +\cs_new_protected:Npn \regex_replacement_g_digits:NN #1#2 { - \prop_get:NnNTF \l_regex_success_submatches_prop - { #1 < } \l_regex_tmpa_tl + \token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N { - \prop_get:NnNTF \l_regex_success_submatches_prop - { #1 > } \l_regex_tmpb_tl - {#2} - {#3} + \if_num:w \c_one < 1#2 \exp_stop_f: + \int_set:Nn \l_regex_tmpa_int { \c_ten * \l_regex_tmpa_int + #2 } + \exp_after:wN \use_i:nnn + \exp_after:wN \regex_replacement_g_digits:NN + \else: + \if_charcode:w \c_rbrace_str #2 + \exp_args:No \regex_replacement_put_submatch:n + { \int_use:N \l_regex_tmpa_int } + \exp_after:wN \exp_after:wN \exp_after:wN \use_none:nn + \else: + \exp_after:wN \exp_after:wN + \exp_after:wN \regex_replacement_error:NNN + \exp_after:wN \exp_after:wN \exp_after:wN g + \fi: + \fi: } - {#3} + { \regex_replacement_error:NNN g } + #1 #2 + } +\cs_new_protected_nopar:Npn \regex_replacement_c:w #1#2 + { + \token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N + { + \cs_if_exist_use:cF { regex_replacement_c_#2:w } + { \regex_replacement_error:NNN c #1#2 } + } + { \regex_replacement_error:NNN c #1#2 } + } +\cs_new_protected_nopar:cpn { regex_replacement_c_ \c_lbrace_str :w } + { + \int_incr:N \l_regex_replacement_csnames_int + \regex_replacement_put:n + { \exp_not:n { \exp_after:wN \regex_replacement_exp_not:N \cs:w } } + } +\cs_new_nopar:Npn \regex_replacement_exp_not:N #1 { \exp_not:n {#1} } +\group_begin: + + \char_set_catcode_math_superscript:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_U:w + { \regex_replacement_char:nNN { ^^@ } } + + \char_set_catcode_alignment:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_T:w + { \regex_replacement_char:nNN { ^^@ } } + + \cs_new_protected_nopar:Npn \regex_replacement_c_S:w #1#2 + { + \int_compare:nNnTF { `#2 } = \c_zero + { \regex_replacement_error:NNN c #1#2 } + { + \char_set_lccode:nn {32} { `#2 } + \tl_to_lowercase:n { \regex_replacement_put:n {~} } + } + } + + \char_set_catcode_parameter:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_P:w + { \regex_replacement_char:nNN { ^^@^^@ } } + + \char_set_catcode_other:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_O:w + { \regex_replacement_char:nNN { ^^@ } } + + \char_set_catcode_math_toggle:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_M:w + { \regex_replacement_char:nNN { ^^@ } } + + \char_set_catcode_letter:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_L:w + { \regex_replacement_char:nNN { ^^@ } } + + \char_set_catcode_group_end:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_E:w + { + \int_decr:N \l_regex_nesting_int + \regex_replacement_char:nNN { \if_false: { \fi: ^^@ } + } + + \char_set_catcode_math_subscript:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_D:w + { \regex_replacement_char:nNN { ^^@ } } + + \char_set_catcode_group_begin:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_B:w + { + \int_incr:N \l_regex_nesting_int + \regex_replacement_char:nNN { \exp_after:wN ^^@ \if_false: } \fi: } + } + + \char_set_catcode_active:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_A:w + { \regex_replacement_char:nNN { \exp_not:N ^^@ } } + +\group_end: + +\cs_new_protected_nopar:Npn \regex_replacement_char:nNN #1#2#3 + { + \char_set_lccode:nn \c_zero { `#3 } + \tl_to_lowercase:n + { \regex_replacement_put:n { \exp_not:n {#1} } } } % \end{macrocode} % \end{macro} % \end{macro} +% \end{macro} % % \subsection{User commands} % % \subsubsection{Precompiled pattern} % -% A given pattern is often reused to match many different strings. +% A given pattern is often reused to match many different query token lists. % We thus give a means of storing the NFA corresponding to a given % pattern in a token list variable of the form % \begin{quote} @@ -3179,7 +3770,7 @@ % \cs{tex_toks:D} 0 \{ \meta{instruction0} \} \\ % \ldots{} \\ % \cs{tex_toks:D} $n$ \{ \meta{instruction$\sb{n}$} \} \\ -% \cs{s_regex_stop} +% \cs{s_stop} % \end{quote} % where $n$ is the number of states in the NFA, % and the various \meta{instruction$\sb{i}$} control @@ -3191,672 +3782,678 @@ % contents of \meta{nfa var} may overwrite data which is used % elsewhere, unless everything is done carefully in a group. % -% \begin{macro}{\regex_set:Nn} % \begin{macro}{\regex_gset:Nn} +% \begin{macro}{\regex_const:Nn} +% \begin{macro}{\regex_set:Nn} % \begin{macro}[aux]{\regex_set_aux:NNn} -% \begin{macro}[aux]{\regex_set_aux:n} -% \begin{macro}[aux]{\regex_nfa:Nw} -% Within a group, build the NFA corresponding to the given regular -% expression, with submatch tracking. Then save the contents of all -% relevant \tn{toks} registers into \cs{g_regex_tmpa_tl}, then -% transferred to the user's tl variable. -% The auxiliary \cs{regex_nfa:Nw} is not protected: this ensures that -% the NFA will properly be replaced by an error message in expansion -% contexts. +% The three user functions only differ with which function is used +% to assign the pre-compiled regular expression to the user's variable. +% Internally, they all first build the NFA corresponding to the regex, +% then store the contents of all the necessary \tn{toks} registers +% in the user's variable. % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_set:Nn - { \regex_set_aux:NNn \tl_set_eq:NN } + { \regex_set_aux:NNn \tl_set:Nn } \cs_new_protected_nopar:Npn \regex_gset:Nn - { \regex_set_aux:NNn \tl_gset_eq:NN } + { \regex_set_aux:NNn \tl_gset:Nn } +\cs_new_protected_nopar:Npn \regex_const:Nn + { \regex_set_aux:NNn \tl_const:Nn } \cs_new_protected:Npn \regex_set_aux:NNn #1#2#3 { \group_begin: \regex_build:n {#3} - \tl_gset:Nx \g_regex_tmpa_tl - { - \exp_not:N \regex_nfa:Nw \exp_not:N #2 - \l_regex_max_state_int - = \int_use:N \l_regex_max_state_int - \l_regex_capturing_group_int - = \int_use:N \l_regex_capturing_group_int - \token_if_eq_meaning:NNT - \regex_match_loop_case_hook: - \regex_match_loop_caseless_hook: - { - \cs_set_eq:NN \regex_match_loop_case_hook: - \regex_match_loop_caseless_hook: - } - \prg_stepwise_function:nnnN - {1} {1} {\l_regex_max_state_int} - \regex_set_aux:n - \s_regex_stop - } + \cs_set_nopar:Npx \regex_tmp:w + { #1 \exp_not:N #2 { \regex_set_aux:N #2 } } + \exp_after:wN \group_end: - #1 #2 \g_regex_tmpa_tl + \regex_tmp:w } -\cs_new_nopar:Npn \regex_set_aux:n #1 - { \tex_toks:D #1 { \tex_the:D \tex_toks:D #1 } } -\cs_new:Npn \regex_nfa:Nw #1 #2 \s_regex_stop - { \msg_expandable_kernel_error:nnn { regex } { nfa-misused } {#1} } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} -% \end{macro} % -% \begin{macro}{\regex_const:Nn} -% The same idea as for setting, but using the new function. -% \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_const:Nn - { \regex_set_aux:NNn \cs_new_eq:NN } -% \end{macrocode} -% \end{macro} -% -% \begin{macro}[int,TF]{\regex_check_nfa:N} -% If a token list variable starts with \cs{regex_nfa:Nw}, -% then it most likely holds the data for a precompiled pattern. +% \begin{macro}[int, rEXP]{\regex_set_aux:N} +% \begin{macro}[aux, rEXP]{\regex_set_aux:n} +% \begin{macro}[aux, EXP]{\regex_nfa:Nw} +% Within a group, build the NFA corresponding to the given regular +% expression, with submatch tracking. Then save the contents of all +% relevant \tn{toks} registers into the variable outside the group. +% The auxiliary \cs{regex_nfa:Nw} is not protected: this ensures that +% the NFA will properly be replaced by an error message in expansion +% contexts. % \begin{macrocode} -\prg_new_protected_conditional:Npnn \regex_check_nfa:N #1 { T , TF } - { \exp_after:wN \regex_check_nfa_aux:Nw #1 \q_stop } -\cs_new:Npn \regex_check_nfa_aux:Nw #1 #2 \q_stop +\cs_new:Npn \regex_set_aux:N #1 + { + \exp_not:n { \regex_nfa:Nw #1 } + \l_regex_max_state_int + = \int_use:N \l_regex_max_state_int + \l_regex_capturing_group_int + = \int_use:N \l_regex_capturing_group_int + \token_if_eq_meaning:NNT + \regex_match_loop_case_hook: + \regex_match_loop_caseless_hook: + { + \cs_set_eq:NN \regex_match_loop_case_hook: + \regex_match_loop_caseless_hook: + } + \prg_stepwise_function:nnnN + {0} {1} {\l_regex_max_state_int - \c_one } + \regex_set_aux:n + \s_stop + } +\cs_new_nopar:Npn \regex_set_aux:n #1 + { \tex_toks:D #1 { \tex_the:D \tex_toks:D #1 } } +\cs_new:Npn \regex_nfa:Nw #1 { - \if_meaning:w \regex_nfa:Nw #1 - \prg_return_true: - \else: - \msg_error:nnx { regex } { not-nfa } { \token_to_str:N #1 } - \prg_return_false: - \fi: + \msg_expandable_kernel_error:nnn { regex } { nfa-misused } {#1} + \use_none_delimit_by_s_stop:w } % \end{macrocode} % \end{macro} +% \end{macro} +% \end{macro} % % \begin{macro}[int]{\regex_use:N} % No error-checking. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_use:N #1 - { \exp_after:wN \use_none:nn #1 } +\cs_new_protected_nopar:Npn \regex_use:N + { \exp_last_unbraced:No \use_none:nn } % \end{macrocode} % \end{macro} % -% \subsubsection{Generic auxiliary functions} -% -% \begin{macro}[aux]{\regex_user_aux:n} -% This is an auxiliary used by most user functions. -% The first and second arguments control whether we should track -% submatches, and whether we track one or multiple submatches. -% Everything is done within a group, so that |#1| can perform -% \enquote{unsafe} assignments. Most user functions return a -% result using \cs{group_insert_after:N}. +% \begin{macro}[int, rEXP]{\regex_to_str:N} +% \begin{macro}[int, rEXP]{\regex_to_str_aux:w} +% Of course, we could simply set \cs{regex_to_str:N} equal +% to \cs{tl_to_str:N}. After all, regex variables are in particular +% token list variables. We do some more processing to start each +% line with \cs{tex_toks:D} instead of a single very long line. % \begin{macrocode} -\cs_new_protected:Npn \regex_user_aux:n #1 +\cs_new_nopar:Npn \regex_to_str:N #1 { - \group_begin: - \tl_clear:N \l_regex_every_match_tl - #1 - \group_end: + \exp_after:wN \regex_to_str_aux:Nw #1 + \tex_toks:D \q_stop \prg_map_break: \tex_toks:D + \prg_break_point:n { } + } +\cs_new_nopar:Npn \regex_to_str_aux:Nw #1#2\tex_toks:D + { + \use_none_delimit_by_q_stop:w #2 \q_stop + \tl_to_str:n {#1#2} \iow_newline: + \regex_to_str_aux:Nw \tex_toks:D } % \end{macrocode} % \end{macro} +% \end{macro} +% +% \subsubsection{Generic auxiliary functions} +% +% Most of \pkg{l3regex}'s work is done within a group. % -% \begin{macro}[aux]{\regex_return_after_group:} -% Most of \pkg{l3regex}'s work is done within a group. +% \begin{macro}[aux]{\regex_aux_return:} % This function triggers either \cs{prg_return_false:} % or \cs{prg_return_true:} as appropriate to whether a % match was found or not. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_return_after_group: +\cs_new_protected_nopar:Npn \regex_aux_return: { - \if_meaning:w \c_true_bool \l_regex_success_bool - \group_insert_after:N \prg_return_true: + \if_meaning:w \c_true_bool \g_regex_success_bool + \prg_return_true: \else: - \group_insert_after:N \prg_return_false: - \fi: - } -% \end{macrocode} -% \end{macro} -% -% \begin{macro}[aux]{\regex_extract_after_group:N} -% Extract submatches, and store them in the user-given variable -% after the group has ended. -% \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_extract_after_group:N #1 - { - \if_meaning:w \c_true_bool \l_regex_success_bool - \regex_extract: - \group_insert_after:N \seq_set_eq:NN - \group_insert_after:N #1 - \group_insert_after:N \g_regex_submatches_seq + \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_count_after_group:N} -% Same procedure as \cs{regex_extract_after_group:N}, but simpler -% since the match counting has already taken place. +% \begin{macro}{\regex_aux_build_match:nn} +% This auxiliary is used by user functions whose \meta{regex} argument +% is given as an explicit regular expression within braces. In that +% case, we need to build the automaton corresponding to that regular +% expression, then perform the matching on the given token list |#2|. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_count_after_group:N #1 +\cs_new_protected:Npn \regex_aux_build_match:nn #1#2 { - \group_insert_after:N \int_set_eq:NN - \group_insert_after:N #1 - \group_insert_after:N \g_regex_match_count_int + \regex_build:n {#1} + \regex_match:n {#2} } % \end{macrocode} % \end{macro} % -% \subsubsection{Matching} -% -% \begin{macro}[TF]{\regex_match:nn} -% \begin{macro}[TF]{\regex_match:Nn} -% We don't track submatches. Then either build the NFA corresponding -% to the regular expression, or use a precompiled pattern. Then match, -% using the internal \cs{regex_match:n}. Finally return the result -% after closing the group. +% \begin{macro}{\regex_aux_use_match:Nn} +% This auxiliary is used by user functions whose \meta{regex} argument +% is given as a pre-compiled regex variable. We make sure that +% the token list variable indeed is an automaton (by testing the +% first token). If not, the match is deemed unsuccessful, after +% raising an error. If we have an automaton, \enquote{use} it, +% then perform the matching on the given token list |#2|. % \begin{macrocode} -\prg_new_protected_conditional:Npnn \regex_match:nn #1#2 { T , F , TF } +\cs_new_protected:Npn \regex_aux_use_match:Nn #1#2 { - \regex_user_aux:n + \exp_args:No \tl_if_head_eq_meaning:nNTF {#1} \regex_nfa:Nw { - \regex_disable_submatches: - \regex_build:n {#1} + \regex_use:N #1 \regex_match:n {#2} - \regex_return_after_group: } - } -\prg_new_protected_conditional:Npnn \regex_match:Nn #1#2 { T , F , TF } - { - \regex_check_nfa:NTF #1 { - \regex_user_aux:n - { - \regex_disable_submatches: - \regex_use:N #1 - \regex_match:n {#2} - \regex_return_after_group: - } + \msg_kernel_error:nnx { regex } { not-nfa } { \token_to_str:N #1 } + \bool_gset_false:N \g_regex_success_bool } - { \prg_return_false: } } % \end{macrocode} % \end{macro} -% \end{macro} % -% \begin{macro}{\regex_count:nnN} -% \begin{macro}{\regex_count:NnN} -% Instead of aborting once the first \enquote{best match} is found, -% we repeat the search. The code is such that the search will not -% start on the same character, hence avoiding infinite loops. +% \begin{macro} +% { +% \regex_extract_once:nnN, \regex_extract_once:NnN, +% \regex_extract_all:nnN, \regex_extract_all:NnN, +% \regex_replace_once:nnN, \regex_replace_once:NnN, +% \regex_replace_all:nnN, \regex_replace_all:NnN, +% \regex_split:nnN, \regex_split:NnN +% } +% \begin{macro}[TF] +% { +% \regex_extract_once:nnN, \regex_extract_once:NnN, +% \regex_extract_all:nnN, \regex_extract_all:NnN, +% \regex_replace_once:nnN, \regex_replace_once:NnN, +% \regex_replace_all:nnN, \regex_replace_all:NnN, +% \regex_split:nnN, \regex_split:NnN +% } +% We define here $40$ user functions, following a common pattern +% in terms of an auxiliary such as \cs{regex_extract_once_aux:NnnN} +% (those auxiliaries are defined in the coming sections). +% The arguments handed to the auxiliary are +% \cs{regex_aux_build_match:nn} or \cs{regex_aux_use_match:Nn}, +% followed by the three arguments of the user function. +% The conditionals call \cs{regex_aux_return:} to return +% either \texttt{true} or \texttt{false} once matching +% has been performed. % \begin{macrocode} -\cs_new_protected:Npn \regex_count:nnN #1#2#3 +\cs_set_protected:Npn \regex_tmp:w #1#2#3 { - \regex_user_aux:n + \cs_new_protected_nopar:Npn #1 + { #3 \regex_aux_build_match:nn } + \cs_new_protected_nopar:Npn #2 + { #3 \regex_aux_use_match:Nn } + \prg_new_protected_conditional:Npnn #1 ##1##2##3 { T , F , TF } { - \regex_disable_submatches: - \int_gzero:N \g_regex_match_count_int - \tl_set:Nn \l_regex_every_match_tl - { - \int_gincr:N \g_regex_match_count_int - \regex_match_once: - } - \regex_build:n {#1} - \regex_match:n {#2} - \regex_count_after_group:N #3 + #3 \regex_aux_build_match:nn {##1} {##2} ##3 + \regex_aux_return: } - } -\cs_new_protected:Npn \regex_count:NnN #1#2#3 - { - \regex_check_nfa:NT #1 + \prg_new_protected_conditional:Npnn #2 ##1##2##3 { T , F , TF } { - \regex_user_aux:n - { - \regex_disable_submatches: - \int_gzero:N \g_regex_match_count_int - \tl_set:Nn \l_regex_every_match_tl - { - \int_gincr:N \g_regex_match_count_int - \regex_match_once: - } - \regex_use:N #1 - \regex_match:n {#2} - \regex_count_after_group:N #3 - } + #3 \regex_aux_use_match:Nn {##1} {##2} ##3 + \regex_aux_return: } } +\tl_map_inline:nn + { + { extract_once } { extract_all } + { replace_once } { replace_all } + { split } + } + { + \exp_args:Nccc \regex_tmp:w + { regex_#1:nnN } { regex_#1:NnN } { regex_#1_aux:NnnN } + } % \end{macrocode} % \end{macro} % \end{macro} % -% \subsubsection{Submatch extraction} +% \subsubsection{Submatches, once the correct match is found} +% +% \begin{macro}[int]{\regex_extract:} +% \begin{macro}[aux]{\regex_extract_aux_b:wn} +% \begin{macro}[aux]{\regex_extract_aux_e:wn} % -% \begin{macro}{\regex_extract_once:nnN} -% \begin{macro}{\regex_extract_once:NnN} -% \begin{macro}[TF]{\regex_extract_once:nnN} -% \begin{macro}[TF]{\regex_extract_once:NnN} % \begin{macrocode} -\cs_new_protected:Npn \regex_extract_once:nnN #1#2#3 +\cs_new_protected_nopar:Npn \regex_extract: { - \regex_user_aux:n - { - \regex_build:n {#1} - \regex_match:n {#2} - \regex_extract_after_group:N #3 - } + \int_set_eq:NN \l_regex_submatch_start_int \l_regex_submatch_int + \if_meaning:w \c_true_bool \g_regex_success_bool + \prg_replicate:nn \l_regex_capturing_group_int + { + \tex_skip:D \l_regex_submatch_int \c_zero sp \scan_stop: + \int_incr:N \l_regex_submatch_int + } + \prop_map_inline:Nn \l_regex_success_submatches_prop + { + \if_num:w ##1 \c_max_int + \exp_after:wN \regex_extract_aux_b:wn \int_use:N + \else: + \exp_after:wN \regex_extract_aux_e:wn \int_use:N + \fi: + \int_eval:w \l_regex_submatch_start_int + ##1 {##2} + } + \fi: } -\prg_new_protected_conditional:Npnn \regex_extract_once:nnN #1#2#3 - { T , F , TF } +\cs_new_protected_nopar:Npn \regex_extract_aux_b:wn #1 < #2 { - \regex_user_aux:n - { - \regex_build:n {#1} - \regex_match:n {#2} - \regex_extract_after_group:N #3 - \regex_return_after_group: - } + \tex_skip:D #1 = #2 sp + plus \etex_gluestretch:D \tex_skip:D #1 \scan_stop: } -\cs_new_protected:Npn \regex_extract_once:NnN #1#2#3 +\cs_new_protected_nopar:Npn \regex_extract_aux_e:wn #1 > #2 { - \regex_check_nfa:NT #1 - { - \regex_user_aux:n - { - \regex_use:N #1 - \regex_match:n {#2} - \regex_extract_after_group:N #3 - } - } - } -\prg_new_protected_conditional:Npnn \regex_extract_once:NnN #1#2#3 - { T , F , TF } - { - \regex_check_nfa:NTF #1 - { - \regex_user_aux:n - { - \regex_use:N #1 - \regex_match:n {#2} - \regex_extract_after_group:N #3 - \regex_return_after_group: - } - } - { \prg_return_false: } + \tex_skip:D #1 + = 1 \tex_skip:D #1 plus #2 sp \scan_stop: } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} -% \end{macro} % -% \begin{macro}{\regex_extract_all:nnN} -% \begin{macro}{\regex_extract_all:NnN} -% \begin{macro}[aux]{\regex_extract_all_aux:} -% \begin{macro}[aux]{\regex_extract_all_after_group:N} -% Similarly to \cs{regex_count:nnN} functions, -% recurse through matches of the pattern. Then we do -% something slightly different, extracting submatches. -% Submatches are not extracted if the pattern matched -% an empty string at the start of the match attempt -% (to avoid adding spurious empty items to the resulting -% sequence). +% \begin{macro}{\regex_group_end_extract_seq:N} +% \begin{macro}[aux]{\regex_extract_seq_aux:n} +% \begin{macro}[aux]{\regex_extract_seq_aux:ww} % \begin{macrocode} -\cs_new_protected:Npn \regex_extract_all:nnN #1#2#3 +\cs_new_protected:Npn \regex_group_end_extract_seq:N #1 { - \regex_user_aux:n - { - \seq_gclear:N \g_regex_extract_all_seq - \tl_set:Nn \l_regex_every_match_tl { \regex_extract_all_aux: } - \regex_build:n {#1} - \regex_match:n {#2} - \regex_extract_all_after_group:N #3 - } + \cs_set_eq:NN \seq_item:n \scan_stop: + \flag_lower:N \l_regex_group_begin_flag + \flag_lower:N \l_regex_group_end_flag + \tl_set:Nx \l_regex_tmpa_tl + { + \prg_stepwise_function:nnnN + \l_regex_max_state_int + \c_one + { \l_regex_submatch_int - \c_one } + \regex_extract_seq_aux:n + } + \flag_test:NF \l_regex_group_begin_flag + { \flag_test:NF \l_regex_group_end_flag { \use_none:nnn } } + \msg_kernel_error:nn { regex } { sequence-unbalanced } + \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl } + \exp_args:NNNo \group_end: + \tl_set:Nn #1 \l_regex_tmpa_tl } -\cs_new_protected:Npn \regex_extract_all:NnN #1#2#3 +\cs_new_nopar:Npn \regex_extract_seq_aux:n #1 { - \regex_check_nfa:NT #1 + \seq_item:n { - \regex_user_aux:n - { - \seq_gclear:N \g_regex_extract_all_seq - \tl_set:Nn \l_regex_every_match_tl { \regex_extract_all_aux: } - \regex_use:N #1 - \regex_match:n {#2} - \regex_extract_all_after_group:N #3 - } + \exp_after:wN \regex_extract_seq_aux:ww + \int_value:w \regex_submatch_nesting_aux:n {#1} ; #1; } } -\cs_new_protected_nopar:Npn \regex_extract_all_aux: - { - \regex_extract: - \seq_gconcat:NNN \g_regex_extract_all_seq - \g_regex_extract_all_seq \g_regex_submatches_seq - \regex_match_once: - } -\cs_new_protected_nopar:Npn \regex_extract_all_after_group:N #1 - { - \seq_if_empty:NF \g_regex_extract_all_seq - { - \regex_extract: - \group_insert_after:N \seq_set_eq:NN - \group_insert_after:N #1 - \group_insert_after:N \g_regex_extract_all_seq - } +\cs_new_nopar:Npn \regex_extract_seq_aux:ww #1; #2; + { %^^A todo: use flags + \if_num:w #1 < \c_zero + \flag_raise:N \l_regex_group_end_flag + \prg_replicate:nn {-#1} { \exp_not:n { { \if_false: } \fi: } } + \fi: + \regex_query_submatch:w #2; + \if_num:w #1 > \c_zero + \flag_raise:N \l_regex_group_begin_flag + \prg_replicate:nn {#1} { \exp_not:n { \if_false: { \fi: } } } + \fi: } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} -% \end{macro} % -% \subsubsection{Splitting a string by matches of a regex} +% \subsubsection{Matching} % -% \begin{macro}{\regex_split:nnN} -% \begin{macro}{\regex_split:NnN} -% \begin{macro}[aux]{\regex_split_aux:} -% \begin{macro}[aux]{\regex_split_after_group:N} -% Similarly to \cs{regex_count:nnN} functions, -% recurse through matches of the pattern. Then we do -% something slightly different, extracting submatches. -% Submatches are not extracted if the pattern matched -% an empty string at the start of the match attempt -% (to avoid adding spurious empty items to the resulting -% sequence). +% \begin{macro}[TF]{\regex_match:nn} +% \begin{macro}[TF]{\regex_match:Nn} +% \begin{macro}{\regex_match_aux:n} +% We don't track submatches. Then either build the NFA corresponding +% to the regular expression, or use a precompiled pattern. Then match, +% using the internal \cs{regex_match:n}. Finally return the result +% after closing the group. % \begin{macrocode} -\cs_new_protected:Npn \regex_split:nnN #1#2#3 +\prg_new_protected_conditional:Npnn \regex_match:nn #1#2 { T , F , TF } { - \regex_user_aux:n - { - \seq_gclear:N \g_regex_split_seq - \tl_set:Nn \l_regex_every_match_tl { \regex_split_aux: } - \regex_build:n {#1} - \regex_match:n {#2} - \regex_split_after_group:N #3 - } + \regex_match_aux:n + { \regex_aux_build_match:nn {#1} {#2} } } -\cs_new_protected:Npn \regex_split:NnN #1#2#3 +\prg_new_protected_conditional:Npnn \regex_match:Nn #1#2 { T , F , TF } { - \regex_check_nfa:NT #1 - { - \regex_user_aux:n - { - \seq_gclear:N \g_regex_split_seq - \tl_set:Nn \l_regex_every_match_tl { \regex_split_aux: } - \regex_use:N #1 - \regex_match:n {#2} - \regex_split_after_group:N #3 - } - } + \regex_match_aux:n + { \regex_aux_use_match:Nn #1 {#2} } } -\cs_new_protected_nopar:Npn \regex_split_aux: +\cs_new_protected:Npn \regex_match_aux:n #1 { - \int_compare:nNnF \l_regex_start_step_int = \l_regex_success_step_int - { - \regex_extract: - \seq_pop:NN \g_regex_submatches_seq \l_regex_tmpa_tl - \regex_extract_aux:nTF {0} - { - \seq_gput_left:Nx \g_regex_submatches_seq - { - \regex_query_substr:NN - \l_regex_start_step_int \l_regex_tmpa_tl - } - } - { \msg_error:nn { regex } { internal } } - \seq_gconcat:NNN \g_regex_split_seq - \g_regex_split_seq \g_regex_submatches_seq - } - \regex_match_once: + \group_begin: + \tl_clear:N \l_regex_every_match_tl + \regex_disable_submatches: + #1 + \group_end: + \regex_aux_return: } -\cs_new_protected_nopar:Npn \regex_split_after_group:N #1 +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \begin{macro}{\regex_count:nnN} +% \begin{macro}{\regex_count:NnN} +% \begin{macro}{\regex_count_aux:n} +% Instead of aborting once the first \enquote{longest match} is found, +% we repeat the search. The code is such that the search will not +% start on the same character, hence avoiding infinite loops. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \regex_count:nnN + { \regex_count_aux:NnnN \regex_aux_build_match:nn } +\cs_new_protected_nopar:Npn \regex_count:NnN + { \regex_count_aux:NnnN \regex_aux_use_match:Nn } +\cs_new_protected:Npn \regex_count_aux:NnnN #1#2#3#4 { - \int_compare:nNnTF \l_regex_start_step_int = \l_regex_current_step_int - { - \bool_if:NF \l_regex_success_empty_bool - { \seq_gput_right:Nn \g_regex_split_seq { } } - } - { - \seq_gput_right:Nx \g_regex_split_seq - { - \regex_query_substr:NN - \l_regex_start_step_int \l_regex_current_step_int - } - } - \group_insert_after:N \seq_set_eq:NN - \group_insert_after:N #1 - \group_insert_after:N \g_regex_split_seq + \group_begin: + \regex_disable_submatches: + \int_zero:N \l_regex_match_count_int + \tl_set:Nn \l_regex_every_match_tl + { + \int_incr:N \l_regex_match_count_int + \regex_match_once: + } + #1 {#2} {#3} + \exp_args:NNNo + \group_end: + \int_set:Nn #4 { \int_use:N \l_regex_match_count_int } } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} -% \end{macro} % -% \subsubsection{String replacement} +% \subsubsection{Submatch extraction} % -% \begin{macro}[int]{\regex_replacement:n} -% \begin{macro}[aux]{\regex_replacement_escaped:N} -% \begin{macro}[aux]{\regex_replacement_submatch:w} -% \begin{macro}[aux]{\regex_replacement_submatch_aux:nN} +% \begin{macro}[aux]{\regex_extract_once_aux:NnnN} +% As announced, here comes the auxiliary for extracting one match. +% Since we only want one match, \cs{l_regex_every_match_tl} +% is empty, and does not trigger the matching code again. +% After matching, \cs{regex_extract:} extracts submatches +% into various \tn{skip} registers, and those are then concatenated +% into a sequence by \cs{regex_group_end_extract_seq:N}. That function +% is also responsible for closing the group. % \begin{macrocode} -\cs_new_protected:Npn \regex_replacement:n #1 - { - \str_aux_escape:NNNn - \prg_do_nothing: - \regex_replacement_escaped:N - \prg_do_nothing: - {#1} - \tl_set_eq:NN \l_regex_replacement_tl \g_str_result_tl - \tl_set:Nx \l_regex_replacement_tl - { \l_regex_replacement_tl \prg_do_nothing: } - } -\cs_new_nopar:Npn \regex_replacement_escaped:N #1 +\cs_new_protected:Npn \regex_extract_once_aux:NnnN #1#2#3#4 { - \regex_token_if_other_digit:NTF #1 - { - \exp_not:n - { \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } } {#1} - } - { - \token_if_eq_charcode:NNTF g #1 - { \exp_not:N \regex_replacement_submatch:w } - { #1 } - } + \group_begin: + \tl_set:Nn \l_regex_every_match_tl { \regex_extract: } + #1 {#2} {#3} + \regex_group_end_extract_seq:N #4 } -\cs_new_nopar:Npn \regex_replacement_submatch:w #1 +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_extract_all_aux:NnnN} +%^^A todo: update doc +% The set of submatches will be built progressively +% in \cs{l_regex_result_seq}. For each match, extract +% the submatches, and concatenate that to the right of +% the result sequence, then start matching again. +% Finally, copy the result in the user's sequence variable. +% \begin{macrocode} +\cs_new_protected:Npn \regex_extract_all_aux:NnnN #1#2#3#4 { - \exp_after:wN \token_if_eq_meaning:NNTF \c_lbrace_str #1 - { \regex_replacement_submatch_aux:nN { } } - { - \regex_token_if_other_digit:NTF #1 - { \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } } - { \msg_expandable_kernel_error:nn { regex } { g-misused } } - #1 - } - } -\cs_new_nopar:Npn \regex_replacement_submatch_aux:nN #1 #2 + \group_begin: + \tl_set:Nn \l_regex_every_match_tl + { \regex_extract: \regex_match_once: } + #1 {#2} {#3} + \regex_group_end_extract_seq:N #4 + } +% \end{macrocode} +% \end{macro} +% +% \subsubsection{Splitting a token list by matches of a regex} +% +% \begin{macro}[aux]{\regex_split_aux:NnnN} +% Recurse through the matches, and for each, do the following. +% Extract the submatches into various \tn{skip} registers, +% then replace the match |\0|, which should not be kept in the +% final result, and replace it by the part of the token list +% before the match. +% This process must be inhibited to avoid creating empty items +% if the regex matched an empty token list at the place where +% the match attempt started. +% After the last successful match, we need to add to the result +% the part of the token list after the last match, unless the +% last match was empty and at the very end. +% Finally, \cs{regex_group_end_extract_seq:N} builds a sequence +% from all the \tn{skip} registers, and assigns it to |#4| +% after closing the group. +% \begin{macrocode} +\cs_new_protected:Npn \regex_split_aux:NnnN #1#2#3#4 { - \regex_token_if_other_digit:NTF #2 - { \regex_replacement_submatch_aux:nN {#1#2} } - { - \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } {#1} - \exp_after:wN \token_if_eq_meaning:NNF \c_rbrace_str #2 - { - \msg_expandable_kernel_error:nn { regex } { g-misused } - #2 - } - } + \group_begin: + \tl_set:Nn \l_regex_every_match_tl + { + \if_num:w \l_regex_start_step_int < \l_regex_success_step_int + \regex_extract: + \tex_skip:D \l_regex_submatch_start_int + = \l_regex_start_step_int sp + plus \tex_skip:D \l_regex_submatch_start_int \scan_stop: + \fi: + \regex_match_once: + } + #1 {#2} {#3} + \tex_skip:D \l_regex_submatch_int + = \l_regex_start_step_int sp + plus \l_regex_current_step_int sp \scan_stop: + \int_incr:N \l_regex_submatch_int + \if_num:w \l_regex_start_step_int = \l_regex_current_step_int + \if_meaning:w \c_true_bool \l_regex_success_empty_bool + \int_decr:N \l_regex_submatch_int + \fi: + \fi: + \regex_group_end_extract_seq:N #4 } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}[aux]{\regex_replace_after_group:N} -% I wonder why I implemented things with \tn{aftergroup} -% rather than simply placing that code after the explicit -% \cs{group_end:} that I have. (General remark.) +% \subsubsection{Replacement} +% +% \begin{macro}[aux]{\regex_replace_once_aux:NnnN} +% The replacement text is analysed by \cs{regex_replacement:n}, +% which defines \cs{regex_replacement_tl:n} to expand to the +% replaced token list, assuming that submatches are stored in +% various \tn{skip} registers, as done by \cs{regex_extract:}. +% If there is a match, we grab +% the parts before and after it, and get the result +% by \texttt{x}-expanding twice. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_replace_after_group:N #1 +\cs_new_protected:Npn \regex_replace_once_aux:NnnN #1#2#3#4 + { + \group_begin: + \tl_clear:N \l_regex_every_match_tl + \regex_replacement:n {#3} + \exp_args:Nno #1 {#2} #4 + \if_meaning:w \c_true_bool \g_regex_success_bool + \regex_extract: + \int_set:Nn \l_regex_tmpa_int + { \regex_nesting:n { \l_regex_submatch_start_int } } + \if_num:w \l_regex_tmpa_int = \c_zero + \else: + \msg_kernel_error:nnx { regex } { replace-unbalanced } + { \l_regex_tmpa_int } + \fi: + \tl_set:Nx \l_regex_tmpa_tl + { + \if_num:w \l_regex_tmpa_int < \c_zero + \prg_replicate:nn { - \l_regex_tmpa_int } + { \exp_not:n { { \if_false: } \fi: } } + \fi: + \regex_query_substr:nn + { \l_regex_min_step_int } + { \tex_skip:D \l_regex_submatch_start_int } + \regex_replacement_tl:n { \l_regex_submatch_start_int } + \regex_query_substr:nn + { \etex_gluestretch:D \tex_skip:D \l_regex_submatch_start_int } + { \l_regex_max_step_int } + \if_num:w \l_regex_tmpa_int > \c_zero + \prg_replicate:nn { \l_regex_tmpa_int } + { \exp_not:n { \if_false: { \fi: } } } + \fi: + } + \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl } + \exp_args:NNNo \group_end: + \tl_set:Nn #4 \l_regex_tmpa_tl + \else: + \group_end: + \fi: + } +\cs_new_protected_nopar:Npn \regex_replace_once_aux:w #1; { - \group_insert_after:N \tl_set:Nx - \group_insert_after:N #1 - \group_insert_after:N { - \group_insert_after:N \tl_to_str:N - \group_insert_after:N \g_regex_replaced_str - \group_insert_after:N } } % \end{macrocode} % \end{macro} % -% \begin{macro}{\regex_replace_once:nnN} -% \begin{macro}{\regex_replace_once:NnN} -% \begin{macro}[TF]{\regex_replace_once:nnN} -% \begin{macro}[TF]{\regex_replace_once:NnN} -% \begin{macro}[aux]{\regex_replace_once_aux:Nn} +% \begin{macro}[aux]{\regex_replace_all_aux:NnnN} +% For every match, extract submatches, and add the part before +% the beginning of the match, as well as the replacement, +% to the result. After the last match, extract the end +% of the token list, and add it to the replaced token list. % \begin{macrocode} -\cs_new_protected:Npn \regex_replace_once_aux:Nn #1#2 +\cs_new_protected:Npn \regex_replace_all_aux:NnnN #1#2#3#4 { \group_begin: - \regex_replace_after_group:N #1 - \tl_clear:N \l_regex_every_match_tl - #2 - \exp_args:No \regex_match:n {#1} - \regex_extract: - \regex_extract_aux:nTF {0} + \tl_set:Nn \l_regex_every_match_tl { - \cs_set_nopar:Npn \regex_tmp:w - { \str_skip_do:nn { \l_regex_tmpb_tl } { } } - \tl_gset:Nx \g_regex_replaced_str - { - \regex_query_substr:NN \c_zero \l_regex_tmpa_tl - \l_regex_replacement_tl - \exp_after:wN \regex_tmp:w \l_regex_query_other_str - } + \regex_extract: + \tex_skip:D \l_regex_submatch_start_int + = \tex_the:D \tex_skip:D \l_regex_submatch_start_int + minus \l_regex_start_step_int sp \scan_stop: + \regex_match_once: } - { \tl_gset_eq:NN \g_regex_replaced_str \l_regex_query_other_str } - \group_end: + \regex_replacement:n {#3} + \exp_args:Nno #1 {#2} #4 + \int_set:Nn \l_regex_tmpa_int + { + 0 + \prg_stepwise_function:nnnN + \l_regex_max_state_int + \l_regex_capturing_group_int + { \l_regex_submatch_int - \c_one } + \regex_nesting:n + } + \if_num:w \l_regex_tmpa_int = \c_zero + \else: + \msg_kernel_error:nnx { regex } { replace-unbalanced } + { \l_regex_tmpa_int } + \fi: + \tl_set:Nx \l_regex_tmpa_tl + { + \if_num:w \l_regex_tmpa_int < \c_zero + \prg_replicate:nn { - \l_regex_tmpa_int } + { \exp_not:n { { \if_false: } \fi: } } + \fi: + \prg_stepwise_function:nnnN + \l_regex_max_state_int + \l_regex_capturing_group_int + { \l_regex_submatch_int - \c_one } + \regex_replace_all_aux:n + \regex_query_substr:nn + \l_regex_start_step_int \l_regex_max_step_int + \if_num:w \l_regex_tmpa_int > \c_zero + \prg_replicate:nn { \l_regex_tmpa_int } + { \exp_not:n { \if_false: { \fi: } } } + \fi: + } + \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl } + \exp_args:NNNo \group_end: + \tl_set:Nn #4 \l_regex_tmpa_tl } -\cs_new_protected:Npn \regex_replace_once:nnN #1#2#3 +\cs_new_nopar:Npn \regex_replace_all_aux:n #1 { - \regex_replace_once_aux:Nn #3 - { - \regex_build:n {#1} - \regex_replacement:n {#2} - } + \regex_query_substr:nn + { \etex_glueshrink:D \tex_skip:D #1 } { \tex_skip:D #1 } + \regex_replacement_tl:n {#1} } -\cs_new_protected:Npn \regex_replace_once:NnN #1#2#3 +% \end{macrocode} +% \end{macro} +% +% \subsection{Messages} +% +% \begin{macrocode} +\msg_kernel_new:nnnn { regex } { sequence-unbalanced } { - \regex_replace_once_aux:Nn #3 + Missing~ + \flag_test:NTF \l_regex_group_end_flag { - \regex_use:N #1 - \regex_replacement:n {#2} + left~ + \flag_test:NTF \l_regex_group_begin_flag + { and~right~braces } { brace } } + { right~brace } + \ inserted~in~extracted~match. } -\prg_new_protected_conditional:Npnn \regex_replace_once:nnN #1#2#3 {T,F,TF} { - \regex_replace_once_aux:Nn #3 - { - \regex_build:n {#1} - \regex_replacement:n {#2} - \regex_return_after_group: - } + LaTeX~was~asked~to~extract~submatches~or~split~a~token~list~ + according~to~a~given~regular~expression,~but~some~of~the~resulting~ + items~were~not~balanced. } -\prg_new_protected_conditional:Npnn \regex_replace_once:NnN #1#2#3 {T,F,TF} +\msg_kernel_new:nnnn { regex } { replace-unbalanced } + { The~result~of~a~replacement~does~not~have~balanced~braces. } { - \regex_replace_once_aux:Nn #3 + LaTeX~was~asked~to~do~some~regular~expression~replacement,~ + and~the~resulting~token~list~would~not~have~the~same~number~ + of~begin-group~and~end-group~tokens. \\ \\ + \ \ \ \ + \prg_case_int:nnn {#1} { - \regex_use:N #1 - \regex_replacement:n {#2} - \regex_return_after_group: + { -1 } { A~left~brace~was } + { 1 } { A~right~brace~was } } + { + \int_abs:n {#1} ~ + \int_compare:nNnTF {#1} < \c_zero { left } { right } ~ + braces ~ were + } + \ inserted. } % \end{macrocode} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}{\regex_replace_all:nnN} -% \begin{macro}{\regex_replace_all:NnN} -% \begin{macro}[aux]{\regex_replace_all_aux:Nn} -% \begin{macro}[aux]{\regex_replace_all_aux:} % \begin{macrocode} -\cs_new_protected:Npn \regex_replace_all_aux:Nn #1#2 +\msg_kernel_new:nnnn { regex } { missing-rparen } + { Missing~right~parenthesis~added~in~regular~expression. } { - \group_begin: - \regex_replace_after_group:N #1 - \tl_set:Nn \l_regex_every_match_tl { \regex_replace_all_aux: } - \tl_gclear:N \g_regex_replaced_str - #2 - \exp_args:No \regex_match:n {#1} - \cs_set_nopar:Npn \regex_tmp:w - { \str_skip_do:nn { \l_regex_start_step_int } { } } - \tl_gput_right:Nx \g_regex_replaced_str - { \exp_after:wN \regex_tmp:w \l_regex_query_other_str } - \group_end: + LaTeX~was~given~a~regular~expression~with~\int_eval:n{#1}~ + more~left~parenthes\int_compare:nTF{#1=1}{i}{e}s~than~right~ + parenthes\int_compare:nTF{#1=1}{i}{e}s. } -\cs_new_protected_nopar:Npn \regex_replace_all_aux: +\msg_kernel_new:nnnn { regex } { extra-rparen } + { Extra~right~parenthesis~ignored~in~regular~expression. } { - \regex_extract: - \regex_extract_aux:nTF {0} - { - \tl_gput_right:Nx \g_regex_replaced_str - { - \regex_query_substr:NN - \l_regex_start_step_int \l_regex_tmpa_tl - \l_regex_replacement_tl - } - } - { \msg_error:nn { regex } { internal } } - \regex_match_once: + LaTeX~came~across~a~closing~parenthesis~when~no~submatch~group~ + was~open.~The~parenthesis~will~be~ignored. } -\cs_new_protected:Npn \regex_replace_all:nnN #1#2#3 +\msg_kernel_new:nnnn { regex } { backwards-range } + { Range~[#1-#2]~out~of~order~in~character~class. } { - \regex_replace_all_aux:Nn #3 - { - \regex_build:n {#1} - \regex_replacement:n {#2} - } + In~ranges~of~characters~[x-y]~appearing~in~character~classes,~ + the~first~character~code~must~not~be~larger~than~the~second.~ + Here,~#1~has~character~code~\int_eval:n {`#1},~while~#2~has~ + character~code~\int_eval:n {`#2}. } -\cs_new_protected:Npn \regex_replace_all:NnN #1#2#3 +\msg_kernel_new:nnnn { regex } { unsupported } + { Unsupported~construction~`#1'. } { - \regex_replace_all_aux:Nn #3 - { - \regex_use:N #1 - \regex_replacement:n {#2} - } + The~construction~`#1'~is~not~supported~by~the~LaTeX~ + regular~expression~module.~Perhaps~some~character~should~ + have~been~escaped? + } +\msg_kernel_new:nnnn { regex } { not-nfa } + { This~is~not~a~regular~expression~variable. } + { + LaTeX~was~expecting~`#1'~to~be~a~regular~expression~variable.\\ + This~control~sequence~is~not~a~regex~variable.~It's~current~meaning~ + is~\\\\ + \ \ \ \ \token_to_str:N #1 = \token_to_meaning:N #1 . } % \end{macrocode} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} % -% \subsection{Messages} -% -% Negative codes are specific to the \LaTeX3 implementation. -% Other error codes match with the PCRE codes (not all of -% the PCRE errors can occur, since many constructions are -% not supported). % \begin{macrocode} -% \msg_new:nnn { regex } { -999 } { File~not~found } -\msg_new:nnn { regex } { -998 } { Unsupported~construct } -\msg_new:nnn { regex } { -997 } - { The~regular~expression~is~too~large~(32768~states). } -\msg_new:nnn { regex } { 1 } { \iow_char:N\\~at~end~of~pattern } +% \msg_new:nnn { regex } { 1 } { \iow_char:N\\~at~end~of~pattern } % \msg_new:nnn { regex } { 2 } { \iow_char:N\\c~at~end~of~pattern } -\msg_new:nnn { regex } { 4 } - { Numbers~out~of~order~in~\iow_char:N\{\iow_char\}~quantifier. } -\msg_new:nnn { regex } { 6 } - { Missing~terminating~\iow_char:N\]~for~character~class } -\msg_new:nnn { regex } { 7 } - { Invalid~escape~sequence~in~character~class } -\msg_new:nnn { regex } { 8 } - { Range~out~of~order~in~character~class } -\msg_new:nnn { regex } { 22 } { Mismatched~parentheses } -\msg_new:nnn { regex } { 34 } - { Character~value~in~\iow_char:N\\x{...}~sequence~is~too~large } +% \msg_new:nnn { regex } { 4 } +% { Numbers~out~of~order~in~\iow_char:N\{\iow_char\}~quantifier. } +% \msg_new:nnn { regex } { 6 } +% { Missing~terminating~\iow_char:N\]~for~character~class } +% \msg_new:nnn { regex } { 7 } +% { Invalid~escape~sequence~in~character~class } +% \msg_new:nnn { regex } { 34 } +% { Character~value~in~\iow_char:N\\x{...}~sequence~is~too~large } % \msg_new:nnn { regex } { 44 } { Invalid~UTF-8~string } % \msg_new:nnn { regex } { 46 } % { Malformed~\iow_char:N\\P~or\iow_char:N\\p~sequence } @@ -3867,18 +4464,46 @@ % \end{macrocode} % % \begin{macrocode} -\msg_new:nnn { regex } { not-nfa } +% \end{macrocode} +% +% \begin{macrocode} +\msg_kernel_new:nnnn { regex } { build-c } + { Misused~\iow_char:N\\c~or\iow_char:N\\C~command~in~regular~expression. } + { + The~\iow_char:N\\C~escape~sequence~can~be~followed~by~ + one~of~the~letters~ABCDELMOPSTU.~The~\iow_char:N\\c~ + escape~sequence~can~also~be~followed~by~a~brace~group. + } +\msg_kernel_new:nnnn { regex } { unknown-option } + { Unknown~option~`#1'~for~regular~expressions. } + { + LaTeX~came~across~something~like~`(?#1)'~in~a~regular~expression,~ + but~the~option~`#1'~is~not~known.~It~will~be~ignored. + } +\msg_kernel_new:nnnn { regex } { invalid-in-option } + { Invalid~character~in~option~of~a~regular~expression. } + { + The~character~or~escape~sequence~`#1'~is~not~defined~ + as~an~option~within~regular~expressions. + } +\msg_kernel_new:nnnn { regex } { replacement-g } + { Missing~brace~for~the~\iow_char:N\\g~construction~in~a~replacement~text. } + { + In~the~replacement~text~for~a~regular~expression~search,~ + submatches~are~represented~either~as~\iow_char:N \\g{dd..d},~ + or~\\d,~where~`d'~are~single~digits.~Here,~a~brace~is~missing. + } +\msg_kernel_new:nnnn { regex } { replacement-c } + { Something~is~wrong~with~the~replacement~text. } { - LaTeX~was~expecting~a~regular~expression~variable.\\ - Instead,~LaTeX~found~'#1'. + The~\iow_char:N\\c~escape~sequence~can~only~be~followed~by~ + ABCDELMN~or~by~a~brace. } % \end{macrocode} % % \begin{macrocode} \msg_kernel_new:nnn { regex } { nfa-misused } { Automaton~#1 used~incorrectly. } -\msg_kernel_new:nnn { regex } { g-misused } - { \g misused~in~replacement~text. } % \end{macrocode} % % \begin{macrocode} |