summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx')
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx1099
1 files changed, 661 insertions, 438 deletions
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx
index 3b26d43f4e5..ddfbc79de23 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx
@@ -23,8 +23,8 @@
%<*driver|package>
% The version of expl3 required is tested as early as possible, as
% some really old versions do not define \ProvidesExplPackage.
-\RequirePackage{expl3}[2017/04/01]
-%<package>\@ifpackagelater{expl3}{2017/04/01}
+\RequirePackage{expl3}[2017/05/13]
+%<package>\@ifpackagelater{expl3}{2017/05/13}
%<package> {}
%<package> {%
%<package> \PackageError{l3regex}{Support package l3kernel too old}
@@ -59,7 +59,7 @@
% }^^A
% }
%
-% \date{Released 2017/04/01}
+% \date{Released 2017/05/13}
%
% \maketitle
%
@@ -352,21 +352,37 @@
% \subsection{Syntax of the replacement text}
%
% Most of the features described in regular expressions do not make
-% sense within the replacement text. Escaped characters are supported
-% as inside regular expressions. The whole match is accessed as~|\0|,
-% and the first~$9$ submatches are accessed as |\1|, \ldots{},~|\9|.
-% Further submatches are accessed through |\g{|\meta{number}|}| where
-% \meta{number} is any non-negative integer. If there are fewer than
-% \meta{number} capturing groups, the submatch is empty.
+% sense within the replacement text. Backslash introduces various
+% special constructions:
+% \begin{itemize}
+% \item |\0| is the whole match;
+% \item |\1|, |\2|, \ldots{}, |\9| or |\g{|\meta{number}|}| are the
+% submatches (empty if there are fewer than \meta{number} capturing
+% groups);
+% \item \verb*|\ | inserts a space (spaces are ignored when not
+% escaped);
+% \item |\a|, |\e|, |\f|, |\n|, |\r|, |\t|, |\xhh|, |\x{hhh}|
+% correspond to single characters as in regular expressions;
+% \item |\c|\Arg{cs~name} inserts a control sequence;
+% \item |\c|\meta{category}\meta{character} (see below);
+% \item |\u|\Arg{tl~var~name} inserts the contents of the
+% \meta{tl~var} (see below).
+% \end{itemize}
+% Characters other than backslash and space are simply inserted in the
+% result (but since the replacement text is first converted to a string,
+% one should also escape characters that are special for \TeX{}, for
+% instance use~|\#|). Non-alphanumeric characters can always be safely
+% escaped with a backslash.
%
% For instance,
% \begin{verbatim}
% \tl_set:Nn \l_my_tl { Hello,~world! }
-% \regex_replace_all:nnN { ([er]?l|o) . } { \(\0\-\-\1\) } \l_my_tl
+% \regex_replace_all:nnN { ([er]?l|o) . } { (\0--\1) } \l_my_tl
% \end{verbatim}
% results in \cs{l_my_tl} holding |H(ell--el)(o,--o) w(or--o)(ld--l)!|
%
-% Submatches keep the same category codes as in the original token list.
+% Submatches always keep the same category codes as in the original
+% token list.
% The characters inserted by the replacement have category code $12$
% (other) by default, with the exception of space characters. Spaces
% inserted through \verb*|\ | have category code $10$, while spaces
@@ -374,21 +390,25 @@
% The escape sequence |\c| allows to insert characters
% with arbitrary category codes, as well as control sequences.
% \begin{l3regex-syntax}
-% \item[\\cXY] Produces the character~|Y| (which can be given as an
-% escape sequence such as~|\t| for tab, or |\(| or~|\)| for a
-% parenthesis) with category code~|X|, which must be one of
-% |CBEMTPUDSLOA|.
+% \item[\\cX(\ldots{})] Produces the characters \enquote{\ldots{}} with
+% category~|X|, which must be one of |CBEMTPUDSLOA| as in regular
+% expressions. Parentheses are optional for a single character (which
+% can be an escape sequence). This can be nested, for instance
+% |\cL(Hello\cS\ world)!|
% \item[\\c\Arg{text}] Produces the control sequence with csname
% \meta{text}. The \meta{text} may contain references to the
-% submatches |\0|, |\1|, \emph{etc.}
+% submatches |\0|, |\1|, and so on, as in the example for |\u| below.
% \end{l3regex-syntax}
%
% The escape sequence |\u|\Arg{tl~var~name} allows to insert the
% contents of the token list with name \meta{tl~var~name} directly into
-% the replacement, avoiding the need to escape special characters.
-% Within the construction |\c|\Arg{text}, the |\u|~escape sequence only
-% expands its argument once, in effect performing \cs{tl_to_str:v}.
-% Submatches can be used within the argument of |\u|. For instance,
+% the replacement, giving an easier control of category codes.
+% Within |\c{|\ldots{}|}| and |\u{|\ldots{}|}| constructions, the |\u|
+% and |\c|~escape sequences perform \cs{tl_to_str:v}, namely extract the
+% value of the control sequence and turn it into a string.
+%
+% Matches can be used within the arguments of |\c| and |\u|. For
+% instance,
% \begin{verbatim}
% \tl_set:Nn \l_my_one_tl { first }
% \tl_set:Nn \l_my_two_tl { \emph{second} }
@@ -611,7 +631,8 @@
% \item Currently, |a{\x34}| is recognized as |a{4}|.
% \item Cleaner error reporting in the replacement phase.
% \item Add tracing information.
-% \item Detect attempts to use back-references.
+% \item Detect attempts to use back-references and other
+% non-implemented syntax.
% \item Test for the maximum register \cs{c_max_register_int}.
% \item Find out whether the fact that |\W| and friends match the
% end-marker leads to bugs. Possibly update \cs{__regex_item_reverse:n}.
@@ -622,23 +643,19 @@
%
% Code improvements to come.
% \begin{itemize}
-% \item Change \tn{skip} to \tn{dimen} for the array of active
-% threads, and shift the array of submatch informations so that it
-% starts at \tn{skip}$0$.
-% \item Optimize |\c{abc}| for matching a specific control sequence.
+% \item Shift arrays so that the useful information starts at
+% position~$1$.
% \item Only build \c{...} once.
-% \item Use \tn{skip} for the left and right state stacks when
+% \item Use arrays for the left and right state stacks when
% compiling a regex.
% \item Should \cs{__regex_action_free_group:n} only be used for greedy
% |{n,}| quantifier? (I think not.)
% \item Quantifiers for |\u| and assertions.
-% \item Improve digit grabbing for the |\g| escape in replacement.
-% Allow arbitrary integer expressions for all those numbers?
% \item When matching, keep track of an explicit stack of
% \texttt{current_state} and \texttt{current_submatches}.
% \item If possible, when a state is reused by the same thread, kill
% other subthreads.
-% \item Use \tn{dimen} registers rather than \cs{l__regex_balance_tl}
+% \item Use an array rather than \cs{l__regex_balance_tl}
% to build \cs{__regex_replacement_balance_one_match:n}.
% \item Reduce the number of epsilon-transitions in alternatives.
% \item Optimize simple strings: use less states (|abcade| should give
@@ -650,8 +667,6 @@
% \item Optimize the use of \cs{int_step_...} functions.
% \item Groups don't capture within regexes for csnames; optimize and
% document.
-% \item Decide and document what |\c{\c{...}}| should do in the
-% replacement text, similar questions for |\u|.
% \item Better \enquote{show} for anchors, properties, and catcode tests.
% \item Does |\K| really need a new state for itself?
% \item When compiling, use a boolean \texttt{in_cs} and less magic
@@ -664,7 +679,6 @@
% The following features are likely to be implemented at some point
% in the future.
% \begin{itemize}
-% \item Allow |\cL(abc)| in replacement text.
% \item General look-ahead/behind assertions.
% \item Regex matching on external files.
% \item Conditional subpatterns with look ahead/behind: \enquote{if
@@ -680,13 +694,11 @@
% This requires to manipulate a lot of data, probably using tree-boxes.
% \end{itemize}
%
-% The following features of \textsc{pcre} or Perl will probably not be
+% The following features of \textsc{pcre} or Perl may or may not be
% implemented.
% \begin{itemize}
% \item |\ddd|, matching the character with octal code \texttt{ddd};
-% \item Callout with |(?C...)|, we cannot run arbitrary user code
-% during the matching, because the regex code uses registers in an
-% unsafe way;
+% \item Callout with |(?C...)|;
% \item Conditional subpatterns (other than with a look-ahead or
% look-behind condition): this is non-regular, isn't it?
% \item Named subpatterns: \TeX{} programmers have lived so far
@@ -705,9 +717,7 @@
% non-backtracking algorithm, and difficult to implement.
% \item Subroutine calls: this syntactic sugar is difficult to include
% in a non-backtracking algorithm, in particular because the
-% corresponding group should be treated as atomic. Also, we cannot
-% afford to run user code within the regular expression matching,
-% because of our \enquote{misuse} of registers.
+% corresponding group should be treated as atomic.
% \item Recursion: this is a non-regular feature.
% \item Back-references: non-regular feature, this requires
% backtracking, which is prohibitively slow.
@@ -734,9 +744,9 @@
%
% \begin{macrocode}
%<*package>
-\ProvidesExplPackage{l3regex}{2017/04/01}{}
+\ProvidesExplPackage{l3regex}{2017/05/13}{}
{L3 Experimental regular expressions}
-\RequirePackage{l3tl-build, l3tl-analysis, l3str-convert}
+\RequirePackage{l3tl-build, l3tl-analysis, l3intarray}
%</package>
% \end{macrocode}
%
@@ -754,7 +764,7 @@
% \item (Compiling.) Analyse the regex, finding invalid input, and
% convert it to an internal representation.
% \item (Building.) Convert the compiled regex to a non-deterministic
-% finite automaton (\textsc{nfa}) with roughly $n$ states which
+% finite automaton (\textsc{nfa}) with $O(n)$ states which
% accepts precisely token lists matching that regex.
% \item (Matching.) Loop through the query token list one token (one
% \enquote{position}) at a time, exploring in parallel every
@@ -787,46 +797,42 @@
% unique id for all the steps of the matching algorithm.
% \end{itemize}
%
-% To achieve a good performance, we abuse \TeX{}'s registers in two
-% ways. We access registers directly by number rather than tying them
-% to control sequence using \cs{int_new:N} and other allocation
-% functions. And we store integers in \tn{dimen} registers in scaled
-% points (\texttt{sp}), using \TeX{}'s implicit conversion from
-% dimensions to integers in some contexts. Specifically, the registers
-% are used as follows. When compiling, \tn{toks} registers are used
-% under the hood by functions from the \pkg{l3tl-build} module. When
-% building,
+% We use \pkg{l3intarray} to manipulate arrays of integers (stored into
+% some dimension registers in scaled points). We also abuse \TeX{}'s
+% \tn{toks} registers, by accessing them directly by number rather than
+% tying them to control sequence using the \tn{newtoks} allocation
+% functions. Specifically, these arrays and \tn{toks} are used as
+% follows. When compiling, \tn{toks} registers are used under the hood
+% by functions from the \pkg{l3tl-build} module. When building,
+% \tn{toks}\meta{state} holds the tests and actions to perform in the
+% \meta{state} of the \textsc{nfa}. When matching,
% \begin{itemize}
-% \item \tn{toks}\meta{state} holds the tests and actions to perform
-% in the \meta{state} of the \textsc{nfa}.
-% \item (Not implemented yet.)
-% \tn{skip}$i$ has the form \meta{group id} \texttt{plus}
-% \meta{left state} \texttt{minus} \meta{right state}.
-% \end{itemize}
-% When matching,
-% \begin{itemize}
-% \item \tn{dimen}\meta{state} is equal to the last \meta{step} in
-% which the \meta{state} was active.
-% \item (Currently, we use \tn{skip} instead of \tn{dimen}.)
-% \tn{dimen}\meta{thread}, with $\texttt{min_active} \leq
-% \meta{thread} < \texttt{max_active}$, is equal to the
-% \meta{state} in which the \meta{thread} currently is. The
+% \item \cs{g_@@_state_active_intarray} holds the last \meta{step} in
+% which each \meta{state} was active.
+% \item \cs{g_@@_thread_state_intarray} maps each \meta{thread} (with
+% $\texttt{min_active} \leq \meta{thread} < \texttt{max_active}$) to
+% the \meta{state} in which the \meta{thread} currently is. The
% \meta{threads} or ordered starting from the best to the least
% preferred.
% \item \tn{toks}\meta{thread} holds the submatch information for the
% \meta{thread}, as the contents of a property list.
-% \item \tn{muskip}\meta{position} holds as its main and stretch
-% components the character and category code of the token at this
+% \item \cs{g_@@_charcode_intarray} and \cs{g_@@_catcode_intarray} hold the
+% character codes and category codes of tokens at each
% \meta{position} in the query.
+% \item \cs{g_@@_balance_intarray} holds the balance of begin-group and
+% end-group character tokens which appear before that point in the
+% token list.
% \item \tn{toks}\meta{position} holds \meta{tokens} which \texttt{o}-
% and \texttt{x}-expand to the \meta{position}-th token in the query.
-% \item \tn{skip} registers hold the value of end-points of all
-% submatches as would be extracted by the \cs{regex_extract}
-% functions. Since smaller \tn{skip} registers are used, the minimum
-% index is twice \texttt{max_state}, and the used registers go up to
-% \cs{l_@@_submatch_int}. They are organized in blocks of
-% \texttt{capturing_group}, each block corresponding to one match
-% with all its submatches stored in consecutive \tn{skip}s.
+% \item \cs{g_@@_submatch_prev_intarray}, \cs{g_@@_submatch_begin_intarray}
+% and \cs{g_@@_submatch_end_intarray} hold, for each submatch (as would
+% be extracted by \cs{regex_extract_all:nnN}), the place where the
+% submatch started to be looked for and its two end-points. For
+% historical reasons, the minimum index is twice \texttt{max_state},
+% and the used registers go up to \cs{l_@@_submatch_int}. They are
+% organized in blocks of \cs{l_@@_capturing_group_int} entries, each
+% block corresponding to one match with all its submatches stored in
+% consecutive entries.
% \end{itemize}
% \tn{count} registers are not abused, which means that we can safely
% use named integers in this module. Note that \tn{box} registers are
@@ -851,6 +857,89 @@
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}[int]{\@@_standard_escapechar:}
+% Make the \tn{escapechar} into the standard backslash.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_standard_escapechar:
+ { \int_set:Nn \tex_escapechar:D { `\\ } }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[int, EXP]{\@@_toks_use:w}
+% Unpack a \tn{toks} given its number.
+% \begin{macrocode}
+\cs_new:Npn \@@_toks_use:w { \tex_the:D \tex_toks:D }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[int]{\@@_toks_clear:N, \@@_toks_set:Nn, \@@_toks_set:No}
+% Empty a \tn{toks} or set it to a value, given its number.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_toks_clear:N #1
+ { \tex_toks:D #1 { } }
+\cs_new_eq:NN \@@_toks_set:Nn \tex_toks:D
+\cs_new_protected:Npn \@@_toks_set:No #1
+ { \@@_toks_set:Nn #1 \exp_after:wN }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[int]{\@@_toks_memcpy:NNn}
+% Copy |#3| \tn{toks} registers from |#2| onwards to |#1| onwards,
+% like |C|'s |memcpy|.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_toks_memcpy:NNn #1#2#3
+ {
+ \prg_replicate:nn {#3}
+ {
+ \tex_toks:D #1 = \tex_toks:D #2
+ \int_incr:N #1
+ \int_incr:N #2
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[int]{\@@_toks_put_left:Nx}
+% \begin{macro}[int]{\@@_toks_put_right:Nx, \@@_toks_put_right:Nn}
+% During the building phase we wish to add \texttt{x}-expanded
+% material to \tn{toks}, either to the left or to the right. The
+% expansion is done \enquote{by hand} for optimization (these
+% operations are used quite a lot). The \texttt{Nn} version of
+% \cs{@@_toks_put_right:Nx} is provided because it is more
+% efficient than \texttt{x}-expanding with \cs{exp_not:n}.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_toks_put_left:Nx #1#2
+ {
+ \cs_set:Npx \@@_tmp:w { #2 }
+ \tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN
+ { \exp_after:wN \@@_tmp:w \tex_the:D \tex_toks:D #1 }
+ }
+\cs_new_protected:Npn \@@_toks_put_right:Nx #1#2
+ {
+ \cs_set:Npx \@@_tmp:w {#2}
+ \tex_toks:D #1 \exp_after:wN
+ { \tex_the:D \tex_toks:D \exp_after:wN #1 \@@_tmp:w }
+ }
+\cs_new_protected:Npn \@@_toks_put_right:Nn #1#2
+ { \tex_toks:D #1 \exp_after:wN { \tex_the:D \tex_toks:D #1 #2 } }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int, rEXP]{\@@_current_cs_to_str:}
+% Expands to the string representation of the token (known to be a
+% control sequence) at the current position \cs{l_@@_current_pos_int}.
+% It should only be used in \texttt{x}-expansion to avoid losing a
+% leading space.
+% \begin{macrocode}
+\cs_new:Npn \@@_current_cs_to_str:
+ {
+ \exp_after:wN \exp_after:wN \exp_after:wN \cs_to_str:N
+ \tex_the:D \tex_toks:D \l_@@_current_pos_int
+ }
+% \end{macrocode}
+% \end{macro}
+%
% \subsubsection{Constants and variables}
%
% \begin{macro}[aux]{\@@_tmp:w}
@@ -894,14 +983,24 @@
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\l_@@_balance_int}
+% \begin{variable}{\g_@@_charcode_intarray, \g_@@_catcode_intarray, \g_@@_balance_intarray}
% The first thing we do when matching is to go once through the query
-% token list and store the information for each token as \tn{muskip}
-% and \tn{toks} registers. During this phase, \cs{l_@@_balance_int}
-% counts the balance of begin-group and end-group character tokens
-% which appear before a given point in the token list, and we store it
-% as the shrink component of each \tn{muskip} register. This variable
-% is also used to keep track of the balance in the replacement text.
+% token list and store the information for each token into
+% \cs{g_@@_charcode_intarray}, \cs{g_@@_catcode_intarray} and \tn{toks}
+% registers. We also store the balance of begin-group/end-group
+% characters into \cs{g_@@_balance_intarray}.
+% \begin{macrocode}
+\__intarray_new:Nn \g_@@_charcode_intarray { 65536 }
+\__intarray_new:Nn \g_@@_catcode_intarray { 65536 }
+\__intarray_new:Nn \g_@@_balance_intarray { 65536 }
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_@@_balance_int}
+% During this phase, \cs{l_@@_balance_int} counts the balance of
+% begin-group and end-group character tokens which appear before a
+% given point in the token list. This variable is also used to keep
+% track of the balance in the replacement text.
% \begin{macrocode}
\int_new:N \l_@@_balance_int
% \end{macrocode}
@@ -918,6 +1017,20 @@
%
% \subsubsection{Testing characters}
%
+% \begin{macro}{\c_@@_ascii_min_int, \c_@@_ascii_max_control_int, \c_@@_ascii_max_int}
+% \begin{macrocode}
+\int_const:Nn \c_@@_ascii_min_int { 0 }
+\int_const:Nn \c_@@_ascii_max_control_int { 31 }
+\int_const:Nn \c_@@_ascii_max_int { 127 }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{variable}{\c_@@_ascii_lower_int}
+% \begin{macrocode}
+\int_const:Nn \c_@@_ascii_lower_int { `a - `A }
+% \end{macrocode}
+% \end{variable}
+%
% \begin{macro}[int]{\@@_break_point:TF}
% \begin{macro}[int]{\@@_break_true:w}
% When testing whether a character of the query token list matches
@@ -1027,12 +1140,12 @@
\if_int_compare:w \l_@@_current_char_int > `Z \exp_stop_f:
\if_int_compare:w \l_@@_current_char_int > `z \exp_stop_f: \else:
\if_int_compare:w \l_@@_current_char_int < `a \exp_stop_f: \else:
- \int_sub:Nn \l_@@_case_changed_char_int { \c__str_ascii_lower_int }
+ \int_sub:Nn \l_@@_case_changed_char_int { \c_@@_ascii_lower_int }
\fi:
\fi:
\else:
\if_int_compare:w \l_@@_current_char_int < `A \exp_stop_f: \else:
- \int_add:Nn \l_@@_case_changed_char_int { \c__str_ascii_lower_int }
+ \int_add:Nn \l_@@_case_changed_char_int { \c_@@_ascii_lower_int }
\fi:
\fi:
}
@@ -1099,12 +1212,7 @@
\int_compare:nNnTF \l_@@_current_catcode_int = 0
{
\tl_set:Nx \l_@@_internal_a_tl
- {
- \scan_stop:
- \exp_after:wN \exp_after:wN \exp_after:wN \cs_to_str:N
- \tex_the:D \tex_toks:D \l_@@_current_pos_int
- \scan_stop:
- }
+ { \scan_stop: \@@_current_cs_to_str: \scan_stop: }
\tl_if_in:noTF { \scan_stop: #1 \scan_stop: } \l_@@_internal_a_tl
{ \@@_break_true:w } { }
}
@@ -1128,12 +1236,7 @@
\int_compare:nNnT \l_@@_current_catcode_int = 0
{
\group_begin:
- \tl_set:Nx \l_@@_cs_name_tl
- {
- \exp_after:wN \exp_after:wN
- \exp_after:wN \cs_to_str:N
- \tex_the:D \tex_toks:D \l_@@_current_pos_int
- }
+ \tl_set:Nx \l_@@_cs_name_tl { \@@_current_cs_to_str: }
\@@_single_match:
\@@_disable_submatches:
\@@_build_for_cs:n {#1}
@@ -1214,16 +1317,16 @@
\cs_new_protected:Npn \@@_posix_ascii:
{
\@@_item_caseful_range:nn
- \c__str_ascii_min_int
- \c__str_ascii_max_int
+ \c_@@_ascii_min_int
+ \c_@@_ascii_max_int
}
\cs_new_eq:NN \@@_posix_blank: \@@_prop_h:
\cs_new_protected:Npn \@@_posix_cntrl:
{
\@@_item_caseful_range:nn
- \c__str_ascii_min_int
- \c__str_ascii_max_control_int
- \@@_item_caseful_equal:n \c__str_ascii_max_int
+ \c_@@_ascii_min_int
+ \c_@@_ascii_max_control_int
+ \@@_item_caseful_equal:n \c_@@_ascii_max_int
}
\cs_new_eq:NN \@@_posix_digit: \@@_prop_d:
\cs_new_protected:Npn \@@_posix_graph:
@@ -1301,8 +1404,8 @@
\cs_set:Npn \@@_escape_unescaped:N ##1 { #1 }
\cs_set:Npn \@@_escape_escaped:N ##1 { #2 }
\cs_set:Npn \@@_escape_raw:N ##1 { #3 }
- \int_set:Nn \tex_escapechar:D { `\\ }
- \__str_gset_other:Nn \g_@@_internal_tl { #4 }
+ \@@_standard_escapechar:
+ \tl_gset:Nx \g_@@_internal_tl { \__str_to_other_fast:n {#4} }
\tl_set:Nx \l_@@_internal_b_tl
{
\exp_after:wN \@@_escape_loop:N \g_@@_internal_tl
@@ -1389,12 +1492,11 @@
%
% \begin{macro}[aux]{\@@_escape_/x:w}
% \begin{macro}[aux]{\@@_escape_x_end:w, \@@_escape_x_large:n}
-% When |\x| is encountered, \cs{@@_escape_x_test:N} is responsible
-% for grabbing some hexadecimal digits, and feeding the result to
-% \cs{@@_escape_x_end:w}. If the number is $<256$, then it is
-% turned into a byte and fed to \cs{@@_escape_raw:N}. Otherwise,
-% interrupt the assignment, and either produce an error, or use a
-% standard \tn{lowercase} trick depending on the precise value.
+% When |\x| is encountered, \cs{@@_escape_x_test:N} is responsible for
+% grabbing some hexadecimal digits, and feeding the result to
+% \cs{@@_escape_x_end:w}. If the number is too big interrupt the
+% assignment and produce an error, otherwise call \cs{@@_escape_raw:N}
+% on the corresponding character token.
% \begin{macrocode}
\cs_new:cpn { @@_escape_/x:w } \@@_escape_loop:N
{
@@ -1403,40 +1505,23 @@
}
\cs_new:Npn \@@_escape_x_end:w #1 ;
{
- \int_compare:nNnTF {#1} > \c__str_max_byte_int
- { \@@_escape_x_large:n {#1} }
+ \int_compare:nNnTF {#1} > \c_max_char_int
+ {
+ \if_false: { \fi: }
+ \__tl_build_one:o \l_@@_internal_b_tl
+ \__msg_kernel_error:nnx { regex } { x-overflow } {#1}
+ \tl_set:Nx \l_@@_internal_b_tl
+ { \if_false: } \fi:
+ }
{
\exp_last_unbraced:Nf \@@_escape_raw:N
- { \__str_output_byte:n {#1} }
+ { \char_generate:nn {#1} { 12 } }
}
}
-\group_begin:
- \char_set_catcode_other:n { 0 }
- \cs_new:Npn \@@_escape_x_large:n #1
- {
- \if_false: { \fi: }
- \__tl_build_one:o \l_@@_internal_b_tl
- \int_compare:nNnTF {#1} > \c_max_char_int
- {
- \__msg_kernel_error:nnx { regex } { x-overflow } {#1}
- \tl_set:Nx \l_@@_internal_b_tl
- { \if_false: } \fi:
- }
- {
- \char_set_lccode:nn { 0 } {#1}
- \tex_lowercase:D
- {
- \tl_set:Nx \l_@@_internal_b_tl
- { \if_false: } \fi:
- \@@_escape_raw:N ^^@
- }
- }
- }
-\group_end:
% \end{macrocode}
% \end{macro}
% \end{macro}
-% ^^A todo: use char_generate in above and rest of file
+%
% \begin{macro}[aux]{\@@_escape_x_test:N, \@@_escape_x_testii:N}
% Find out whether the first character is a left brace (allowing any
% number of hexadecimal digits), or not (allowing up to two
@@ -1461,7 +1546,7 @@
\if_charcode:w \c_left_brace_str #1
\exp_after:wN \@@_escape_x_loop:N
\else:
- \__str_hexadecimal_use:NTF #1
+ \@@_hexadecimal_use:NTF #1
{ \exp_after:wN \@@_escape_x:N }
{ ; \exp_after:wN \@@_escape_loop:N \exp_after:wN #1 }
\fi:
@@ -1476,7 +1561,7 @@
{
\str_if_eq_x:nnTF {#1} { break } { ; }
{
- \__str_hexadecimal_use:NTF #1
+ \@@_hexadecimal_use:NTF #1
{ ; \@@_escape_loop:N }
{ ; \@@_escape_loop:N #1 }
}
@@ -1494,7 +1579,7 @@
\str_if_eq_x:nnTF {#1} { break }
{ ; \@@_escape_x_loop_error:n { } {#1} }
{
- \__str_hexadecimal_use:NTF #1
+ \@@_hexadecimal_use:NTF #1
{ \@@_escape_x_loop:N }
{
\token_if_eq_charcode:NNTF \c_space_token #1
@@ -1520,6 +1605,35 @@
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}[aux, rEXP]{\@@_hexadecimal_use:NTF}
+% \TeX{} detects uppercase hexadecimal digits for us but not the
+% lowercase letters, which we need to detect and replace by their
+% uppercase counterpart.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \@@_hexadecimal_use:N #1 { TF }
+ {
+ \if_int_compare:w 1 < "1 \token_to_str:N #1 \exp_stop_f:
+ #1 \prg_return_true:
+ \else:
+ \if_case:w \__int_eval:w
+ \exp_after:wN ` \token_to_str:N #1 - `a
+ \__int_eval_end:
+ A
+ \or: B
+ \or: C
+ \or: D
+ \or: E
+ \or: F
+ \else:
+ \prg_return_false:
+ \exp_after:wN \use_none:n
+ \fi:
+ \prg_return_true:
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
% \begin{macro}[EXP, aux]
% {\@@_char_if_alphanumeric:NTF, \@@_char_if_special:NTF}
% These two tests are used in the first pass when parsing a regular
@@ -1549,7 +1663,7 @@
{
\if_int_compare:w `#1 > `Z \exp_stop_f:
\if_int_compare:w `#1 > `z \exp_stop_f:
- \if_int_compare:w `#1 < \c__str_ascii_max_int
+ \if_int_compare:w `#1 < \c_@@_ascii_max_int
\prg_return_true: \else: \prg_return_false: \fi:
\else:
\if_int_compare:w `#1 < `a \exp_stop_f:
@@ -1991,7 +2105,7 @@
\cs_new_protected:Npn \@@_compile:n #1
{
\@@_compile:w
- \int_set:Nn \tex_escapechar:D { `\\ }
+ \@@_standard_escapechar:
\int_set_eq:NN \l_@@_mode_int \c_@@_outer_mode_int
\@@_escape_use:nnnn
{
@@ -2915,7 +3029,7 @@
{
\if_int_odd:w \__int_eval:w \l_@@_catcodes_int / #1 \__int_eval_end:
\else:
- \tex_advance:D \l_@@_catcodes_int #1
+ \int_add:Nn \l_@@_catcodes_int {#1}
\fi:
}
\cs_new_protected:Npn \@@_compile_c_lbrack_end:
@@ -3105,8 +3219,8 @@
% \begin{macrocode}
\cs_new_protected:Npn \@@_compile_u_in_cs:
{
- \exp_args:NNo \__str_gset_other:Nn \g_@@_internal_tl
- { \l_@@_internal_a_tl }
+ \tl_gset:Nx \g_@@_internal_tl
+ { \exp_args:No \__str_to_other_fast:n { \l_@@_internal_a_tl } }
\__tl_build_one:x
{
\tl_map_function:NN \g_@@_internal_tl
@@ -3396,10 +3510,11 @@
% \begin{variable}{\l_@@_min_state_int, \l_@@_max_state_int}
% The last state that was allocated is $\cs{l_@@_max_state_int}-1$,
% so that \cs{l_@@_max_state_int} always points to a free state.
-% The \texttt{min_state} variable is always $0$, but is included to
-% avoid hard-coding this value.
+% The \texttt{min_state} variable is $1$, but is included to
+% avoid hard-coding this value everywhere.
% \begin{macrocode}
\int_new:N \l_@@_min_state_int
+\int_set:Nn \l_@@_min_state_int { 1 }
\int_new:N \l_@@_max_state_int
% \end{macrocode}
% \end{variable}
@@ -3489,7 +3604,7 @@
\cs_new_protected:Npn \@@_build:N #1
{
%<trace> \trace_push:nnn { regex } { 1 } { @@_build }
- \int_set:Nn \tex_escapechar:D { `\\ }
+ \@@_standard_escapechar:
\int_zero:N \l_@@_capturing_group_int
\int_set_eq:NN \l_@@_max_state_int \l_@@_min_state_int
\@@_build_new_state:
@@ -3557,33 +3672,6 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[int]{\@@_toks_put_left:Nx}
-% \begin{macro}[int]{\@@_toks_put_right:Nx, \@@_toks_put_right:Nn}
-% During the building phase we wish to add \texttt{x}-expanded
-% material to \tn{toks}, either to the left or to the right. The
-% expansion is done \enquote{by hand} for optimization (these
-% operations are used quite a lot). The \texttt{Nn} version of
-% \cs{@@_toks_put_right:Nx} is provided because it is more
-% efficient than \texttt{x}-expanding with \cs{exp_not:n}.
-% \begin{macrocode}
-\cs_new_protected:Npn \@@_toks_put_left:Nx #1#2
- {
- \cs_set:Npx \@@_tmp:w { #2 }
- \tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN
- { \exp_after:wN \@@_tmp:w \tex_the:D \tex_toks:D #1 }
- }
-\cs_new_protected:Npn \@@_toks_put_right:Nx #1#2
- {
- \cs_set:Npx \@@_tmp:w {#2}
- \tex_toks:D #1 \exp_after:wN
- { \tex_the:D \tex_toks:D \exp_after:wN #1 \@@_tmp:w }
- }
-\cs_new_protected:Npn \@@_toks_put_right:Nn #1#2
- { \tex_toks:D #1 \exp_after:wN { \tex_the:D \tex_toks:D #1 #2 } }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-%
% \begin{macro}[int]
% {
% \@@_build_transition_left:NNN,
@@ -3620,7 +3708,7 @@
\int_eval:n { \l_@@_max_state_int + 1 }
}
%</trace>
- \tex_toks:D \l_@@_max_state_int { }
+ \@@_toks_clear:N \l_@@_max_state_int
\int_set_eq:NN \l_@@_left_state_int \l_@@_right_state_int
\int_set_eq:NN \l_@@_right_state_int \l_@@_max_state_int
\int_incr:N \l_@@_max_state_int
@@ -3933,15 +4021,12 @@
( #1 - 1 )
* ( \l_@@_internal_b_int - \l_@@_internal_a_int )
}
- \tex_advance:D \l_@@_right_state_int \l_@@_internal_c_int
- \tex_advance:D \l_@@_max_state_int \l_@@_internal_c_int
- \prg_replicate:nn \l_@@_internal_c_int
- {
- \tex_toks:D \l_@@_internal_b_int
- = \tex_toks:D \l_@@_internal_a_int
- \int_incr:N \l_@@_internal_a_int
- \int_incr:N \l_@@_internal_b_int
- }
+ \int_add:Nn \l_@@_right_state_int { \l_@@_internal_c_int }
+ \int_add:Nn \l_@@_max_state_int { \l_@@_internal_c_int }
+ \@@_toks_memcpy:NNn
+ \l_@@_internal_b_int
+ \l_@@_internal_a_int
+ \l_@@_internal_c_int
\fi:
}
% \end{macrocode}
@@ -4132,7 +4217,7 @@
% transitions, the instruction at the new state of the \textsc{nfa} is
% performed immediately. When a transition consumes a character, the
% new state is appended to a list of \enquote{active states}, stored in
-% \tn{skip} registers: this thread will be active again when the next
+% \cs{g_@@_thread_state_intarray}: this thread will be active again when the next
% token is read from the query. At every step (for each token in the
% query), we unpack that list of active states and the corresponding
% submatch props, and empty those.
@@ -4176,7 +4261,7 @@
% }
% The tokens in the query are indexed from \texttt{min_pos} for the
% first to $\texttt{max_pos}-1$ for the last, and their information is
-% stored in \tn{muskip} and \tn{toks} registers with those numbers. We
+% stored in several arrays and \tn{toks} registers with those numbers. We
% don't start from $0$ because the \tn{toks} registers with low
% numbers are used to hold the states of the \textsc{nfa}. We match
% without backtracking, keeping all threads in lockstep at the
@@ -4243,15 +4328,15 @@
%
% \begin{variable}{\l_@@_step_int}
% This integer, always even, is increased every time a character in
-% the query is read, and not reset when doing multiple matches. For
-% each \meta{state} in the \textsc{nfa} we store in
-% \tn{dimen}\meta{state} the last step in which this state was
-% encountered. This lets us break infinite loops by not visiting the
-% same state twice in the same step. In fact, \tn{dimen}\meta{state}
-% is equal \texttt{step} when we have started performing the
-% operations of \tn{toks}\meta{state}, but not finished yet. However,
-% once we finish, we set \tn{dimen}\meta{state} to
-% $\text{\texttt{step}}+1$. This is needed to track submatches
+% the query is read, and not reset when doing multiple matches. We
+% store in \cs{g_@@_state_active_intarray} the last step in which each
+% \meta{state} in the \textsc{nfa} was encountered. This lets us break
+% infinite loops by not visiting the same state twice in the same
+% step. In fact, the step we store is equal to \texttt{step} when we
+% have started performing the operations of \tn{toks}\meta{state}, but
+% not finished yet. However, once we finish, we store
+% $\text{\texttt{step}}+1$ in \cs{g_@@_state_active_intarray}. This is
+% needed to track submatches
% properly (see building phase). The \texttt{step} is also used to
% attach each set of submatch information to a given iteration (and
% automatically discard it when it corresponds to a past step).
@@ -4261,8 +4346,8 @@
% \end{variable}
%
% \begin{variable}{\l_@@_min_active_int, \l_@@_max_active_int}
-% All the currently active states are kept in order of precedence in
-% the \tn{skip} registers, and the corresponding submatches in the
+% All the currently active threads are kept in order of precedence in
+% \cs{g_@@_thread_state_intarray}, and the corresponding submatches in the
% \tn{toks}. For our purposes, those serve as an array, indexed from
% \texttt{min_active} (inclusive) to \texttt{max_active} (excluded).
% At the start of every step, the whole array is unpacked, so that the
@@ -4274,6 +4359,17 @@
% \end{macrocode}
% \end{variable}
%
+% \begin{variable}{\g_@@_state_active_intarray, \g_@@_thread_state_intarray}
+% \cs{g_@@_state_active_intarray} stores the last \meta{step} in which
+% each \meta{state} was active. \cs{g_@@_thread_state_intarray} stores
+% threads that will be considered in the next step, more precisely the
+% states in which these threads are.
+% \begin{macrocode}
+\__intarray_new:Nn \g_@@_state_active_intarray { 65536 }
+\__intarray_new:Nn \g_@@_thread_state_intarray { 65536 }
+% \end{macrocode}
+% \end{variable}
+%
% \begin{variable}{\l_@@_every_match_tl}
% Every time a match is found, this token list is used. For single
% matching, the token list is empty. For multiple matching, the token
@@ -4332,11 +4428,11 @@
% \subsubsection{Matching: framework}
%
% \begin{macro}[int]{\@@_match:n}
-% First store the query into \tn{toks} and \tn{muskip} registers (see
+% First store the query into \tn{toks} registers and arrays (see
% \cs{@@_query_set:nnn}). Then initialize the variables that should
% be set once for each user function (even for multiple
% matches). Namely, the overall matching is not yet successful; none of
-% the states should be marked as visited (\tn{dimen} registers), and
+% the states should be marked as visited (\cs{g_@@_state_active_intarray}), and
% we start at step $0$; we pretend that there was a previous match
% ending at the start of the query, which was not empty (to avoid
% smothering an empty match at the start). Once all this is set up, we
@@ -4358,12 +4454,13 @@
\bool_gset_false:N \g_@@_success_bool
\int_step_inline:nnnn
\l_@@_min_state_int { 1 } { \l_@@_max_state_int - 1 }
- { \tex_dimen:D ##1 ~ 1 sp \scan_stop: }
+ { \__intarray_gset_fast:Nnn \g_@@_state_active_intarray {##1} { 1 } }
\int_set_eq:NN \l_@@_min_active_int \l_@@_max_state_int
\int_zero:N \l_@@_step_int
\int_set_eq:NN \l_@@_success_pos_int \l_@@_min_pos_int
- \int_set:Nn \l_@@_submatch_int
+ \int_set:Nn \l_@@_min_submatch_int
{ 2 * \l_@@_max_state_int }
+ \int_set_eq:NN \l_@@_submatch_int \l_@@_min_submatch_int
\bool_set_false:N \l_@@_empty_success_bool
\@@_match_once:
%<trace> \trace_pop:nnx { regex } { 1 } { @@_match }
@@ -4433,7 +4530,7 @@
% \end{macro}
%
% \begin{macro}[aux]{\@@_match_loop:}
-% \begin{macro}[aux, rEXP]{\@@_match_one_active:w}
+% \begin{macro}[aux, rEXP]{\@@_match_one_active:n}
% At each new position, set some variables and get the new character
% and category from the query. Then unpack the array of active
% threads, and clear it by resetting its length
@@ -4448,7 +4545,7 @@
% \begin{macrocode}
\cs_new_protected:Npn \@@_match_loop:
{
- \tex_advance:D \l_@@_step_int 2 \exp_stop_f:
+ \int_add:Nn \l_@@_step_int { 2 }
\int_incr:N \l_@@_current_pos_int
\int_set_eq:NN \l_@@_last_char_int \l_@@_current_char_int
\int_set_eq:NN \l_@@_case_changed_char_int \c_max_int
@@ -4456,8 +4553,11 @@
\use:x
{
\int_set_eq:NN \l_@@_max_active_int \l_@@_min_active_int
- \exp_after:wN \@@_match_one_active:w
- \int_use:N \l_@@_min_active_int ;
+ \int_step_function:nnnN
+ { \l_@@_min_active_int }
+ { 1 }
+ { \l_@@_max_active_int - 1 }
+ \@@_match_one_active:n
}
\__prg_break_point:
\bool_set_false:N \l_@@_fresh_thread_bool %^^A was arg of break_point:n
@@ -4467,15 +4567,11 @@
\fi:
\fi:
}
-\cs_new:Npn \@@_match_one_active:w #1;
+\cs_new:Npn \@@_match_one_active:n #1
{
- \if_int_compare:w #1 < \l_@@_max_active_int
- \@@_use_state_and_submatches:nn
- { \__int_value:w \tex_skip:D #1 }
- { \tex_the:D \tex_toks:D #1 }
- \exp_after:wN \@@_match_one_active:w
- \__int_value:w \__int_eval:w #1 + 1 \exp_after:wN ;
- \fi:
+ \@@_use_state_and_submatches:nn
+ { \__intarray_item_fast:Nn \g_@@_thread_state_intarray {#1} }
+ { \@@_toks_use:w #1 }
}
% \end{macrocode}
% \end{macro}
@@ -4485,18 +4581,18 @@
% The arguments are: tokens that \texttt{o} and \texttt{x} expand to
% one token of the query, the catcode, and the character code. Store
% those, and the current brace balance (used later to check for
-% overall brace balance) in a \tn{muskip} register and a \tn{toks},
+% overall brace balance) in a \tn{toks} register and some arrays,
% then update the \texttt{balance}.
% \begin{macrocode}
\cs_new_protected:Npn \@@_query_set:nnn #1#2#3
{
- \tex_muskip:D \l_@@_current_pos_int
- = \etex_gluetomu:D
- #3 sp
- plus #2 sp
- minus \l_@@_balance_int sp
- \scan_stop:
- \tex_toks:D \l_@@_current_pos_int {#1}
+ \__intarray_gset_fast:Nnn \g_@@_charcode_intarray
+ { \l_@@_current_pos_int } {#3}
+ \__intarray_gset_fast:Nnn \g_@@_catcode_intarray
+ { \l_@@_current_pos_int } {#2}
+ \__intarray_gset_fast:Nnn \g_@@_balance_intarray
+ { \l_@@_current_pos_int } { \l_@@_balance_int }
+ \@@_toks_set:Nn \l_@@_current_pos_int {#1}
\int_incr:N \l_@@_current_pos_int
\if_case:w #2 \exp_stop_f:
\or: \int_incr:N \l_@@_balance_int
@@ -4507,17 +4603,17 @@
% \end{macro}
%
% \begin{macro}[aux]{\@@_query_get:}
-% Extract the current character and category codes from the
-% \tn{muskip} register of the current position: those are the main and
-% the stretch components, and we need a conversion to avoid \TeX{}'s
-% \enquote{incompatible glue units} error.
+% Extract the current character and category codes at the current
+% position from the appropriate arrays.
% \begin{macrocode}
\cs_new_protected:Npn \@@_query_get:
{
\l_@@_current_char_int
- = \etex_mutoglue:D \tex_muskip:D \l_@@_current_pos_int
- \l_@@_current_catcode_int = \etex_gluestretch:D
- \etex_mutoglue:D \tex_muskip:D \l_@@_current_pos_int
+ = \__intarray_item_fast:Nn \g_@@_charcode_intarray
+ { \l_@@_current_pos_int } \scan_stop:
+ \l_@@_current_catcode_int
+ = \__intarray_item_fast:Nn \g_@@_catcode_intarray
+ { \l_@@_current_pos_int } \scan_stop:
}
% \end{macrocode}
% \end{macro}
@@ -4537,11 +4633,11 @@
%<*trace>
\trace:nnx { regex } { 2 } { state~\int_use:N \l_@@_current_state_int }
%</trace>
- \tex_dimen:D \l_@@_current_state_int
- = \l_@@_step_int sp \scan_stop:
- \tex_the:D \tex_toks:D \l_@@_current_state_int
- \tex_dimen:D \l_@@_current_state_int
- = \__int_eval:w \l_@@_step_int + 1 \__int_eval_end: sp \scan_stop:
+ \__intarray_gset_fast:Nnn \g_@@_state_active_intarray
+ { \l_@@_current_state_int } { \l_@@_step_int }
+ \@@_toks_use:w \l_@@_current_state_int
+ \__intarray_gset_fast:Nnn \g_@@_state_active_intarray
+ { \l_@@_current_state_int } { \l_@@_step_int + 1 }
}
% \end{macrocode}
% \end{macro}
@@ -4555,7 +4651,9 @@
\cs_new_protected:Npn \@@_use_state_and_submatches:nn #1 #2
{
\int_set:Nn \l_@@_current_state_int {#1}
- \if_int_compare:w \tex_dimen:D \l_@@_current_state_int
+ \if_int_compare:w
+ \__intarray_item_fast:Nn \g_@@_state_active_intarray
+ { \l_@@_current_state_int }
< \l_@@_step_int
\tl_set:Nn \l_@@_current_submatches_prop {#2}
\exp_after:wN \@@_use_state:
@@ -4610,7 +4708,10 @@
\int_add:Nn \l_@@_current_state_int {#2}
\exp_not:n
{
- \if_int_compare:w \tex_dimen:D \l_@@_current_state_int #1
+ \if_int_compare:w
+ \__intarray_item_fast:Nn \g_@@_state_active_intarray
+ { \l_@@_current_state_int }
+ #1
\exp_after:wN \@@_use_state:
\fi:
}
@@ -4626,7 +4727,7 @@
%
% \begin{macro}[int]{\@@_action_cost:n}
% A transition which consumes the current character and shifts the
-% state by |#1|. The resulting state is stored in the \tn{skip} array
+% state by |#1|. The resulting state is stored in the appropriate array
% for use at the next position, and we also store the current
% submatches.
% \begin{macrocode}
@@ -4640,22 +4741,20 @@
%
% \begin{macro}[int]{\@@_store_state:n}
% \begin{macro}[aux]{\@@_store_submatches:}
-% Put the given state in the array of \tn{skip} registers (converted
-% to a dimension in scaled points), and increment the length of the
-% array. Then store the current submatch in the This is done by
-% increasing the pointer \cs{l_@@_max_active_int}, and converting
-% the integer to a dimension (suitable for a \tn{skip} assignment) in
-% scaled points.
+% Put the given state in \cs{g_@@_thread_state_intarray}, and increment
+% the length of the array. Also store the current submatch in the
+% appropriate \tn{toks}.
% \begin{macrocode}
\cs_new_protected:Npn \@@_store_state:n #1
{
\@@_store_submatches:
- \tex_skip:D \l_@@_max_active_int = #1 sp \scan_stop:
+ \__intarray_gset_fast:Nnn \g_@@_thread_state_intarray
+ { \l_@@_max_active_int } {#1}
\int_incr:N \l_@@_max_active_int
}
\cs_new_protected:Npn \@@_store_submatches:
{
- \tex_toks:D \l_@@_max_active_int \exp_after:wN
+ \@@_toks_set:No \l_@@_max_active_int
{ \l_@@_current_submatches_prop }
}
% \end{macrocode}
@@ -4729,6 +4828,15 @@
% \end{macrocode}
% \end{variable}
%
+% \begin{variable}{\l_@@_replacement_category_tl, \l_@@_replacement_category_seq}
+% This sequence of letters is used to correctly restore categories in
+% nested constructions such as |\cL(abc\cD(_)d)|.
+% \begin{macrocode}
+\tl_new:N \l_@@_replacement_category_tl
+\seq_new:N \l_@@_replacement_category_seq
+% \end{macrocode}
+% \end{variable}
+%
% \begin{variable}{\l_@@_balance_tl}
% This token list holds the replacement text for
% \cs{@@_replacement_balance_one_match:n} while it is being built
@@ -4739,8 +4847,9 @@
% \end{variable}
%
% \begin{macro}[aux, rEXP]{\@@_replacement_balance_one_match:n}
-% This expects as an argument the first index of a range of \tn{skip}
-% registers which hold the submatch information for a given match. It
+% This expects as an argument the first index of a set of entries in
+% \cs{g_@@_submatch_begin_intarray} (and related arrays) which hold the
+% submatch information for a given match. It
% can be used within an integer expression to obtain the brace balance
% incurred by performing the replacement on that match. This combines
% the braces lost by removing the match, braces added by all the
@@ -4764,13 +4873,13 @@
% with all possible arguments (one call for each match), as well as
% the range from the end of the last match to the end of the string,
% will produce the fully replaced token list. The initialization does
-% not matter, but we set it as for an empty replacement.
+% not matter, but (as an example) we set it as for an empty replacement.
% \begin{macrocode}
\cs_new:Npn \@@_replacement_do_one_match:n #1
{
\@@_query_range:nn
- { \etex_glueshrink:D \tex_skip:D #1 }
- { \tex_skip:D #1 }
+ { \__intarray_item_fast:Nn \g_@@_submatch_prev_intarray {#1} }
+ { \__intarray_item_fast:Nn \g_@@_submatch_begin_intarray {#1} }
}
% \end{macrocode}
% \end{macro}
@@ -4816,7 +4925,7 @@
\else:
\exp_after:wN \__prg_break:
\fi:
- \tex_the:D \tex_toks:D #1 \exp_stop_f:
+ \@@_toks_use:w #1 \exp_stop_f:
\exp_after:wN \@@_query_range_loop:ww
\__int_value:w \__int_eval:w #1 + 1 ; #2 ;
}
@@ -4825,17 +4934,13 @@
% \end{macro}
%
% \begin{macro}[int]{\@@_query_submatch:n}
-% When this function is called, \tn{skip}$i$ holds the start and end
-% positions for the $i$-th overall submatch as its main and stretch
-% components. In the case of repeated matches, submatches from all the
-% matches are put one after the other in blocks of
-% \cs{l_@@_capturing_group_int} \tn{skip} registers.
+% Find the start and end positions for a given submatch (of a given match).
% \begin{macrocode}
\cs_new:Npn \@@_query_submatch:n #1
{
\@@_query_range:nn
- { \tex_skip:D \__int_eval:w #1 }
- { \etex_gluestretch:D \tex_skip:D \__int_eval:w #1 }
+ { \__intarray_item_fast:Nn \g_@@_submatch_begin_intarray {#1} }
+ { \__intarray_item_fast:Nn \g_@@_submatch_end_intarray {#1} }
}
% \end{macrocode}
% \end{macro}
@@ -4843,21 +4948,31 @@
% \begin{macro}[rEXP]{\@@_submatch_balance:n}
% Every user function must result in a balanced token list (unbalanced
% token lists cannot be stored by TeX). When we unpacked the query, we
-% kept track of the brace balance as the shrink component of
-% \tn{muskip} registers, hence the contribution from a given range is
-% the difference between the shrink components of
-% \tn{muskip}\meta{max~pos} and \tn{muskip}\meta{min~pos}. For the
-% $i$-th submatch, the end-points of the range are the main and
-% stretch components of \tn{skip}$i$. The trailing \cs{scan_stop:} is
-% gobbled by \cs{etex_muexpr:D}, and the whole expression can be cast
-% safely to an integer (no trailing expansion).
+% kept track of the brace balance, hence the contribution from a given
+% range is the difference between the brace balances at the
+% \meta{max~pos} and \meta{min~pos}. These two positions are found in
+% the corresponding \enquote{submatch} arrays.
+%^^A todo: understand when these int_compare are needed
% \begin{macrocode}
\cs_new_protected:Npn \@@_submatch_balance:n #1
{
- \etex_glueshrink:D \etex_mutoglue:D \etex_muexpr:D
- \tex_muskip:D \etex_gluestretch:D \tex_skip:D #1
- - \tex_muskip:D \tex_skip:D #1
- \scan_stop:
+ \__int_eval:w
+ \int_compare:nNnTF
+ { \__intarray_item_fast:Nn \g_@@_submatch_end_intarray {#1} } = 0
+ { 0 }
+ {
+ \__intarray_item_fast:Nn \g_@@_balance_intarray
+ { \__intarray_item_fast:Nn \g_@@_submatch_end_intarray {#1} }
+ }
+ -
+ \int_compare:nNnTF
+ { \__intarray_item_fast:Nn \g_@@_submatch_begin_intarray {#1} } = 0
+ { 0 }
+ {
+ \__intarray_item_fast:Nn \g_@@_balance_intarray
+ { \__intarray_item_fast:Nn \g_@@_submatch_begin_intarray {#1} }
+ }
+ \__int_eval_end:
}
% \end{macrocode}
% \end{macro}
@@ -4886,10 +5001,14 @@
\@@_escape_use:nnnn
{
\if_charcode:w \c_right_brace_str ##1
- \@@_replacement_rbrace:N \else: \__tl_build_one:n \fi: ##1
+ \@@_replacement_rbrace:N
+ \else:
+ \@@_replacement_normal:n
+ \fi:
+ ##1
}
{ \@@_replacement_escaped:N ##1 }
- { \__tl_build_one:n ##1 }
+ { \@@_replacement_normal:n ##1 }
{#1}
\prg_do_nothing: \prg_do_nothing:
\if_int_compare:w \l_@@_replacement_csnames_int > 0 \exp_stop_f:
@@ -4898,6 +5017,12 @@
\__tl_build_one:x
{ \prg_replicate:nn \l_@@_replacement_csnames_int \cs_end: }
\fi:
+ \seq_if_empty:NF \l_@@_replacement_category_seq
+ {
+ \__msg_kernel_error:nnx { regex } { replacement-missing-rparen }
+ { \seq_count:N \l_@@_replacement_category_seq }
+ \seq_clear:N \l_@@_replacement_category_seq
+ }
\cs_gset:Npx \@@_replacement_balance_one_match:n ##1
{
+ \int_use:N \l_@@_balance_int
@@ -4913,8 +5038,8 @@
\cs_set:Npn \@@_replacement_do_one_match:n ##1
{
\@@_query_range:nn
- { \etex_glueshrink:D \tex_skip:D ##1 }
- { \tex_skip:D ##1 }
+ { \__intarray_item_fast:Nn \g_@@_submatch_prev_intarray {##1} }
+ { \__intarray_item_fast:Nn \g_@@_submatch_begin_intarray {##1} }
#1
}
}
@@ -4922,6 +5047,35 @@
% \end{macro}
% \end{macro}
%
+% \begin{macro}[aux]{\@@_replacement_normal:n}
+% Most characters are simply sent to the output by
+% \cs{__tl_build_one:n}, unless a particular category code has been
+% requested: then \cs{@@_replacement_c_A:w} or a similar auxiliary is
+% called. One exception is right parentheses, which restore the
+% category code in place before the group started. Note that the
+% sequence is non-empty there: it contains an empty entry
+% corresponding to the initial value of
+% \cs{l_@@_replacement_category_tl}.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_replacement_normal:n #1
+ {
+ \tl_if_empty:NTF \l_@@_replacement_category_tl
+ { \__tl_build_one:n {#1} }
+ { % (
+ \token_if_eq_charcode:NNTF #1 )
+ {
+ \seq_pop:NN \l_@@_replacement_category_seq
+ \l_@@_replacement_category_tl
+ }
+ {
+ \use:c { @@_replacement_c_ \l_@@_replacement_category_tl :w }
+ \@@_replacement_normal:n {#1}
+ }
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+%
% \begin{macro}[aux]{\@@_replacement_escaped:N}
% As in parsing a regular expression, we use an auxiliary built from
% |#1| if defined. Otherwise, check for escaped digits (standing from
@@ -4935,7 +5089,8 @@
\if_int_compare:w 1 < 1#1 \exp_stop_f:
\@@_replacement_put_submatch:n {#1}
\else:
- \__tl_build_one:o { \token_to_str:N #1 }
+ \exp_args:No \@@_replacement_normal:n
+ { \token_to_str:N #1 }
\fi:
}
}
@@ -4967,34 +5122,34 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\@@_replacement_g:w, \@@_replacement_g_digits:NN}
-% An ugly method to grab digits for the |\g| escape sequence. At the
-% end of the run of digits, check that it ends with a right brace.
+% \begin{macro}[aux]{\@@_replacement_g:w}
+% \begin{macro}[aux,rEXP]{\@@_replacement_g_digits:NN}
+% Grab digits for the |\g| escape sequence in a primitive assignment
+% to the integer \cs{l_@@_internal_a_int}. At the end of the run of
+% digits, check that it ends with a right brace.
% \begin{macrocode}
\cs_new_protected:Npn \@@_replacement_g:w #1#2
{
- \str_if_eq_x:nnTF { #1#2 } { \__tl_build_one:n \c_left_brace_str }
- {
- \int_zero:N \l_@@_internal_a_int
- \@@_replacement_g_digits:NN
- }
+ \str_if_eq_x:nnTF { #1#2 } { \@@_replacement_normal:n \c_left_brace_str }
+ { \l_@@_internal_a_int = \@@_replacement_g_digits:NN }
{ \@@_replacement_error:NNN g #1 #2 }
}
-\cs_new_protected:Npn \@@_replacement_g_digits:NN #1#2
+\cs_new:Npn \@@_replacement_g_digits:NN #1#2
{
- \token_if_eq_meaning:NNTF #1 \__tl_build_one:n
+ \token_if_eq_meaning:NNTF #1 \@@_replacement_normal:n
{
\if_int_compare:w 1 < 1#2 \exp_stop_f:
- \int_set:Nn \l_@@_internal_a_int
- { 10 * \l_@@_internal_a_int + #2 }
+ #2
\exp_after:wN \use_i:nnn
\exp_after:wN \@@_replacement_g_digits:NN
\else:
+ \exp_stop_f:
\exp_after:wN \@@_replacement_error:NNN
\exp_after:wN g
\fi:
}
{
+ \exp_stop_f:
\if_meaning:w \@@_replacement_rbrace:N #1
\exp_args:No \@@_replacement_put_submatch:n
{ \int_use:N \l_@@_internal_a_int }
@@ -5008,41 +5163,52 @@
}
% \end{macrocode}
% \end{macro}
+% \end{macro}
%
% \subsubsection{Csnames in replacement}
%
% \begin{macro}[aux]{\@@_replacement_c:w}
-% \begin{macro}[aux]+\@@_replacement_c_{:w+
-% |\c| can be followed by a left brace, or by a letter for which we
-% have defined a way to produce that category of characters. The
-% appropriate definitions for catcodes are introduced later. For
-% control sequences, if we are within a control sequence, convert
-% the token list to a string, otherwise simply prevent expansion,
-% with a weird cross-over between \cs{exp_not:n} and \cs{exp_not:N}
-% (see this helper's description for an explanation).
+% |\c| may only be followed by an unescaped character. If followed by
+% a left brace, start a control sequence by calling an auxiliary
+% common with |\u|. Otherwise test whether the category is known; if
+% it is not, complain.
% \begin{macrocode}
\cs_new_protected:Npn \@@_replacement_c:w #1#2
{
- \token_if_eq_meaning:NNTF #1 \__tl_build_one:n
+ \token_if_eq_meaning:NNTF #1 \@@_replacement_normal:n
{
- \cs_if_exist_use:cF { @@_replacement_c_#2:w }
- { \@@_replacement_error:NNN c #1#2 }
+ \exp_after:wN \token_if_eq_charcode:NNTF \c_left_brace_str #2
+ { \@@_replacement_cu_aux:Nw \@@_replacement_exp_not:N }
+ {
+ \cs_if_exist:cTF { @@_replacement_c_#2:w }
+ { \@@_replacement_cat:NNN #2 }
+ { \@@_replacement_error:NNN c #1#2 }
+ }
}
{ \@@_replacement_error:NNN c #1#2 }
}
-\cs_new_protected:cpn { @@_replacement_c_ \c_left_brace_str :w }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\@@_replacement_cu_aux:Nw}
+% Start a control sequence with \cs{cs:w}, which will be protected
+% from expansion by |#1| (either \cs{@@_replacement_exp_not:N} or
+% \cs{exp_not:V}), or turned to a string by \cs{tl_to_str:V} if inside
+% another csname construction |\c| or |\u|. We use \cs{tl_to_str:V}
+% rather than \cs{tl_to_str:N} to deal with integers and other
+% registers.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_replacement_cu_aux:Nw #1
{
\if_case:w \l_@@_replacement_csnames_int
- \__tl_build_one:n
- { \exp_not:n { \exp_after:wN \@@_replacement_exp_not:N \cs:w } }
+ \__tl_build_one:n { \exp_not:n { \exp_after:wN #1 \cs:w } }
\else:
- \__tl_build_one:n { \exp_not:n { \exp_after:wN \tl_to_str:N \cs:w } }
+ \__tl_build_one:n { \exp_not:n { \exp_after:wN \tl_to_str:V \cs:w } }
\fi:
\int_incr:N \l_@@_replacement_csnames_int
}
% \end{macrocode}
% \end{macro}
-% \end{macro}
%
% \begin{macro}[aux]{\@@_replacement_u:w}
% Check that |\u| is followed by a left brace. If so, start a control
@@ -5051,15 +5217,8 @@
% \begin{macrocode}
\cs_new_protected:Npn \@@_replacement_u:w #1#2
{
- \str_if_eq_x:nnTF { #1#2 } { \__tl_build_one:n \c_left_brace_str }
- {
- \if_case:w \l_@@_replacement_csnames_int
- \__tl_build_one:n { \exp_not:n { \exp_after:wN \exp_not:V \cs:w } }
- \else:
- \__tl_build_one:n { \exp_not:n { \exp_after:wN \tl_to_str:V \cs:w } }
- \fi:
- \int_incr:N \l_@@_replacement_csnames_int
- }
+ \str_if_eq_x:nnTF { #1#2 } { \@@_replacement_normal:n \c_left_brace_str }
+ { \@@_replacement_cu_aux:Nw \exp_not:V }
{ \@@_replacement_error:NNN u #1#2 }
}
% \end{macrocode}
@@ -5076,7 +5235,7 @@
\__tl_build_one:n \cs_end:
\int_decr:N \l_@@_replacement_csnames_int
\else:
- \__tl_build_one:n #1
+ \@@_replacement_normal:n {#1}
\fi:
}
% \end{macrocode}
@@ -5084,6 +5243,39 @@
%
% \subsubsection{Characters in replacement}
%
+% \begin{macro}[aux]{\@@_replacement_cat:NNN}
+% Here, |#1| is a letter among |BEMTPUDSLOA| and |#2#3| denote the
+% next character. Complain if we reach the end of the replacement or
+% if the construction appears inside |\c{|\ldots{}|}| or
+% |\u{|\ldots{}|}|, and detect the case of a parenthesis. In that
+% case, store the current category in a sequence and switch to a new
+% one.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_replacement_cat:NNN #1#2#3
+ {
+ \token_if_eq_meaning:NNTF \prg_do_nothing: #3
+ { \__msg_kernel_error:nn { regex } { replacement-catcode-end } }
+ {
+ \int_compare:nNnTF { \l_@@_replacement_csnames_int } > 0
+ {
+ \__msg_kernel_error:nnnn
+ { regex } { replacement-catcode-in-cs } {#1} {#3}
+ #2 #3
+ }
+ {
+ \str_if_eq:nnTF { #2 #3 } { \@@_replacement_normal:n ( } % )
+ {
+ \seq_push:NV \l_@@_replacement_category_seq
+ \l_@@_replacement_category_tl
+ \tl_set:Nn \l_@@_replacement_category_tl {#1}
+ }
+ { \use:c { @@_replacement_c_#1:w } #2 #3 }
+ }
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+%
% We will need to change the category code of the null character many
% times, hence work in a group. The catcode-specific macros below are
% defined in alphabetical order; if you are trying to understand the
@@ -5099,16 +5291,13 @@
% wrapper for our purposes. The first argument is the null character
% with various catcodes. The second and third arguments are grabbed
% from the input stream: |#3| is the character whose character code to
-% reproduce.
+% reproduce. We could use \cs{char_generate:nn} but only for some
+% catcodes (active characters and spaces are not supported).
% \begin{macrocode}
\cs_new_protected:Npn \@@_replacement_char:nNN #1#2#3
{
- \if_meaning:w \prg_do_nothing: #3
- \__msg_kernel_error:nn { regex } { replacement-catcode-end }
- \else:
- \tex_lccode:D 0 = `#3 \scan_stop:
- \tex_lowercase:D { \__tl_build_one:n {#1} }
- \fi:
+ \tex_lccode:D 0 = `#3 \scan_stop:
+ \tex_lowercase:D { \__tl_build_one:n {#1} }
}
% \end{macrocode}
% \end{macro}
@@ -5234,15 +5423,11 @@
% \begin{macrocode}
\cs_new_protected:Npn \@@_replacement_c_S:w #1#2
{
- \if_meaning:w \prg_do_nothing: #2
- \__msg_kernel_error:nn { regex } { replacement-catcode-end }
- \else:
- \if_int_compare:w `#2 = 0 \exp_stop_f:
- \__msg_kernel_error:nn { regex } { replacement-null-space }
- \fi:
- \tex_lccode:D `\ = `#2 \scan_stop:
- \tex_lowercase:D { \__tl_build_one:n {~} }
+ \if_int_compare:w `#2 = 0 \exp_stop_f:
+ \__msg_kernel_error:nn { regex } { replacement-null-space }
\fi:
+ \tex_lccode:D `\ = `#2 \scan_stop:
+ \tex_lowercase:D { \__tl_build_one:n {~} }
}
% \end{macrocode}
% \end{macro}
@@ -5433,23 +5618,32 @@
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\l_@@_submatch_int, \l_@@_zeroth_submatch_int}
-% The end-points of each submatch are stored as main and stretch
-% components of \tn{skip}\meta{submatch}, where \meta{submatch} ranges
-% from \cs{l_@@_max_state_int} (inclusive) to
+% \begin{variable}{\l_@@_min_submatch_int, \l_@@_submatch_int, \l_@@_zeroth_submatch_int}
+% The end-points of each submatch are stored in two arrays whose index \meta{submatch} ranges
+% from \cs{l_@@_min_submatch_int} (inclusive) to
% \cs{l_@@_submatch_int} (exclusive). Each successful match comes
% with a $0$-th submatch (the full match), and one match for each
% capturing group: submatches corresponding to the last successful
-% match are labelled starting at
-% \texttt{zeroth_submatch}. Additionally, the shrink component of this
-% $0$-th submatch is the position at which that match attempt started:
-% this is used for splitting and replacements.
+% match are labelled starting at \texttt{zeroth_submatch}. The entry
+% \cs{l_@@_zeroth_submatch_int} in \cs{g_@@_submatch_prev_intarray} holds
+% the position at which that match attempt started: this is used for
+% splitting and replacements.
% \begin{macrocode}
+\int_new:N \l_@@_min_submatch_int
\int_new:N \l_@@_submatch_int
\int_new:N \l_@@_zeroth_submatch_int
% \end{macrocode}
% \end{variable}
%
+% \begin{variable}{\g_@@_submatch_prev_intarray, \g_@@_submatch_begin_intarray, \g_@@_submatch_end_intarray}
+% Hold the place where the match attempt begun and the end-points of each submatch.
+% \begin{macrocode}
+\__intarray_new:Nn \g_@@_submatch_prev_intarray { 65536 }
+\__intarray_new:Nn \g_@@_submatch_begin_intarray { 65536 }
+\__intarray_new:Nn \g_@@_submatch_end_intarray { 65536 }
+% \end{macrocode}
+% \end{variable}
+%
% \begin{macro}[aux]{\@@_return:}
% This function triggers either \cs{prg_return_false:} or
% \cs{prg_return_true:} as appropriate to whether a match was found or
@@ -5543,8 +5737,7 @@
% match, store the last part of the token list, which ranges from the
% start of the match attempt to the end of the query. This step is
% inhibited if the last match was empty and at the very end: decrement
-% \cs{l_@@_submatch_int}, which controls which \tn{skip} registers
-% will be used.
+% \cs{l_@@_submatch_int}, which controls which matches will be used.
% \begin{macrocode}
\cs_new_protected:Npn \@@_split:nnN #1#2#3
{
@@ -5553,16 +5746,30 @@
{
\if_int_compare:w \l_@@_start_pos_int < \l_@@_success_pos_int
\@@_extract:
- \tex_skip:D \l_@@_zeroth_submatch_int
- = \l_@@_start_pos_int sp
- plus \tex_skip:D \l_@@_zeroth_submatch_int \scan_stop:
+ \__intarray_gset_fast:Nnn \g_@@_submatch_prev_intarray
+ { \l_@@_zeroth_submatch_int } { 0 }
+ \__intarray_gset_fast:Nnn \g_@@_submatch_end_intarray
+ { \l_@@_zeroth_submatch_int }
+ {
+ \__intarray_item_fast:Nn \g_@@_submatch_begin_intarray
+ { \l_@@_zeroth_submatch_int }
+ }
+ \__intarray_gset_fast:Nnn \g_@@_submatch_begin_intarray
+ { \l_@@_zeroth_submatch_int }
+ { \l_@@_start_pos_int }
\fi:
}
#1
\@@_match:n {#2}
%<assert>\assert_int:n { \l_@@_current_pos_int = \l_@@_max_pos_int }
- \tex_skip:D \l_@@_submatch_int
- = \l_@@_start_pos_int sp plus \l_@@_max_pos_int sp \scan_stop:
+ \__intarray_gset_fast:Nnn \g_@@_submatch_prev_intarray
+ { \l_@@_submatch_int } { 0 }
+ \__intarray_gset_fast:Nnn \g_@@_submatch_end_intarray
+ { \l_@@_submatch_int }
+ { \l_@@_max_pos_int }
+ \__intarray_gset_fast:Nnn \g_@@_submatch_begin_intarray
+ { \l_@@_submatch_int }
+ { \l_@@_start_pos_int }
\int_incr:N \l_@@_submatch_int
\if_meaning:w \c_true_bool \l_@@_empty_success_bool
\if_int_compare:w \l_@@_start_pos_int = \l_@@_max_pos_int
@@ -5575,8 +5782,8 @@
% \end{macro}
%
% \begin{macro}[aux]{\@@_group_end_extract_seq:N}
-% The end-points of submatches are stored as the main and stretch
-% components of \tn{skip} registers from \cs{l_@@_max_state_int} to
+% The end-points of submatches are stored as entries of two arrays
+% from \cs{l_@@_min_submatch_int} to
% \cs{l_@@_submatch_int} (exclusive). Extract the relevant ranges
% into \cs{l_@@_internal_a_tl}. We detect unbalanced results using
% the two flags \texttt{@@_begin} and \texttt{@@_end}, raised
@@ -5593,7 +5800,7 @@
{
\s__seq
\int_step_function:nnnN
- { 2 * \l_@@_max_state_int }
+ { \l_@@_min_submatch_int }
{ 1 }
{ \l_@@_submatch_int - 1 }
\@@_extract_seq_aux:n
@@ -5649,14 +5856,14 @@
% {\@@_extract:, \@@_extract_b:wn, \@@_extract_e:wn}
% Our task here is to extract from the property list
% \cs{l_@@_success_submatches_prop} the list of end-points of
-% submatches, and store them in \tn{skip} registers, from
+% submatches, and store them in appropriate array entries, from
% \cs{l_@@_zeroth_submatch_int} upwards. We begin by emptying those
-% \tn{skip} registers. Then for each \meta{key}--\meta{value} pair in
-% the property list update the appropriate \tn{skip} component. This
+% entries. Then for each \meta{key}--\meta{value} pair in
+% the property list update the appropriate entry. This
% is somewhat a hack: the \meta{key} is a non-negative integer
% followed by |<| or |>|, which we use in a comparison to $-1$. At the
% end, store the information about the position at which the match
-% attempt started, as a shrink component.
+% attempt started, in \cs{g_@@_submatch_prev_intarray}.
% \begin{macrocode}
\cs_new_protected:Npn \@@_extract:
{
@@ -5664,7 +5871,12 @@
\int_set_eq:NN \l_@@_zeroth_submatch_int \l_@@_submatch_int
\prg_replicate:nn \l_@@_capturing_group_int
{
- \tex_skip:D \l_@@_submatch_int 0 sp \scan_stop:
+ \__intarray_gset_fast:Nnn \g_@@_submatch_begin_intarray
+ { \l_@@_submatch_int } { 0 }
+ \__intarray_gset_fast:Nnn \g_@@_submatch_end_intarray
+ { \l_@@_submatch_int } { 0 }
+ \__intarray_gset_fast:Nnn \g_@@_submatch_prev_intarray
+ { \l_@@_submatch_int } { 0 }
\int_incr:N \l_@@_submatch_int
}
\prop_map_inline:Nn \l_@@_success_submatches_prop
@@ -5676,21 +5888,14 @@
\fi:
\__int_eval:w \l_@@_zeroth_submatch_int + ##1 {##2}
}
- \tex_skip:D \l_@@_zeroth_submatch_int
- = \tex_the:D \tex_skip:D \l_@@_zeroth_submatch_int
- minus \l_@@_start_pos_int sp \scan_stop:
+ \__intarray_gset_fast:Nnn \g_@@_submatch_prev_intarray
+ { \l_@@_zeroth_submatch_int } { \l_@@_start_pos_int }
\fi:
}
\cs_new_protected:Npn \@@_extract_b:wn #1 < #2
- {
- \tex_skip:D #1 = #2 sp
- plus \etex_gluestretch:D \tex_skip:D #1 \scan_stop:
- }
+ { \__intarray_gset_fast:Nnn \g_@@_submatch_begin_intarray {#1} {#2} }
\cs_new_protected:Npn \@@_extract_e:wn #1 > #2
- {
- \tex_skip:D #1
- = 1 \tex_skip:D #1 plus #2 sp \scan_stop:
- }
+ { \__intarray_gset_fast:Nnn \g_@@_submatch_end_intarray {#1} {#2} }
% \end{macrocode}
% \end{macro}
%
@@ -5730,7 +5935,10 @@
{
\@@_replacement_do_one_match:n { \l_@@_zeroth_submatch_int }
\@@_query_range:nn
- { \etex_gluestretch:D \tex_skip:D \l_@@_zeroth_submatch_int }
+ {
+ \__intarray_item_fast:Nn \g_@@_submatch_end_intarray
+ { \l_@@_zeroth_submatch_int }
+ }
{ \l_@@_max_pos_int }
}
\@@_group_end_replace:N #3
@@ -5741,12 +5949,11 @@
%
% \begin{macro}[aux]{\@@_replace_all:nnN}
% Match multiple times, and for every match, extract submatches and
-% additionally store the position at which the match attempt started
-% (as the shrink component of a \tn{skip} register). The \tn{skip}
-% registers from \cs{l_@@_max_state_int} to
+% additionally store the position at which the match attempt started.
+% The entries from \cs{l_@@_min_submatch_int} to
% \cs{l_@@_submatch_int} hold information about submatches of every
% match in order; each match corresponds to
-% \cs{l_@@_capturing_group_int} consecutive \tn{skip} registers.
+% \cs{l_@@_capturing_group_int} consecutive entries.
% Compute the brace balance corresponding to doing all the
% replacements: this is the sum of brace balances for replacing each
% match. Join together the replacement texts for each match (including
@@ -5763,7 +5970,7 @@
{
0
\int_step_function:nnnN
- { 2 * \l_@@_max_state_int }
+ { \l_@@_min_submatch_int }
\l_@@_capturing_group_int
{ \l_@@_submatch_int - 1 }
\@@_replacement_balance_one_match:n
@@ -5771,7 +5978,7 @@
\tl_set:Nx \l_@@_internal_a_tl
{
\int_step_function:nnnN
- { 2 * \l_@@_max_state_int }
+ { \l_@@_min_submatch_int }
\l_@@_capturing_group_int
{ \l_@@_submatch_int - 1 }
\@@_replacement_do_one_match:n
@@ -5825,23 +6032,23 @@
% Messages for the preparsing phase.
% \begin{macrocode}
\__msg_kernel_new:nnnn { regex } { trailing-backslash }
- { Trailing~escape~character~`\iow_char:N\\'. }
+ { Trailing~escape~character~'\iow_char:N\\'. }
{
A~regular~expression~or~its~replacement~text~ends~with~
- the~escape~character~`\iow_char:N\\'.~It~will~be~ignored.
+ the~escape~character~'\iow_char:N\\'.~It~will~be~ignored.
}
\__msg_kernel_new:nnnn { regex } { x-missing-rbrace }
- { Missing~closing~brace~in~`\iow_char:N\\x'~hexadecimal~sequence. }
+ { Missing~closing~brace~in~'\iow_char:N\\x'~hexadecimal~sequence. }
{
You~wrote~something~like~
- `\iow_char:N\\x\{...#1'.~
+ '\iow_char:N\\x\{...#1'.~
The~closing~brace~is~missing.
}
\__msg_kernel_new:nnnn { regex } { x-overflow }
- { Character~code~`#1'~too~large~in~`\iow_char:N\\x'~hexadecimal~sequence. }
+ { Character~code~'#1'~too~large~in~'\iow_char:N\\x'~hexadecimal~sequence. }
{
You~wrote~something~like~
- `\iow_char:N\\x\{\int_to_Hex:n{#1}\}'.~
+ '\iow_char:N\\x\{\int_to_Hex:n{#1}\}'.~
The~character~code~#1~is~larger~than~
the~maximum~value~\int_use:N \c_max_char_int.
}
@@ -5850,11 +6057,11 @@
% Invalid quantifier.
% \begin{macrocode}
\__msg_kernel_new:nnnn { regex } { invalid-quantifier }
- { Braced~quantifier~`#1'~may~not~be~followed~by~`#2'. }
+ { Braced~quantifier~'#1'~may~not~be~followed~by~'#2'. }
{
- The~character~`#2'~is~invalid~in~the~braced~quantifier~`#1'.~
- The~only~valid~quantifiers~are~`*',~`?',~`+',~`{<int>}',~
- `{<min>,}'~and~`{<min>,<max>}',~optionally~followed~by~`?'.
+ The~character~'#2'~is~invalid~in~the~braced~quantifier~'#1'.~
+ The~only~valid~quantifiers~are~'*',~'?',~'+',~'{<int>}',~
+ '{<min>,}'~and~'{<min>,<max>}',~optionally~followed~by~'?'.
}
% \end{macrocode}
%
@@ -5865,7 +6072,7 @@
{ Missing~right~bracket~inserted~in~regular~expression. }
{
LaTeX~was~given~a~regular~expression~where~a~character~class~
- was~started~with~`[',~but~the~matching~`]'~is~missing.
+ was~started~with~'[',~but~the~matching~']'~is~missing.
}
\__msg_kernel_new:nnnn { regex } { missing-rparen }
{
@@ -5889,7 +6096,7 @@
% \begin{macrocode}
\__msg_kernel_new:nnnn { regex } { bad-escape }
{
- Invalid~escape~`\iow_char:N\\#1'~
+ Invalid~escape~'\iow_char:N\\#1'~
\@@_if_in_cs:TF { within~a~control~sequence. }
{
\@@_if_in_class:TF
@@ -5898,16 +6105,16 @@
}
}
{
- The~escape~sequence~`\iow_char:N\\#1'~may~not~appear~
+ The~escape~sequence~'\iow_char:N\\#1'~may~not~appear~
\@@_if_in_cs:TF
{
within~a~control~sequence~test~introduced~by~
- `\iow_char:N\\c\iow_char:N\{'.
+ '\iow_char:N\\c\iow_char:N\{'.
}
{
\@@_if_in_class:TF
{ within~a~character~class~ }
- { following~a~category~test~such~as~`\iow_char:N\\cL'~ }
+ { following~a~category~test~such~as~'\iow_char:N\\cL'~ }
because~it~does~not~match~exactly~one~character.
}
}
@@ -5916,96 +6123,96 @@
% Range errors.
% \begin{macrocode}
\__msg_kernel_new:nnnn { regex } { range-missing-end }
- { Invalid~end-point~for~range~`#1-#2'~in~character~class. }
+ { Invalid~end-point~for~range~'#1-#2'~in~character~class. }
{
- The~end-point~`#2'~of~the~range~`#1-#2'~may~not~serve~as~an~
+ The~end-point~'#2'~of~the~range~'#1-#2'~may~not~serve~as~an~
end-point~for~a~range:~alphanumeric~characters~should~not~be~
escaped,~and~non-alphanumeric~characters~should~be~escaped.
}
\__msg_kernel_new:nnnn { regex } { range-backwards }
- { Range~`[#1-#2]'~out~of~order~in~character~class. }
+ { Range~'[#1-#2]'~out~of~order~in~character~class. }
{
- In~ranges~of~characters~`[x-y]'~appearing~in~character~classes,~
+ In~ranges~of~characters~'[x-y]'~appearing~in~character~classes,~
the~first~character~code~must~not~be~larger~than~the~second.~
- Here,~`#1'~has~character~code~\int_eval:n {`#1},~while~
- `#2'~has~character~code~\int_eval:n {`#2}.
+ Here,~'#1'~has~character~code~\int_eval:n {`#1},~while~
+ '#2'~has~character~code~\int_eval:n {`#2}.
}
% \end{macrocode}
%
% Errors related to |\c| and |\u|.
% \begin{macrocode}
\__msg_kernel_new:nnnn { regex } { c-bad-mode }
- { Invalid~nested~`\iow_char:N\\c'~escape~in~regular~expression. }
+ { Invalid~nested~'\iow_char:N\\c'~escape~in~regular~expression. }
{
- The~`\iow_char:N\\c'~escape~cannot~be~used~within~
- a~control~sequence~test~`\iow_char:N\\c{...}'.~
- To~combine~several~category~tests,~use~`\iow_char:N\\c[...]'.
+ The~'\iow_char:N\\c'~escape~cannot~be~used~within~
+ a~control~sequence~test~'\iow_char:N\\c{...}'.~
+ To~combine~several~category~tests,~use~'\iow_char:N\\c[...]'.
}
\__msg_kernel_new:nnnn { regex } { c-missing-rbrace }
- { Missing~right~brace~inserted~for~`\iow_char:N\\c'~escape. }
+ { Missing~right~brace~inserted~for~'\iow_char:N\\c'~escape. }
{
LaTeX~was~given~a~regular~expression~where~a~
- `\iow_char:N\\c\iow_char:N\{...'~construction~was~not~ended~
- with~a~closing~brace~`\iow_char:N\}'.
+ '\iow_char:N\\c\iow_char:N\{...'~construction~was~not~ended~
+ with~a~closing~brace~'\iow_char:N\}'.
}
\__msg_kernel_new:nnnn { regex } { c-missing-rbrack }
- { Missing~right~bracket~inserted~for~`\iow_char:N\\c'~escape. }
+ { Missing~right~bracket~inserted~for~'\iow_char:N\\c'~escape. }
{
- A~construction~`\iow_char:N\\c[...'~appears~in~a~
- regular~expression,~but~the~closing~`]'~is~not~present.
+ A~construction~'\iow_char:N\\c[...'~appears~in~a~
+ regular~expression,~but~the~closing~']'~is~not~present.
}
\__msg_kernel_new:nnnn { regex } { c-missing-category }
- { Invalid~character~`#1'~following~`\iow_char:N\\c'~escape. }
+ { Invalid~character~'#1'~following~'\iow_char:N\\c'~escape. }
{
- In~regular~expressions,~the~`\iow_char:N\\c'~escape~sequence~
+ In~regular~expressions,~the~'\iow_char:N\\c'~escape~sequence~
may~only~be~followed~by~a~left~brace,~a~left~bracket,~or~a~
capital~letter~representing~a~character~category,~namely~
- one~of~`ABCDELMOPSTU'.
+ one~of~'ABCDELMOPSTU'.
}
\__msg_kernel_new:nnnn { regex } { c-trailing }
- { Trailing~category~code~escape~`\iow_char:N\\c'... }
+ { Trailing~category~code~escape~'\iow_char:N\\c'... }
{
- A~regular~expression~ends~with~`\iow_char:N\\c'~followed~
+ A~regular~expression~ends~with~'\iow_char:N\\c'~followed~
by~a~letter.~It~will~be~ignored.
}
\__msg_kernel_new:nnnn { regex } { u-missing-lbrace }
- { Missing~left~brace~following~`\iow_char:N\\u'~escape. }
+ { Missing~left~brace~following~'\iow_char:N\\u'~escape. }
{
- The~`\iow_char:N\\u'~escape~sequence~must~be~followed~by~
+ The~'\iow_char:N\\u'~escape~sequence~must~be~followed~by~
a~brace~group~with~the~name~of~the~variable~to~use.
}
\__msg_kernel_new:nnnn { regex } { u-missing-rbrace }
- { Missing~right~brace~inserted~for~`\iow_char:N\\u'~escape. }
+ { Missing~right~brace~inserted~for~'\iow_char:N\\u'~escape. }
{
LaTeX~
\str_if_eq_x:nnTF { } {#2}
{ reached~the~end~of~the~string~ }
- { encountered~an~escaped~alphanumeric~character `\iow_char:N\\#2'~ }
- when~parsing~the~argument~of~an~`\iow_char:N\\u\iow_char:N\{...\}'~escape.
+ { encountered~an~escaped~alphanumeric~character '\iow_char:N\\#2'~ }
+ when~parsing~the~argument~of~an~'\iow_char:N\\u\iow_char:N\{...\}'~escape.
}
% \end{macrocode}
%
% Errors when encountering the \textsc{posix} syntax |[:...:]|.
% \begin{macrocode}
\__msg_kernel_new:nnnn { regex } { posix-unsupported }
- { POSIX~collating~element~`[#1 ~ #1]'~not~supported. }
+ { POSIX~collating~element~'[#1 ~ #1]'~not~supported. }
{
- The~`[.foo.]'~and~`[=bar=]'~syntaxes~have~a~special~meaning~
+ The~'[.foo.]'~and~'[=bar=]'~syntaxes~have~a~special~meaning~
in~POSIX~regular~expressions.~This~is~not~supported~by~LaTeX.~
Maybe~you~forgot~to~escape~a~left~bracket~in~a~character~class?
}
\__msg_kernel_new:nnnn { regex } { posix-unknown }
- { POSIX~class~`[:#1:]'~unknown. }
+ { POSIX~class~'[:#1:]'~unknown. }
{
- `[:#1:]'~is~not~among~the~known~POSIX~classes~
- `[:alnum:]',~`[:alpha:]',~`[:ascii:]',~`[:blank:]',~
- `[:cntrl:]',~`[:digit:]',~`[:graph:]',~`[:lower:]',~
- `[:print:]',~`[:punct:]',~`[:space:]',~`[:upper:]',~
- `[:word:]',~and~`[:xdigit:]'.
+ '[:#1:]'~is~not~among~the~known~POSIX~classes~
+ '[:alnum:]',~'[:alpha:]',~'[:ascii:]',~'[:blank:]',~
+ '[:cntrl:]',~'[:digit:]',~'[:graph:]',~'[:lower:]',~
+ '[:print:]',~'[:punct:]',~'[:space:]',~'[:upper:]',~
+ '[:word:]',~and~'[:xdigit:]'.
}
\__msg_kernel_new:nnnn { regex } { posix-missing-close }
- { Missing~closing~`:]'~for~POSIX~class. }
- { The~POSIX~syntax~`#1'~must~be~followed~by~`:]',~not~`#2'. }
+ { Missing~closing~':]'~for~POSIX~class. }
+ { The~POSIX~syntax~'#1'~must~be~followed~by~':]',~not~'#2'. }
% \end{macrocode}
%
% In various cases, the result of a \pkg{l3regex} operation can leave us
@@ -6025,62 +6232,72 @@
% Error message for unknown options.
% \begin{macrocode}
\__msg_kernel_new:nnnn { regex } { unknown-option }
- { Unknown~option~`#1'~for~regular~expressions. }
+ { Unknown~option~'#1'~for~regular~expressions. }
{
- The~only~available~option~is~`case-insensitive',~toggled~by~
- `(?i)'~and~`(?-i)'.
+ The~only~available~option~is~'case-insensitive',~toggled~by~
+ '(?i)'~and~'(?-i)'.
}
\__msg_kernel_new:nnnn { regex } { special-group-unknown }
- { Unknown~special~group~`#1~...'~in~a~regular~expression. }
+ { Unknown~special~group~'#1~...'~in~a~regular~expression. }
{
- The~only~valid~constructions~starting~with~`(?'~are~
- `(?:~...~)',~`(?|~...~)',~`(?i)',~and~`(?-i)'.
+ The~only~valid~constructions~starting~with~'(?'~are~
+ '(?:~...~)',~'(?|~...~)',~'(?i)',~and~'(?-i)'.
}
% \end{macrocode}
%
% Errors in the replacement text.
% \begin{macrocode}
\__msg_kernel_new:nnnn { regex } { replacement-c }
- { Misused~`\iow_char:N\\c'~command~in~a~replacement~text. }
+ { Misused~'\iow_char:N\\c'~command~in~a~replacement~text. }
{
- In~a~replacement~text,~the~`\iow_char:N\\c'~escape~sequence~
- can~be~followed~by~one~of~the~letters~`ABCDELMOPSTU'~
- or~a~brace~group,~not~by~`#1'.
+ In~a~replacement~text,~the~'\iow_char:N\\c'~escape~sequence~
+ can~be~followed~by~one~of~the~letters~'ABCDELMOPSTU'~
+ or~a~brace~group,~not~by~'#1'.
}
\__msg_kernel_new:nnnn { regex } { replacement-u }
- { Misused~`\iow_char:N\\u'~command~in~a~replacement~text. }
+ { Misused~'\iow_char:N\\u'~command~in~a~replacement~text. }
{
- In~a~replacement~text,~the~`\iow_char:N\\u'~escape~sequence~
+ In~a~replacement~text,~the~'\iow_char:N\\u'~escape~sequence~
must~be~~followed~by~a~brace~group~holding~the~name~of~the~
variable~to~use.
}
\__msg_kernel_new:nnnn { regex } { replacement-g }
{
- Missing~brace~for~the~`\iow_char:N\\g'~construction~
+ Missing~brace~for~the~'\iow_char:N\\g'~construction~
in~a~replacement~text.
}
{
In~the~replacement~text~for~a~regular~expression~search,~
- submatches~are~represented~either~as~`\iow_char:N \\g{dd..d}',~
- or~`\\d',~where~`d'~are~single~digits.~Here,~a~brace~is~missing.
+ submatches~are~represented~either~as~'\iow_char:N \\g{dd..d}',~
+ or~'\\d',~where~'d'~are~single~digits.~Here,~a~brace~is~missing.
}
\__msg_kernel_new:nnnn { regex } { replacement-catcode-end }
{
- Missing~character~for~the~`\iow_char:N\\c<category><character>'~
+ Missing~character~for~the~'\iow_char:N\\c<category><character>'~
construction~in~a~replacement~text.
}
{
- In~a~replacement~text,~the~`\iow_char:N\\c'~escape~sequence~
- can~be~followed~by~one~of~the~letters~`ABCDELMOPSTU'~representing~
+ In~a~replacement~text,~the~'\iow_char:N\\c'~escape~sequence~
+ can~be~followed~by~one~of~the~letters~'ABCDELMOPSTU'~representing~
the~character~category.~Then,~a~character~must~follow.~LaTeX~
reached~the~end~of~the~replacement~when~looking~for~that.
}
+\__msg_kernel_new:nnnn { regex } { replacement-catcode-in-cs }
+ {
+ Category~code~'\iow_char:N\\c#1#3'~ignored~inside~
+ '\iow_char:N\\c\{...\}'~in~a~replacement~text.
+ }
+ {
+ In~a~replacement~text,~the~category~codes~of~the~argument~of~
+ '\iow_char:N\\c\{...\}'~are~ignored~when~building~the~control~
+ sequence~name.
+ }
\__msg_kernel_new:nnnn { regex } { replacement-null-space }
{ TeX~cannot~build~a~space~token~with~character~code~0. }
{
You~asked~for~a~character~token~with~category~space,~
and~character~code~0,~for~instance~through~
- `\iow_char:N\\cS\iow_char:N\\x00'.~
+ '\iow_char:N\\cS\iow_char:N\\x00'.~
This~specific~case~is~impossible~and~will~be~replaced~
by~a~normal~space.
}
@@ -6090,6 +6307,12 @@
There~ \int_compare:nTF { #1 = 1 } { was } { were } ~ #1~
missing~right~\int_compare:nTF { #1 = 1 } { brace } { braces } .
}
+\__msg_kernel_new:nnnn { regex } { replacement-missing-rparen }
+ { Missing~right~parenthesis~inserted~in~replacement~text. }
+ {
+ There~ \int_compare:nTF { #1 = 1 } { was } { were } ~ #1~
+ missing~right~\int_compare:nTF { #1 = 1 } { parenthesis } { parentheses } .
+ }
% \end{macrocode}
%
% \begin{macro}[aux]{\@@_msg_repeated:nnN}
@@ -6136,7 +6359,7 @@
{ \l_@@_max_state_int - 1 }
{
\trace:nnx { regex } { #1 }
- { \iow_char:N \\toks ##1 = { \tex_the:D \tex_toks:D ##1 } }
+ { \iow_char:N \\toks ##1 = { \@@_toks_use:w ##1 } }
}
}
%</trace>