summaryrefslogtreecommitdiff
path: root/macros/latex/contrib/l3kernel/l3regex.dtx
diff options
context:
space:
mode:
Diffstat (limited to 'macros/latex/contrib/l3kernel/l3regex.dtx')
-rw-r--r--macros/latex/contrib/l3kernel/l3regex.dtx1219
1 files changed, 808 insertions, 411 deletions
diff --git a/macros/latex/contrib/l3kernel/l3regex.dtx b/macros/latex/contrib/l3kernel/l3regex.dtx
index 2086b5c3ec..5e64d73980 100644
--- a/macros/latex/contrib/l3kernel/l3regex.dtx
+++ b/macros/latex/contrib/l3kernel/l3regex.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
@@ -713,7 +713,7 @@
% |{n,}| quantifier? (I think not.)
% \item Quantifiers for |\u| and assertions.
% \item When matching, keep track of an explicit stack of
-% \texttt{current_state} and \texttt{current_submatches}.
+% \texttt{curr_state} and \texttt{curr_submatches}.
% \item If possible, when a state is reused by the same thread, kill
% other subthreads.
% \item Use an array rather than \cs[no-index]{l__regex_balance_tl}
@@ -845,12 +845,14 @@
% names).
% \begin{itemize}
% \item \emph{Group}: index of the capturing group, $-1$ for
-% non-capturing groups.
+% non-capturing groups. ^^A start/end index?
% \item \emph{Position}: each token in the query is labelled by an
% integer \meta{position}, with $\texttt{min_pos} - 1 \leq
% \meta{position} \leq \texttt{max_pos}$. The lowest and highest
-% positions correspond to imaginary begin and end markers (with
-% inaccessible category code and character code).
+% positions $\texttt{min_pos} - 1$ and $\texttt{max_pos}$
+% correspond to imaginary begin and end markers (with
+% non-existent category code and character code).
+% $\texttt{max_pos}$ is only set quite late in the processing.
% \item \emph{Query}: the token list to which we apply the regular
% expression.
% \item \emph{State}: each state of the \textsc{nfa} is labelled by an
@@ -865,8 +867,8 @@
% unique id for all the steps of the matching algorithm.
% \end{itemize}
%
-% We use \pkg{l3intarray} to manipulate arrays of integers (stored into
-% some dimension registers in scaled points). We also abuse \TeX{}'s
+% We use \pkg{l3intarray} to manipulate arrays of integers.
+% We also abuse \TeX{}'s
% \tn{toks} registers, by accessing them directly by number rather than
% tying them to control sequence using the \tn{newtoks} allocation
% functions. Specifically, these arrays and \tn{toks} are used as
@@ -876,21 +878,14 @@
% \begin{itemize}
% \item \cs{g_@@_state_active_intarray} holds the last \meta{step} in
% which each \meta{state} was active.
-% \item \cs{g_@@_thread_state_intarray} maps each \meta{thread} (with
-% $\texttt{min_active} \leq \meta{thread} < \texttt{max_active}$) to
-% the \meta{state} in which the \meta{thread} currently is. The
+% \item \cs{g_@@_thread_info_intarray} consists of blocks for each
+% \meta{thread} (with $\texttt{min_thread} \leq \meta{thread} <
+% \texttt{max_thread}$). Each block has
+% $1+2\cs{l_@@_capturing_group_int}$ entries: the \meta{state} in
+% which the \meta{thread} currently is, followed by the beginnings
+% of all submatches, and then the ends of all submatches. The
% \meta{threads} are ordered starting from the best to the least
% preferred.
-% \item \tn{toks}\meta{thread} holds the submatch information for the
-% \meta{thread}, as the contents of a property list.
-% \item \cs{g_@@_charcode_intarray} and \cs{g_@@_catcode_intarray} hold the
-% character codes and category codes of tokens at each
-% \meta{position} in the query.
-% \item \cs{g_@@_balance_intarray} holds the balance of begin-group and
-% end-group character tokens which appear before that point in the
-% token list.
-% \item \tn{toks}\meta{position} holds \meta{tokens} which \texttt{o}-
-% and \texttt{x}-expand to the \meta{position}-th token in the query.
% \item \cs{g_@@_submatch_prev_intarray}, \cs{g_@@_submatch_begin_intarray}
% and \cs{g_@@_submatch_end_intarray} hold, for each submatch (as would
% be extracted by \cs{regex_extract_all:nnN}), the place where the
@@ -901,6 +896,14 @@
% block corresponding to one match with all its submatches stored in
% consecutive entries.
% \end{itemize}
+% When actually building the result,
+% \begin{itemize}
+% \item \tn{toks}\meta{position} holds \meta{tokens} which \texttt{o}-
+% and \texttt{x}-expand to the \meta{position}-th token in the query.
+% \item \cs{g_@@_balance_intarray} holds the balance of begin-group and
+% end-group character tokens which appear before that point in the
+% token list.
+% \end{itemize}
%
% The code is structured as follows. Variables are introduced in the
% relevant section. First we present some generic helper functions. Then
@@ -944,7 +947,7 @@
{ \@@_toks_set:Nn #1 { } }
\cs_new_eq:NN \@@_toks_set:Nn \tex_toks:D
\cs_new_protected:Npn \@@_toks_set:No #1
- { \@@_toks_set:Nn #1 \exp_after:wN }
+ { \tex_toks:D #1 \exp_after:wN }
% \end{macrocode}
% \end{macro}
%
@@ -975,13 +978,13 @@
% \begin{macrocode}
\cs_new_protected:Npn \@@_toks_put_left:Nx #1#2
{
- \cs_set:Npx \@@_tmp:w { #2 }
+ \cs_set_nopar:Npx \@@_tmp:w { #2 }
\tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN
{ \exp_after:wN \@@_tmp:w \tex_the:D \tex_toks:D #1 }
}
\cs_new_protected:Npn \@@_toks_put_right:Nx #1#2
{
- \cs_set:Npx \@@_tmp:w {#2}
+ \cs_set_nopar:Npx \@@_tmp:w {#2}
\tex_toks:D #1 \exp_after:wN
{ \tex_the:D \tex_toks:D \exp_after:wN #1 \@@_tmp:w }
}
@@ -1000,11 +1003,38 @@
\cs_new:Npn \@@_curr_cs_to_str:
{
\exp_after:wN \exp_after:wN \exp_after:wN \cs_to_str:N
- \tex_the:D \tex_toks:D \l_@@_curr_pos_int
+ \l_@@_curr_token_tl
}
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}{\@@_intarray_item:NnF, \@@_intarray_item_aux:nNF}
+% Item of intarray, with a default value.
+% \begin{macrocode}
+\cs_new:Npn \@@_intarray_item:NnF #1#2
+ { \exp_args:Nf \@@_intarray_item_aux:nNF { \int_eval:n {#2} } #1 }
+\cs_new:Npn \@@_intarray_item_aux:nNF #1#2
+ {
+ \if_int_compare:w #1 > \c_zero_int
+ \exp_after:wN \use_i:nn
+ \else:
+ \exp_after:wN \use_ii:nn
+ \fi:
+ { \__kernel_intarray_item:Nn #2 {#1} }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_maplike_break:}
+% Analogous to \cs{tl_map_break:}, this correctly exits
+% \cs{tl_map_inline:nn} and similar constructions and jumps to the
+% matching \cs{prg_break_point:Nn} \cs{@@_maplike_break:} |{| |}|.
+% \begin{macrocode}
+\cs_new:Npn \@@_maplike_break:
+ { \prg_map_break:Nn \@@_maplike_break: { } }
+% \end{macrocode}
+% \end{macro}
+%
% \subsubsection{Constants and variables}
%
% \begin{macro}{\@@_tmp:w}
@@ -1056,19 +1086,6 @@
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\g_@@_charcode_intarray, \g_@@_catcode_intarray, \g_@@_balance_intarray}
-% The first thing we do when matching is to go once through the query
-% token list and store the information for each token into
-% \cs{g_@@_charcode_intarray}, \cs{g_@@_catcode_intarray} and \tn{toks}
-% registers. We also store the balance of begin-group/end-group
-% characters into \cs{g_@@_balance_intarray}.
-% \begin{macrocode}
-\intarray_new:Nn \g_@@_charcode_intarray { 65536 }
-\intarray_new:Nn \g_@@_catcode_intarray { 65536 }
-\intarray_new:Nn \g_@@_balance_intarray { 65536 }
-% \end{macrocode}
-% \end{variable}
-%
% \begin{variable}{\l_@@_balance_int}
% During this phase, \cs{l_@@_balance_int} counts the balance of
% begin-group and end-group character tokens which appear before a
@@ -1079,15 +1096,6 @@
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\l_@@_cs_name_tl}
-% This variable is used in \cs{@@_item_cs:n} to store the csname of
-% the currently-tested token when the regex contains a sub-regex for
-% testing csnames.
-% \begin{macrocode}
-\tl_new:N \l_@@_cs_name_tl
-% \end{macrocode}
-% \end{variable}
-%
% \subsubsection{Testing characters}
%
% \begin{macro}{\c_@@_ascii_min_int, \c_@@_ascii_max_control_int, \c_@@_ascii_max_int}
@@ -1201,7 +1209,7 @@
% \begin{macro}
% {\@@_item_caseless_equal:n, \@@_item_caseless_range:nn}
% For caseless matching, we perform the test both on the
-% \texttt{current_char} and on the \texttt{case_changed_char}. Before
+% \texttt{curr_char} and on the \texttt{case_changed_char}. Before
% doing the second set of tests, we make sure that
% \texttt{case_changed_char} has been computed.
% \begin{macrocode}
@@ -1308,7 +1316,8 @@
%
% \begin{macro}{\@@_item_exact:nn, \@@_item_exact_cs:n}
% This matches an exact \meta{category}-\meta{character code} pair, or
-% an exact control sequence, more precisely one of several possible control sequences.
+% an exact control sequence, more precisely one of several possible
+% control sequences, separated by \cs{scan_stop:}.
% \begin{macrocode}
\cs_new_protected:Npn \@@_item_exact:nn #1#2
{
@@ -1337,24 +1346,19 @@
% Match a control sequence (the argument is a compiled regex).
% First test the catcode of the current token to be zero.
% Then perform the matching test, and break if the csname
-% indeed matches. The three \cs{exp_after:wN} expand the contents
-% of the \tn{toks}\meta{current position} (of the form \cs{exp_not:n}
-% \Arg{control sequence}) to \meta{control sequence}.
-% We store the cs name before building states for the cs, as those
-% states may overlap with toks registers storing the user's input.
+% indeed matches.
% \begin{macrocode}
\cs_new_protected:Npn \@@_item_cs:n #1
{
\int_compare:nNnT \l_@@_curr_catcode_int = 0
{
\group_begin:
- \__kernel_tl_set:Nx \l_@@_cs_name_tl { \@@_curr_cs_to_str: }
\@@_single_match:
\@@_disable_submatches:
\@@_build_for_cs:n {#1}
\bool_set_eq:NN \l_@@_saved_success_bool
\g_@@_success_bool
- \exp_args:NV \@@_match_cs:n \l_@@_cs_name_tl
+ \exp_args:Nx \@@_match_cs:n { \@@_curr_cs_to_str: }
\if_meaning:w \c_true_bool \g_@@_success_bool
\group_insert_after:N \@@_break_true:w
\fi:
@@ -1822,7 +1826,7 @@
% \item \cs{@@_command_K:}
% \item \cs{@@_assertion:Nn} \meta{boolean} \Arg{assertion test},
% where the \meta{assertion test} is \cs{@@_b_test:} or
-% |{|\cs{@@_anchor:N} \meta{integer}|}|
+% \cs{@@_Z_test:} or \cs{@@_A_test:} or \cs{@@_G_test:}
% \end{itemize}
% Tests can be the following:
% \begin{itemize}
@@ -2663,73 +2667,56 @@
%
% \subsubsection{Anchoring and simple assertions}
%
-% \begin{macro}{\@@_compile_anchor:NF}
+% \begin{macro}{\@@_compile_anchor_letter:NNN}
+% \begin{macro}{\@@_compile_/A:, \@@_compile_/G:, \@@_compile_/Z:, \@@_compile_/z:, \@@_compile_/b:, \@@_compile_/B:}
% \begin{macro}+\@@_compile_^:+
-% \begin{macro}{\@@_compile_/A:, \@@_compile_/G:}
% \begin{macro}+\@@_compile_$:+
-% \begin{macro}{\@@_compile_/Z:, \@@_compile_/z:}
-% In modes where assertions are allowed, anchor to the start of the
-% query, the start of the match, or the end of the query, depending on
-% the integer |#1|. In other modes, |#2| treats the character as raw,
-% with an error for escaped letters (|$| is valid in a class, but |\A|
-% is definitely a mistake on the user's part).
+% In modes where assertions are forbidden, anchors such as |\A|
+% produce an error (|\A|~is invalid in classes); otherwise they add an
+% \cs{@@_assertion:Nn} test as appropriate (the only negative
+% assertion is~|\B|). The test functions are defined later. The
+% implementation for
+% |$| and |^| is only different from |\A| etc because these are valid
+% in a class.
% \begin{macrocode}
-\cs_new_protected:Npn \@@_compile_anchor:NF #1#2
+\cs_new_protected:Npn \@@_compile_anchor_letter:NNN #1#2#3
{
- \@@_if_in_class_or_catcode:TF {#2}
+ \@@_if_in_class_or_catcode:TF { \@@_compile_raw_error:N #1 }
{
\tl_build_put_right:Nn \l_@@_build_tl
- { \@@_assertion:Nn \c_true_bool { \@@_anchor:N #1 } }
+ { \@@_assertion:Nn #2 {#3} }
}
}
-\cs_set_protected:Npn \@@_tmp:w #1#2
- {
- \cs_new_protected:cpn { @@_compile_/#1: }
- { \@@_compile_anchor:NF #2 { \@@_compile_raw_error:N #1 } }
- }
-\@@_tmp:w A \l_@@_min_pos_int
-\@@_tmp:w G \l_@@_start_pos_int
-\@@_tmp:w Z \l_@@_max_pos_int
-\@@_tmp:w z \l_@@_max_pos_int
+\cs_new_protected:cpn { @@_compile_/A: }
+ { \@@_compile_anchor_letter:NNN A \c_true_bool \@@_A_test: }
+\cs_new_protected:cpn { @@_compile_/G: }
+ { \@@_compile_anchor_letter:NNN G \c_true_bool \@@_G_test: }
+\cs_new_protected:cpn { @@_compile_/Z: }
+ { \@@_compile_anchor_letter:NNN Z \c_true_bool \@@_Z_test: }
+\cs_new_protected:cpn { @@_compile_/z: }
+ { \@@_compile_anchor_letter:NNN z \c_true_bool \@@_Z_test: }
+\cs_new_protected:cpn { @@_compile_/b: }
+ { \@@_compile_anchor_letter:NNN b \c_true_bool \@@_b_test: }
+\cs_new_protected:cpn { @@_compile_/B: }
+ { \@@_compile_anchor_letter:NNN B \c_false_bool \@@_b_test: }
\cs_set_protected:Npn \@@_tmp:w #1#2
{
\cs_new_protected:cpn { @@_compile_#1: }
- { \@@_compile_anchor:NF #2 { \@@_compile_raw:N #1 } }
+ {
+ \@@_if_in_class_or_catcode:TF { \@@_compile_raw:N #1 }
+ {
+ \tl_build_put_right:Nn \l_@@_build_tl
+ { \@@_assertion:Nn \c_true_bool {#2} }
+ }
+ }
}
-\exp_args:Nx \@@_tmp:w { \iow_char:N \^ } \l_@@_min_pos_int
-\exp_args:Nx \@@_tmp:w { \iow_char:N \$ } \l_@@_max_pos_int
+\exp_args:Nx \@@_tmp:w { \iow_char:N \^ } { \@@_A_test: }
+\exp_args:Nx \@@_tmp:w { \iow_char:N \$ } { \@@_Z_test: }
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
-%
-% \begin{macro}{\@@_compile_/b:, \@@_compile_/B:}
-% Contrarily to |^| and |$|, which could be implemented without really
-% knowing what precedes in the token list, this requires more
-% information, namely, the knowledge of the last character code.
-% \begin{macrocode}
-\cs_new_protected:cpn { @@_compile_/b: }
- {
- \@@_if_in_class_or_catcode:TF
- { \@@_compile_raw_error:N b }
- {
- \tl_build_put_right:Nn \l_@@_build_tl
- { \@@_assertion:Nn \c_true_bool { \@@_b_test: } }
- }
- }
-\cs_new_protected:cpn { @@_compile_/B: }
- {
- \@@_if_in_class_or_catcode:TF
- { \@@_compile_raw_error:N B }
- {
- \tl_build_put_right:Nn \l_@@_build_tl
- { \@@_assertion:Nn \c_false_bool { \@@_b_test: } }
- }
- }
-% \end{macrocode}
-% \end{macro}
%
% \subsubsection{Character classes}
%
@@ -3512,7 +3499,9 @@
{ \bool_if:NF ##1 { negative~ } assertion:~##2 }
}
\cs_set:Npn \@@_b_test: { word~boundary }
- \cs_set_eq:NN \@@_anchor:N \@@_show_anchor_to_str:N
+ \cs_set:Npn \@@_Z_test: { anchor~at~end~(\iow_char:N\\Z) }
+ \cs_set:Npn \@@_A_test: { anchor~at~start~(\iow_char:N\\A) }
+ \cs_set:Npn \@@_G_test: { anchor~at~start~of~match~(\iow_char:N\\G) }
\cs_set_protected:Npn \@@_item_caseful_equal:n ##1
{ \@@_show_one:n { char~code~\int_eval:n{##1} } }
\cs_set_protected:Npn \@@_item_caseful_range:nn ##1##2
@@ -3655,24 +3644,6 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[rEXP]{\@@_show_anchor_to_str:N}
-% The argument is an integer telling us where the anchor is. We
-% convert that to the relevant info.
-% \begin{macrocode}
-\cs_new:Npn \@@_show_anchor_to_str:N #1
- {
- anchor~at~
- \str_case:nnF { #1 }
- {
- { \l_@@_min_pos_int } { start~(\iow_char:N\\A) }
- { \l_@@_start_pos_int } { start~of~match~(\iow_char:N\\G) }
- { \l_@@_max_pos_int } { end~(\iow_char:N\\Z) }
- }
- { <error:~'#1'~not~recognized> }
- }
-% \end{macrocode}
-% \end{macro}
-%
% \begin{macro}{\@@_show_item_catcode:NnT}
% Produce a sequence of categories which the catcode bitmap |#2|
% contains, and show it, indenting the tests on which this catcode
@@ -3757,8 +3728,9 @@
% Each state of the \textsc{nfa} is stored in a \tn{toks}. The
% operations which can appear in the \tn{toks} are
% \begin{itemize}
-% \item \cs{@@_action_start_wildcard:} inserted at the start
-% of the regular expression to make it unanchored.
+% \item \cs{@@_action_start_wildcard:N} \meta{boolean} inserted at the
+% start of the regular expression, where a \texttt{true}
+% \meta{boolean} makes it unanchored.
% \item \cs{@@_action_success:} marks the exit state of the
% \textsc{nfa}.
% \item \cs{@@_action_cost:n} \Arg{shift} is a transition from the
@@ -3772,10 +3744,11 @@
% how they detect and avoid infinite loops. For now, we just need to
% know that the \texttt{group} variant must be used for transitions
% back to the start of a group.
-% \item \cs{@@_action_submatch:n} \Arg{key} where the \meta{key} is
-% a group number followed by |<| or |>| for the beginning or end of
-% group. This causes the current position in the query to be stored
-% as the \meta{key} submatch boundary.
+% \item \cs{@@_action_submatch:nN} \Arg{group} \meta{key} where the
+% \meta{key} is |<| or |>| for the beginning or end of group
+% numbered \meta{group}. This causes the current position in the
+% query to be stored as the \meta{key} submatch boundary.
+% \item One of these actions, within a conditional.
% \end{itemize}
%
% We strive to preserve the following properties while building.
@@ -3793,20 +3766,26 @@
% corresponding end-points of nested groups.
% \end{itemize}
%
-% \begin{macro}{\@@_build:n, \@@_build:N}
+% \begin{macro}{\@@_build:n, \@@_build_aux:Nn, \@@_build:N, \@@_build_aux:NN}
% The \texttt{n}-type function first compiles its argument. Reset some
% variables. Allocate two states, and put a wildcard in state $0$
% (transitions to state $1$ and $0$ state). Then build the regex
% within a (capturing) group numbered $0$ (current
% value of \texttt{capturing_group}). Finally, if the match reaches the
-% last state, it is successful.
+% last state, it is successful. A \texttt{false} boolean for argument
+% |#1| for the auxiliaries will suppress the wildcard and make the
+% match anchored: used for \cs{peek_regex:nTF} and similar.
% \begin{macrocode}
-\cs_new_protected:Npn \@@_build:n #1
+\cs_new_protected:Npn \@@_build:n
+ { \@@_build_aux:Nn \c_true_bool }
+\cs_new_protected:Npn \@@_build:N
+ { \@@_build_aux:NN \c_true_bool }
+\cs_new_protected:Npn \@@_build_aux:Nn #1#2
{
- \@@_compile:n {#1}
- \@@_build:N \l_@@_internal_regex
+ \@@_compile:n {#2}
+ \@@_build_aux:NN #1 \l_@@_internal_regex
}
-\cs_new_protected:Npn \@@_build:N #1
+\cs_new_protected:Npn \@@_build_aux:NN #1#2
{
\@@_standard_escapechar:
\int_zero:N \l_@@_capturing_group_int
@@ -3814,8 +3793,8 @@
\@@_build_new_state:
\@@_build_new_state:
\@@_toks_put_right:Nn \l_@@_left_state_int
- { \@@_action_start_wildcard: }
- \@@_group:nnnN {#1} { 1 } { 0 } \c_false_bool
+ { \@@_action_start_wildcard:N #1 }
+ \@@_group:nnnN {#2} { 1 } { 0 } \c_false_bool
\@@_toks_put_right:Nn \l_@@_right_state_int
{ \@@_action_success: }
}
@@ -3826,19 +3805,14 @@
% The matching code relies on some global intarray variables, but only
% uses a range of their entries. Specifically,
% \begin{itemize}
-% \item \cs{g_@@_state_active_intarray} from \cs{l_@@_min_state_int}
-% to $\cs{l_@@_max_state_int}-1$;
-% \item \cs{g_@@_thread_state_intarray} from \cs{l_@@_min_active_int}
-% to $\cs{l_@@_max_active_int}-1$.
+% \item \cs{g_@@_state_active_intarray} from \cs{l_@@_min_state_int}
+% to $\cs{l_@@_max_state_int}-1$;
% \end{itemize}
-% In fact, some data is stored in \tn{toks} registers (local) in the
-% same ranges so these ranges mustn't overlap. This is done by
-% setting \cs{l_@@_min_active_int} to \cs{l_@@_max_state_int} after
-% building the \textsc{nfa}. Here, in this nested call to the
-% matching code, we need the new versions of these ranges to involve
+% Here, in this nested call to the
+% matching code, we need the new versions of this range to involve
% completely new entries of the intarray variables, so we begin by
% setting (the new) \cs{l_@@_min_state_int} to (the old)
-% \cs{l_@@_max_active_int} to use higher entries.
+% \cs{l_@@_max_state_int} to use higher entries.
%
% When using a regex to match a cs, we don't insert a wildcard, we
% anchor at the end, and since we ignore submatches, there is no need
@@ -3848,8 +3822,7 @@
% \begin{macrocode}
\cs_new_protected:Npn \@@_build_for_cs:n #1
{
- \int_set_eq:NN \l_@@_min_state_int \l_@@_max_active_int
- \int_set_eq:NN \l_@@_max_state_int \l_@@_min_state_int
+ \int_set_eq:NN \l_@@_min_state_int \l_@@_max_state_int
\@@_build_new_state:
\@@_build_new_state:
\@@_push_lr_states:
@@ -3857,7 +3830,7 @@
\@@_pop_lr_states:
\@@_toks_put_right:Nn \l_@@_right_state_int
{
- \if_int_compare:w \l_@@_curr_pos_int = \l_@@_max_pos_int
+ \if_int_compare:w -2 = \l_@@_curr_char_int
\exp_after:wN \@@_action_success:
\fi:
}
@@ -4195,8 +4168,8 @@
\cs_new_protected:Npn \@@_group_submatches:nNN #1#2#3
{
\if_int_compare:w #1 > - 1 \exp_stop_f:
- \@@_toks_put_left:Nx #2 { \@@_action_submatch:n { #1 < } }
- \@@_toks_put_left:Nx #3 { \@@_action_submatch:n { #1 > } }
+ \@@_toks_put_left:Nx #2 { \@@_action_submatch:nN {#1} < }
+ \@@_toks_put_left:Nx #3 { \@@_action_submatch:nN {#1} > }
\fi:
}
% \end{macrocode}
@@ -4345,16 +4318,14 @@
%
% \subsubsection{Others}
%
-% \begin{macro}{\@@_assertion:Nn, \@@_b_test:, \@@_anchor:N}
+% \begin{macro}{\@@_assertion:Nn, \@@_b_test:, \@@_A_test:, \@@_G_test:, \@@_Z_test:}
% Usage: \cs{@@_assertion:Nn} \meta{boolean} \Arg{test}, where the
% \meta{test} is either of the two other functions. Add a free
% transition to a new state, conditionally to the assertion test. The
% \cs{@@_b_test:} test is used by the |\b| and |\B| escape: check
% if the last character was a word character or not, and do the same
% to the current character. The boundary-markers of the string are
-% non-word characters for this purpose. Anchors at the start or end
-% of match use \cs{@@_anchor:N}, with a position controlled by the
-% integer |#1|.
+% non-word characters for this purpose.
% \begin{macrocode}
\cs_new_protected:Npn \@@_assertion:Nn #1#2
{
@@ -4374,12 +4345,6 @@
\bool_if:NT #1 { { } }
}
}
-\cs_new_protected:Npn \@@_anchor:N #1
- {
- \if_int_compare:w #1 = \l_@@_curr_pos_int
- \exp_after:wN \@@_break_true:w
- \fi:
- }
\cs_new_protected:Npn \@@_b_test:
{
\group_begin:
@@ -4389,6 +4354,24 @@
{ \group_end: \@@_item_reverse:n \@@_prop_w: }
{ \group_end: \@@_prop_w: }
}
+\cs_new_protected:Npn \@@_Z_test:
+ {
+ \if_int_compare:w -2 = \l_@@_curr_char_int
+ \exp_after:wN \@@_break_true:w
+ \fi:
+ }
+\cs_new_protected:Npn \@@_A_test:
+ {
+ \if_int_compare:w -2 = \l_@@_last_char_int
+ \exp_after:wN \@@_break_true:w
+ \fi:
+ }
+\cs_new_protected:Npn \@@_G_test:
+ {
+ \if_int_compare:w \l_@@_curr_pos_int = \l_@@_start_pos_int
+ \exp_after:wN \@@_break_true:w
+ \fi:
+ }
% \end{macrocode}
% \end{macro}
%
@@ -4401,7 +4384,7 @@
\@@_build_new_state:
\@@_toks_put_right:Nx \l_@@_left_state_int
{
- \@@_action_submatch:n { 0< }
+ \@@_action_submatch:nN { 0 } <
\bool_set_true:N \l_@@_fresh_thread_bool
\@@_action_free:n
{
@@ -4423,7 +4406,8 @@
% transitions, the instruction at the new state of the \textsc{nfa} is
% performed immediately. When a transition consumes a character, the
% new state is appended to a list of \enquote{active states}, stored in
-% \cs{g_@@_thread_state_intarray}: this thread is made active again when the next
+% \cs{g_@@_thread_info_intarray} (together with submatch information):
+% this thread is made active again when the next
% token is read from the query. At every step (for each token in the
% query), we unpack that list of active states and the corresponding
% submatch props, and empty those.
@@ -4467,11 +4451,10 @@
% }
% The tokens in the query are indexed from \texttt{min_pos} for the
% first to $\texttt{max_pos}-1$ for the last, and their information is
-% stored in several arrays and \tn{toks} registers with those numbers. We
-% don't start from $0$ because the \tn{toks} registers with low
-% numbers are used to hold the states of the \textsc{nfa}. We match
+% stored in several arrays and \tn{toks} registers with those numbers.
+% We match
% without backtracking, keeping all threads in lockstep at the
-% \texttt{current_pos} in the query. The starting point of the current
+% \texttt{curr_pos} in the query. The starting point of the current
% match attempt is \texttt{start_pos}, and \texttt{success_pos},
% updated whenever a thread succeeds, is used as the next starting
% position.
@@ -4488,20 +4471,26 @@
% {
% \l_@@_curr_char_int,
% \l_@@_curr_catcode_int,
+% \l_@@_curr_token_tl,
% \l_@@_last_char_int,
+% \l_@@_last_char_success_int,
% \l_@@_case_changed_char_int
% }
% The character and category codes of the token at the current
-% position; the character code of the token at the previous position;
+% position and a token list expanding to that token; the character
+% code of the token at the previous position;
+% the character code of the token just before a successful match;
% and the character code of the result of changing the case of the
% current token (|A-Z|$\leftrightarrow$|a-z|). This last integer is
% only computed when necessary, and is otherwise \cs{c_max_int}. The
-% \texttt{current_char} variable is also used in various other phases
+% \texttt{curr_char} variable is also used in various other phases
% to hold a character code.
% \begin{macrocode}
\int_new:N \l_@@_curr_char_int
\int_new:N \l_@@_curr_catcode_int
+\tl_new:N \l_@@_curr_token_tl
\int_new:N \l_@@_last_char_int
+\int_new:N \l_@@_last_char_success_int
\int_new:N \l_@@_case_changed_char_int
% \end{macrocode}
% \end{variable}
@@ -4517,18 +4506,17 @@
% \end{variable}
%
% \begin{variable}
-% {\l_@@_curr_submatches_prop, \l_@@_success_submatches_prop}
+% {\l_@@_curr_submatches_tl, \l_@@_success_submatches_tl}
% The submatches for the thread which is currently active are stored
-% in the \texttt{current_submatches} property list variable. This
-% property list is stored by \cs{@@_action_cost:n} into the
-% \tn{toks} register for the target state of the transition, to be
-% retrieved when matching at the next position. When a thread
-% succeeds, this property list is copied to
-% \cs{l_@@_success_submatches_prop}: only the last successful thread
+% in the \texttt{curr_submatches} list, which is almost a comma list,
+% but ends with a comma. This list is stored by \cs{@@_store_state:n}
+% into an intarray variable, to be retrieved when matching at the next
+% position. When a thread succeeds, this list is copied to
+% \cs{l_@@_success_submatches_tl}: only the last successful thread
% remains there.
% \begin{macrocode}
-\prop_new:N \l_@@_curr_submatches_prop
-\prop_new:N \l_@@_success_submatches_prop
+\tl_new:N \l_@@_curr_submatches_tl
+\tl_new:N \l_@@_success_submatches_tl
% \end{macrocode}
% \end{variable}
%
@@ -4551,28 +4539,44 @@
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\l_@@_min_active_int, \l_@@_max_active_int}
+% \begin{variable}{\l_@@_min_thread_int, \l_@@_max_thread_int}
% All the currently active threads are kept in order of precedence in
-% \cs{g_@@_thread_state_intarray}, and the corresponding submatches in the
-% \tn{toks}. For our purposes, those serve as an array, indexed from
-% \texttt{min_active} (inclusive) to \texttt{max_active} (excluded).
-% At the start of every step, the whole array is unpacked, so that the
-% space can immediately be reused, and \texttt{max_active} is reset to
-% \texttt{min_active}, effectively clearing the array.
+% \cs{g_@@_thread_info_intarray} together with the corresponding
+% submatch information. Data in this intarray is organized as blocks
+% from \texttt{min_thread} (included) to \texttt{max_thread}
+% (excluded). At the start of every step, the whole array is
+% unpacked, so that the space can immediately be reused, and
+% \texttt{max_thread} is reset to \texttt{min_thread}, effectively
+% clearing the array.
% \begin{macrocode}
-\int_new:N \l_@@_min_active_int
-\int_new:N \l_@@_max_active_int
+\int_new:N \l_@@_min_thread_int
+\int_new:N \l_@@_max_thread_int
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\g_@@_state_active_intarray, \g_@@_thread_state_intarray}
+% \begin{variable}{\g_@@_state_active_intarray, \g_@@_thread_info_intarray}
% \cs{g_@@_state_active_intarray} stores the last \meta{step} in which
-% each \meta{state} was active. \cs{g_@@_thread_state_intarray} stores
+% each \meta{state} was active. \cs{g_@@_thread_info_intarray} stores
% threads to be considered in the next step, more precisely the
% states in which these threads are.
% \begin{macrocode}
\intarray_new:Nn \g_@@_state_active_intarray { 65536 }
-\intarray_new:Nn \g_@@_thread_state_intarray { 65536 }
+\intarray_new:Nn \g_@@_thread_info_intarray { 65536 }
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_@@_matched_analysis_tl, \l_@@_curr_analysis_tl}
+% The list \cs{l_@@_curr_analysis_tl} consists of a brace group
+% containing three brace groups corresponding to the current token,
+% with the same syntax as \cs{tl_analysis_map_inline:nn}. The list
+% \cs{l_@@_matched_analysis_tl} (constructed under the
+% \texttt{tl\_build} machinery) has one item for each token that has
+% already been treated so far in a given match attempt: each item
+% consists of three brace groups with the same syntax as
+% \cs{tl_analysis_map_inline:nn}.
+% \begin{macrocode}
+\tl_new:N \l_@@_matched_analysis_tl
+\tl_new:N \l_@@_curr_analysis_tl
% \end{macrocode}
% \end{variable}
%
@@ -4635,8 +4639,7 @@
%
% \begin{macro}{\@@_match:n, \@@_match_cs:n}
% \begin{macro}{\@@_match_init:}
-% First store the query into \tn{toks} registers and arrays (see
-% \cs{@@_query_set:nnn}). Then initialize the variables that should
+% Initialize the variables that should
% be set once for each user function (even for multiple
% matches). Namely, the overall matching is not yet successful; none of
% the states should be marked as visited (\cs{g_@@_state_active_intarray}), and
@@ -4647,38 +4650,26 @@
% \begin{macrocode}
\cs_new_protected:Npn \@@_match:n #1
{
- \int_zero:N \l_@@_balance_int
- \int_set:Nn \l_@@_curr_pos_int { 2 * \l_@@_max_state_int }
- \@@_query_set:nnn { } { -1 } { -2 }
- \int_set_eq:NN \l_@@_min_pos_int \l_@@_curr_pos_int
- \tl_analysis_map_inline:nn {#1}
- { \@@_query_set:nnn {##1} {"##3} {##2} }
- \int_set_eq:NN \l_@@_max_pos_int \l_@@_curr_pos_int
- \@@_query_set:nnn { } { -1 } { -2 }
\@@_match_init:
- \@@_match_once:
+ \@@_match_once_init:
+ \tl_analysis_map_inline:nn {#1}
+ { \@@_match_one_token:nnN {##1} {##2} ##3 }
+ \@@_match_one_token:nnN { } { -2 } F
+ \prg_break_point:Nn \@@_maplike_break: { }
}
\cs_new_protected:Npn \@@_match_cs:n #1
{
- \int_zero:N \l_@@_balance_int
- \int_set:Nn \l_@@_curr_pos_int
- {
- \int_max:nn { 2 * \l_@@_max_state_int - \l_@@_min_state_int }
- { \l_@@_max_pos_int }
- + 1
- }
- \@@_query_set:nnn { } { -1 } { -2 }
- \int_set_eq:NN \l_@@_min_pos_int \l_@@_curr_pos_int
+ \int_set_eq:NN \l_@@_min_thread_int \l_@@_max_thread_int
+ \@@_match_init:
+ \@@_match_once_init:
\str_map_inline:nn {#1}
{
- \@@_query_set:nnn { \exp_not:n {##1} }
- { \tl_if_blank:nTF {##1} { 10 } { 12 } }
- { `##1 }
+ \tl_if_blank:nTF {##1}
+ { \@@_match_one_token:nnN {##1} {`##1} A }
+ { \@@_match_one_token:nnN {##1} {`##1} C }
}
- \int_set_eq:NN \l_@@_max_pos_int \l_@@_curr_pos_int
- \@@_query_set:nnn { } { -1 } { -2 }
- \@@_match_init:
- \@@_match_once:
+ \@@_match_one_token:nnN { } { -2 } F
+ \prg_break_point:Nn \@@_maplike_break: { }
}
\cs_new_protected:Npn \@@_match_init:
{
@@ -4689,11 +4680,13 @@
\__kernel_intarray_gset:Nnn
\g_@@_state_active_intarray {##1} { 1 }
}
- \int_set_eq:NN \l_@@_min_active_int \l_@@_max_state_int
\int_zero:N \l_@@_step_int
+ \int_set:Nn \l_@@_min_pos_int { 2 }
\int_set_eq:NN \l_@@_success_pos_int \l_@@_min_pos_int
- \int_set:Nn \l_@@_min_submatch_int
- { 2 * \l_@@_max_state_int }
+ \int_set:Nn \l_@@_last_char_success_int { -2 }
+ \tl_build_begin:N \l_@@_matched_analysis_tl
+ \tl_clear:N \l_@@_curr_analysis_tl
+ \int_set:Nn \l_@@_min_submatch_int { 1 }
\int_set_eq:NN \l_@@_submatch_int \l_@@_min_submatch_int
\bool_set_false:N \l_@@_empty_success_bool
}
@@ -4701,21 +4694,24 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}{\@@_match_once:}
-% This function finds one match, then does some action defined by the
-% \texttt{every_match} token list, which may recursively call
-% \cs{@@_match_once:}. First initialize some variables: set the
+% \begin{macro}{\@@_match_once_init:}
+% This function resets various variables used when finding one match.
+% It is called before the loop through characters, and every time we
+% find a match, before searching for another match (this is controlled
+% by the \texttt{every_match} token list).
+%
+% First initialize some variables: set the
% conditional which detects identical empty matches; this match
% attempt starts at the previous \texttt{success_pos}, is not yet
% successful, and has no submatches yet; clear the array of active
% threads, and put the starting state $0$ in it. We are then almost
% ready to read our first token in the query, but we actually start
-% one position earlier than the start, and \texttt{get} that token, to
-% set \texttt{last_char} properly for word
-% boundaries. Then call \cs{@@_match_loop:}, which runs through the
-% query until the end or until a successful match breaks early.
+% one position earlier than the start because
+% \cs{@@_match_one_token:nnN} increments \cs{l_@@_curr_pos_int} and
+% saves \cs{l_@@_curr_char_int} as the \texttt{last_char} so that word
+% boundaries can be correctly identified.
% \begin{macrocode}
-\cs_new_protected:Npn \@@_match_once:
+\cs_new_protected:Npn \@@_match_once_init:
{
\if_meaning:w \c_true_bool \l_@@_empty_success_bool
\cs_set:Npn \@@_if_two_empty_matches:F
@@ -4728,14 +4724,24 @@
\fi:
\int_set_eq:NN \l_@@_start_pos_int \l_@@_success_pos_int
\bool_set_false:N \l_@@_match_success_bool
- \prop_clear:N \l_@@_curr_submatches_prop
- \int_set_eq:NN \l_@@_max_active_int \l_@@_min_active_int
+ \tl_set:Nx \l_@@_curr_submatches_tl
+ { \prg_replicate:nn { 2 * \l_@@_capturing_group_int } { 0 , } }
+ \int_set_eq:NN \l_@@_max_thread_int \l_@@_min_thread_int
\@@_store_state:n { \l_@@_min_state_int }
\int_set:Nn \l_@@_curr_pos_int
{ \l_@@_start_pos_int - 1 }
- \@@_query_get:
- \@@_match_loop:
- \l_@@_every_match_tl
+ \int_set_eq:NN \l_@@_curr_char_int \l_@@_last_char_success_int
+ \tl_build_get:NN \l_@@_matched_analysis_tl \l_@@_internal_a_tl
+ \exp_args:NNf \@@_match_once_init_aux:
+ \tl_map_inline:nn
+ { \exp_after:wN \l_@@_internal_a_tl \l_@@_curr_analysis_tl }
+ { \@@_match_one_token:nnN ##1 }
+ \prg_break_point:Nn \@@_maplike_break: { }
+ }
+\cs_new_protected:Npn \@@_match_once_init_aux:
+ {
+ \tl_build_clear:N \l_@@_matched_analysis_tl
+ \tl_clear:N \l_@@_curr_analysis_tl
}
% \end{macrocode}
% \end{macro}
@@ -4753,110 +4759,80 @@
\bool_gset_eq:NN
\g_@@_success_bool
\l_@@_match_success_bool
+ \@@_maplike_break:
}
}
\cs_new_protected:Npn \@@_multi_match:n #1
{
\tl_set:Nn \l_@@_every_match_tl
{
- \if_meaning:w \c_true_bool \l_@@_match_success_bool
- \bool_gset_true:N \g_@@_success_bool
- #1
- \exp_after:wN \@@_match_once:
+ \if_meaning:w \c_false_bool \l_@@_match_success_bool
+ \exp_after:wN \@@_maplike_break:
\fi:
+ \bool_gset_true:N \g_@@_success_bool
+ #1
+ \@@_match_once_init:
}
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}{\@@_match_loop:}
+% \begin{macro}{\@@_match_one_token:nnN}
% \begin{macro}[rEXP]{\@@_match_one_active:n}
% At each new position, set some variables and get the new character
% and category from the query. Then unpack the array of active
% threads, and clear it by resetting its length
-% (\texttt{max_active}). This results in a sequence of
-% \cs{@@_use_state_and_submatches:nn} \Arg{state} \Arg{prop}, and
+% (\texttt{max_thread}). This results in a sequence of
+% \cs{@@_use_state_and_submatches:w} \meta{state}|,|\meta{submatch-clist}|;| and
% we consider those states one by one in order. As soon as a thread
% succeeds, exit the step, and, if there are threads to consider at the
% next position, and we have not reached the end of the string,
-% repeat the loop. Otherwise, the last thread that succeeded is what
-% \cs{@@_match_once:} matches. We explain the \texttt{fresh_thread}
-% business when describing \cs{@@_action_wildcard:}.
+% repeat the loop. Otherwise, the last thread that succeeded is the
+% match. We explain the \texttt{fresh_thread} business when
+% describing \cs{@@_action_wildcard:}.
% \begin{macrocode}
-\cs_new_protected:Npn \@@_match_loop:
+\cs_new_protected:Npn \@@_match_one_token:nnN #1#2#3
{
\int_add:Nn \l_@@_step_int { 2 }
\int_incr:N \l_@@_curr_pos_int
\int_set_eq:NN \l_@@_last_char_int \l_@@_curr_char_int
\int_set_eq:NN \l_@@_case_changed_char_int \c_max_int
- \@@_query_get:
+ \tl_set:Nn \l_@@_curr_token_tl {#1}
+ \int_set:Nn \l_@@_curr_char_int {#2}
+ \int_set:Nn \l_@@_curr_catcode_int { "#3 }
+ \tl_build_put_right:Nx \l_@@_matched_analysis_tl
+ { \exp_not:o \l_@@_curr_analysis_tl }
+ \tl_set:Nn \l_@@_curr_analysis_tl { { {#1} {#2} #3 } }
\use:x
{
- \int_set_eq:NN \l_@@_max_active_int \l_@@_min_active_int
+ \int_set_eq:NN \l_@@_max_thread_int \l_@@_min_thread_int
\int_step_function:nnN
- { \l_@@_min_active_int }
- { \l_@@_max_active_int - 1 }
+ { \l_@@_min_thread_int }
+ { \l_@@_max_thread_int - 1 }
\@@_match_one_active:n
}
\prg_break_point:
\bool_set_false:N \l_@@_fresh_thread_bool
- \if_int_compare:w \l_@@_max_active_int > \l_@@_min_active_int
- \if_int_compare:w \l_@@_curr_pos_int < \l_@@_max_pos_int
- \exp_after:wN \exp_after:wN \exp_after:wN \@@_match_loop:
+ \if_int_compare:w \l_@@_max_thread_int > \l_@@_min_thread_int
+ \if_int_compare:w -2 < \l_@@_curr_char_int
+ \exp_after:wN \exp_after:wN \exp_after:wN \use_none:n
\fi:
\fi:
+ \l_@@_every_match_tl
}
\cs_new:Npn \@@_match_one_active:n #1
{
- \@@_use_state_and_submatches:nn
- { \__kernel_intarray_item:Nn \g_@@_thread_state_intarray {#1} }
- { \@@_toks_use:w #1 }
+ \@@_use_state_and_submatches:w
+ \__kernel_intarray_range_to_clist:Nnn
+ \g_@@_thread_info_intarray
+ { 1 + #1 * (\l_@@_capturing_group_int * 2 + 1) }
+ { (1 + #1) * (\l_@@_capturing_group_int * 2 + 1) }
+ ;
}
% \end{macrocode}
% \end{macro}
% \end{macro}
%
-% \begin{macro}{\@@_query_set:nnn}
-% The arguments are: tokens that \texttt{o} and \texttt{x} expand to
-% one token of the query, the catcode, and the character code. Store
-% those, and the current brace balance (used later to check for
-% overall brace balance) in a \tn{toks} register and some arrays,
-% then update the \texttt{balance}.
-% \begin{macrocode}
-\cs_new_protected:Npn \@@_query_set:nnn #1#2#3
- {
- \__kernel_intarray_gset:Nnn \g_@@_charcode_intarray
- { \l_@@_curr_pos_int } {#3}
- \__kernel_intarray_gset:Nnn \g_@@_catcode_intarray
- { \l_@@_curr_pos_int } {#2}
- \__kernel_intarray_gset:Nnn \g_@@_balance_intarray
- { \l_@@_curr_pos_int } { \l_@@_balance_int }
- \@@_toks_set:Nn \l_@@_curr_pos_int {#1}
- \int_incr:N \l_@@_curr_pos_int
- \if_case:w #2 \exp_stop_f:
- \or: \int_incr:N \l_@@_balance_int
- \or: \int_decr:N \l_@@_balance_int
- \fi:
- }
-% \end{macrocode}
-% \end{macro}
-%
-% \begin{macro}{\@@_query_get:}
-% Extract the current character and category codes at the current
-% position from the appropriate arrays.
-% \begin{macrocode}
-\cs_new_protected:Npn \@@_query_get:
- {
- \l_@@_curr_char_int
- = \__kernel_intarray_item:Nn \g_@@_charcode_intarray
- { \l_@@_curr_pos_int } \scan_stop:
- \l_@@_curr_catcode_int
- = \__kernel_intarray_item:Nn \g_@@_catcode_intarray
- { \l_@@_curr_pos_int } \scan_stop:
- }
-% \end{macrocode}
-% \end{macro}
-%
% \subsubsection{Using states of the \textsc{nfa}}
%
% \begin{macro}{\@@_use_state:}
@@ -4879,20 +4855,20 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}{\@@_use_state_and_submatches:nn}
+% \begin{macro}{\@@_use_state_and_submatches:w}
% This function is called as one item in the array of active threads
% after that array has been unpacked for a new step. Update the
-% \texttt{current_state} and \texttt{current_submatches} and use the
+% \texttt{curr_state} and \texttt{curr_submatches} and use the
% state if it has not yet been encountered at this step.
% \begin{macrocode}
-\cs_new_protected:Npn \@@_use_state_and_submatches:nn #1 #2
+\cs_new_protected:Npn \@@_use_state_and_submatches:w #1 , #2 ;
{
\int_set:Nn \l_@@_curr_state_int {#1}
\if_int_compare:w
\__kernel_intarray_item:Nn \g_@@_state_active_intarray
{ \l_@@_curr_state_int }
< \l_@@_step_int
- \tl_set:Nn \l_@@_curr_submatches_prop {#2}
+ \tl_set:Nn \l_@@_curr_submatches_tl { #2 , }
\exp_after:wN \@@_use_state:
\fi:
\scan_stop:
@@ -4902,20 +4878,20 @@
%
% \subsubsection{Actions when matching}
%
-% \begin{macro}{\@@_action_start_wildcard:}
+% \begin{macro}{\@@_action_start_wildcard:N}
% For an unanchored match, state $0$ has a free transition to the next
% and a costly one to itself, to repeat at the next position. To catch
% repeated identical empty matches, we need to know if a successful
% thread corresponds to an empty match. The instruction resetting
% \cs{l_@@_fresh_thread_bool} may be skipped by a successful
-% thread, hence we had to add it to \cs{@@_match_loop:} too.
+% thread, hence we had to add it to \cs{@@_match_one_token:nnN} too.
% \begin{macrocode}
-\cs_new_protected:Npn \@@_action_start_wildcard:
+\cs_new_protected:Npn \@@_action_start_wildcard:N #1
{
\bool_set_true:N \l_@@_fresh_thread_bool
\@@_action_free:n {1}
\bool_set_false:N \l_@@_fresh_thread_bool
- \@@_action_cost:n {0}
+ \bool_if:NT #1 { \@@_action_cost:n {0} }
}
% \end{macrocode}
% \end{macro}
@@ -4954,8 +4930,8 @@
}
\int_set:Nn \l_@@_curr_state_int
{ \int_use:N \l_@@_curr_state_int }
- \tl_set:Nn \exp_not:N \l_@@_curr_submatches_prop
- { \exp_not:o \l_@@_curr_submatches_prop }
+ \tl_set:Nn \exp_not:N \l_@@_curr_submatches_tl
+ { \exp_not:o \l_@@_curr_submatches_tl }
}
}
% \end{macrocode}
@@ -4978,21 +4954,26 @@
%
% \begin{macro}{\@@_store_state:n}
% \begin{macro}{\@@_store_submatches:}
-% Put the given state in \cs{g_@@_thread_state_intarray}, and increment
-% the length of the array. Also store the current submatch in the
-% appropriate \tn{toks}.
+% Put the given state and current submatch information in
+% \cs{g_@@_thread_info_intarray}, and increment the length of the
+% array.
% \begin{macrocode}
\cs_new_protected:Npn \@@_store_state:n #1
{
- \@@_store_submatches:
- \__kernel_intarray_gset:Nnn \g_@@_thread_state_intarray
- { \l_@@_max_active_int } {#1}
- \int_incr:N \l_@@_max_active_int
+ \exp_args:No \@@_store_submatches:nn
+ \l_@@_curr_submatches_tl {#1}
+ \int_incr:N \l_@@_max_thread_int
}
-\cs_new_protected:Npn \@@_store_submatches:
+\cs_new_protected:Npn \@@_store_submatches:nn #1#2
{
- \@@_toks_set:No \l_@@_max_active_int
- { \l_@@_curr_submatches_prop }
+ \__kernel_intarray_gset_range_from_clist:Nnn
+ \g_@@_thread_info_intarray
+ {
+ \@@_int_eval:w
+ 1 + \l_@@_max_thread_int *
+ (\l_@@_capturing_group_int * 2 + 1)
+ }
+ { #2 , #1 }
}
% \end{macrocode}
% \end{macro}
@@ -5006,21 +4987,37 @@
% \begin{macrocode}
\cs_new_protected:Npn \@@_disable_submatches:
{
- \cs_set_protected:Npn \@@_store_submatches: { }
- \cs_set_protected:Npn \@@_action_submatch:n ##1 { }
+ \cs_set_protected:Npn \@@_store_submatches:n ##1 { }
+ \cs_set_protected:Npn \@@_action_submatch:nN ##1##2 { }
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}{\@@_action_submatch:n}
+% \begin{macro}{\@@_action_submatch:nN, \@@_action_submatch_aux:w, \@@_action_submatch_auxii:w, \@@_action_submatch_auxiii:w, \@@_action_submatch_auxiv:w}
% Update the current submatches with the information from the current
% position. Maybe a bottleneck.
% \begin{macrocode}
-\cs_new_protected:Npn \@@_action_submatch:n #1
+\cs_new_protected:Npn \@@_action_submatch:nN #1#2
+ {
+ \exp_after:wN \@@_action_submatch_aux:w
+ \l_@@_curr_submatches_tl ; {#1} #2
+ }
+\cs_new_protected:Npn \@@_action_submatch_aux:w #1 ; #2#3
{
- \prop_put:Nno \l_@@_curr_submatches_prop {#1}
- { \int_use:N \l_@@_curr_pos_int }
+ \tl_set:Nx \l_@@_curr_submatches_tl
+ {
+ \prg_replicate:nn
+ { #2 \if_meaning:w > #3 + \l_@@_capturing_group_int \fi: }
+ { \@@_action_submatch_auxii:w }
+ \@@_action_submatch_auxiii:w
+ #1
+ }
}
+\cs_new:Npn \@@_action_submatch_auxii:w
+ #1 \@@_action_submatch_auxiii:w #2 ,
+ { #2 , #1 \@@_action_submatch_auxiii:w }
+\cs_new:Npn \@@_action_submatch_auxiii:w #1 ,
+ { \int_use:N \l_@@_curr_pos_int , }
% \end{macrocode}
% \end{macro}
%
@@ -5042,8 +5039,10 @@
\bool_set_eq:NN \l_@@_empty_success_bool
\l_@@_fresh_thread_bool
\int_set_eq:NN \l_@@_success_pos_int \l_@@_curr_pos_int
- \prop_set_eq:NN \l_@@_success_submatches_prop
- \l_@@_curr_submatches_prop
+ \int_set_eq:NN \l_@@_last_char_success_int \l_@@_last_char_int
+ \tl_build_clear:N \l_@@_matched_analysis_tl
+ \tl_set_eq:NN \l_@@_success_submatches_tl
+ \l_@@_curr_submatches_tl
\prg_break:
}
}
@@ -5134,6 +5133,14 @@
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}{\@@_replacement_exp_not:V}
+% This is used for the implementation of~|\u|, and it gets redefined
+% for \cs{peek_regex_replace_once:nnTF}.
+% \begin{macrocode}
+\cs_new_eq:NN \@@_replacement_exp_not:V \exp_not:V
+% \end{macrocode}
+% \end{macro}
+%
% \subsubsection{Query and brace balance}
%
% \begin{macro}[rEXP]{\@@_query_range:nn}
@@ -5189,42 +5196,25 @@
% range is the difference between the brace balances at the
% \meta{max~pos} and \meta{min~pos}. These two positions are found in
% the corresponding \enquote{submatch} arrays.
-%^^A todo: understand when these int_compare are needed
% \begin{macrocode}
\cs_new_protected:Npn \@@_submatch_balance:n #1
{
\int_eval:n
- {
- \int_compare:nNnTF
- {
- \__kernel_intarray_item:Nn
- \g_@@_submatch_end_intarray {#1}
- }
- = 0
- { 0 }
- {
- \__kernel_intarray_item:Nn \g_@@_balance_intarray
- {
- \__kernel_intarray_item:Nn
- \g_@@_submatch_end_intarray {#1}
- }
- }
- -
- \int_compare:nNnTF
- {
- \__kernel_intarray_item:Nn
- \g_@@_submatch_begin_intarray {#1}
- }
- = 0
- { 0 }
- {
- \__kernel_intarray_item:Nn \g_@@_balance_intarray
- {
- \__kernel_intarray_item:Nn
- \g_@@_submatch_begin_intarray {#1}
- }
- }
- }
+ {
+ \@@_intarray_item:NnF \g_@@_balance_intarray
+ {
+ \__kernel_intarray_item:Nn
+ \g_@@_submatch_end_intarray {#1}
+ }
+ { 0 }
+ -
+ \@@_intarray_item:NnF \g_@@_balance_intarray
+ {
+ \__kernel_intarray_item:Nn
+ \g_@@_submatch_begin_intarray {#1}
+ }
+ { 0 }
+ }
}
% \end{macrocode}
% \end{macro}
@@ -5305,6 +5295,14 @@
% \end{macro}
% \end{macro}
%
+% \begin{macro}{\@@_replacement_put:n}
+% This gets redefined for \cs{peek_regex_replace_once:nnTF}.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_replacement_put:n
+ { \tl_build_put_right:Nn \l_@@_build_tl }
+% \end{macrocode}
+% \end{macro}
+%
% \begin{macro}{\@@_replacement_normal:n}
% Most characters are simply sent to the output by
% \cs{tl_build_put_right:Nn}, unless a particular category code has been
@@ -5314,11 +5312,13 @@
% sequence is non-empty there: it contains an empty entry
% corresponding to the initial value of
% \cs{l_@@_replacement_category_tl}.
+% The argument |#1| can be a space, otherwise it is a single
+% character.
% \begin{macrocode}
\cs_new_protected:Npn \@@_replacement_normal:n #1
{
\tl_if_empty:NTF \l_@@_replacement_category_tl
- { \tl_build_put_right:Nn \l_@@_build_tl {#1} }
+ { \@@_replacement_put:n {#1} }
{ % (
\token_if_eq_charcode:NNTF #1 )
{
@@ -5361,7 +5361,7 @@
%
% \subsubsection{Submatches}
%
-% \begin{macro}{\@@_replacement_put_submatch:n}
+% \begin{macro}{\@@_replacement_put_submatch:n, \@@_replacement_put_submatch_aux:n}
% Insert a submatch in the replacement text. This is dropped if the
% submatch number is larger than the number of capturing groups.
% Unless the submatch appears inside a |\c{...}| or |\u{...}|
@@ -5374,15 +5374,19 @@
\cs_new_protected:Npn \@@_replacement_put_submatch:n #1
{
\if_int_compare:w #1 < \l_@@_capturing_group_int
- \tl_build_put_right:Nn \l_@@_build_tl
- { \@@_query_submatch:n { \int_eval:n { #1 + ##1 } } }
- \if_int_compare:w \l_@@_replacement_csnames_int = 0 \exp_stop_f:
- \tl_put_right:Nn \l_@@_balance_tl
- {
- + \@@_submatch_balance:n
- { \exp_not:N \int_eval:n { #1 + ##1 } }
- }
- \fi:
+ \@@_replacement_put_submatch_aux:n {#1}
+ \fi:
+ }
+\cs_new_protected:Npn \@@_replacement_put_submatch_aux:n #1
+ {
+ \tl_build_put_right:Nn \l_@@_build_tl
+ { \@@_query_submatch:n { \int_eval:n { #1 + ##1 } } }
+ \if_int_compare:w \l_@@_replacement_csnames_int = 0 \exp_stop_f:
+ \tl_put_right:Nn \l_@@_balance_tl
+ {
+ + \@@_submatch_balance:n
+ { \exp_not:N \int_eval:n { #1 + ##1 } }
+ }
\fi:
}
% \end{macrocode}
@@ -5488,7 +5492,7 @@
{
\@@_two_if_eq:NNNNTF
#1 #2 \@@_replacement_normal:n \c_left_brace_str
- { \@@_replacement_cu_aux:Nw \exp_not:V }
+ { \@@_replacement_cu_aux:Nw \@@_replacement_exp_not:V }
{ \@@_replacement_error:NNN u #1#2 }
}
% \end{macrocode}
@@ -5579,7 +5583,7 @@
\cs_new_protected:Npn \@@_replacement_char:nNN #1#2#3
{
\tex_lccode:D 0 = `#3 \scan_stop:
- \tex_lowercase:D { \tl_build_put_right:Nn \l_@@_build_tl {#1} }
+ \tex_lowercase:D { \@@_replacement_put:n {#1} }
}
% \end{macrocode}
% \end{macro}
@@ -5625,7 +5629,7 @@
\cs_new_protected:Npn \@@_replacement_c_C:w #1#2
{
\tl_build_put_right:Nn \l_@@_build_tl
- { \exp_not:N \exp_not:N \exp_not:c {#2} }
+ { \exp_not:N \@@_replacement_exp_not:N \exp_not:c {#2} }
}
% \end{macrocode}
% \end{macro}
@@ -5712,7 +5716,7 @@
\__kernel_msg_error:nn { kernel } { replacement-null-space }
\fi:
\tex_lccode:D `\ = `#2 \scan_stop:
- \tex_lowercase:D { \tl_build_put_right:Nn \l_@@_build_tl {~} }
+ \tex_lowercase:D { \@@_replacement_put:n {~} }
}
% \end{macrocode}
% \end{macro}
@@ -5940,6 +5944,14 @@
% \end{macrocode}
% \end{variable}
%
+% \begin{variable}{\g_@@_balance_intarray}
+% The first thing we do when matching is to store the balance of
+% begin-group/end-group characters into \cs{g_@@_balance_intarray}.
+% \begin{macrocode}
+\intarray_new:Nn \g_@@_balance_intarray { 65536 }
+% \end{macrocode}
+% \end{variable}
+%
% \begin{macro}{\@@_return:}
% This function triggers either \cs{prg_return_false:} or
% \cs{prg_return_true:} as appropriate to whether a match was found or
@@ -5956,6 +5968,36 @@
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}{\@@_query_set:n, \@@_query_set_aux:nN}
+% To easily extract subsets of the input once we found the positions
+% at which to cut, store the input tokens one by one into successive
+% \tn{toks} registers. Also store the brace balance (used to check
+% for overall brace balance) in an array.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_query_set:n #1
+ {
+ \int_zero:N \l_@@_balance_int
+ \int_zero:N \l_@@_curr_pos_int
+ \@@_query_set_aux:nN { } F
+ \tl_analysis_map_inline:nn {#1}
+ { \@@_query_set_aux:nN {##1} ##3 }
+ \@@_query_set_aux:nN { } F
+ \int_set_eq:NN \l_@@_max_pos_int \l_@@_curr_pos_int
+ }
+\cs_new_protected:Npn \@@_query_set_aux:nN #1#2
+ {
+ \int_incr:N \l_@@_curr_pos_int
+ \@@_toks_set:Nn \l_@@_curr_pos_int {#1}
+ \__kernel_intarray_gset:Nnn \g_@@_balance_intarray
+ { \l_@@_curr_pos_int } { \l_@@_balance_int }
+ \if_case:w "#2 \exp_stop_f:
+ \or: \int_incr:N \l_@@_balance_int
+ \or: \int_decr:N \l_@@_balance_int
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
% \subsubsection{Matching}
%
% \begin{macro}{\@@_if_match:nn}
@@ -6011,6 +6053,7 @@
#1
\@@_match:n {#2}
\@@_extract:
+ \@@_query_set:n {#2}
\@@_group_end_extract_seq:N #3
}
\cs_new_protected:Npn \@@_extract_all:nnN #1#2#3
@@ -6019,6 +6062,7 @@
\@@_multi_match:n { \@@_extract: }
#1
\@@_match:n {#2}
+ \@@_query_set:n {#2}
\@@_group_end_extract_seq:N #3
}
% \end{macrocode}
@@ -6058,7 +6102,7 @@
}
#1
\@@_match:n {#2}
-%<assert>\assert_int:n { \l_@@_curr_pos_int = \l_@@_max_pos_int }
+ \@@_query_set:n {#2}
\__kernel_intarray_gset:Nnn \g_@@_submatch_prev_intarray
{ \l_@@_submatch_int } { 0 }
\__kernel_intarray_gset:Nnn \g_@@_submatch_end_intarray
@@ -6143,18 +6187,15 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}
-% {\@@_extract:, \@@_extract_b:wn, \@@_extract_e:wn}
-% Our task here is to extract from the property list
-% \cs{l_@@_success_submatches_prop} the list of end-points of
-% submatches, and store them in appropriate array entries, from
-% \cs{l_@@_zeroth_submatch_int} upwards. We begin by emptying those
-% entries. Then for each \meta{key}--\meta{value} pair in
-% the property list update the appropriate entry. This
-% is somewhat a hack: the \meta{key} is a non-negative integer
-% followed by |<| or |>|, which we use in a comparison to $-1$. At the
-% end, store the information about the position at which the match
-% attempt started, in \cs{g_@@_submatch_prev_intarray}.
+% \begin{macro}{\@@_extract:}
+% Our task here is to store the list of end-points of submatches, and
+% store them in appropriate array entries, from
+% \cs{l_@@_zeroth_submatch_int} upwards. First, we store in
+% \cs{g_@@_submatch_prev_intarray} the position at which the match
+% attempt started. We extract the rest from the comma list
+% \cs{l_@@_success_submatches_tl}, which starts with entries to be
+% stored in \cs{g_@@_submatch_begin_intarray} and continues with
+% entries for \cs{g_@@_submatch_end_intarray}.
% \begin{macrocode}
\cs_new_protected:Npn \@@_extract:
{
@@ -6162,34 +6203,26 @@
\int_set_eq:NN \l_@@_zeroth_submatch_int \l_@@_submatch_int
\prg_replicate:nn \l_@@_capturing_group_int
{
- \__kernel_intarray_gset:Nnn \g_@@_submatch_begin_intarray
- { \l_@@_submatch_int } { 0 }
- \__kernel_intarray_gset:Nnn \g_@@_submatch_end_intarray
- { \l_@@_submatch_int } { 0 }
\__kernel_intarray_gset:Nnn \g_@@_submatch_prev_intarray
{ \l_@@_submatch_int } { 0 }
\int_incr:N \l_@@_submatch_int
}
- \prop_map_inline:Nn \l_@@_success_submatches_prop
+ \__kernel_intarray_gset:Nnn \g_@@_submatch_prev_intarray
+ { \l_@@_zeroth_submatch_int } { \l_@@_start_pos_int }
+ \int_zero:N \l_@@_internal_a_int
+ \clist_map_inline:Nn \l_@@_success_submatches_tl
{
- \if_int_compare:w ##1 - 1 \exp_stop_f:
- \exp_after:wN \@@_extract_e:wn \int_value:w
+ \if_int_compare:w \l_@@_internal_a_int < \l_@@_capturing_group_int
+ \__kernel_intarray_gset:Nnn \g_@@_submatch_begin_intarray
+ { \@@_int_eval:w \l_@@_zeroth_submatch_int + \l_@@_internal_a_int } {##1}
\else:
- \exp_after:wN \@@_extract_b:wn \int_value:w
+ \__kernel_intarray_gset:Nnn \g_@@_submatch_end_intarray
+ { \@@_int_eval:w \l_@@_zeroth_submatch_int + \l_@@_internal_a_int - \l_@@_capturing_group_int } {##1}
\fi:
- \@@_int_eval:w \l_@@_zeroth_submatch_int + ##1 {##2}
+ \int_incr:N \l_@@_internal_a_int
}
- \__kernel_intarray_gset:Nnn \g_@@_submatch_prev_intarray
- { \l_@@_zeroth_submatch_int } { \l_@@_start_pos_int }
\fi:
}
-\cs_new_protected:Npn \@@_extract_b:wn #1 < #2
- {
- \__kernel_intarray_gset:Nnn
- \g_@@_submatch_begin_intarray {#1} {#2}
- }
-\cs_new_protected:Npn \@@_extract_e:wn #1 > #2
- { \__kernel_intarray_gset:Nnn \g_@@_submatch_end_intarray {#1} {#2} }
% \end{macrocode}
% \end{macro}
%
@@ -6214,12 +6247,13 @@
\group_begin:
\@@_single_match:
#1
- \@@_replacement:n {#2}
- \exp_args:No \@@_match:n { #3 }
+ \exp_args:No \@@_match:n {#3}
\if_meaning:w \c_false_bool \g_@@_success_bool
\group_end:
\else:
\@@_extract:
+ \exp_args:No \@@_query_set:n {#3}
+ \@@_replacement:n {#2}
\int_set:Nn \l_@@_balance_int
{
\@@_replacement_balance_one_match:n
@@ -6259,8 +6293,9 @@
\group_begin:
\@@_multi_match:n { \@@_extract: }
#1
- \@@_replacement:n {#2}
\exp_args:No \@@_match:n {#3}
+ \exp_args:No \@@_query_set:n {#3}
+ \@@_replacement:n {#2}
\int_set:Nn \l_@@_balance_int
{
0
@@ -6320,7 +6355,371 @@
% \end{macrocode}
% \end{macro}
%
-% \subsubsection{Storing and showing compiled patterns}
+% \subsubsection{Peeking ahead}
+%
+% \begin{variable}{\l_@@_peek_true_tl, \l_@@_peek_false_tl}
+% True/false code arguments of \cs{peek_regex:nTF} or similar.
+% \begin{macrocode}
+\tl_new:N \l_@@_peek_true_tl
+\tl_new:N \l_@@_peek_false_tl
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_@@_replacement_tl}
+% When peeking in \cs{peek_regex_replace_once:nnTF} we need to store
+% the replacement text.
+% \begin{macrocode}
+\tl_new:N \l_@@_replacement_tl
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_@@_input_tl}
+% \begin{macro}{\@@_input_item:n}
+% Stores each token found as \cs{@@_input_item:n} \Arg{tokens}, where
+% the \meta{tokens} \texttt{o}-expand to the token found, as for
+% \cs{tl_analysis_map_inline:nn}.
+% \begin{macrocode}
+\tl_new:N \l_@@_input_tl
+\cs_new_eq:NN \@@_input_item:n ?
+% \end{macrocode}
+% \end{macro}
+% \end{variable}
+%
+% \begin{macro}[TF]
+% {\peek_regex:n, \peek_regex:N, \peek_regex_remove_once:n, \peek_regex_remove_once:N}
+% The |T| and |F| functions just call the corresponding |TF| function.
+% The four |TF| functions differ along two axes: whether to remove the
+% token or not, distinguished by using \cs{@@_peek_end:} or
+% \cs{@@_peek_remove_end:n} (the latter case needs an argument, as we
+% will see), and whether the regex has to be compiled or is already in
+% an |N|-type variable, distinguished by calling \cs{@@_build_aux:Nn}
+% or \cs{@@_build_aux:NN}. The first argument of these functions is
+% \cs{c_false_bool} to indicate that there should be no implicit
+% insertion of a wildcard at the start of the pattern: otherwise the
+% code would keep looking further into the input stream until matching
+% the regex.
+% \begin{macrocode}
+\cs_new_protected:Npn \peek_regex:nTF #1
+ {
+ \@@_peek:nnTF
+ { \@@_build_aux:Nn \c_false_bool {#1} }
+ { \@@_peek_end: }
+ }
+\cs_new_protected:Npn \peek_regex:nT #1#2
+ { \peek_regex:nTF {#1} {#2} { } }
+\cs_new_protected:Npn \peek_regex:nF #1 { \peek_regex:nTF {#1} { } }
+\cs_new_protected:Npn \peek_regex:NTF #1
+ {
+ \@@_peek:nnTF
+ { \@@_build_aux:NN \c_false_bool #1 }
+ { \@@_peek_end: }
+ }
+\cs_new_protected:Npn \peek_regex:NT #1#2
+ { \peek_regex:NTF #1 {#2} { } }
+\cs_new_protected:Npn \peek_regex:NF #1 { \peek_regex:NTF {#1} { } }
+\cs_new_protected:Npn \peek_regex_remove_once:nTF #1
+ {
+ \@@_peek:nnTF
+ { \@@_build_aux:Nn \c_false_bool {#1} }
+ { \@@_peek_remove_end:n {##1} }
+ }
+\cs_new_protected:Npn \peek_regex_remove_once:nT #1#2
+ { \peek_regex_remove_once:nTF {#1} {#2} { } }
+\cs_new_protected:Npn \peek_regex_remove_once:nF #1
+ { \peek_regex_remove_once:nTF {#1} { } }
+\cs_new_protected:Npn \peek_regex_remove_once:NTF #1
+ {
+ \@@_peek:nnTF
+ { \@@_build_aux:NN \c_false_bool #1 }
+ { \@@_peek_remove_end:n {##1} }
+ }
+\cs_new_protected:Npn \peek_regex_remove_once:NT #1#2
+ { \peek_regex_remove_once:NTF #1 {#2} { } }
+\cs_new_protected:Npn \peek_regex_remove_once:NF #1
+ { \peek_regex_remove_once:NTF #1 { } }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_peek:nnTF, \@@_peek_aux:nnTF}
+% Store the user's true/false codes (plus \cs{group_end:}) into two
+% token lists. Then build the automaton with |#1|, without submatch
+% tracking, and aiming for a single match. Then start matching by
+% setting up a few variables like for any regex matching like
+% \cs{regex_match:nnTF}, with the addition of \cs{l_@@_input_tl}
+% that keeps track of the tokens seen, to reinsert them at the
+% end. Instead of \cs{tl_analysis_map_inline:nn} on the input, we
+% call \cs{peek_analysis_map_inline:n} to go through tokens in the
+% input stream. Since \cs{@@_match_one_token:nnN} calls
+% \cs{@@_maplike_break:} we need to catch that and break the
+% \cs{peek_analysis_map_inline:n} loop instead.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek:nnTF #1
+ {
+ \@@_peek_aux:nnTF
+ {
+ \@@_disable_submatches:
+ #1
+ }
+ }
+\cs_new_protected:Npn \@@_peek_aux:nnTF #1#2#3#4
+ {
+ \group_begin:
+ \tl_set:Nn \l_@@_peek_true_tl { \group_end: #3 }
+ \tl_set:Nn \l_@@_peek_false_tl { \group_end: #4 }
+ \@@_single_match:
+ #1
+ \@@_match_init:
+ \tl_build_clear:N \l_@@_input_tl
+ \@@_match_once_init:
+ \peek_analysis_map_inline:n
+ {
+ \tl_build_put_right:Nn \l_@@_input_tl
+ { \@@_input_item:n {##1} }
+ \@@_match_one_token:nnN {##1} {##2} ##3
+ \use_none:nnn
+ \prg_break_point:Nn \@@_maplike_break:
+ { \peek_analysis_map_break:n {#2} }
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_peek_end:, \@@_peek_remove_end:n}
+% Once the regex matches (or permanently fails to match) we call
+% \cs{@@_peek_end:}, or \cs{@@_peek_remove_end:n} with argument the
+% last token seen (or rather tokens that \texttt{o}-expand and
+% \texttt{x}-expand to it). For \cs{peek_regex:nTF} we reinsert
+% tokens seen by calling \cs{@@_peek_reinsert:N} regardless of the
+% result of the match. For \cs{peek_regex_remove_once:nTF} we reinsert the
+% tokens seen only if the match failed; otherwise we just reinsert the
+% tokens~|#1|, with one expansion.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_end:
+ {
+ \bool_if:NTF \g_@@_success_bool
+ { \@@_peek_reinsert:N \l_@@_peek_true_tl }
+ { \@@_peek_reinsert:N \l_@@_peek_false_tl }
+ }
+\cs_new_protected:Npn \@@_peek_remove_end:n #1
+ {
+ \bool_if:NTF \g_@@_success_bool
+ { \exp_after:wN \l_@@_peek_true_tl #1 }
+ { \@@_peek_reinsert:N \l_@@_peek_false_tl }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_peek_reinsert:N, \@@_reinsert_item:n}
+% Insert the true/false code |#1|, followed by the tokens found, which
+% were stored in \cs{l_@@_input_tl}. For this, loop through that
+% token list using \cs{@@_reinsert_item:n}, which expands |#1| once to
+% get a single token, and jumps over it to expand what follows, with
+% suitable \cs{exp:w} and \cs{exp_end:}. We cannot just use
+% \cs{use:e} on the whole token list because the result may be
+% unbalanced, which would stop the primitive prematurely, or let it
+% continue beyond where we would like.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_reinsert:N #1
+ {
+ \tl_build_end:N \l_@@_input_tl
+ \cs_set_eq:NN \@@_input_item:n \@@_reinsert_item:n
+ \exp_after:wN #1 \exp:w \l_@@_input_tl \exp_end:
+ }
+\cs_new_protected:Npn \@@_reinsert_item:n #1
+ {
+ \exp_after:wN \exp_after:wN
+ \exp_after:wN \exp_end:
+ \exp_after:wN \exp_after:wN
+ #1
+ \exp:w
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[noTF]
+% {\peek_regex_replace_once:nn, \peek_regex_replace_once:Nn}
+% Similar to \cs{peek_regex:nTF} above.
+% \begin{macrocode}
+\cs_new_protected:Npn \peek_regex_replace_once:nnTF #1
+ { \@@_peek_replace:nnTF { \@@_build_aux:Nn \c_false_bool {#1} } }
+\cs_new_protected:Npn \peek_regex_replace_once:nnT #1#2#3
+ { \peek_regex_replace_once:nnTF {#1} {#2} {#3} { } }
+\cs_new_protected:Npn \peek_regex_replace_once:nnF #1#2
+ { \peek_regex_replace_once:nnTF {#1} {#2} { } }
+\cs_new_protected:Npn \peek_regex_replace_once:nn #1#2
+ { \peek_regex_replace_once:nnTF {#1} {#2} { } { } }
+\cs_new_protected:Npn \peek_regex_replace_once:NnTF #1
+ { \@@_peek_replace:nnTF { \@@_build_aux:NN \c_false_bool #1 } }
+\cs_new_protected:Npn \peek_regex_replace_once:NnT #1#2#3
+ { \peek_regex_replace_once:NnTF #1 {#2} {#3} { } }
+\cs_new_protected:Npn \peek_regex_replace_once:NnF #1#2
+ { \peek_regex_replace_once:NnTF #1 {#2} { } }
+\cs_new_protected:Npn \peek_regex_replace_once:Nn #1#2
+ { \peek_regex_replace_once:NnTF #1 {#2} { } { } }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_peek_replace:nnTF}
+% Same as \cs{@@_peek:nnTF} (used for \cs{peek_regex:nTF} above), but
+% without disabling submatches, and with a different end. The
+% replacement text |#2| is stored, to be analyzed later.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_replace:nnTF #1#2
+ {
+ \tl_set:Nn \l_@@_replacement_tl {#2}
+ \@@_peek_aux:nnTF {#1} { \@@_peek_replace_end: }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_peek_replace_end:}
+% If the match failed \cs{@@_peek_reinsert:N} reinserts the tokens
+% found. Otherwise, finish storing the submatch information using
+% \cs{@@_extract:}, and store the input into \tn{toks}. Redefine a
+% few auxiliaries to change slightly their expansion behaviour as
+% explained below. Analyse the replacement text with
+% \cs{@@_replacement:n}, which as usual defines
+% \cs{@@_replacement_do_one_match:n} to insert the tokens from the
+% start of the match attempt to the beginning of the match, followed
+% by the replacement text. The \cs{use:x} expands for instance the
+% trailing \cs{@@_query_range:nn} down to a sequence of
+% \cs{@@_reinsert_item:n} \Arg{tokens} where \meta{tokens}
+% \texttt{o}-expand to a single token that we want to insert. After
+% \texttt{x}-expansion, \cs{use:x} does \cs{use:n}, so we have
+% \cs{exp_after:wN} \cs{l_@@_peek_true_tl} \cs{exp:w} \ldots{}
+% \cs{exp_end:}. This is set up such as to obtain
+% \cs{l_@@_peek_true_tl} followed by the replaced tokens (possibly
+% unbalanced) in the input stream.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_replace_end:
+ {
+ \bool_if:NTF \g_@@_success_bool
+ {
+ \@@_extract:
+ \@@_query_set_from_input_tl:
+ \cs_set_eq:NN \@@_replacement_put:n \@@_peek_replacement_put:n
+ \cs_set_eq:NN \@@_replacement_put_submatch_aux:n
+ \@@_peek_replacement_put_submatch_aux:n
+ \cs_set_eq:NN \@@_input_item:n \@@_reinsert_item:n
+ \cs_set_eq:NN \@@_replacement_exp_not:N \@@_peek_replacement_token:n
+ \cs_set_eq:NN \@@_replacement_exp_not:V \@@_peek_replacement_var:N
+ \exp_args:No \@@_replacement:n { \l_@@_replacement_tl }
+ \use:x
+ {
+ \exp_not:n { \exp_after:wN \l_@@_peek_true_tl \exp:w }
+ \@@_replacement_do_one_match:n
+ { \l_@@_zeroth_submatch_int }
+ \@@_query_range:nn
+ {
+ \__kernel_intarray_item:Nn \g_@@_submatch_end_intarray
+ { \l_@@_zeroth_submatch_int }
+ }
+ { \l_@@_max_pos_int }
+ \exp_end:
+ }
+ }
+ { \@@_peek_reinsert:N \l_@@_peek_false_tl }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_query_set_from_input_tl:, \@@_query_set_item:n}
+% The input was stored into \cs{l_@@_input_tl} as successive items
+% \cs{@@_input_item:n} \Arg{tokens}. Store that in successive
+% \tn{toks}. It's not clear whether the empty entries before and
+% after are both useful.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_query_set_from_input_tl:
+ {
+ \tl_build_end:N \l_@@_input_tl
+ \int_zero:N \l_@@_curr_pos_int
+ \cs_set_eq:NN \@@_input_item:n \@@_query_set_item:n
+ \@@_query_set_item:n { }
+ \l_@@_input_tl
+ \@@_query_set_item:n { }
+ \int_set_eq:NN \l_@@_max_pos_int \l_@@_curr_pos_int
+ }
+\cs_new_protected:Npn \@@_query_set_item:n #1
+ {
+ \int_incr:N \l_@@_curr_pos_int
+ \@@_toks_set:Nn \l_@@_curr_pos_int { \@@_input_item:n {#1} }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_peek_replacement_put:n}
+% While building the replacement function
+% \cs{@@_replacement_do_one_match:n}, we often want to put simple
+% material, given as |#1|, whose \texttt{x}-expansion
+% \texttt{o}-expands to a single token. Normally we can just add the
+% token to \cs{l_@@_build_tl}, but for
+% \cs{peek_regex_replace_once:nnTF} we eventually want to do some
+% strange expansion that is basically using \cs{exp_after:wN} to jump
+% through numerous tokens (we cannot use \texttt{x}-expansion like for
+% \cs{regex_replace_once:nnNTF} because it is ok for the result to be
+% unbalanced since we insert it in the input stream rather than
+% storing it. When within a csname we don't do any such shenanigan
+% because \cs{cs:w} \ldots{} \cs{cs_end:} does all the expansion we
+% need.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_replacement_put:n #1
+ {
+ \if_case:w \l_@@_replacement_csnames_int
+ \tl_build_put_right:Nn \l_@@_build_tl
+ { \exp_not:N \@@_reinsert_item:n {#1} }
+ \else:
+ \tl_build_put_right:Nn \l_@@_build_tl {#1}
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_peek_replacement_token:n}
+% When hit with \cs{exp:w}, \cs{@@_peek_replacement_token:n}
+% \Arg{token} stops \cs{exp_end:} and does \cs{exp_after:wN}
+% \meta{token} \cs{exp:w} to continue expansion after it.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_replacement_token:n #1
+ { \exp_after:wN \exp_end: \exp_after:wN #1 \exp:w }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_peek_replacement_put_submatch_aux:n}
+% While analyzing the replacement we also have to insert submatches
+% found in the query. Since query items \cs{@@_input_item:n}
+% \Arg{tokens} expand correctly only when surrounded by \cs{exp:w}
+% \ldots{} \cs{exp_end:}, and since these expansion controls are not
+% there within csnames (because \cs{cs:w} \ldots{} \cs{cs_end:} make
+% them unnecessary in most cases), we have to put \cs{exp:w} and
+% \cs{exp_end:} by hand here.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_replacement_put_submatch_aux:n #1
+ {
+ \if_case:w \l_@@_replacement_csnames_int
+ \tl_build_put_right:Nn \l_@@_build_tl
+ { \@@_query_submatch:n { \int_eval:n { #1 + ##1 } } }
+ \else:
+ \tl_build_put_right:Nn \l_@@_build_tl
+ { \exp:w \@@_query_submatch:n { \int_eval:n { #1 + ##1 } } \exp_end: }
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_peek_replacement_var:N}
+% This is used for |\u| outside csnames. It makes sure to continue
+% expansion with \cs{exp:w} before expanding the variable~|#1| and
+% stopping the \cs{exp:w} that precedes.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_replacement_var:N #1
+ {
+ \exp_after:wN \exp_last_unbraced:NV
+ \exp_after:wN \exp_end:
+ \exp_after:wN #1
+ \exp:w
+ }
+% \end{macrocode}
+% \end{macro}
%
% \subsection{Messages}
%
@@ -6726,8 +7125,6 @@
%^^A NOT IMPLEMENTED
%^^A \p{xx} a character with the xx property
%^^A \P{xx} a character without the xx property
-%^^A [[:xxx:]] positive POSIX named set
-%^^A [[:^xxx:]] negative POSIX named set
%^^A (?=...) positive look ahead
%^^A (?!...) negative look ahead
%^^A (?<=...) positive look behind