diff options
Diffstat (limited to 'Master/texmf-dist/source/latex/l3kernel/l3regex.dtx')
-rw-r--r-- | Master/texmf-dist/source/latex/l3kernel/l3regex.dtx | 1219 |
1 files changed, 808 insertions, 411 deletions
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3regex.dtx b/Master/texmf-dist/source/latex/l3kernel/l3regex.dtx index 2086b5c3ec0..5e64d739805 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3regex.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3regex.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % @@ -713,7 +713,7 @@ % |{n,}| quantifier? (I think not.) % \item Quantifiers for |\u| and assertions. % \item When matching, keep track of an explicit stack of -% \texttt{current_state} and \texttt{current_submatches}. +% \texttt{curr_state} and \texttt{curr_submatches}. % \item If possible, when a state is reused by the same thread, kill % other subthreads. % \item Use an array rather than \cs[no-index]{l__regex_balance_tl} @@ -845,12 +845,14 @@ % names). % \begin{itemize} % \item \emph{Group}: index of the capturing group, $-1$ for -% non-capturing groups. +% non-capturing groups. ^^A start/end index? % \item \emph{Position}: each token in the query is labelled by an % integer \meta{position}, with $\texttt{min_pos} - 1 \leq % \meta{position} \leq \texttt{max_pos}$. The lowest and highest -% positions correspond to imaginary begin and end markers (with -% inaccessible category code and character code). +% positions $\texttt{min_pos} - 1$ and $\texttt{max_pos}$ +% correspond to imaginary begin and end markers (with +% non-existent category code and character code). +% $\texttt{max_pos}$ is only set quite late in the processing. % \item \emph{Query}: the token list to which we apply the regular % expression. % \item \emph{State}: each state of the \textsc{nfa} is labelled by an @@ -865,8 +867,8 @@ % unique id for all the steps of the matching algorithm. % \end{itemize} % -% We use \pkg{l3intarray} to manipulate arrays of integers (stored into -% some dimension registers in scaled points). We also abuse \TeX{}'s +% We use \pkg{l3intarray} to manipulate arrays of integers. +% We also abuse \TeX{}'s % \tn{toks} registers, by accessing them directly by number rather than % tying them to control sequence using the \tn{newtoks} allocation % functions. Specifically, these arrays and \tn{toks} are used as @@ -876,21 +878,14 @@ % \begin{itemize} % \item \cs{g_@@_state_active_intarray} holds the last \meta{step} in % which each \meta{state} was active. -% \item \cs{g_@@_thread_state_intarray} maps each \meta{thread} (with -% $\texttt{min_active} \leq \meta{thread} < \texttt{max_active}$) to -% the \meta{state} in which the \meta{thread} currently is. The +% \item \cs{g_@@_thread_info_intarray} consists of blocks for each +% \meta{thread} (with $\texttt{min_thread} \leq \meta{thread} < +% \texttt{max_thread}$). Each block has +% $1+2\cs{l_@@_capturing_group_int}$ entries: the \meta{state} in +% which the \meta{thread} currently is, followed by the beginnings +% of all submatches, and then the ends of all submatches. The % \meta{threads} are ordered starting from the best to the least % preferred. -% \item \tn{toks}\meta{thread} holds the submatch information for the -% \meta{thread}, as the contents of a property list. -% \item \cs{g_@@_charcode_intarray} and \cs{g_@@_catcode_intarray} hold the -% character codes and category codes of tokens at each -% \meta{position} in the query. -% \item \cs{g_@@_balance_intarray} holds the balance of begin-group and -% end-group character tokens which appear before that point in the -% token list. -% \item \tn{toks}\meta{position} holds \meta{tokens} which \texttt{o}- -% and \texttt{x}-expand to the \meta{position}-th token in the query. % \item \cs{g_@@_submatch_prev_intarray}, \cs{g_@@_submatch_begin_intarray} % and \cs{g_@@_submatch_end_intarray} hold, for each submatch (as would % be extracted by \cs{regex_extract_all:nnN}), the place where the @@ -901,6 +896,14 @@ % block corresponding to one match with all its submatches stored in % consecutive entries. % \end{itemize} +% When actually building the result, +% \begin{itemize} +% \item \tn{toks}\meta{position} holds \meta{tokens} which \texttt{o}- +% and \texttt{x}-expand to the \meta{position}-th token in the query. +% \item \cs{g_@@_balance_intarray} holds the balance of begin-group and +% end-group character tokens which appear before that point in the +% token list. +% \end{itemize} % % The code is structured as follows. Variables are introduced in the % relevant section. First we present some generic helper functions. Then @@ -944,7 +947,7 @@ { \@@_toks_set:Nn #1 { } } \cs_new_eq:NN \@@_toks_set:Nn \tex_toks:D \cs_new_protected:Npn \@@_toks_set:No #1 - { \@@_toks_set:Nn #1 \exp_after:wN } + { \tex_toks:D #1 \exp_after:wN } % \end{macrocode} % \end{macro} % @@ -975,13 +978,13 @@ % \begin{macrocode} \cs_new_protected:Npn \@@_toks_put_left:Nx #1#2 { - \cs_set:Npx \@@_tmp:w { #2 } + \cs_set_nopar:Npx \@@_tmp:w { #2 } \tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN { \exp_after:wN \@@_tmp:w \tex_the:D \tex_toks:D #1 } } \cs_new_protected:Npn \@@_toks_put_right:Nx #1#2 { - \cs_set:Npx \@@_tmp:w {#2} + \cs_set_nopar:Npx \@@_tmp:w {#2} \tex_toks:D #1 \exp_after:wN { \tex_the:D \tex_toks:D \exp_after:wN #1 \@@_tmp:w } } @@ -1000,11 +1003,38 @@ \cs_new:Npn \@@_curr_cs_to_str: { \exp_after:wN \exp_after:wN \exp_after:wN \cs_to_str:N - \tex_the:D \tex_toks:D \l_@@_curr_pos_int + \l_@@_curr_token_tl } % \end{macrocode} % \end{macro} % +% \begin{macro}{\@@_intarray_item:NnF, \@@_intarray_item_aux:nNF} +% Item of intarray, with a default value. +% \begin{macrocode} +\cs_new:Npn \@@_intarray_item:NnF #1#2 + { \exp_args:Nf \@@_intarray_item_aux:nNF { \int_eval:n {#2} } #1 } +\cs_new:Npn \@@_intarray_item_aux:nNF #1#2 + { + \if_int_compare:w #1 > \c_zero_int + \exp_after:wN \use_i:nn + \else: + \exp_after:wN \use_ii:nn + \fi: + { \__kernel_intarray_item:Nn #2 {#1} } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_maplike_break:} +% Analogous to \cs{tl_map_break:}, this correctly exits +% \cs{tl_map_inline:nn} and similar constructions and jumps to the +% matching \cs{prg_break_point:Nn} \cs{@@_maplike_break:} |{| |}|. +% \begin{macrocode} +\cs_new:Npn \@@_maplike_break: + { \prg_map_break:Nn \@@_maplike_break: { } } +% \end{macrocode} +% \end{macro} +% % \subsubsection{Constants and variables} % % \begin{macro}{\@@_tmp:w} @@ -1056,19 +1086,6 @@ % \end{macrocode} % \end{variable} % -% \begin{variable}{\g_@@_charcode_intarray, \g_@@_catcode_intarray, \g_@@_balance_intarray} -% The first thing we do when matching is to go once through the query -% token list and store the information for each token into -% \cs{g_@@_charcode_intarray}, \cs{g_@@_catcode_intarray} and \tn{toks} -% registers. We also store the balance of begin-group/end-group -% characters into \cs{g_@@_balance_intarray}. -% \begin{macrocode} -\intarray_new:Nn \g_@@_charcode_intarray { 65536 } -\intarray_new:Nn \g_@@_catcode_intarray { 65536 } -\intarray_new:Nn \g_@@_balance_intarray { 65536 } -% \end{macrocode} -% \end{variable} -% % \begin{variable}{\l_@@_balance_int} % During this phase, \cs{l_@@_balance_int} counts the balance of % begin-group and end-group character tokens which appear before a @@ -1079,15 +1096,6 @@ % \end{macrocode} % \end{variable} % -% \begin{variable}{\l_@@_cs_name_tl} -% This variable is used in \cs{@@_item_cs:n} to store the csname of -% the currently-tested token when the regex contains a sub-regex for -% testing csnames. -% \begin{macrocode} -\tl_new:N \l_@@_cs_name_tl -% \end{macrocode} -% \end{variable} -% % \subsubsection{Testing characters} % % \begin{macro}{\c_@@_ascii_min_int, \c_@@_ascii_max_control_int, \c_@@_ascii_max_int} @@ -1201,7 +1209,7 @@ % \begin{macro} % {\@@_item_caseless_equal:n, \@@_item_caseless_range:nn} % For caseless matching, we perform the test both on the -% \texttt{current_char} and on the \texttt{case_changed_char}. Before +% \texttt{curr_char} and on the \texttt{case_changed_char}. Before % doing the second set of tests, we make sure that % \texttt{case_changed_char} has been computed. % \begin{macrocode} @@ -1308,7 +1316,8 @@ % % \begin{macro}{\@@_item_exact:nn, \@@_item_exact_cs:n} % This matches an exact \meta{category}-\meta{character code} pair, or -% an exact control sequence, more precisely one of several possible control sequences. +% an exact control sequence, more precisely one of several possible +% control sequences, separated by \cs{scan_stop:}. % \begin{macrocode} \cs_new_protected:Npn \@@_item_exact:nn #1#2 { @@ -1337,24 +1346,19 @@ % Match a control sequence (the argument is a compiled regex). % First test the catcode of the current token to be zero. % Then perform the matching test, and break if the csname -% indeed matches. The three \cs{exp_after:wN} expand the contents -% of the \tn{toks}\meta{current position} (of the form \cs{exp_not:n} -% \Arg{control sequence}) to \meta{control sequence}. -% We store the cs name before building states for the cs, as those -% states may overlap with toks registers storing the user's input. +% indeed matches. % \begin{macrocode} \cs_new_protected:Npn \@@_item_cs:n #1 { \int_compare:nNnT \l_@@_curr_catcode_int = 0 { \group_begin: - \__kernel_tl_set:Nx \l_@@_cs_name_tl { \@@_curr_cs_to_str: } \@@_single_match: \@@_disable_submatches: \@@_build_for_cs:n {#1} \bool_set_eq:NN \l_@@_saved_success_bool \g_@@_success_bool - \exp_args:NV \@@_match_cs:n \l_@@_cs_name_tl + \exp_args:Nx \@@_match_cs:n { \@@_curr_cs_to_str: } \if_meaning:w \c_true_bool \g_@@_success_bool \group_insert_after:N \@@_break_true:w \fi: @@ -1822,7 +1826,7 @@ % \item \cs{@@_command_K:} % \item \cs{@@_assertion:Nn} \meta{boolean} \Arg{assertion test}, % where the \meta{assertion test} is \cs{@@_b_test:} or -% |{|\cs{@@_anchor:N} \meta{integer}|}| +% \cs{@@_Z_test:} or \cs{@@_A_test:} or \cs{@@_G_test:} % \end{itemize} % Tests can be the following: % \begin{itemize} @@ -2663,73 +2667,56 @@ % % \subsubsection{Anchoring and simple assertions} % -% \begin{macro}{\@@_compile_anchor:NF} +% \begin{macro}{\@@_compile_anchor_letter:NNN} +% \begin{macro}{\@@_compile_/A:, \@@_compile_/G:, \@@_compile_/Z:, \@@_compile_/z:, \@@_compile_/b:, \@@_compile_/B:} % \begin{macro}+\@@_compile_^:+ -% \begin{macro}{\@@_compile_/A:, \@@_compile_/G:} % \begin{macro}+\@@_compile_$:+ -% \begin{macro}{\@@_compile_/Z:, \@@_compile_/z:} -% In modes where assertions are allowed, anchor to the start of the -% query, the start of the match, or the end of the query, depending on -% the integer |#1|. In other modes, |#2| treats the character as raw, -% with an error for escaped letters (|$| is valid in a class, but |\A| -% is definitely a mistake on the user's part). +% In modes where assertions are forbidden, anchors such as |\A| +% produce an error (|\A|~is invalid in classes); otherwise they add an +% \cs{@@_assertion:Nn} test as appropriate (the only negative +% assertion is~|\B|). The test functions are defined later. The +% implementation for +% |$| and |^| is only different from |\A| etc because these are valid +% in a class. % \begin{macrocode} -\cs_new_protected:Npn \@@_compile_anchor:NF #1#2 +\cs_new_protected:Npn \@@_compile_anchor_letter:NNN #1#2#3 { - \@@_if_in_class_or_catcode:TF {#2} + \@@_if_in_class_or_catcode:TF { \@@_compile_raw_error:N #1 } { \tl_build_put_right:Nn \l_@@_build_tl - { \@@_assertion:Nn \c_true_bool { \@@_anchor:N #1 } } + { \@@_assertion:Nn #2 {#3} } } } -\cs_set_protected:Npn \@@_tmp:w #1#2 - { - \cs_new_protected:cpn { @@_compile_/#1: } - { \@@_compile_anchor:NF #2 { \@@_compile_raw_error:N #1 } } - } -\@@_tmp:w A \l_@@_min_pos_int -\@@_tmp:w G \l_@@_start_pos_int -\@@_tmp:w Z \l_@@_max_pos_int -\@@_tmp:w z \l_@@_max_pos_int +\cs_new_protected:cpn { @@_compile_/A: } + { \@@_compile_anchor_letter:NNN A \c_true_bool \@@_A_test: } +\cs_new_protected:cpn { @@_compile_/G: } + { \@@_compile_anchor_letter:NNN G \c_true_bool \@@_G_test: } +\cs_new_protected:cpn { @@_compile_/Z: } + { \@@_compile_anchor_letter:NNN Z \c_true_bool \@@_Z_test: } +\cs_new_protected:cpn { @@_compile_/z: } + { \@@_compile_anchor_letter:NNN z \c_true_bool \@@_Z_test: } +\cs_new_protected:cpn { @@_compile_/b: } + { \@@_compile_anchor_letter:NNN b \c_true_bool \@@_b_test: } +\cs_new_protected:cpn { @@_compile_/B: } + { \@@_compile_anchor_letter:NNN B \c_false_bool \@@_b_test: } \cs_set_protected:Npn \@@_tmp:w #1#2 { \cs_new_protected:cpn { @@_compile_#1: } - { \@@_compile_anchor:NF #2 { \@@_compile_raw:N #1 } } + { + \@@_if_in_class_or_catcode:TF { \@@_compile_raw:N #1 } + { + \tl_build_put_right:Nn \l_@@_build_tl + { \@@_assertion:Nn \c_true_bool {#2} } + } + } } -\exp_args:Nx \@@_tmp:w { \iow_char:N \^ } \l_@@_min_pos_int -\exp_args:Nx \@@_tmp:w { \iow_char:N \$ } \l_@@_max_pos_int +\exp_args:Nx \@@_tmp:w { \iow_char:N \^ } { \@@_A_test: } +\exp_args:Nx \@@_tmp:w { \iow_char:N \$ } { \@@_Z_test: } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} -% \end{macro} -% -% \begin{macro}{\@@_compile_/b:, \@@_compile_/B:} -% Contrarily to |^| and |$|, which could be implemented without really -% knowing what precedes in the token list, this requires more -% information, namely, the knowledge of the last character code. -% \begin{macrocode} -\cs_new_protected:cpn { @@_compile_/b: } - { - \@@_if_in_class_or_catcode:TF - { \@@_compile_raw_error:N b } - { - \tl_build_put_right:Nn \l_@@_build_tl - { \@@_assertion:Nn \c_true_bool { \@@_b_test: } } - } - } -\cs_new_protected:cpn { @@_compile_/B: } - { - \@@_if_in_class_or_catcode:TF - { \@@_compile_raw_error:N B } - { - \tl_build_put_right:Nn \l_@@_build_tl - { \@@_assertion:Nn \c_false_bool { \@@_b_test: } } - } - } -% \end{macrocode} -% \end{macro} % % \subsubsection{Character classes} % @@ -3512,7 +3499,9 @@ { \bool_if:NF ##1 { negative~ } assertion:~##2 } } \cs_set:Npn \@@_b_test: { word~boundary } - \cs_set_eq:NN \@@_anchor:N \@@_show_anchor_to_str:N + \cs_set:Npn \@@_Z_test: { anchor~at~end~(\iow_char:N\\Z) } + \cs_set:Npn \@@_A_test: { anchor~at~start~(\iow_char:N\\A) } + \cs_set:Npn \@@_G_test: { anchor~at~start~of~match~(\iow_char:N\\G) } \cs_set_protected:Npn \@@_item_caseful_equal:n ##1 { \@@_show_one:n { char~code~\int_eval:n{##1} } } \cs_set_protected:Npn \@@_item_caseful_range:nn ##1##2 @@ -3655,24 +3644,6 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}[rEXP]{\@@_show_anchor_to_str:N} -% The argument is an integer telling us where the anchor is. We -% convert that to the relevant info. -% \begin{macrocode} -\cs_new:Npn \@@_show_anchor_to_str:N #1 - { - anchor~at~ - \str_case:nnF { #1 } - { - { \l_@@_min_pos_int } { start~(\iow_char:N\\A) } - { \l_@@_start_pos_int } { start~of~match~(\iow_char:N\\G) } - { \l_@@_max_pos_int } { end~(\iow_char:N\\Z) } - } - { <error:~'#1'~not~recognized> } - } -% \end{macrocode} -% \end{macro} -% % \begin{macro}{\@@_show_item_catcode:NnT} % Produce a sequence of categories which the catcode bitmap |#2| % contains, and show it, indenting the tests on which this catcode @@ -3757,8 +3728,9 @@ % Each state of the \textsc{nfa} is stored in a \tn{toks}. The % operations which can appear in the \tn{toks} are % \begin{itemize} -% \item \cs{@@_action_start_wildcard:} inserted at the start -% of the regular expression to make it unanchored. +% \item \cs{@@_action_start_wildcard:N} \meta{boolean} inserted at the +% start of the regular expression, where a \texttt{true} +% \meta{boolean} makes it unanchored. % \item \cs{@@_action_success:} marks the exit state of the % \textsc{nfa}. % \item \cs{@@_action_cost:n} \Arg{shift} is a transition from the @@ -3772,10 +3744,11 @@ % how they detect and avoid infinite loops. For now, we just need to % know that the \texttt{group} variant must be used for transitions % back to the start of a group. -% \item \cs{@@_action_submatch:n} \Arg{key} where the \meta{key} is -% a group number followed by |<| or |>| for the beginning or end of -% group. This causes the current position in the query to be stored -% as the \meta{key} submatch boundary. +% \item \cs{@@_action_submatch:nN} \Arg{group} \meta{key} where the +% \meta{key} is |<| or |>| for the beginning or end of group +% numbered \meta{group}. This causes the current position in the +% query to be stored as the \meta{key} submatch boundary. +% \item One of these actions, within a conditional. % \end{itemize} % % We strive to preserve the following properties while building. @@ -3793,20 +3766,26 @@ % corresponding end-points of nested groups. % \end{itemize} % -% \begin{macro}{\@@_build:n, \@@_build:N} +% \begin{macro}{\@@_build:n, \@@_build_aux:Nn, \@@_build:N, \@@_build_aux:NN} % The \texttt{n}-type function first compiles its argument. Reset some % variables. Allocate two states, and put a wildcard in state $0$ % (transitions to state $1$ and $0$ state). Then build the regex % within a (capturing) group numbered $0$ (current % value of \texttt{capturing_group}). Finally, if the match reaches the -% last state, it is successful. +% last state, it is successful. A \texttt{false} boolean for argument +% |#1| for the auxiliaries will suppress the wildcard and make the +% match anchored: used for \cs{peek_regex:nTF} and similar. % \begin{macrocode} -\cs_new_protected:Npn \@@_build:n #1 +\cs_new_protected:Npn \@@_build:n + { \@@_build_aux:Nn \c_true_bool } +\cs_new_protected:Npn \@@_build:N + { \@@_build_aux:NN \c_true_bool } +\cs_new_protected:Npn \@@_build_aux:Nn #1#2 { - \@@_compile:n {#1} - \@@_build:N \l_@@_internal_regex + \@@_compile:n {#2} + \@@_build_aux:NN #1 \l_@@_internal_regex } -\cs_new_protected:Npn \@@_build:N #1 +\cs_new_protected:Npn \@@_build_aux:NN #1#2 { \@@_standard_escapechar: \int_zero:N \l_@@_capturing_group_int @@ -3814,8 +3793,8 @@ \@@_build_new_state: \@@_build_new_state: \@@_toks_put_right:Nn \l_@@_left_state_int - { \@@_action_start_wildcard: } - \@@_group:nnnN {#1} { 1 } { 0 } \c_false_bool + { \@@_action_start_wildcard:N #1 } + \@@_group:nnnN {#2} { 1 } { 0 } \c_false_bool \@@_toks_put_right:Nn \l_@@_right_state_int { \@@_action_success: } } @@ -3826,19 +3805,14 @@ % The matching code relies on some global intarray variables, but only % uses a range of their entries. Specifically, % \begin{itemize} -% \item \cs{g_@@_state_active_intarray} from \cs{l_@@_min_state_int} -% to $\cs{l_@@_max_state_int}-1$; -% \item \cs{g_@@_thread_state_intarray} from \cs{l_@@_min_active_int} -% to $\cs{l_@@_max_active_int}-1$. +% \item \cs{g_@@_state_active_intarray} from \cs{l_@@_min_state_int} +% to $\cs{l_@@_max_state_int}-1$; % \end{itemize} -% In fact, some data is stored in \tn{toks} registers (local) in the -% same ranges so these ranges mustn't overlap. This is done by -% setting \cs{l_@@_min_active_int} to \cs{l_@@_max_state_int} after -% building the \textsc{nfa}. Here, in this nested call to the -% matching code, we need the new versions of these ranges to involve +% Here, in this nested call to the +% matching code, we need the new versions of this range to involve % completely new entries of the intarray variables, so we begin by % setting (the new) \cs{l_@@_min_state_int} to (the old) -% \cs{l_@@_max_active_int} to use higher entries. +% \cs{l_@@_max_state_int} to use higher entries. % % When using a regex to match a cs, we don't insert a wildcard, we % anchor at the end, and since we ignore submatches, there is no need @@ -3848,8 +3822,7 @@ % \begin{macrocode} \cs_new_protected:Npn \@@_build_for_cs:n #1 { - \int_set_eq:NN \l_@@_min_state_int \l_@@_max_active_int - \int_set_eq:NN \l_@@_max_state_int \l_@@_min_state_int + \int_set_eq:NN \l_@@_min_state_int \l_@@_max_state_int \@@_build_new_state: \@@_build_new_state: \@@_push_lr_states: @@ -3857,7 +3830,7 @@ \@@_pop_lr_states: \@@_toks_put_right:Nn \l_@@_right_state_int { - \if_int_compare:w \l_@@_curr_pos_int = \l_@@_max_pos_int + \if_int_compare:w -2 = \l_@@_curr_char_int \exp_after:wN \@@_action_success: \fi: } @@ -4195,8 +4168,8 @@ \cs_new_protected:Npn \@@_group_submatches:nNN #1#2#3 { \if_int_compare:w #1 > - 1 \exp_stop_f: - \@@_toks_put_left:Nx #2 { \@@_action_submatch:n { #1 < } } - \@@_toks_put_left:Nx #3 { \@@_action_submatch:n { #1 > } } + \@@_toks_put_left:Nx #2 { \@@_action_submatch:nN {#1} < } + \@@_toks_put_left:Nx #3 { \@@_action_submatch:nN {#1} > } \fi: } % \end{macrocode} @@ -4345,16 +4318,14 @@ % % \subsubsection{Others} % -% \begin{macro}{\@@_assertion:Nn, \@@_b_test:, \@@_anchor:N} +% \begin{macro}{\@@_assertion:Nn, \@@_b_test:, \@@_A_test:, \@@_G_test:, \@@_Z_test:} % Usage: \cs{@@_assertion:Nn} \meta{boolean} \Arg{test}, where the % \meta{test} is either of the two other functions. Add a free % transition to a new state, conditionally to the assertion test. The % \cs{@@_b_test:} test is used by the |\b| and |\B| escape: check % if the last character was a word character or not, and do the same % to the current character. The boundary-markers of the string are -% non-word characters for this purpose. Anchors at the start or end -% of match use \cs{@@_anchor:N}, with a position controlled by the -% integer |#1|. +% non-word characters for this purpose. % \begin{macrocode} \cs_new_protected:Npn \@@_assertion:Nn #1#2 { @@ -4374,12 +4345,6 @@ \bool_if:NT #1 { { } } } } -\cs_new_protected:Npn \@@_anchor:N #1 - { - \if_int_compare:w #1 = \l_@@_curr_pos_int - \exp_after:wN \@@_break_true:w - \fi: - } \cs_new_protected:Npn \@@_b_test: { \group_begin: @@ -4389,6 +4354,24 @@ { \group_end: \@@_item_reverse:n \@@_prop_w: } { \group_end: \@@_prop_w: } } +\cs_new_protected:Npn \@@_Z_test: + { + \if_int_compare:w -2 = \l_@@_curr_char_int + \exp_after:wN \@@_break_true:w + \fi: + } +\cs_new_protected:Npn \@@_A_test: + { + \if_int_compare:w -2 = \l_@@_last_char_int + \exp_after:wN \@@_break_true:w + \fi: + } +\cs_new_protected:Npn \@@_G_test: + { + \if_int_compare:w \l_@@_curr_pos_int = \l_@@_start_pos_int + \exp_after:wN \@@_break_true:w + \fi: + } % \end{macrocode} % \end{macro} % @@ -4401,7 +4384,7 @@ \@@_build_new_state: \@@_toks_put_right:Nx \l_@@_left_state_int { - \@@_action_submatch:n { 0< } + \@@_action_submatch:nN { 0 } < \bool_set_true:N \l_@@_fresh_thread_bool \@@_action_free:n { @@ -4423,7 +4406,8 @@ % transitions, the instruction at the new state of the \textsc{nfa} is % performed immediately. When a transition consumes a character, the % new state is appended to a list of \enquote{active states}, stored in -% \cs{g_@@_thread_state_intarray}: this thread is made active again when the next +% \cs{g_@@_thread_info_intarray} (together with submatch information): +% this thread is made active again when the next % token is read from the query. At every step (for each token in the % query), we unpack that list of active states and the corresponding % submatch props, and empty those. @@ -4467,11 +4451,10 @@ % } % The tokens in the query are indexed from \texttt{min_pos} for the % first to $\texttt{max_pos}-1$ for the last, and their information is -% stored in several arrays and \tn{toks} registers with those numbers. We -% don't start from $0$ because the \tn{toks} registers with low -% numbers are used to hold the states of the \textsc{nfa}. We match +% stored in several arrays and \tn{toks} registers with those numbers. +% We match % without backtracking, keeping all threads in lockstep at the -% \texttt{current_pos} in the query. The starting point of the current +% \texttt{curr_pos} in the query. The starting point of the current % match attempt is \texttt{start_pos}, and \texttt{success_pos}, % updated whenever a thread succeeds, is used as the next starting % position. @@ -4488,20 +4471,26 @@ % { % \l_@@_curr_char_int, % \l_@@_curr_catcode_int, +% \l_@@_curr_token_tl, % \l_@@_last_char_int, +% \l_@@_last_char_success_int, % \l_@@_case_changed_char_int % } % The character and category codes of the token at the current -% position; the character code of the token at the previous position; +% position and a token list expanding to that token; the character +% code of the token at the previous position; +% the character code of the token just before a successful match; % and the character code of the result of changing the case of the % current token (|A-Z|$\leftrightarrow$|a-z|). This last integer is % only computed when necessary, and is otherwise \cs{c_max_int}. The -% \texttt{current_char} variable is also used in various other phases +% \texttt{curr_char} variable is also used in various other phases % to hold a character code. % \begin{macrocode} \int_new:N \l_@@_curr_char_int \int_new:N \l_@@_curr_catcode_int +\tl_new:N \l_@@_curr_token_tl \int_new:N \l_@@_last_char_int +\int_new:N \l_@@_last_char_success_int \int_new:N \l_@@_case_changed_char_int % \end{macrocode} % \end{variable} @@ -4517,18 +4506,17 @@ % \end{variable} % % \begin{variable} -% {\l_@@_curr_submatches_prop, \l_@@_success_submatches_prop} +% {\l_@@_curr_submatches_tl, \l_@@_success_submatches_tl} % The submatches for the thread which is currently active are stored -% in the \texttt{current_submatches} property list variable. This -% property list is stored by \cs{@@_action_cost:n} into the -% \tn{toks} register for the target state of the transition, to be -% retrieved when matching at the next position. When a thread -% succeeds, this property list is copied to -% \cs{l_@@_success_submatches_prop}: only the last successful thread +% in the \texttt{curr_submatches} list, which is almost a comma list, +% but ends with a comma. This list is stored by \cs{@@_store_state:n} +% into an intarray variable, to be retrieved when matching at the next +% position. When a thread succeeds, this list is copied to +% \cs{l_@@_success_submatches_tl}: only the last successful thread % remains there. % \begin{macrocode} -\prop_new:N \l_@@_curr_submatches_prop -\prop_new:N \l_@@_success_submatches_prop +\tl_new:N \l_@@_curr_submatches_tl +\tl_new:N \l_@@_success_submatches_tl % \end{macrocode} % \end{variable} % @@ -4551,28 +4539,44 @@ % \end{macrocode} % \end{variable} % -% \begin{variable}{\l_@@_min_active_int, \l_@@_max_active_int} +% \begin{variable}{\l_@@_min_thread_int, \l_@@_max_thread_int} % All the currently active threads are kept in order of precedence in -% \cs{g_@@_thread_state_intarray}, and the corresponding submatches in the -% \tn{toks}. For our purposes, those serve as an array, indexed from -% \texttt{min_active} (inclusive) to \texttt{max_active} (excluded). -% At the start of every step, the whole array is unpacked, so that the -% space can immediately be reused, and \texttt{max_active} is reset to -% \texttt{min_active}, effectively clearing the array. +% \cs{g_@@_thread_info_intarray} together with the corresponding +% submatch information. Data in this intarray is organized as blocks +% from \texttt{min_thread} (included) to \texttt{max_thread} +% (excluded). At the start of every step, the whole array is +% unpacked, so that the space can immediately be reused, and +% \texttt{max_thread} is reset to \texttt{min_thread}, effectively +% clearing the array. % \begin{macrocode} -\int_new:N \l_@@_min_active_int -\int_new:N \l_@@_max_active_int +\int_new:N \l_@@_min_thread_int +\int_new:N \l_@@_max_thread_int % \end{macrocode} % \end{variable} % -% \begin{variable}{\g_@@_state_active_intarray, \g_@@_thread_state_intarray} +% \begin{variable}{\g_@@_state_active_intarray, \g_@@_thread_info_intarray} % \cs{g_@@_state_active_intarray} stores the last \meta{step} in which -% each \meta{state} was active. \cs{g_@@_thread_state_intarray} stores +% each \meta{state} was active. \cs{g_@@_thread_info_intarray} stores % threads to be considered in the next step, more precisely the % states in which these threads are. % \begin{macrocode} \intarray_new:Nn \g_@@_state_active_intarray { 65536 } -\intarray_new:Nn \g_@@_thread_state_intarray { 65536 } +\intarray_new:Nn \g_@@_thread_info_intarray { 65536 } +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_@@_matched_analysis_tl, \l_@@_curr_analysis_tl} +% The list \cs{l_@@_curr_analysis_tl} consists of a brace group +% containing three brace groups corresponding to the current token, +% with the same syntax as \cs{tl_analysis_map_inline:nn}. The list +% \cs{l_@@_matched_analysis_tl} (constructed under the +% \texttt{tl\_build} machinery) has one item for each token that has +% already been treated so far in a given match attempt: each item +% consists of three brace groups with the same syntax as +% \cs{tl_analysis_map_inline:nn}. +% \begin{macrocode} +\tl_new:N \l_@@_matched_analysis_tl +\tl_new:N \l_@@_curr_analysis_tl % \end{macrocode} % \end{variable} % @@ -4635,8 +4639,7 @@ % % \begin{macro}{\@@_match:n, \@@_match_cs:n} % \begin{macro}{\@@_match_init:} -% First store the query into \tn{toks} registers and arrays (see -% \cs{@@_query_set:nnn}). Then initialize the variables that should +% Initialize the variables that should % be set once for each user function (even for multiple % matches). Namely, the overall matching is not yet successful; none of % the states should be marked as visited (\cs{g_@@_state_active_intarray}), and @@ -4647,38 +4650,26 @@ % \begin{macrocode} \cs_new_protected:Npn \@@_match:n #1 { - \int_zero:N \l_@@_balance_int - \int_set:Nn \l_@@_curr_pos_int { 2 * \l_@@_max_state_int } - \@@_query_set:nnn { } { -1 } { -2 } - \int_set_eq:NN \l_@@_min_pos_int \l_@@_curr_pos_int - \tl_analysis_map_inline:nn {#1} - { \@@_query_set:nnn {##1} {"##3} {##2} } - \int_set_eq:NN \l_@@_max_pos_int \l_@@_curr_pos_int - \@@_query_set:nnn { } { -1 } { -2 } \@@_match_init: - \@@_match_once: + \@@_match_once_init: + \tl_analysis_map_inline:nn {#1} + { \@@_match_one_token:nnN {##1} {##2} ##3 } + \@@_match_one_token:nnN { } { -2 } F + \prg_break_point:Nn \@@_maplike_break: { } } \cs_new_protected:Npn \@@_match_cs:n #1 { - \int_zero:N \l_@@_balance_int - \int_set:Nn \l_@@_curr_pos_int - { - \int_max:nn { 2 * \l_@@_max_state_int - \l_@@_min_state_int } - { \l_@@_max_pos_int } - + 1 - } - \@@_query_set:nnn { } { -1 } { -2 } - \int_set_eq:NN \l_@@_min_pos_int \l_@@_curr_pos_int + \int_set_eq:NN \l_@@_min_thread_int \l_@@_max_thread_int + \@@_match_init: + \@@_match_once_init: \str_map_inline:nn {#1} { - \@@_query_set:nnn { \exp_not:n {##1} } - { \tl_if_blank:nTF {##1} { 10 } { 12 } } - { `##1 } + \tl_if_blank:nTF {##1} + { \@@_match_one_token:nnN {##1} {`##1} A } + { \@@_match_one_token:nnN {##1} {`##1} C } } - \int_set_eq:NN \l_@@_max_pos_int \l_@@_curr_pos_int - \@@_query_set:nnn { } { -1 } { -2 } - \@@_match_init: - \@@_match_once: + \@@_match_one_token:nnN { } { -2 } F + \prg_break_point:Nn \@@_maplike_break: { } } \cs_new_protected:Npn \@@_match_init: { @@ -4689,11 +4680,13 @@ \__kernel_intarray_gset:Nnn \g_@@_state_active_intarray {##1} { 1 } } - \int_set_eq:NN \l_@@_min_active_int \l_@@_max_state_int \int_zero:N \l_@@_step_int + \int_set:Nn \l_@@_min_pos_int { 2 } \int_set_eq:NN \l_@@_success_pos_int \l_@@_min_pos_int - \int_set:Nn \l_@@_min_submatch_int - { 2 * \l_@@_max_state_int } + \int_set:Nn \l_@@_last_char_success_int { -2 } + \tl_build_begin:N \l_@@_matched_analysis_tl + \tl_clear:N \l_@@_curr_analysis_tl + \int_set:Nn \l_@@_min_submatch_int { 1 } \int_set_eq:NN \l_@@_submatch_int \l_@@_min_submatch_int \bool_set_false:N \l_@@_empty_success_bool } @@ -4701,21 +4694,24 @@ % \end{macro} % \end{macro} % -% \begin{macro}{\@@_match_once:} -% This function finds one match, then does some action defined by the -% \texttt{every_match} token list, which may recursively call -% \cs{@@_match_once:}. First initialize some variables: set the +% \begin{macro}{\@@_match_once_init:} +% This function resets various variables used when finding one match. +% It is called before the loop through characters, and every time we +% find a match, before searching for another match (this is controlled +% by the \texttt{every_match} token list). +% +% First initialize some variables: set the % conditional which detects identical empty matches; this match % attempt starts at the previous \texttt{success_pos}, is not yet % successful, and has no submatches yet; clear the array of active % threads, and put the starting state $0$ in it. We are then almost % ready to read our first token in the query, but we actually start -% one position earlier than the start, and \texttt{get} that token, to -% set \texttt{last_char} properly for word -% boundaries. Then call \cs{@@_match_loop:}, which runs through the -% query until the end or until a successful match breaks early. +% one position earlier than the start because +% \cs{@@_match_one_token:nnN} increments \cs{l_@@_curr_pos_int} and +% saves \cs{l_@@_curr_char_int} as the \texttt{last_char} so that word +% boundaries can be correctly identified. % \begin{macrocode} -\cs_new_protected:Npn \@@_match_once: +\cs_new_protected:Npn \@@_match_once_init: { \if_meaning:w \c_true_bool \l_@@_empty_success_bool \cs_set:Npn \@@_if_two_empty_matches:F @@ -4728,14 +4724,24 @@ \fi: \int_set_eq:NN \l_@@_start_pos_int \l_@@_success_pos_int \bool_set_false:N \l_@@_match_success_bool - \prop_clear:N \l_@@_curr_submatches_prop - \int_set_eq:NN \l_@@_max_active_int \l_@@_min_active_int + \tl_set:Nx \l_@@_curr_submatches_tl + { \prg_replicate:nn { 2 * \l_@@_capturing_group_int } { 0 , } } + \int_set_eq:NN \l_@@_max_thread_int \l_@@_min_thread_int \@@_store_state:n { \l_@@_min_state_int } \int_set:Nn \l_@@_curr_pos_int { \l_@@_start_pos_int - 1 } - \@@_query_get: - \@@_match_loop: - \l_@@_every_match_tl + \int_set_eq:NN \l_@@_curr_char_int \l_@@_last_char_success_int + \tl_build_get:NN \l_@@_matched_analysis_tl \l_@@_internal_a_tl + \exp_args:NNf \@@_match_once_init_aux: + \tl_map_inline:nn + { \exp_after:wN \l_@@_internal_a_tl \l_@@_curr_analysis_tl } + { \@@_match_one_token:nnN ##1 } + \prg_break_point:Nn \@@_maplike_break: { } + } +\cs_new_protected:Npn \@@_match_once_init_aux: + { + \tl_build_clear:N \l_@@_matched_analysis_tl + \tl_clear:N \l_@@_curr_analysis_tl } % \end{macrocode} % \end{macro} @@ -4753,110 +4759,80 @@ \bool_gset_eq:NN \g_@@_success_bool \l_@@_match_success_bool + \@@_maplike_break: } } \cs_new_protected:Npn \@@_multi_match:n #1 { \tl_set:Nn \l_@@_every_match_tl { - \if_meaning:w \c_true_bool \l_@@_match_success_bool - \bool_gset_true:N \g_@@_success_bool - #1 - \exp_after:wN \@@_match_once: + \if_meaning:w \c_false_bool \l_@@_match_success_bool + \exp_after:wN \@@_maplike_break: \fi: + \bool_gset_true:N \g_@@_success_bool + #1 + \@@_match_once_init: } } % \end{macrocode} % \end{macro} % -% \begin{macro}{\@@_match_loop:} +% \begin{macro}{\@@_match_one_token:nnN} % \begin{macro}[rEXP]{\@@_match_one_active:n} % At each new position, set some variables and get the new character % and category from the query. Then unpack the array of active % threads, and clear it by resetting its length -% (\texttt{max_active}). This results in a sequence of -% \cs{@@_use_state_and_submatches:nn} \Arg{state} \Arg{prop}, and +% (\texttt{max_thread}). This results in a sequence of +% \cs{@@_use_state_and_submatches:w} \meta{state}|,|\meta{submatch-clist}|;| and % we consider those states one by one in order. As soon as a thread % succeeds, exit the step, and, if there are threads to consider at the % next position, and we have not reached the end of the string, -% repeat the loop. Otherwise, the last thread that succeeded is what -% \cs{@@_match_once:} matches. We explain the \texttt{fresh_thread} -% business when describing \cs{@@_action_wildcard:}. +% repeat the loop. Otherwise, the last thread that succeeded is the +% match. We explain the \texttt{fresh_thread} business when +% describing \cs{@@_action_wildcard:}. % \begin{macrocode} -\cs_new_protected:Npn \@@_match_loop: +\cs_new_protected:Npn \@@_match_one_token:nnN #1#2#3 { \int_add:Nn \l_@@_step_int { 2 } \int_incr:N \l_@@_curr_pos_int \int_set_eq:NN \l_@@_last_char_int \l_@@_curr_char_int \int_set_eq:NN \l_@@_case_changed_char_int \c_max_int - \@@_query_get: + \tl_set:Nn \l_@@_curr_token_tl {#1} + \int_set:Nn \l_@@_curr_char_int {#2} + \int_set:Nn \l_@@_curr_catcode_int { "#3 } + \tl_build_put_right:Nx \l_@@_matched_analysis_tl + { \exp_not:o \l_@@_curr_analysis_tl } + \tl_set:Nn \l_@@_curr_analysis_tl { { {#1} {#2} #3 } } \use:x { - \int_set_eq:NN \l_@@_max_active_int \l_@@_min_active_int + \int_set_eq:NN \l_@@_max_thread_int \l_@@_min_thread_int \int_step_function:nnN - { \l_@@_min_active_int } - { \l_@@_max_active_int - 1 } + { \l_@@_min_thread_int } + { \l_@@_max_thread_int - 1 } \@@_match_one_active:n } \prg_break_point: \bool_set_false:N \l_@@_fresh_thread_bool - \if_int_compare:w \l_@@_max_active_int > \l_@@_min_active_int - \if_int_compare:w \l_@@_curr_pos_int < \l_@@_max_pos_int - \exp_after:wN \exp_after:wN \exp_after:wN \@@_match_loop: + \if_int_compare:w \l_@@_max_thread_int > \l_@@_min_thread_int + \if_int_compare:w -2 < \l_@@_curr_char_int + \exp_after:wN \exp_after:wN \exp_after:wN \use_none:n \fi: \fi: + \l_@@_every_match_tl } \cs_new:Npn \@@_match_one_active:n #1 { - \@@_use_state_and_submatches:nn - { \__kernel_intarray_item:Nn \g_@@_thread_state_intarray {#1} } - { \@@_toks_use:w #1 } + \@@_use_state_and_submatches:w + \__kernel_intarray_range_to_clist:Nnn + \g_@@_thread_info_intarray + { 1 + #1 * (\l_@@_capturing_group_int * 2 + 1) } + { (1 + #1) * (\l_@@_capturing_group_int * 2 + 1) } + ; } % \end{macrocode} % \end{macro} % \end{macro} % -% \begin{macro}{\@@_query_set:nnn} -% The arguments are: tokens that \texttt{o} and \texttt{x} expand to -% one token of the query, the catcode, and the character code. Store -% those, and the current brace balance (used later to check for -% overall brace balance) in a \tn{toks} register and some arrays, -% then update the \texttt{balance}. -% \begin{macrocode} -\cs_new_protected:Npn \@@_query_set:nnn #1#2#3 - { - \__kernel_intarray_gset:Nnn \g_@@_charcode_intarray - { \l_@@_curr_pos_int } {#3} - \__kernel_intarray_gset:Nnn \g_@@_catcode_intarray - { \l_@@_curr_pos_int } {#2} - \__kernel_intarray_gset:Nnn \g_@@_balance_intarray - { \l_@@_curr_pos_int } { \l_@@_balance_int } - \@@_toks_set:Nn \l_@@_curr_pos_int {#1} - \int_incr:N \l_@@_curr_pos_int - \if_case:w #2 \exp_stop_f: - \or: \int_incr:N \l_@@_balance_int - \or: \int_decr:N \l_@@_balance_int - \fi: - } -% \end{macrocode} -% \end{macro} -% -% \begin{macro}{\@@_query_get:} -% Extract the current character and category codes at the current -% position from the appropriate arrays. -% \begin{macrocode} -\cs_new_protected:Npn \@@_query_get: - { - \l_@@_curr_char_int - = \__kernel_intarray_item:Nn \g_@@_charcode_intarray - { \l_@@_curr_pos_int } \scan_stop: - \l_@@_curr_catcode_int - = \__kernel_intarray_item:Nn \g_@@_catcode_intarray - { \l_@@_curr_pos_int } \scan_stop: - } -% \end{macrocode} -% \end{macro} -% % \subsubsection{Using states of the \textsc{nfa}} % % \begin{macro}{\@@_use_state:} @@ -4879,20 +4855,20 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}{\@@_use_state_and_submatches:nn} +% \begin{macro}{\@@_use_state_and_submatches:w} % This function is called as one item in the array of active threads % after that array has been unpacked for a new step. Update the -% \texttt{current_state} and \texttt{current_submatches} and use the +% \texttt{curr_state} and \texttt{curr_submatches} and use the % state if it has not yet been encountered at this step. % \begin{macrocode} -\cs_new_protected:Npn \@@_use_state_and_submatches:nn #1 #2 +\cs_new_protected:Npn \@@_use_state_and_submatches:w #1 , #2 ; { \int_set:Nn \l_@@_curr_state_int {#1} \if_int_compare:w \__kernel_intarray_item:Nn \g_@@_state_active_intarray { \l_@@_curr_state_int } < \l_@@_step_int - \tl_set:Nn \l_@@_curr_submatches_prop {#2} + \tl_set:Nn \l_@@_curr_submatches_tl { #2 , } \exp_after:wN \@@_use_state: \fi: \scan_stop: @@ -4902,20 +4878,20 @@ % % \subsubsection{Actions when matching} % -% \begin{macro}{\@@_action_start_wildcard:} +% \begin{macro}{\@@_action_start_wildcard:N} % For an unanchored match, state $0$ has a free transition to the next % and a costly one to itself, to repeat at the next position. To catch % repeated identical empty matches, we need to know if a successful % thread corresponds to an empty match. The instruction resetting % \cs{l_@@_fresh_thread_bool} may be skipped by a successful -% thread, hence we had to add it to \cs{@@_match_loop:} too. +% thread, hence we had to add it to \cs{@@_match_one_token:nnN} too. % \begin{macrocode} -\cs_new_protected:Npn \@@_action_start_wildcard: +\cs_new_protected:Npn \@@_action_start_wildcard:N #1 { \bool_set_true:N \l_@@_fresh_thread_bool \@@_action_free:n {1} \bool_set_false:N \l_@@_fresh_thread_bool - \@@_action_cost:n {0} + \bool_if:NT #1 { \@@_action_cost:n {0} } } % \end{macrocode} % \end{macro} @@ -4954,8 +4930,8 @@ } \int_set:Nn \l_@@_curr_state_int { \int_use:N \l_@@_curr_state_int } - \tl_set:Nn \exp_not:N \l_@@_curr_submatches_prop - { \exp_not:o \l_@@_curr_submatches_prop } + \tl_set:Nn \exp_not:N \l_@@_curr_submatches_tl + { \exp_not:o \l_@@_curr_submatches_tl } } } % \end{macrocode} @@ -4978,21 +4954,26 @@ % % \begin{macro}{\@@_store_state:n} % \begin{macro}{\@@_store_submatches:} -% Put the given state in \cs{g_@@_thread_state_intarray}, and increment -% the length of the array. Also store the current submatch in the -% appropriate \tn{toks}. +% Put the given state and current submatch information in +% \cs{g_@@_thread_info_intarray}, and increment the length of the +% array. % \begin{macrocode} \cs_new_protected:Npn \@@_store_state:n #1 { - \@@_store_submatches: - \__kernel_intarray_gset:Nnn \g_@@_thread_state_intarray - { \l_@@_max_active_int } {#1} - \int_incr:N \l_@@_max_active_int + \exp_args:No \@@_store_submatches:nn + \l_@@_curr_submatches_tl {#1} + \int_incr:N \l_@@_max_thread_int } -\cs_new_protected:Npn \@@_store_submatches: +\cs_new_protected:Npn \@@_store_submatches:nn #1#2 { - \@@_toks_set:No \l_@@_max_active_int - { \l_@@_curr_submatches_prop } + \__kernel_intarray_gset_range_from_clist:Nnn + \g_@@_thread_info_intarray + { + \@@_int_eval:w + 1 + \l_@@_max_thread_int * + (\l_@@_capturing_group_int * 2 + 1) + } + { #2 , #1 } } % \end{macrocode} % \end{macro} @@ -5006,21 +4987,37 @@ % \begin{macrocode} \cs_new_protected:Npn \@@_disable_submatches: { - \cs_set_protected:Npn \@@_store_submatches: { } - \cs_set_protected:Npn \@@_action_submatch:n ##1 { } + \cs_set_protected:Npn \@@_store_submatches:n ##1 { } + \cs_set_protected:Npn \@@_action_submatch:nN ##1##2 { } } % \end{macrocode} % \end{macro} % -% \begin{macro}{\@@_action_submatch:n} +% \begin{macro}{\@@_action_submatch:nN, \@@_action_submatch_aux:w, \@@_action_submatch_auxii:w, \@@_action_submatch_auxiii:w, \@@_action_submatch_auxiv:w} % Update the current submatches with the information from the current % position. Maybe a bottleneck. % \begin{macrocode} -\cs_new_protected:Npn \@@_action_submatch:n #1 +\cs_new_protected:Npn \@@_action_submatch:nN #1#2 + { + \exp_after:wN \@@_action_submatch_aux:w + \l_@@_curr_submatches_tl ; {#1} #2 + } +\cs_new_protected:Npn \@@_action_submatch_aux:w #1 ; #2#3 { - \prop_put:Nno \l_@@_curr_submatches_prop {#1} - { \int_use:N \l_@@_curr_pos_int } + \tl_set:Nx \l_@@_curr_submatches_tl + { + \prg_replicate:nn + { #2 \if_meaning:w > #3 + \l_@@_capturing_group_int \fi: } + { \@@_action_submatch_auxii:w } + \@@_action_submatch_auxiii:w + #1 + } } +\cs_new:Npn \@@_action_submatch_auxii:w + #1 \@@_action_submatch_auxiii:w #2 , + { #2 , #1 \@@_action_submatch_auxiii:w } +\cs_new:Npn \@@_action_submatch_auxiii:w #1 , + { \int_use:N \l_@@_curr_pos_int , } % \end{macrocode} % \end{macro} % @@ -5042,8 +5039,10 @@ \bool_set_eq:NN \l_@@_empty_success_bool \l_@@_fresh_thread_bool \int_set_eq:NN \l_@@_success_pos_int \l_@@_curr_pos_int - \prop_set_eq:NN \l_@@_success_submatches_prop - \l_@@_curr_submatches_prop + \int_set_eq:NN \l_@@_last_char_success_int \l_@@_last_char_int + \tl_build_clear:N \l_@@_matched_analysis_tl + \tl_set_eq:NN \l_@@_success_submatches_tl + \l_@@_curr_submatches_tl \prg_break: } } @@ -5134,6 +5133,14 @@ % \end{macrocode} % \end{macro} % +% \begin{macro}{\@@_replacement_exp_not:V} +% This is used for the implementation of~|\u|, and it gets redefined +% for \cs{peek_regex_replace_once:nnTF}. +% \begin{macrocode} +\cs_new_eq:NN \@@_replacement_exp_not:V \exp_not:V +% \end{macrocode} +% \end{macro} +% % \subsubsection{Query and brace balance} % % \begin{macro}[rEXP]{\@@_query_range:nn} @@ -5189,42 +5196,25 @@ % range is the difference between the brace balances at the % \meta{max~pos} and \meta{min~pos}. These two positions are found in % the corresponding \enquote{submatch} arrays. -%^^A todo: understand when these int_compare are needed % \begin{macrocode} \cs_new_protected:Npn \@@_submatch_balance:n #1 { \int_eval:n - { - \int_compare:nNnTF - { - \__kernel_intarray_item:Nn - \g_@@_submatch_end_intarray {#1} - } - = 0 - { 0 } - { - \__kernel_intarray_item:Nn \g_@@_balance_intarray - { - \__kernel_intarray_item:Nn - \g_@@_submatch_end_intarray {#1} - } - } - - - \int_compare:nNnTF - { - \__kernel_intarray_item:Nn - \g_@@_submatch_begin_intarray {#1} - } - = 0 - { 0 } - { - \__kernel_intarray_item:Nn \g_@@_balance_intarray - { - \__kernel_intarray_item:Nn - \g_@@_submatch_begin_intarray {#1} - } - } - } + { + \@@_intarray_item:NnF \g_@@_balance_intarray + { + \__kernel_intarray_item:Nn + \g_@@_submatch_end_intarray {#1} + } + { 0 } + - + \@@_intarray_item:NnF \g_@@_balance_intarray + { + \__kernel_intarray_item:Nn + \g_@@_submatch_begin_intarray {#1} + } + { 0 } + } } % \end{macrocode} % \end{macro} @@ -5305,6 +5295,14 @@ % \end{macro} % \end{macro} % +% \begin{macro}{\@@_replacement_put:n} +% This gets redefined for \cs{peek_regex_replace_once:nnTF}. +% \begin{macrocode} +\cs_new_protected:Npn \@@_replacement_put:n + { \tl_build_put_right:Nn \l_@@_build_tl } +% \end{macrocode} +% \end{macro} +% % \begin{macro}{\@@_replacement_normal:n} % Most characters are simply sent to the output by % \cs{tl_build_put_right:Nn}, unless a particular category code has been @@ -5314,11 +5312,13 @@ % sequence is non-empty there: it contains an empty entry % corresponding to the initial value of % \cs{l_@@_replacement_category_tl}. +% The argument |#1| can be a space, otherwise it is a single +% character. % \begin{macrocode} \cs_new_protected:Npn \@@_replacement_normal:n #1 { \tl_if_empty:NTF \l_@@_replacement_category_tl - { \tl_build_put_right:Nn \l_@@_build_tl {#1} } + { \@@_replacement_put:n {#1} } { % ( \token_if_eq_charcode:NNTF #1 ) { @@ -5361,7 +5361,7 @@ % % \subsubsection{Submatches} % -% \begin{macro}{\@@_replacement_put_submatch:n} +% \begin{macro}{\@@_replacement_put_submatch:n, \@@_replacement_put_submatch_aux:n} % Insert a submatch in the replacement text. This is dropped if the % submatch number is larger than the number of capturing groups. % Unless the submatch appears inside a |\c{...}| or |\u{...}| @@ -5374,15 +5374,19 @@ \cs_new_protected:Npn \@@_replacement_put_submatch:n #1 { \if_int_compare:w #1 < \l_@@_capturing_group_int - \tl_build_put_right:Nn \l_@@_build_tl - { \@@_query_submatch:n { \int_eval:n { #1 + ##1 } } } - \if_int_compare:w \l_@@_replacement_csnames_int = 0 \exp_stop_f: - \tl_put_right:Nn \l_@@_balance_tl - { - + \@@_submatch_balance:n - { \exp_not:N \int_eval:n { #1 + ##1 } } - } - \fi: + \@@_replacement_put_submatch_aux:n {#1} + \fi: + } +\cs_new_protected:Npn \@@_replacement_put_submatch_aux:n #1 + { + \tl_build_put_right:Nn \l_@@_build_tl + { \@@_query_submatch:n { \int_eval:n { #1 + ##1 } } } + \if_int_compare:w \l_@@_replacement_csnames_int = 0 \exp_stop_f: + \tl_put_right:Nn \l_@@_balance_tl + { + + \@@_submatch_balance:n + { \exp_not:N \int_eval:n { #1 + ##1 } } + } \fi: } % \end{macrocode} @@ -5488,7 +5492,7 @@ { \@@_two_if_eq:NNNNTF #1 #2 \@@_replacement_normal:n \c_left_brace_str - { \@@_replacement_cu_aux:Nw \exp_not:V } + { \@@_replacement_cu_aux:Nw \@@_replacement_exp_not:V } { \@@_replacement_error:NNN u #1#2 } } % \end{macrocode} @@ -5579,7 +5583,7 @@ \cs_new_protected:Npn \@@_replacement_char:nNN #1#2#3 { \tex_lccode:D 0 = `#3 \scan_stop: - \tex_lowercase:D { \tl_build_put_right:Nn \l_@@_build_tl {#1} } + \tex_lowercase:D { \@@_replacement_put:n {#1} } } % \end{macrocode} % \end{macro} @@ -5625,7 +5629,7 @@ \cs_new_protected:Npn \@@_replacement_c_C:w #1#2 { \tl_build_put_right:Nn \l_@@_build_tl - { \exp_not:N \exp_not:N \exp_not:c {#2} } + { \exp_not:N \@@_replacement_exp_not:N \exp_not:c {#2} } } % \end{macrocode} % \end{macro} @@ -5712,7 +5716,7 @@ \__kernel_msg_error:nn { kernel } { replacement-null-space } \fi: \tex_lccode:D `\ = `#2 \scan_stop: - \tex_lowercase:D { \tl_build_put_right:Nn \l_@@_build_tl {~} } + \tex_lowercase:D { \@@_replacement_put:n {~} } } % \end{macrocode} % \end{macro} @@ -5940,6 +5944,14 @@ % \end{macrocode} % \end{variable} % +% \begin{variable}{\g_@@_balance_intarray} +% The first thing we do when matching is to store the balance of +% begin-group/end-group characters into \cs{g_@@_balance_intarray}. +% \begin{macrocode} +\intarray_new:Nn \g_@@_balance_intarray { 65536 } +% \end{macrocode} +% \end{variable} +% % \begin{macro}{\@@_return:} % This function triggers either \cs{prg_return_false:} or % \cs{prg_return_true:} as appropriate to whether a match was found or @@ -5956,6 +5968,36 @@ % \end{macrocode} % \end{macro} % +% \begin{macro}{\@@_query_set:n, \@@_query_set_aux:nN} +% To easily extract subsets of the input once we found the positions +% at which to cut, store the input tokens one by one into successive +% \tn{toks} registers. Also store the brace balance (used to check +% for overall brace balance) in an array. +% \begin{macrocode} +\cs_new_protected:Npn \@@_query_set:n #1 + { + \int_zero:N \l_@@_balance_int + \int_zero:N \l_@@_curr_pos_int + \@@_query_set_aux:nN { } F + \tl_analysis_map_inline:nn {#1} + { \@@_query_set_aux:nN {##1} ##3 } + \@@_query_set_aux:nN { } F + \int_set_eq:NN \l_@@_max_pos_int \l_@@_curr_pos_int + } +\cs_new_protected:Npn \@@_query_set_aux:nN #1#2 + { + \int_incr:N \l_@@_curr_pos_int + \@@_toks_set:Nn \l_@@_curr_pos_int {#1} + \__kernel_intarray_gset:Nnn \g_@@_balance_intarray + { \l_@@_curr_pos_int } { \l_@@_balance_int } + \if_case:w "#2 \exp_stop_f: + \or: \int_incr:N \l_@@_balance_int + \or: \int_decr:N \l_@@_balance_int + \fi: + } +% \end{macrocode} +% \end{macro} +% % \subsubsection{Matching} % % \begin{macro}{\@@_if_match:nn} @@ -6011,6 +6053,7 @@ #1 \@@_match:n {#2} \@@_extract: + \@@_query_set:n {#2} \@@_group_end_extract_seq:N #3 } \cs_new_protected:Npn \@@_extract_all:nnN #1#2#3 @@ -6019,6 +6062,7 @@ \@@_multi_match:n { \@@_extract: } #1 \@@_match:n {#2} + \@@_query_set:n {#2} \@@_group_end_extract_seq:N #3 } % \end{macrocode} @@ -6058,7 +6102,7 @@ } #1 \@@_match:n {#2} -%<assert>\assert_int:n { \l_@@_curr_pos_int = \l_@@_max_pos_int } + \@@_query_set:n {#2} \__kernel_intarray_gset:Nnn \g_@@_submatch_prev_intarray { \l_@@_submatch_int } { 0 } \__kernel_intarray_gset:Nnn \g_@@_submatch_end_intarray @@ -6143,18 +6187,15 @@ % \end{macrocode} % \end{macro} % -% \begin{macro} -% {\@@_extract:, \@@_extract_b:wn, \@@_extract_e:wn} -% Our task here is to extract from the property list -% \cs{l_@@_success_submatches_prop} the list of end-points of -% submatches, and store them in appropriate array entries, from -% \cs{l_@@_zeroth_submatch_int} upwards. We begin by emptying those -% entries. Then for each \meta{key}--\meta{value} pair in -% the property list update the appropriate entry. This -% is somewhat a hack: the \meta{key} is a non-negative integer -% followed by |<| or |>|, which we use in a comparison to $-1$. At the -% end, store the information about the position at which the match -% attempt started, in \cs{g_@@_submatch_prev_intarray}. +% \begin{macro}{\@@_extract:} +% Our task here is to store the list of end-points of submatches, and +% store them in appropriate array entries, from +% \cs{l_@@_zeroth_submatch_int} upwards. First, we store in +% \cs{g_@@_submatch_prev_intarray} the position at which the match +% attempt started. We extract the rest from the comma list +% \cs{l_@@_success_submatches_tl}, which starts with entries to be +% stored in \cs{g_@@_submatch_begin_intarray} and continues with +% entries for \cs{g_@@_submatch_end_intarray}. % \begin{macrocode} \cs_new_protected:Npn \@@_extract: { @@ -6162,34 +6203,26 @@ \int_set_eq:NN \l_@@_zeroth_submatch_int \l_@@_submatch_int \prg_replicate:nn \l_@@_capturing_group_int { - \__kernel_intarray_gset:Nnn \g_@@_submatch_begin_intarray - { \l_@@_submatch_int } { 0 } - \__kernel_intarray_gset:Nnn \g_@@_submatch_end_intarray - { \l_@@_submatch_int } { 0 } \__kernel_intarray_gset:Nnn \g_@@_submatch_prev_intarray { \l_@@_submatch_int } { 0 } \int_incr:N \l_@@_submatch_int } - \prop_map_inline:Nn \l_@@_success_submatches_prop + \__kernel_intarray_gset:Nnn \g_@@_submatch_prev_intarray + { \l_@@_zeroth_submatch_int } { \l_@@_start_pos_int } + \int_zero:N \l_@@_internal_a_int + \clist_map_inline:Nn \l_@@_success_submatches_tl { - \if_int_compare:w ##1 - 1 \exp_stop_f: - \exp_after:wN \@@_extract_e:wn \int_value:w + \if_int_compare:w \l_@@_internal_a_int < \l_@@_capturing_group_int + \__kernel_intarray_gset:Nnn \g_@@_submatch_begin_intarray + { \@@_int_eval:w \l_@@_zeroth_submatch_int + \l_@@_internal_a_int } {##1} \else: - \exp_after:wN \@@_extract_b:wn \int_value:w + \__kernel_intarray_gset:Nnn \g_@@_submatch_end_intarray + { \@@_int_eval:w \l_@@_zeroth_submatch_int + \l_@@_internal_a_int - \l_@@_capturing_group_int } {##1} \fi: - \@@_int_eval:w \l_@@_zeroth_submatch_int + ##1 {##2} + \int_incr:N \l_@@_internal_a_int } - \__kernel_intarray_gset:Nnn \g_@@_submatch_prev_intarray - { \l_@@_zeroth_submatch_int } { \l_@@_start_pos_int } \fi: } -\cs_new_protected:Npn \@@_extract_b:wn #1 < #2 - { - \__kernel_intarray_gset:Nnn - \g_@@_submatch_begin_intarray {#1} {#2} - } -\cs_new_protected:Npn \@@_extract_e:wn #1 > #2 - { \__kernel_intarray_gset:Nnn \g_@@_submatch_end_intarray {#1} {#2} } % \end{macrocode} % \end{macro} % @@ -6214,12 +6247,13 @@ \group_begin: \@@_single_match: #1 - \@@_replacement:n {#2} - \exp_args:No \@@_match:n { #3 } + \exp_args:No \@@_match:n {#3} \if_meaning:w \c_false_bool \g_@@_success_bool \group_end: \else: \@@_extract: + \exp_args:No \@@_query_set:n {#3} + \@@_replacement:n {#2} \int_set:Nn \l_@@_balance_int { \@@_replacement_balance_one_match:n @@ -6259,8 +6293,9 @@ \group_begin: \@@_multi_match:n { \@@_extract: } #1 - \@@_replacement:n {#2} \exp_args:No \@@_match:n {#3} + \exp_args:No \@@_query_set:n {#3} + \@@_replacement:n {#2} \int_set:Nn \l_@@_balance_int { 0 @@ -6320,7 +6355,371 @@ % \end{macrocode} % \end{macro} % -% \subsubsection{Storing and showing compiled patterns} +% \subsubsection{Peeking ahead} +% +% \begin{variable}{\l_@@_peek_true_tl, \l_@@_peek_false_tl} +% True/false code arguments of \cs{peek_regex:nTF} or similar. +% \begin{macrocode} +\tl_new:N \l_@@_peek_true_tl +\tl_new:N \l_@@_peek_false_tl +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_@@_replacement_tl} +% When peeking in \cs{peek_regex_replace_once:nnTF} we need to store +% the replacement text. +% \begin{macrocode} +\tl_new:N \l_@@_replacement_tl +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_@@_input_tl} +% \begin{macro}{\@@_input_item:n} +% Stores each token found as \cs{@@_input_item:n} \Arg{tokens}, where +% the \meta{tokens} \texttt{o}-expand to the token found, as for +% \cs{tl_analysis_map_inline:nn}. +% \begin{macrocode} +\tl_new:N \l_@@_input_tl +\cs_new_eq:NN \@@_input_item:n ? +% \end{macrocode} +% \end{macro} +% \end{variable} +% +% \begin{macro}[TF] +% {\peek_regex:n, \peek_regex:N, \peek_regex_remove_once:n, \peek_regex_remove_once:N} +% The |T| and |F| functions just call the corresponding |TF| function. +% The four |TF| functions differ along two axes: whether to remove the +% token or not, distinguished by using \cs{@@_peek_end:} or +% \cs{@@_peek_remove_end:n} (the latter case needs an argument, as we +% will see), and whether the regex has to be compiled or is already in +% an |N|-type variable, distinguished by calling \cs{@@_build_aux:Nn} +% or \cs{@@_build_aux:NN}. The first argument of these functions is +% \cs{c_false_bool} to indicate that there should be no implicit +% insertion of a wildcard at the start of the pattern: otherwise the +% code would keep looking further into the input stream until matching +% the regex. +% \begin{macrocode} +\cs_new_protected:Npn \peek_regex:nTF #1 + { + \@@_peek:nnTF + { \@@_build_aux:Nn \c_false_bool {#1} } + { \@@_peek_end: } + } +\cs_new_protected:Npn \peek_regex:nT #1#2 + { \peek_regex:nTF {#1} {#2} { } } +\cs_new_protected:Npn \peek_regex:nF #1 { \peek_regex:nTF {#1} { } } +\cs_new_protected:Npn \peek_regex:NTF #1 + { + \@@_peek:nnTF + { \@@_build_aux:NN \c_false_bool #1 } + { \@@_peek_end: } + } +\cs_new_protected:Npn \peek_regex:NT #1#2 + { \peek_regex:NTF #1 {#2} { } } +\cs_new_protected:Npn \peek_regex:NF #1 { \peek_regex:NTF {#1} { } } +\cs_new_protected:Npn \peek_regex_remove_once:nTF #1 + { + \@@_peek:nnTF + { \@@_build_aux:Nn \c_false_bool {#1} } + { \@@_peek_remove_end:n {##1} } + } +\cs_new_protected:Npn \peek_regex_remove_once:nT #1#2 + { \peek_regex_remove_once:nTF {#1} {#2} { } } +\cs_new_protected:Npn \peek_regex_remove_once:nF #1 + { \peek_regex_remove_once:nTF {#1} { } } +\cs_new_protected:Npn \peek_regex_remove_once:NTF #1 + { + \@@_peek:nnTF + { \@@_build_aux:NN \c_false_bool #1 } + { \@@_peek_remove_end:n {##1} } + } +\cs_new_protected:Npn \peek_regex_remove_once:NT #1#2 + { \peek_regex_remove_once:NTF #1 {#2} { } } +\cs_new_protected:Npn \peek_regex_remove_once:NF #1 + { \peek_regex_remove_once:NTF #1 { } } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_peek:nnTF, \@@_peek_aux:nnTF} +% Store the user's true/false codes (plus \cs{group_end:}) into two +% token lists. Then build the automaton with |#1|, without submatch +% tracking, and aiming for a single match. Then start matching by +% setting up a few variables like for any regex matching like +% \cs{regex_match:nnTF}, with the addition of \cs{l_@@_input_tl} +% that keeps track of the tokens seen, to reinsert them at the +% end. Instead of \cs{tl_analysis_map_inline:nn} on the input, we +% call \cs{peek_analysis_map_inline:n} to go through tokens in the +% input stream. Since \cs{@@_match_one_token:nnN} calls +% \cs{@@_maplike_break:} we need to catch that and break the +% \cs{peek_analysis_map_inline:n} loop instead. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek:nnTF #1 + { + \@@_peek_aux:nnTF + { + \@@_disable_submatches: + #1 + } + } +\cs_new_protected:Npn \@@_peek_aux:nnTF #1#2#3#4 + { + \group_begin: + \tl_set:Nn \l_@@_peek_true_tl { \group_end: #3 } + \tl_set:Nn \l_@@_peek_false_tl { \group_end: #4 } + \@@_single_match: + #1 + \@@_match_init: + \tl_build_clear:N \l_@@_input_tl + \@@_match_once_init: + \peek_analysis_map_inline:n + { + \tl_build_put_right:Nn \l_@@_input_tl + { \@@_input_item:n {##1} } + \@@_match_one_token:nnN {##1} {##2} ##3 + \use_none:nnn + \prg_break_point:Nn \@@_maplike_break: + { \peek_analysis_map_break:n {#2} } + } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_peek_end:, \@@_peek_remove_end:n} +% Once the regex matches (or permanently fails to match) we call +% \cs{@@_peek_end:}, or \cs{@@_peek_remove_end:n} with argument the +% last token seen (or rather tokens that \texttt{o}-expand and +% \texttt{x}-expand to it). For \cs{peek_regex:nTF} we reinsert +% tokens seen by calling \cs{@@_peek_reinsert:N} regardless of the +% result of the match. For \cs{peek_regex_remove_once:nTF} we reinsert the +% tokens seen only if the match failed; otherwise we just reinsert the +% tokens~|#1|, with one expansion. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_end: + { + \bool_if:NTF \g_@@_success_bool + { \@@_peek_reinsert:N \l_@@_peek_true_tl } + { \@@_peek_reinsert:N \l_@@_peek_false_tl } + } +\cs_new_protected:Npn \@@_peek_remove_end:n #1 + { + \bool_if:NTF \g_@@_success_bool + { \exp_after:wN \l_@@_peek_true_tl #1 } + { \@@_peek_reinsert:N \l_@@_peek_false_tl } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_peek_reinsert:N, \@@_reinsert_item:n} +% Insert the true/false code |#1|, followed by the tokens found, which +% were stored in \cs{l_@@_input_tl}. For this, loop through that +% token list using \cs{@@_reinsert_item:n}, which expands |#1| once to +% get a single token, and jumps over it to expand what follows, with +% suitable \cs{exp:w} and \cs{exp_end:}. We cannot just use +% \cs{use:e} on the whole token list because the result may be +% unbalanced, which would stop the primitive prematurely, or let it +% continue beyond where we would like. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_reinsert:N #1 + { + \tl_build_end:N \l_@@_input_tl + \cs_set_eq:NN \@@_input_item:n \@@_reinsert_item:n + \exp_after:wN #1 \exp:w \l_@@_input_tl \exp_end: + } +\cs_new_protected:Npn \@@_reinsert_item:n #1 + { + \exp_after:wN \exp_after:wN + \exp_after:wN \exp_end: + \exp_after:wN \exp_after:wN + #1 + \exp:w + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[noTF] +% {\peek_regex_replace_once:nn, \peek_regex_replace_once:Nn} +% Similar to \cs{peek_regex:nTF} above. +% \begin{macrocode} +\cs_new_protected:Npn \peek_regex_replace_once:nnTF #1 + { \@@_peek_replace:nnTF { \@@_build_aux:Nn \c_false_bool {#1} } } +\cs_new_protected:Npn \peek_regex_replace_once:nnT #1#2#3 + { \peek_regex_replace_once:nnTF {#1} {#2} {#3} { } } +\cs_new_protected:Npn \peek_regex_replace_once:nnF #1#2 + { \peek_regex_replace_once:nnTF {#1} {#2} { } } +\cs_new_protected:Npn \peek_regex_replace_once:nn #1#2 + { \peek_regex_replace_once:nnTF {#1} {#2} { } { } } +\cs_new_protected:Npn \peek_regex_replace_once:NnTF #1 + { \@@_peek_replace:nnTF { \@@_build_aux:NN \c_false_bool #1 } } +\cs_new_protected:Npn \peek_regex_replace_once:NnT #1#2#3 + { \peek_regex_replace_once:NnTF #1 {#2} {#3} { } } +\cs_new_protected:Npn \peek_regex_replace_once:NnF #1#2 + { \peek_regex_replace_once:NnTF #1 {#2} { } } +\cs_new_protected:Npn \peek_regex_replace_once:Nn #1#2 + { \peek_regex_replace_once:NnTF #1 {#2} { } { } } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_peek_replace:nnTF} +% Same as \cs{@@_peek:nnTF} (used for \cs{peek_regex:nTF} above), but +% without disabling submatches, and with a different end. The +% replacement text |#2| is stored, to be analyzed later. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_replace:nnTF #1#2 + { + \tl_set:Nn \l_@@_replacement_tl {#2} + \@@_peek_aux:nnTF {#1} { \@@_peek_replace_end: } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_peek_replace_end:} +% If the match failed \cs{@@_peek_reinsert:N} reinserts the tokens +% found. Otherwise, finish storing the submatch information using +% \cs{@@_extract:}, and store the input into \tn{toks}. Redefine a +% few auxiliaries to change slightly their expansion behaviour as +% explained below. Analyse the replacement text with +% \cs{@@_replacement:n}, which as usual defines +% \cs{@@_replacement_do_one_match:n} to insert the tokens from the +% start of the match attempt to the beginning of the match, followed +% by the replacement text. The \cs{use:x} expands for instance the +% trailing \cs{@@_query_range:nn} down to a sequence of +% \cs{@@_reinsert_item:n} \Arg{tokens} where \meta{tokens} +% \texttt{o}-expand to a single token that we want to insert. After +% \texttt{x}-expansion, \cs{use:x} does \cs{use:n}, so we have +% \cs{exp_after:wN} \cs{l_@@_peek_true_tl} \cs{exp:w} \ldots{} +% \cs{exp_end:}. This is set up such as to obtain +% \cs{l_@@_peek_true_tl} followed by the replaced tokens (possibly +% unbalanced) in the input stream. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_replace_end: + { + \bool_if:NTF \g_@@_success_bool + { + \@@_extract: + \@@_query_set_from_input_tl: + \cs_set_eq:NN \@@_replacement_put:n \@@_peek_replacement_put:n + \cs_set_eq:NN \@@_replacement_put_submatch_aux:n + \@@_peek_replacement_put_submatch_aux:n + \cs_set_eq:NN \@@_input_item:n \@@_reinsert_item:n + \cs_set_eq:NN \@@_replacement_exp_not:N \@@_peek_replacement_token:n + \cs_set_eq:NN \@@_replacement_exp_not:V \@@_peek_replacement_var:N + \exp_args:No \@@_replacement:n { \l_@@_replacement_tl } + \use:x + { + \exp_not:n { \exp_after:wN \l_@@_peek_true_tl \exp:w } + \@@_replacement_do_one_match:n + { \l_@@_zeroth_submatch_int } + \@@_query_range:nn + { + \__kernel_intarray_item:Nn \g_@@_submatch_end_intarray + { \l_@@_zeroth_submatch_int } + } + { \l_@@_max_pos_int } + \exp_end: + } + } + { \@@_peek_reinsert:N \l_@@_peek_false_tl } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_query_set_from_input_tl:, \@@_query_set_item:n} +% The input was stored into \cs{l_@@_input_tl} as successive items +% \cs{@@_input_item:n} \Arg{tokens}. Store that in successive +% \tn{toks}. It's not clear whether the empty entries before and +% after are both useful. +% \begin{macrocode} +\cs_new_protected:Npn \@@_query_set_from_input_tl: + { + \tl_build_end:N \l_@@_input_tl + \int_zero:N \l_@@_curr_pos_int + \cs_set_eq:NN \@@_input_item:n \@@_query_set_item:n + \@@_query_set_item:n { } + \l_@@_input_tl + \@@_query_set_item:n { } + \int_set_eq:NN \l_@@_max_pos_int \l_@@_curr_pos_int + } +\cs_new_protected:Npn \@@_query_set_item:n #1 + { + \int_incr:N \l_@@_curr_pos_int + \@@_toks_set:Nn \l_@@_curr_pos_int { \@@_input_item:n {#1} } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_peek_replacement_put:n} +% While building the replacement function +% \cs{@@_replacement_do_one_match:n}, we often want to put simple +% material, given as |#1|, whose \texttt{x}-expansion +% \texttt{o}-expands to a single token. Normally we can just add the +% token to \cs{l_@@_build_tl}, but for +% \cs{peek_regex_replace_once:nnTF} we eventually want to do some +% strange expansion that is basically using \cs{exp_after:wN} to jump +% through numerous tokens (we cannot use \texttt{x}-expansion like for +% \cs{regex_replace_once:nnNTF} because it is ok for the result to be +% unbalanced since we insert it in the input stream rather than +% storing it. When within a csname we don't do any such shenanigan +% because \cs{cs:w} \ldots{} \cs{cs_end:} does all the expansion we +% need. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_replacement_put:n #1 + { + \if_case:w \l_@@_replacement_csnames_int + \tl_build_put_right:Nn \l_@@_build_tl + { \exp_not:N \@@_reinsert_item:n {#1} } + \else: + \tl_build_put_right:Nn \l_@@_build_tl {#1} + \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_peek_replacement_token:n} +% When hit with \cs{exp:w}, \cs{@@_peek_replacement_token:n} +% \Arg{token} stops \cs{exp_end:} and does \cs{exp_after:wN} +% \meta{token} \cs{exp:w} to continue expansion after it. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_replacement_token:n #1 + { \exp_after:wN \exp_end: \exp_after:wN #1 \exp:w } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_peek_replacement_put_submatch_aux:n} +% While analyzing the replacement we also have to insert submatches +% found in the query. Since query items \cs{@@_input_item:n} +% \Arg{tokens} expand correctly only when surrounded by \cs{exp:w} +% \ldots{} \cs{exp_end:}, and since these expansion controls are not +% there within csnames (because \cs{cs:w} \ldots{} \cs{cs_end:} make +% them unnecessary in most cases), we have to put \cs{exp:w} and +% \cs{exp_end:} by hand here. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_replacement_put_submatch_aux:n #1 + { + \if_case:w \l_@@_replacement_csnames_int + \tl_build_put_right:Nn \l_@@_build_tl + { \@@_query_submatch:n { \int_eval:n { #1 + ##1 } } } + \else: + \tl_build_put_right:Nn \l_@@_build_tl + { \exp:w \@@_query_submatch:n { \int_eval:n { #1 + ##1 } } \exp_end: } + \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_peek_replacement_var:N} +% This is used for |\u| outside csnames. It makes sure to continue +% expansion with \cs{exp:w} before expanding the variable~|#1| and +% stopping the \cs{exp:w} that precedes. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_replacement_var:N #1 + { + \exp_after:wN \exp_last_unbraced:NV + \exp_after:wN \exp_end: + \exp_after:wN #1 + \exp:w + } +% \end{macrocode} +% \end{macro} % % \subsection{Messages} % @@ -6726,8 +7125,6 @@ %^^A NOT IMPLEMENTED %^^A \p{xx} a character with the xx property %^^A \P{xx} a character without the xx property -%^^A [[:xxx:]] positive POSIX named set -%^^A [[:^xxx:]] negative POSIX named set %^^A (?=...) positive look ahead %^^A (?!...) negative look ahead %^^A (?<=...) positive look behind |