diff options
Diffstat (limited to 'Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx')
| -rw-r--r-- | Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx | 1099 |
1 files changed, 661 insertions, 438 deletions
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx index 3b26d43f4e5..ddfbc79de23 100644 --- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx @@ -23,8 +23,8 @@ %<*driver|package> % The version of expl3 required is tested as early as possible, as % some really old versions do not define \ProvidesExplPackage. -\RequirePackage{expl3}[2017/04/01] -%<package>\@ifpackagelater{expl3}{2017/04/01} +\RequirePackage{expl3}[2017/05/13] +%<package>\@ifpackagelater{expl3}{2017/05/13} %<package> {} %<package> {% %<package> \PackageError{l3regex}{Support package l3kernel too old} @@ -59,7 +59,7 @@ % }^^A % } % -% \date{Released 2017/04/01} +% \date{Released 2017/05/13} % % \maketitle % @@ -352,21 +352,37 @@ % \subsection{Syntax of the replacement text} % % Most of the features described in regular expressions do not make -% sense within the replacement text. Escaped characters are supported -% as inside regular expressions. The whole match is accessed as~|\0|, -% and the first~$9$ submatches are accessed as |\1|, \ldots{},~|\9|. -% Further submatches are accessed through |\g{|\meta{number}|}| where -% \meta{number} is any non-negative integer. If there are fewer than -% \meta{number} capturing groups, the submatch is empty. +% sense within the replacement text. Backslash introduces various +% special constructions: +% \begin{itemize} +% \item |\0| is the whole match; +% \item |\1|, |\2|, \ldots{}, |\9| or |\g{|\meta{number}|}| are the +% submatches (empty if there are fewer than \meta{number} capturing +% groups); +% \item \verb*|\ | inserts a space (spaces are ignored when not +% escaped); +% \item |\a|, |\e|, |\f|, |\n|, |\r|, |\t|, |\xhh|, |\x{hhh}| +% correspond to single characters as in regular expressions; +% \item |\c|\Arg{cs~name} inserts a control sequence; +% \item |\c|\meta{category}\meta{character} (see below); +% \item |\u|\Arg{tl~var~name} inserts the contents of the +% \meta{tl~var} (see below). +% \end{itemize} +% Characters other than backslash and space are simply inserted in the +% result (but since the replacement text is first converted to a string, +% one should also escape characters that are special for \TeX{}, for +% instance use~|\#|). Non-alphanumeric characters can always be safely +% escaped with a backslash. % % For instance, % \begin{verbatim} % \tl_set:Nn \l_my_tl { Hello,~world! } -% \regex_replace_all:nnN { ([er]?l|o) . } { \(\0\-\-\1\) } \l_my_tl +% \regex_replace_all:nnN { ([er]?l|o) . } { (\0--\1) } \l_my_tl % \end{verbatim} % results in \cs{l_my_tl} holding |H(ell--el)(o,--o) w(or--o)(ld--l)!| % -% Submatches keep the same category codes as in the original token list. +% Submatches always keep the same category codes as in the original +% token list. % The characters inserted by the replacement have category code $12$ % (other) by default, with the exception of space characters. Spaces % inserted through \verb*|\ | have category code $10$, while spaces @@ -374,21 +390,25 @@ % The escape sequence |\c| allows to insert characters % with arbitrary category codes, as well as control sequences. % \begin{l3regex-syntax} -% \item[\\cXY] Produces the character~|Y| (which can be given as an -% escape sequence such as~|\t| for tab, or |\(| or~|\)| for a -% parenthesis) with category code~|X|, which must be one of -% |CBEMTPUDSLOA|. +% \item[\\cX(\ldots{})] Produces the characters \enquote{\ldots{}} with +% category~|X|, which must be one of |CBEMTPUDSLOA| as in regular +% expressions. Parentheses are optional for a single character (which +% can be an escape sequence). This can be nested, for instance +% |\cL(Hello\cS\ world)!| % \item[\\c\Arg{text}] Produces the control sequence with csname % \meta{text}. The \meta{text} may contain references to the -% submatches |\0|, |\1|, \emph{etc.} +% submatches |\0|, |\1|, and so on, as in the example for |\u| below. % \end{l3regex-syntax} % % The escape sequence |\u|\Arg{tl~var~name} allows to insert the % contents of the token list with name \meta{tl~var~name} directly into -% the replacement, avoiding the need to escape special characters. -% Within the construction |\c|\Arg{text}, the |\u|~escape sequence only -% expands its argument once, in effect performing \cs{tl_to_str:v}. -% Submatches can be used within the argument of |\u|. For instance, +% the replacement, giving an easier control of category codes. +% Within |\c{|\ldots{}|}| and |\u{|\ldots{}|}| constructions, the |\u| +% and |\c|~escape sequences perform \cs{tl_to_str:v}, namely extract the +% value of the control sequence and turn it into a string. +% +% Matches can be used within the arguments of |\c| and |\u|. For +% instance, % \begin{verbatim} % \tl_set:Nn \l_my_one_tl { first } % \tl_set:Nn \l_my_two_tl { \emph{second} } @@ -611,7 +631,8 @@ % \item Currently, |a{\x34}| is recognized as |a{4}|. % \item Cleaner error reporting in the replacement phase. % \item Add tracing information. -% \item Detect attempts to use back-references. +% \item Detect attempts to use back-references and other +% non-implemented syntax. % \item Test for the maximum register \cs{c_max_register_int}. % \item Find out whether the fact that |\W| and friends match the % end-marker leads to bugs. Possibly update \cs{__regex_item_reverse:n}. @@ -622,23 +643,19 @@ % % Code improvements to come. % \begin{itemize} -% \item Change \tn{skip} to \tn{dimen} for the array of active -% threads, and shift the array of submatch informations so that it -% starts at \tn{skip}$0$. -% \item Optimize |\c{abc}| for matching a specific control sequence. +% \item Shift arrays so that the useful information starts at +% position~$1$. % \item Only build \c{...} once. -% \item Use \tn{skip} for the left and right state stacks when +% \item Use arrays for the left and right state stacks when % compiling a regex. % \item Should \cs{__regex_action_free_group:n} only be used for greedy % |{n,}| quantifier? (I think not.) % \item Quantifiers for |\u| and assertions. -% \item Improve digit grabbing for the |\g| escape in replacement. -% Allow arbitrary integer expressions for all those numbers? % \item When matching, keep track of an explicit stack of % \texttt{current_state} and \texttt{current_submatches}. % \item If possible, when a state is reused by the same thread, kill % other subthreads. -% \item Use \tn{dimen} registers rather than \cs{l__regex_balance_tl} +% \item Use an array rather than \cs{l__regex_balance_tl} % to build \cs{__regex_replacement_balance_one_match:n}. % \item Reduce the number of epsilon-transitions in alternatives. % \item Optimize simple strings: use less states (|abcade| should give @@ -650,8 +667,6 @@ % \item Optimize the use of \cs{int_step_...} functions. % \item Groups don't capture within regexes for csnames; optimize and % document. -% \item Decide and document what |\c{\c{...}}| should do in the -% replacement text, similar questions for |\u|. % \item Better \enquote{show} for anchors, properties, and catcode tests. % \item Does |\K| really need a new state for itself? % \item When compiling, use a boolean \texttt{in_cs} and less magic @@ -664,7 +679,6 @@ % The following features are likely to be implemented at some point % in the future. % \begin{itemize} -% \item Allow |\cL(abc)| in replacement text. % \item General look-ahead/behind assertions. % \item Regex matching on external files. % \item Conditional subpatterns with look ahead/behind: \enquote{if @@ -680,13 +694,11 @@ % This requires to manipulate a lot of data, probably using tree-boxes. % \end{itemize} % -% The following features of \textsc{pcre} or Perl will probably not be +% The following features of \textsc{pcre} or Perl may or may not be % implemented. % \begin{itemize} % \item |\ddd|, matching the character with octal code \texttt{ddd}; -% \item Callout with |(?C...)|, we cannot run arbitrary user code -% during the matching, because the regex code uses registers in an -% unsafe way; +% \item Callout with |(?C...)|; % \item Conditional subpatterns (other than with a look-ahead or % look-behind condition): this is non-regular, isn't it? % \item Named subpatterns: \TeX{} programmers have lived so far @@ -705,9 +717,7 @@ % non-backtracking algorithm, and difficult to implement. % \item Subroutine calls: this syntactic sugar is difficult to include % in a non-backtracking algorithm, in particular because the -% corresponding group should be treated as atomic. Also, we cannot -% afford to run user code within the regular expression matching, -% because of our \enquote{misuse} of registers. +% corresponding group should be treated as atomic. % \item Recursion: this is a non-regular feature. % \item Back-references: non-regular feature, this requires % backtracking, which is prohibitively slow. @@ -734,9 +744,9 @@ % % \begin{macrocode} %<*package> -\ProvidesExplPackage{l3regex}{2017/04/01}{} +\ProvidesExplPackage{l3regex}{2017/05/13}{} {L3 Experimental regular expressions} -\RequirePackage{l3tl-build, l3tl-analysis, l3str-convert} +\RequirePackage{l3tl-build, l3tl-analysis, l3intarray} %</package> % \end{macrocode} % @@ -754,7 +764,7 @@ % \item (Compiling.) Analyse the regex, finding invalid input, and % convert it to an internal representation. % \item (Building.) Convert the compiled regex to a non-deterministic -% finite automaton (\textsc{nfa}) with roughly $n$ states which +% finite automaton (\textsc{nfa}) with $O(n)$ states which % accepts precisely token lists matching that regex. % \item (Matching.) Loop through the query token list one token (one % \enquote{position}) at a time, exploring in parallel every @@ -787,46 +797,42 @@ % unique id for all the steps of the matching algorithm. % \end{itemize} % -% To achieve a good performance, we abuse \TeX{}'s registers in two -% ways. We access registers directly by number rather than tying them -% to control sequence using \cs{int_new:N} and other allocation -% functions. And we store integers in \tn{dimen} registers in scaled -% points (\texttt{sp}), using \TeX{}'s implicit conversion from -% dimensions to integers in some contexts. Specifically, the registers -% are used as follows. When compiling, \tn{toks} registers are used -% under the hood by functions from the \pkg{l3tl-build} module. When -% building, +% We use \pkg{l3intarray} to manipulate arrays of integers (stored into +% some dimension registers in scaled points). We also abuse \TeX{}'s +% \tn{toks} registers, by accessing them directly by number rather than +% tying them to control sequence using the \tn{newtoks} allocation +% functions. Specifically, these arrays and \tn{toks} are used as +% follows. When compiling, \tn{toks} registers are used under the hood +% by functions from the \pkg{l3tl-build} module. When building, +% \tn{toks}\meta{state} holds the tests and actions to perform in the +% \meta{state} of the \textsc{nfa}. When matching, % \begin{itemize} -% \item \tn{toks}\meta{state} holds the tests and actions to perform -% in the \meta{state} of the \textsc{nfa}. -% \item (Not implemented yet.) -% \tn{skip}$i$ has the form \meta{group id} \texttt{plus} -% \meta{left state} \texttt{minus} \meta{right state}. -% \end{itemize} -% When matching, -% \begin{itemize} -% \item \tn{dimen}\meta{state} is equal to the last \meta{step} in -% which the \meta{state} was active. -% \item (Currently, we use \tn{skip} instead of \tn{dimen}.) -% \tn{dimen}\meta{thread}, with $\texttt{min_active} \leq -% \meta{thread} < \texttt{max_active}$, is equal to the -% \meta{state} in which the \meta{thread} currently is. The +% \item \cs{g_@@_state_active_intarray} holds the last \meta{step} in +% which each \meta{state} was active. +% \item \cs{g_@@_thread_state_intarray} maps each \meta{thread} (with +% $\texttt{min_active} \leq \meta{thread} < \texttt{max_active}$) to +% the \meta{state} in which the \meta{thread} currently is. The % \meta{threads} or ordered starting from the best to the least % preferred. % \item \tn{toks}\meta{thread} holds the submatch information for the % \meta{thread}, as the contents of a property list. -% \item \tn{muskip}\meta{position} holds as its main and stretch -% components the character and category code of the token at this +% \item \cs{g_@@_charcode_intarray} and \cs{g_@@_catcode_intarray} hold the +% character codes and category codes of tokens at each % \meta{position} in the query. +% \item \cs{g_@@_balance_intarray} holds the balance of begin-group and +% end-group character tokens which appear before that point in the +% token list. % \item \tn{toks}\meta{position} holds \meta{tokens} which \texttt{o}- % and \texttt{x}-expand to the \meta{position}-th token in the query. -% \item \tn{skip} registers hold the value of end-points of all -% submatches as would be extracted by the \cs{regex_extract} -% functions. Since smaller \tn{skip} registers are used, the minimum -% index is twice \texttt{max_state}, and the used registers go up to -% \cs{l_@@_submatch_int}. They are organized in blocks of -% \texttt{capturing_group}, each block corresponding to one match -% with all its submatches stored in consecutive \tn{skip}s. +% \item \cs{g_@@_submatch_prev_intarray}, \cs{g_@@_submatch_begin_intarray} +% and \cs{g_@@_submatch_end_intarray} hold, for each submatch (as would +% be extracted by \cs{regex_extract_all:nnN}), the place where the +% submatch started to be looked for and its two end-points. For +% historical reasons, the minimum index is twice \texttt{max_state}, +% and the used registers go up to \cs{l_@@_submatch_int}. They are +% organized in blocks of \cs{l_@@_capturing_group_int} entries, each +% block corresponding to one match with all its submatches stored in +% consecutive entries. % \end{itemize} % \tn{count} registers are not abused, which means that we can safely % use named integers in this module. Note that \tn{box} registers are @@ -851,6 +857,89 @@ % \end{macrocode} % \end{macro} % +% \begin{macro}[int]{\@@_standard_escapechar:} +% Make the \tn{escapechar} into the standard backslash. +% \begin{macrocode} +\cs_new_protected:Npn \@@_standard_escapechar: + { \int_set:Nn \tex_escapechar:D { `\\ } } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int, EXP]{\@@_toks_use:w} +% Unpack a \tn{toks} given its number. +% \begin{macrocode} +\cs_new:Npn \@@_toks_use:w { \tex_the:D \tex_toks:D } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int]{\@@_toks_clear:N, \@@_toks_set:Nn, \@@_toks_set:No} +% Empty a \tn{toks} or set it to a value, given its number. +% \begin{macrocode} +\cs_new_protected:Npn \@@_toks_clear:N #1 + { \tex_toks:D #1 { } } +\cs_new_eq:NN \@@_toks_set:Nn \tex_toks:D +\cs_new_protected:Npn \@@_toks_set:No #1 + { \@@_toks_set:Nn #1 \exp_after:wN } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int]{\@@_toks_memcpy:NNn} +% Copy |#3| \tn{toks} registers from |#2| onwards to |#1| onwards, +% like |C|'s |memcpy|. +% \begin{macrocode} +\cs_new_protected:Npn \@@_toks_memcpy:NNn #1#2#3 + { + \prg_replicate:nn {#3} + { + \tex_toks:D #1 = \tex_toks:D #2 + \int_incr:N #1 + \int_incr:N #2 + } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int]{\@@_toks_put_left:Nx} +% \begin{macro}[int]{\@@_toks_put_right:Nx, \@@_toks_put_right:Nn} +% During the building phase we wish to add \texttt{x}-expanded +% material to \tn{toks}, either to the left or to the right. The +% expansion is done \enquote{by hand} for optimization (these +% operations are used quite a lot). The \texttt{Nn} version of +% \cs{@@_toks_put_right:Nx} is provided because it is more +% efficient than \texttt{x}-expanding with \cs{exp_not:n}. +% \begin{macrocode} +\cs_new_protected:Npn \@@_toks_put_left:Nx #1#2 + { + \cs_set:Npx \@@_tmp:w { #2 } + \tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN + { \exp_after:wN \@@_tmp:w \tex_the:D \tex_toks:D #1 } + } +\cs_new_protected:Npn \@@_toks_put_right:Nx #1#2 + { + \cs_set:Npx \@@_tmp:w {#2} + \tex_toks:D #1 \exp_after:wN + { \tex_the:D \tex_toks:D \exp_after:wN #1 \@@_tmp:w } + } +\cs_new_protected:Npn \@@_toks_put_right:Nn #1#2 + { \tex_toks:D #1 \exp_after:wN { \tex_the:D \tex_toks:D #1 #2 } } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int, rEXP]{\@@_current_cs_to_str:} +% Expands to the string representation of the token (known to be a +% control sequence) at the current position \cs{l_@@_current_pos_int}. +% It should only be used in \texttt{x}-expansion to avoid losing a +% leading space. +% \begin{macrocode} +\cs_new:Npn \@@_current_cs_to_str: + { + \exp_after:wN \exp_after:wN \exp_after:wN \cs_to_str:N + \tex_the:D \tex_toks:D \l_@@_current_pos_int + } +% \end{macrocode} +% \end{macro} +% % \subsubsection{Constants and variables} % % \begin{macro}[aux]{\@@_tmp:w} @@ -894,14 +983,24 @@ % \end{macrocode} % \end{variable} % -% \begin{variable}{\l_@@_balance_int} +% \begin{variable}{\g_@@_charcode_intarray, \g_@@_catcode_intarray, \g_@@_balance_intarray} % The first thing we do when matching is to go once through the query -% token list and store the information for each token as \tn{muskip} -% and \tn{toks} registers. During this phase, \cs{l_@@_balance_int} -% counts the balance of begin-group and end-group character tokens -% which appear before a given point in the token list, and we store it -% as the shrink component of each \tn{muskip} register. This variable -% is also used to keep track of the balance in the replacement text. +% token list and store the information for each token into +% \cs{g_@@_charcode_intarray}, \cs{g_@@_catcode_intarray} and \tn{toks} +% registers. We also store the balance of begin-group/end-group +% characters into \cs{g_@@_balance_intarray}. +% \begin{macrocode} +\__intarray_new:Nn \g_@@_charcode_intarray { 65536 } +\__intarray_new:Nn \g_@@_catcode_intarray { 65536 } +\__intarray_new:Nn \g_@@_balance_intarray { 65536 } +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_@@_balance_int} +% During this phase, \cs{l_@@_balance_int} counts the balance of +% begin-group and end-group character tokens which appear before a +% given point in the token list. This variable is also used to keep +% track of the balance in the replacement text. % \begin{macrocode} \int_new:N \l_@@_balance_int % \end{macrocode} @@ -918,6 +1017,20 @@ % % \subsubsection{Testing characters} % +% \begin{macro}{\c_@@_ascii_min_int, \c_@@_ascii_max_control_int, \c_@@_ascii_max_int} +% \begin{macrocode} +\int_const:Nn \c_@@_ascii_min_int { 0 } +\int_const:Nn \c_@@_ascii_max_control_int { 31 } +\int_const:Nn \c_@@_ascii_max_int { 127 } +% \end{macrocode} +% \end{macro} +% +% \begin{variable}{\c_@@_ascii_lower_int} +% \begin{macrocode} +\int_const:Nn \c_@@_ascii_lower_int { `a - `A } +% \end{macrocode} +% \end{variable} +% % \begin{macro}[int]{\@@_break_point:TF} % \begin{macro}[int]{\@@_break_true:w} % When testing whether a character of the query token list matches @@ -1027,12 +1140,12 @@ \if_int_compare:w \l_@@_current_char_int > `Z \exp_stop_f: \if_int_compare:w \l_@@_current_char_int > `z \exp_stop_f: \else: \if_int_compare:w \l_@@_current_char_int < `a \exp_stop_f: \else: - \int_sub:Nn \l_@@_case_changed_char_int { \c__str_ascii_lower_int } + \int_sub:Nn \l_@@_case_changed_char_int { \c_@@_ascii_lower_int } \fi: \fi: \else: \if_int_compare:w \l_@@_current_char_int < `A \exp_stop_f: \else: - \int_add:Nn \l_@@_case_changed_char_int { \c__str_ascii_lower_int } + \int_add:Nn \l_@@_case_changed_char_int { \c_@@_ascii_lower_int } \fi: \fi: } @@ -1099,12 +1212,7 @@ \int_compare:nNnTF \l_@@_current_catcode_int = 0 { \tl_set:Nx \l_@@_internal_a_tl - { - \scan_stop: - \exp_after:wN \exp_after:wN \exp_after:wN \cs_to_str:N - \tex_the:D \tex_toks:D \l_@@_current_pos_int - \scan_stop: - } + { \scan_stop: \@@_current_cs_to_str: \scan_stop: } \tl_if_in:noTF { \scan_stop: #1 \scan_stop: } \l_@@_internal_a_tl { \@@_break_true:w } { } } @@ -1128,12 +1236,7 @@ \int_compare:nNnT \l_@@_current_catcode_int = 0 { \group_begin: - \tl_set:Nx \l_@@_cs_name_tl - { - \exp_after:wN \exp_after:wN - \exp_after:wN \cs_to_str:N - \tex_the:D \tex_toks:D \l_@@_current_pos_int - } + \tl_set:Nx \l_@@_cs_name_tl { \@@_current_cs_to_str: } \@@_single_match: \@@_disable_submatches: \@@_build_for_cs:n {#1} @@ -1214,16 +1317,16 @@ \cs_new_protected:Npn \@@_posix_ascii: { \@@_item_caseful_range:nn - \c__str_ascii_min_int - \c__str_ascii_max_int + \c_@@_ascii_min_int + \c_@@_ascii_max_int } \cs_new_eq:NN \@@_posix_blank: \@@_prop_h: \cs_new_protected:Npn \@@_posix_cntrl: { \@@_item_caseful_range:nn - \c__str_ascii_min_int - \c__str_ascii_max_control_int - \@@_item_caseful_equal:n \c__str_ascii_max_int + \c_@@_ascii_min_int + \c_@@_ascii_max_control_int + \@@_item_caseful_equal:n \c_@@_ascii_max_int } \cs_new_eq:NN \@@_posix_digit: \@@_prop_d: \cs_new_protected:Npn \@@_posix_graph: @@ -1301,8 +1404,8 @@ \cs_set:Npn \@@_escape_unescaped:N ##1 { #1 } \cs_set:Npn \@@_escape_escaped:N ##1 { #2 } \cs_set:Npn \@@_escape_raw:N ##1 { #3 } - \int_set:Nn \tex_escapechar:D { `\\ } - \__str_gset_other:Nn \g_@@_internal_tl { #4 } + \@@_standard_escapechar: + \tl_gset:Nx \g_@@_internal_tl { \__str_to_other_fast:n {#4} } \tl_set:Nx \l_@@_internal_b_tl { \exp_after:wN \@@_escape_loop:N \g_@@_internal_tl @@ -1389,12 +1492,11 @@ % % \begin{macro}[aux]{\@@_escape_/x:w} % \begin{macro}[aux]{\@@_escape_x_end:w, \@@_escape_x_large:n} -% When |\x| is encountered, \cs{@@_escape_x_test:N} is responsible -% for grabbing some hexadecimal digits, and feeding the result to -% \cs{@@_escape_x_end:w}. If the number is $<256$, then it is -% turned into a byte and fed to \cs{@@_escape_raw:N}. Otherwise, -% interrupt the assignment, and either produce an error, or use a -% standard \tn{lowercase} trick depending on the precise value. +% When |\x| is encountered, \cs{@@_escape_x_test:N} is responsible for +% grabbing some hexadecimal digits, and feeding the result to +% \cs{@@_escape_x_end:w}. If the number is too big interrupt the +% assignment and produce an error, otherwise call \cs{@@_escape_raw:N} +% on the corresponding character token. % \begin{macrocode} \cs_new:cpn { @@_escape_/x:w } \@@_escape_loop:N { @@ -1403,40 +1505,23 @@ } \cs_new:Npn \@@_escape_x_end:w #1 ; { - \int_compare:nNnTF {#1} > \c__str_max_byte_int - { \@@_escape_x_large:n {#1} } + \int_compare:nNnTF {#1} > \c_max_char_int + { + \if_false: { \fi: } + \__tl_build_one:o \l_@@_internal_b_tl + \__msg_kernel_error:nnx { regex } { x-overflow } {#1} + \tl_set:Nx \l_@@_internal_b_tl + { \if_false: } \fi: + } { \exp_last_unbraced:Nf \@@_escape_raw:N - { \__str_output_byte:n {#1} } + { \char_generate:nn {#1} { 12 } } } } -\group_begin: - \char_set_catcode_other:n { 0 } - \cs_new:Npn \@@_escape_x_large:n #1 - { - \if_false: { \fi: } - \__tl_build_one:o \l_@@_internal_b_tl - \int_compare:nNnTF {#1} > \c_max_char_int - { - \__msg_kernel_error:nnx { regex } { x-overflow } {#1} - \tl_set:Nx \l_@@_internal_b_tl - { \if_false: } \fi: - } - { - \char_set_lccode:nn { 0 } {#1} - \tex_lowercase:D - { - \tl_set:Nx \l_@@_internal_b_tl - { \if_false: } \fi: - \@@_escape_raw:N ^^@ - } - } - } -\group_end: % \end{macrocode} % \end{macro} % \end{macro} -% ^^A todo: use char_generate in above and rest of file +% % \begin{macro}[aux]{\@@_escape_x_test:N, \@@_escape_x_testii:N} % Find out whether the first character is a left brace (allowing any % number of hexadecimal digits), or not (allowing up to two @@ -1461,7 +1546,7 @@ \if_charcode:w \c_left_brace_str #1 \exp_after:wN \@@_escape_x_loop:N \else: - \__str_hexadecimal_use:NTF #1 + \@@_hexadecimal_use:NTF #1 { \exp_after:wN \@@_escape_x:N } { ; \exp_after:wN \@@_escape_loop:N \exp_after:wN #1 } \fi: @@ -1476,7 +1561,7 @@ { \str_if_eq_x:nnTF {#1} { break } { ; } { - \__str_hexadecimal_use:NTF #1 + \@@_hexadecimal_use:NTF #1 { ; \@@_escape_loop:N } { ; \@@_escape_loop:N #1 } } @@ -1494,7 +1579,7 @@ \str_if_eq_x:nnTF {#1} { break } { ; \@@_escape_x_loop_error:n { } {#1} } { - \__str_hexadecimal_use:NTF #1 + \@@_hexadecimal_use:NTF #1 { \@@_escape_x_loop:N } { \token_if_eq_charcode:NNTF \c_space_token #1 @@ -1520,6 +1605,35 @@ % \end{macrocode} % \end{macro} % +% \begin{macro}[aux, rEXP]{\@@_hexadecimal_use:NTF} +% \TeX{} detects uppercase hexadecimal digits for us but not the +% lowercase letters, which we need to detect and replace by their +% uppercase counterpart. +% \begin{macrocode} +\prg_new_conditional:Npnn \@@_hexadecimal_use:N #1 { TF } + { + \if_int_compare:w 1 < "1 \token_to_str:N #1 \exp_stop_f: + #1 \prg_return_true: + \else: + \if_case:w \__int_eval:w + \exp_after:wN ` \token_to_str:N #1 - `a + \__int_eval_end: + A + \or: B + \or: C + \or: D + \or: E + \or: F + \else: + \prg_return_false: + \exp_after:wN \use_none:n + \fi: + \prg_return_true: + \fi: + } +% \end{macrocode} +% \end{macro} +% % \begin{macro}[EXP, aux] % {\@@_char_if_alphanumeric:NTF, \@@_char_if_special:NTF} % These two tests are used in the first pass when parsing a regular @@ -1549,7 +1663,7 @@ { \if_int_compare:w `#1 > `Z \exp_stop_f: \if_int_compare:w `#1 > `z \exp_stop_f: - \if_int_compare:w `#1 < \c__str_ascii_max_int + \if_int_compare:w `#1 < \c_@@_ascii_max_int \prg_return_true: \else: \prg_return_false: \fi: \else: \if_int_compare:w `#1 < `a \exp_stop_f: @@ -1991,7 +2105,7 @@ \cs_new_protected:Npn \@@_compile:n #1 { \@@_compile:w - \int_set:Nn \tex_escapechar:D { `\\ } + \@@_standard_escapechar: \int_set_eq:NN \l_@@_mode_int \c_@@_outer_mode_int \@@_escape_use:nnnn { @@ -2915,7 +3029,7 @@ { \if_int_odd:w \__int_eval:w \l_@@_catcodes_int / #1 \__int_eval_end: \else: - \tex_advance:D \l_@@_catcodes_int #1 + \int_add:Nn \l_@@_catcodes_int {#1} \fi: } \cs_new_protected:Npn \@@_compile_c_lbrack_end: @@ -3105,8 +3219,8 @@ % \begin{macrocode} \cs_new_protected:Npn \@@_compile_u_in_cs: { - \exp_args:NNo \__str_gset_other:Nn \g_@@_internal_tl - { \l_@@_internal_a_tl } + \tl_gset:Nx \g_@@_internal_tl + { \exp_args:No \__str_to_other_fast:n { \l_@@_internal_a_tl } } \__tl_build_one:x { \tl_map_function:NN \g_@@_internal_tl @@ -3396,10 +3510,11 @@ % \begin{variable}{\l_@@_min_state_int, \l_@@_max_state_int} % The last state that was allocated is $\cs{l_@@_max_state_int}-1$, % so that \cs{l_@@_max_state_int} always points to a free state. -% The \texttt{min_state} variable is always $0$, but is included to -% avoid hard-coding this value. +% The \texttt{min_state} variable is $1$, but is included to +% avoid hard-coding this value everywhere. % \begin{macrocode} \int_new:N \l_@@_min_state_int +\int_set:Nn \l_@@_min_state_int { 1 } \int_new:N \l_@@_max_state_int % \end{macrocode} % \end{variable} @@ -3489,7 +3604,7 @@ \cs_new_protected:Npn \@@_build:N #1 { %<trace> \trace_push:nnn { regex } { 1 } { @@_build } - \int_set:Nn \tex_escapechar:D { `\\ } + \@@_standard_escapechar: \int_zero:N \l_@@_capturing_group_int \int_set_eq:NN \l_@@_max_state_int \l_@@_min_state_int \@@_build_new_state: @@ -3557,33 +3672,6 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}[int]{\@@_toks_put_left:Nx} -% \begin{macro}[int]{\@@_toks_put_right:Nx, \@@_toks_put_right:Nn} -% During the building phase we wish to add \texttt{x}-expanded -% material to \tn{toks}, either to the left or to the right. The -% expansion is done \enquote{by hand} for optimization (these -% operations are used quite a lot). The \texttt{Nn} version of -% \cs{@@_toks_put_right:Nx} is provided because it is more -% efficient than \texttt{x}-expanding with \cs{exp_not:n}. -% \begin{macrocode} -\cs_new_protected:Npn \@@_toks_put_left:Nx #1#2 - { - \cs_set:Npx \@@_tmp:w { #2 } - \tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN - { \exp_after:wN \@@_tmp:w \tex_the:D \tex_toks:D #1 } - } -\cs_new_protected:Npn \@@_toks_put_right:Nx #1#2 - { - \cs_set:Npx \@@_tmp:w {#2} - \tex_toks:D #1 \exp_after:wN - { \tex_the:D \tex_toks:D \exp_after:wN #1 \@@_tmp:w } - } -\cs_new_protected:Npn \@@_toks_put_right:Nn #1#2 - { \tex_toks:D #1 \exp_after:wN { \tex_the:D \tex_toks:D #1 #2 } } -% \end{macrocode} -% \end{macro} -% \end{macro} -% % \begin{macro}[int] % { % \@@_build_transition_left:NNN, @@ -3620,7 +3708,7 @@ \int_eval:n { \l_@@_max_state_int + 1 } } %</trace> - \tex_toks:D \l_@@_max_state_int { } + \@@_toks_clear:N \l_@@_max_state_int \int_set_eq:NN \l_@@_left_state_int \l_@@_right_state_int \int_set_eq:NN \l_@@_right_state_int \l_@@_max_state_int \int_incr:N \l_@@_max_state_int @@ -3933,15 +4021,12 @@ ( #1 - 1 ) * ( \l_@@_internal_b_int - \l_@@_internal_a_int ) } - \tex_advance:D \l_@@_right_state_int \l_@@_internal_c_int - \tex_advance:D \l_@@_max_state_int \l_@@_internal_c_int - \prg_replicate:nn \l_@@_internal_c_int - { - \tex_toks:D \l_@@_internal_b_int - = \tex_toks:D \l_@@_internal_a_int - \int_incr:N \l_@@_internal_a_int - \int_incr:N \l_@@_internal_b_int - } + \int_add:Nn \l_@@_right_state_int { \l_@@_internal_c_int } + \int_add:Nn \l_@@_max_state_int { \l_@@_internal_c_int } + \@@_toks_memcpy:NNn + \l_@@_internal_b_int + \l_@@_internal_a_int + \l_@@_internal_c_int \fi: } % \end{macrocode} @@ -4132,7 +4217,7 @@ % transitions, the instruction at the new state of the \textsc{nfa} is % performed immediately. When a transition consumes a character, the % new state is appended to a list of \enquote{active states}, stored in -% \tn{skip} registers: this thread will be active again when the next +% \cs{g_@@_thread_state_intarray}: this thread will be active again when the next % token is read from the query. At every step (for each token in the % query), we unpack that list of active states and the corresponding % submatch props, and empty those. @@ -4176,7 +4261,7 @@ % } % The tokens in the query are indexed from \texttt{min_pos} for the % first to $\texttt{max_pos}-1$ for the last, and their information is -% stored in \tn{muskip} and \tn{toks} registers with those numbers. We +% stored in several arrays and \tn{toks} registers with those numbers. We % don't start from $0$ because the \tn{toks} registers with low % numbers are used to hold the states of the \textsc{nfa}. We match % without backtracking, keeping all threads in lockstep at the @@ -4243,15 +4328,15 @@ % % \begin{variable}{\l_@@_step_int} % This integer, always even, is increased every time a character in -% the query is read, and not reset when doing multiple matches. For -% each \meta{state} in the \textsc{nfa} we store in -% \tn{dimen}\meta{state} the last step in which this state was -% encountered. This lets us break infinite loops by not visiting the -% same state twice in the same step. In fact, \tn{dimen}\meta{state} -% is equal \texttt{step} when we have started performing the -% operations of \tn{toks}\meta{state}, but not finished yet. However, -% once we finish, we set \tn{dimen}\meta{state} to -% $\text{\texttt{step}}+1$. This is needed to track submatches +% the query is read, and not reset when doing multiple matches. We +% store in \cs{g_@@_state_active_intarray} the last step in which each +% \meta{state} in the \textsc{nfa} was encountered. This lets us break +% infinite loops by not visiting the same state twice in the same +% step. In fact, the step we store is equal to \texttt{step} when we +% have started performing the operations of \tn{toks}\meta{state}, but +% not finished yet. However, once we finish, we store +% $\text{\texttt{step}}+1$ in \cs{g_@@_state_active_intarray}. This is +% needed to track submatches % properly (see building phase). The \texttt{step} is also used to % attach each set of submatch information to a given iteration (and % automatically discard it when it corresponds to a past step). @@ -4261,8 +4346,8 @@ % \end{variable} % % \begin{variable}{\l_@@_min_active_int, \l_@@_max_active_int} -% All the currently active states are kept in order of precedence in -% the \tn{skip} registers, and the corresponding submatches in the +% All the currently active threads are kept in order of precedence in +% \cs{g_@@_thread_state_intarray}, and the corresponding submatches in the % \tn{toks}. For our purposes, those serve as an array, indexed from % \texttt{min_active} (inclusive) to \texttt{max_active} (excluded). % At the start of every step, the whole array is unpacked, so that the @@ -4274,6 +4359,17 @@ % \end{macrocode} % \end{variable} % +% \begin{variable}{\g_@@_state_active_intarray, \g_@@_thread_state_intarray} +% \cs{g_@@_state_active_intarray} stores the last \meta{step} in which +% each \meta{state} was active. \cs{g_@@_thread_state_intarray} stores +% threads that will be considered in the next step, more precisely the +% states in which these threads are. +% \begin{macrocode} +\__intarray_new:Nn \g_@@_state_active_intarray { 65536 } +\__intarray_new:Nn \g_@@_thread_state_intarray { 65536 } +% \end{macrocode} +% \end{variable} +% % \begin{variable}{\l_@@_every_match_tl} % Every time a match is found, this token list is used. For single % matching, the token list is empty. For multiple matching, the token @@ -4332,11 +4428,11 @@ % \subsubsection{Matching: framework} % % \begin{macro}[int]{\@@_match:n} -% First store the query into \tn{toks} and \tn{muskip} registers (see +% First store the query into \tn{toks} registers and arrays (see % \cs{@@_query_set:nnn}). Then initialize the variables that should % be set once for each user function (even for multiple % matches). Namely, the overall matching is not yet successful; none of -% the states should be marked as visited (\tn{dimen} registers), and +% the states should be marked as visited (\cs{g_@@_state_active_intarray}), and % we start at step $0$; we pretend that there was a previous match % ending at the start of the query, which was not empty (to avoid % smothering an empty match at the start). Once all this is set up, we @@ -4358,12 +4454,13 @@ \bool_gset_false:N \g_@@_success_bool \int_step_inline:nnnn \l_@@_min_state_int { 1 } { \l_@@_max_state_int - 1 } - { \tex_dimen:D ##1 ~ 1 sp \scan_stop: } + { \__intarray_gset_fast:Nnn \g_@@_state_active_intarray {##1} { 1 } } \int_set_eq:NN \l_@@_min_active_int \l_@@_max_state_int \int_zero:N \l_@@_step_int \int_set_eq:NN \l_@@_success_pos_int \l_@@_min_pos_int - \int_set:Nn \l_@@_submatch_int + \int_set:Nn \l_@@_min_submatch_int { 2 * \l_@@_max_state_int } + \int_set_eq:NN \l_@@_submatch_int \l_@@_min_submatch_int \bool_set_false:N \l_@@_empty_success_bool \@@_match_once: %<trace> \trace_pop:nnx { regex } { 1 } { @@_match } @@ -4433,7 +4530,7 @@ % \end{macro} % % \begin{macro}[aux]{\@@_match_loop:} -% \begin{macro}[aux, rEXP]{\@@_match_one_active:w} +% \begin{macro}[aux, rEXP]{\@@_match_one_active:n} % At each new position, set some variables and get the new character % and category from the query. Then unpack the array of active % threads, and clear it by resetting its length @@ -4448,7 +4545,7 @@ % \begin{macrocode} \cs_new_protected:Npn \@@_match_loop: { - \tex_advance:D \l_@@_step_int 2 \exp_stop_f: + \int_add:Nn \l_@@_step_int { 2 } \int_incr:N \l_@@_current_pos_int \int_set_eq:NN \l_@@_last_char_int \l_@@_current_char_int \int_set_eq:NN \l_@@_case_changed_char_int \c_max_int @@ -4456,8 +4553,11 @@ \use:x { \int_set_eq:NN \l_@@_max_active_int \l_@@_min_active_int - \exp_after:wN \@@_match_one_active:w - \int_use:N \l_@@_min_active_int ; + \int_step_function:nnnN + { \l_@@_min_active_int } + { 1 } + { \l_@@_max_active_int - 1 } + \@@_match_one_active:n } \__prg_break_point: \bool_set_false:N \l_@@_fresh_thread_bool %^^A was arg of break_point:n @@ -4467,15 +4567,11 @@ \fi: \fi: } -\cs_new:Npn \@@_match_one_active:w #1; +\cs_new:Npn \@@_match_one_active:n #1 { - \if_int_compare:w #1 < \l_@@_max_active_int - \@@_use_state_and_submatches:nn - { \__int_value:w \tex_skip:D #1 } - { \tex_the:D \tex_toks:D #1 } - \exp_after:wN \@@_match_one_active:w - \__int_value:w \__int_eval:w #1 + 1 \exp_after:wN ; - \fi: + \@@_use_state_and_submatches:nn + { \__intarray_item_fast:Nn \g_@@_thread_state_intarray {#1} } + { \@@_toks_use:w #1 } } % \end{macrocode} % \end{macro} @@ -4485,18 +4581,18 @@ % The arguments are: tokens that \texttt{o} and \texttt{x} expand to % one token of the query, the catcode, and the character code. Store % those, and the current brace balance (used later to check for -% overall brace balance) in a \tn{muskip} register and a \tn{toks}, +% overall brace balance) in a \tn{toks} register and some arrays, % then update the \texttt{balance}. % \begin{macrocode} \cs_new_protected:Npn \@@_query_set:nnn #1#2#3 { - \tex_muskip:D \l_@@_current_pos_int - = \etex_gluetomu:D - #3 sp - plus #2 sp - minus \l_@@_balance_int sp - \scan_stop: - \tex_toks:D \l_@@_current_pos_int {#1} + \__intarray_gset_fast:Nnn \g_@@_charcode_intarray + { \l_@@_current_pos_int } {#3} + \__intarray_gset_fast:Nnn \g_@@_catcode_intarray + { \l_@@_current_pos_int } {#2} + \__intarray_gset_fast:Nnn \g_@@_balance_intarray + { \l_@@_current_pos_int } { \l_@@_balance_int } + \@@_toks_set:Nn \l_@@_current_pos_int {#1} \int_incr:N \l_@@_current_pos_int \if_case:w #2 \exp_stop_f: \or: \int_incr:N \l_@@_balance_int @@ -4507,17 +4603,17 @@ % \end{macro} % % \begin{macro}[aux]{\@@_query_get:} -% Extract the current character and category codes from the -% \tn{muskip} register of the current position: those are the main and -% the stretch components, and we need a conversion to avoid \TeX{}'s -% \enquote{incompatible glue units} error. +% Extract the current character and category codes at the current +% position from the appropriate arrays. % \begin{macrocode} \cs_new_protected:Npn \@@_query_get: { \l_@@_current_char_int - = \etex_mutoglue:D \tex_muskip:D \l_@@_current_pos_int - \l_@@_current_catcode_int = \etex_gluestretch:D - \etex_mutoglue:D \tex_muskip:D \l_@@_current_pos_int + = \__intarray_item_fast:Nn \g_@@_charcode_intarray + { \l_@@_current_pos_int } \scan_stop: + \l_@@_current_catcode_int + = \__intarray_item_fast:Nn \g_@@_catcode_intarray + { \l_@@_current_pos_int } \scan_stop: } % \end{macrocode} % \end{macro} @@ -4537,11 +4633,11 @@ %<*trace> \trace:nnx { regex } { 2 } { state~\int_use:N \l_@@_current_state_int } %</trace> - \tex_dimen:D \l_@@_current_state_int - = \l_@@_step_int sp \scan_stop: - \tex_the:D \tex_toks:D \l_@@_current_state_int - \tex_dimen:D \l_@@_current_state_int - = \__int_eval:w \l_@@_step_int + 1 \__int_eval_end: sp \scan_stop: + \__intarray_gset_fast:Nnn \g_@@_state_active_intarray + { \l_@@_current_state_int } { \l_@@_step_int } + \@@_toks_use:w \l_@@_current_state_int + \__intarray_gset_fast:Nnn \g_@@_state_active_intarray + { \l_@@_current_state_int } { \l_@@_step_int + 1 } } % \end{macrocode} % \end{macro} @@ -4555,7 +4651,9 @@ \cs_new_protected:Npn \@@_use_state_and_submatches:nn #1 #2 { \int_set:Nn \l_@@_current_state_int {#1} - \if_int_compare:w \tex_dimen:D \l_@@_current_state_int + \if_int_compare:w + \__intarray_item_fast:Nn \g_@@_state_active_intarray + { \l_@@_current_state_int } < \l_@@_step_int \tl_set:Nn \l_@@_current_submatches_prop {#2} \exp_after:wN \@@_use_state: @@ -4610,7 +4708,10 @@ \int_add:Nn \l_@@_current_state_int {#2} \exp_not:n { - \if_int_compare:w \tex_dimen:D \l_@@_current_state_int #1 + \if_int_compare:w + \__intarray_item_fast:Nn \g_@@_state_active_intarray + { \l_@@_current_state_int } + #1 \exp_after:wN \@@_use_state: \fi: } @@ -4626,7 +4727,7 @@ % % \begin{macro}[int]{\@@_action_cost:n} % A transition which consumes the current character and shifts the -% state by |#1|. The resulting state is stored in the \tn{skip} array +% state by |#1|. The resulting state is stored in the appropriate array % for use at the next position, and we also store the current % submatches. % \begin{macrocode} @@ -4640,22 +4741,20 @@ % % \begin{macro}[int]{\@@_store_state:n} % \begin{macro}[aux]{\@@_store_submatches:} -% Put the given state in the array of \tn{skip} registers (converted -% to a dimension in scaled points), and increment the length of the -% array. Then store the current submatch in the This is done by -% increasing the pointer \cs{l_@@_max_active_int}, and converting -% the integer to a dimension (suitable for a \tn{skip} assignment) in -% scaled points. +% Put the given state in \cs{g_@@_thread_state_intarray}, and increment +% the length of the array. Also store the current submatch in the +% appropriate \tn{toks}. % \begin{macrocode} \cs_new_protected:Npn \@@_store_state:n #1 { \@@_store_submatches: - \tex_skip:D \l_@@_max_active_int = #1 sp \scan_stop: + \__intarray_gset_fast:Nnn \g_@@_thread_state_intarray + { \l_@@_max_active_int } {#1} \int_incr:N \l_@@_max_active_int } \cs_new_protected:Npn \@@_store_submatches: { - \tex_toks:D \l_@@_max_active_int \exp_after:wN + \@@_toks_set:No \l_@@_max_active_int { \l_@@_current_submatches_prop } } % \end{macrocode} @@ -4729,6 +4828,15 @@ % \end{macrocode} % \end{variable} % +% \begin{variable}{\l_@@_replacement_category_tl, \l_@@_replacement_category_seq} +% This sequence of letters is used to correctly restore categories in +% nested constructions such as |\cL(abc\cD(_)d)|. +% \begin{macrocode} +\tl_new:N \l_@@_replacement_category_tl +\seq_new:N \l_@@_replacement_category_seq +% \end{macrocode} +% \end{variable} +% % \begin{variable}{\l_@@_balance_tl} % This token list holds the replacement text for % \cs{@@_replacement_balance_one_match:n} while it is being built @@ -4739,8 +4847,9 @@ % \end{variable} % % \begin{macro}[aux, rEXP]{\@@_replacement_balance_one_match:n} -% This expects as an argument the first index of a range of \tn{skip} -% registers which hold the submatch information for a given match. It +% This expects as an argument the first index of a set of entries in +% \cs{g_@@_submatch_begin_intarray} (and related arrays) which hold the +% submatch information for a given match. It % can be used within an integer expression to obtain the brace balance % incurred by performing the replacement on that match. This combines % the braces lost by removing the match, braces added by all the @@ -4764,13 +4873,13 @@ % with all possible arguments (one call for each match), as well as % the range from the end of the last match to the end of the string, % will produce the fully replaced token list. The initialization does -% not matter, but we set it as for an empty replacement. +% not matter, but (as an example) we set it as for an empty replacement. % \begin{macrocode} \cs_new:Npn \@@_replacement_do_one_match:n #1 { \@@_query_range:nn - { \etex_glueshrink:D \tex_skip:D #1 } - { \tex_skip:D #1 } + { \__intarray_item_fast:Nn \g_@@_submatch_prev_intarray {#1} } + { \__intarray_item_fast:Nn \g_@@_submatch_begin_intarray {#1} } } % \end{macrocode} % \end{macro} @@ -4816,7 +4925,7 @@ \else: \exp_after:wN \__prg_break: \fi: - \tex_the:D \tex_toks:D #1 \exp_stop_f: + \@@_toks_use:w #1 \exp_stop_f: \exp_after:wN \@@_query_range_loop:ww \__int_value:w \__int_eval:w #1 + 1 ; #2 ; } @@ -4825,17 +4934,13 @@ % \end{macro} % % \begin{macro}[int]{\@@_query_submatch:n} -% When this function is called, \tn{skip}$i$ holds the start and end -% positions for the $i$-th overall submatch as its main and stretch -% components. In the case of repeated matches, submatches from all the -% matches are put one after the other in blocks of -% \cs{l_@@_capturing_group_int} \tn{skip} registers. +% Find the start and end positions for a given submatch (of a given match). % \begin{macrocode} \cs_new:Npn \@@_query_submatch:n #1 { \@@_query_range:nn - { \tex_skip:D \__int_eval:w #1 } - { \etex_gluestretch:D \tex_skip:D \__int_eval:w #1 } + { \__intarray_item_fast:Nn \g_@@_submatch_begin_intarray {#1} } + { \__intarray_item_fast:Nn \g_@@_submatch_end_intarray {#1} } } % \end{macrocode} % \end{macro} @@ -4843,21 +4948,31 @@ % \begin{macro}[rEXP]{\@@_submatch_balance:n} % Every user function must result in a balanced token list (unbalanced % token lists cannot be stored by TeX). When we unpacked the query, we -% kept track of the brace balance as the shrink component of -% \tn{muskip} registers, hence the contribution from a given range is -% the difference between the shrink components of -% \tn{muskip}\meta{max~pos} and \tn{muskip}\meta{min~pos}. For the -% $i$-th submatch, the end-points of the range are the main and -% stretch components of \tn{skip}$i$. The trailing \cs{scan_stop:} is -% gobbled by \cs{etex_muexpr:D}, and the whole expression can be cast -% safely to an integer (no trailing expansion). +% kept track of the brace balance, hence the contribution from a given +% range is the difference between the brace balances at the +% \meta{max~pos} and \meta{min~pos}. These two positions are found in +% the corresponding \enquote{submatch} arrays. +%^^A todo: understand when these int_compare are needed % \begin{macrocode} \cs_new_protected:Npn \@@_submatch_balance:n #1 { - \etex_glueshrink:D \etex_mutoglue:D \etex_muexpr:D - \tex_muskip:D \etex_gluestretch:D \tex_skip:D #1 - - \tex_muskip:D \tex_skip:D #1 - \scan_stop: + \__int_eval:w + \int_compare:nNnTF + { \__intarray_item_fast:Nn \g_@@_submatch_end_intarray {#1} } = 0 + { 0 } + { + \__intarray_item_fast:Nn \g_@@_balance_intarray + { \__intarray_item_fast:Nn \g_@@_submatch_end_intarray {#1} } + } + - + \int_compare:nNnTF + { \__intarray_item_fast:Nn \g_@@_submatch_begin_intarray {#1} } = 0 + { 0 } + { + \__intarray_item_fast:Nn \g_@@_balance_intarray + { \__intarray_item_fast:Nn \g_@@_submatch_begin_intarray {#1} } + } + \__int_eval_end: } % \end{macrocode} % \end{macro} @@ -4886,10 +5001,14 @@ \@@_escape_use:nnnn { \if_charcode:w \c_right_brace_str ##1 - \@@_replacement_rbrace:N \else: \__tl_build_one:n \fi: ##1 + \@@_replacement_rbrace:N + \else: + \@@_replacement_normal:n + \fi: + ##1 } { \@@_replacement_escaped:N ##1 } - { \__tl_build_one:n ##1 } + { \@@_replacement_normal:n ##1 } {#1} \prg_do_nothing: \prg_do_nothing: \if_int_compare:w \l_@@_replacement_csnames_int > 0 \exp_stop_f: @@ -4898,6 +5017,12 @@ \__tl_build_one:x { \prg_replicate:nn \l_@@_replacement_csnames_int \cs_end: } \fi: + \seq_if_empty:NF \l_@@_replacement_category_seq + { + \__msg_kernel_error:nnx { regex } { replacement-missing-rparen } + { \seq_count:N \l_@@_replacement_category_seq } + \seq_clear:N \l_@@_replacement_category_seq + } \cs_gset:Npx \@@_replacement_balance_one_match:n ##1 { + \int_use:N \l_@@_balance_int @@ -4913,8 +5038,8 @@ \cs_set:Npn \@@_replacement_do_one_match:n ##1 { \@@_query_range:nn - { \etex_glueshrink:D \tex_skip:D ##1 } - { \tex_skip:D ##1 } + { \__intarray_item_fast:Nn \g_@@_submatch_prev_intarray {##1} } + { \__intarray_item_fast:Nn \g_@@_submatch_begin_intarray {##1} } #1 } } @@ -4922,6 +5047,35 @@ % \end{macro} % \end{macro} % +% \begin{macro}[aux]{\@@_replacement_normal:n} +% Most characters are simply sent to the output by +% \cs{__tl_build_one:n}, unless a particular category code has been +% requested: then \cs{@@_replacement_c_A:w} or a similar auxiliary is +% called. One exception is right parentheses, which restore the +% category code in place before the group started. Note that the +% sequence is non-empty there: it contains an empty entry +% corresponding to the initial value of +% \cs{l_@@_replacement_category_tl}. +% \begin{macrocode} +\cs_new_protected:Npn \@@_replacement_normal:n #1 + { + \tl_if_empty:NTF \l_@@_replacement_category_tl + { \__tl_build_one:n {#1} } + { % ( + \token_if_eq_charcode:NNTF #1 ) + { + \seq_pop:NN \l_@@_replacement_category_seq + \l_@@_replacement_category_tl + } + { + \use:c { @@_replacement_c_ \l_@@_replacement_category_tl :w } + \@@_replacement_normal:n {#1} + } + } + } +% \end{macrocode} +% \end{macro} +% % \begin{macro}[aux]{\@@_replacement_escaped:N} % As in parsing a regular expression, we use an auxiliary built from % |#1| if defined. Otherwise, check for escaped digits (standing from @@ -4935,7 +5089,8 @@ \if_int_compare:w 1 < 1#1 \exp_stop_f: \@@_replacement_put_submatch:n {#1} \else: - \__tl_build_one:o { \token_to_str:N #1 } + \exp_args:No \@@_replacement_normal:n + { \token_to_str:N #1 } \fi: } } @@ -4967,34 +5122,34 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\@@_replacement_g:w, \@@_replacement_g_digits:NN} -% An ugly method to grab digits for the |\g| escape sequence. At the -% end of the run of digits, check that it ends with a right brace. +% \begin{macro}[aux]{\@@_replacement_g:w} +% \begin{macro}[aux,rEXP]{\@@_replacement_g_digits:NN} +% Grab digits for the |\g| escape sequence in a primitive assignment +% to the integer \cs{l_@@_internal_a_int}. At the end of the run of +% digits, check that it ends with a right brace. % \begin{macrocode} \cs_new_protected:Npn \@@_replacement_g:w #1#2 { - \str_if_eq_x:nnTF { #1#2 } { \__tl_build_one:n \c_left_brace_str } - { - \int_zero:N \l_@@_internal_a_int - \@@_replacement_g_digits:NN - } + \str_if_eq_x:nnTF { #1#2 } { \@@_replacement_normal:n \c_left_brace_str } + { \l_@@_internal_a_int = \@@_replacement_g_digits:NN } { \@@_replacement_error:NNN g #1 #2 } } -\cs_new_protected:Npn \@@_replacement_g_digits:NN #1#2 +\cs_new:Npn \@@_replacement_g_digits:NN #1#2 { - \token_if_eq_meaning:NNTF #1 \__tl_build_one:n + \token_if_eq_meaning:NNTF #1 \@@_replacement_normal:n { \if_int_compare:w 1 < 1#2 \exp_stop_f: - \int_set:Nn \l_@@_internal_a_int - { 10 * \l_@@_internal_a_int + #2 } + #2 \exp_after:wN \use_i:nnn \exp_after:wN \@@_replacement_g_digits:NN \else: + \exp_stop_f: \exp_after:wN \@@_replacement_error:NNN \exp_after:wN g \fi: } { + \exp_stop_f: \if_meaning:w \@@_replacement_rbrace:N #1 \exp_args:No \@@_replacement_put_submatch:n { \int_use:N \l_@@_internal_a_int } @@ -5008,41 +5163,52 @@ } % \end{macrocode} % \end{macro} +% \end{macro} % % \subsubsection{Csnames in replacement} % % \begin{macro}[aux]{\@@_replacement_c:w} -% \begin{macro}[aux]+\@@_replacement_c_{:w+ -% |\c| can be followed by a left brace, or by a letter for which we -% have defined a way to produce that category of characters. The -% appropriate definitions for catcodes are introduced later. For -% control sequences, if we are within a control sequence, convert -% the token list to a string, otherwise simply prevent expansion, -% with a weird cross-over between \cs{exp_not:n} and \cs{exp_not:N} -% (see this helper's description for an explanation). +% |\c| may only be followed by an unescaped character. If followed by +% a left brace, start a control sequence by calling an auxiliary +% common with |\u|. Otherwise test whether the category is known; if +% it is not, complain. % \begin{macrocode} \cs_new_protected:Npn \@@_replacement_c:w #1#2 { - \token_if_eq_meaning:NNTF #1 \__tl_build_one:n + \token_if_eq_meaning:NNTF #1 \@@_replacement_normal:n { - \cs_if_exist_use:cF { @@_replacement_c_#2:w } - { \@@_replacement_error:NNN c #1#2 } + \exp_after:wN \token_if_eq_charcode:NNTF \c_left_brace_str #2 + { \@@_replacement_cu_aux:Nw \@@_replacement_exp_not:N } + { + \cs_if_exist:cTF { @@_replacement_c_#2:w } + { \@@_replacement_cat:NNN #2 } + { \@@_replacement_error:NNN c #1#2 } + } } { \@@_replacement_error:NNN c #1#2 } } -\cs_new_protected:cpn { @@_replacement_c_ \c_left_brace_str :w } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\@@_replacement_cu_aux:Nw} +% Start a control sequence with \cs{cs:w}, which will be protected +% from expansion by |#1| (either \cs{@@_replacement_exp_not:N} or +% \cs{exp_not:V}), or turned to a string by \cs{tl_to_str:V} if inside +% another csname construction |\c| or |\u|. We use \cs{tl_to_str:V} +% rather than \cs{tl_to_str:N} to deal with integers and other +% registers. +% \begin{macrocode} +\cs_new_protected:Npn \@@_replacement_cu_aux:Nw #1 { \if_case:w \l_@@_replacement_csnames_int - \__tl_build_one:n - { \exp_not:n { \exp_after:wN \@@_replacement_exp_not:N \cs:w } } + \__tl_build_one:n { \exp_not:n { \exp_after:wN #1 \cs:w } } \else: - \__tl_build_one:n { \exp_not:n { \exp_after:wN \tl_to_str:N \cs:w } } + \__tl_build_one:n { \exp_not:n { \exp_after:wN \tl_to_str:V \cs:w } } \fi: \int_incr:N \l_@@_replacement_csnames_int } % \end{macrocode} % \end{macro} -% \end{macro} % % \begin{macro}[aux]{\@@_replacement_u:w} % Check that |\u| is followed by a left brace. If so, start a control @@ -5051,15 +5217,8 @@ % \begin{macrocode} \cs_new_protected:Npn \@@_replacement_u:w #1#2 { - \str_if_eq_x:nnTF { #1#2 } { \__tl_build_one:n \c_left_brace_str } - { - \if_case:w \l_@@_replacement_csnames_int - \__tl_build_one:n { \exp_not:n { \exp_after:wN \exp_not:V \cs:w } } - \else: - \__tl_build_one:n { \exp_not:n { \exp_after:wN \tl_to_str:V \cs:w } } - \fi: - \int_incr:N \l_@@_replacement_csnames_int - } + \str_if_eq_x:nnTF { #1#2 } { \@@_replacement_normal:n \c_left_brace_str } + { \@@_replacement_cu_aux:Nw \exp_not:V } { \@@_replacement_error:NNN u #1#2 } } % \end{macrocode} @@ -5076,7 +5235,7 @@ \__tl_build_one:n \cs_end: \int_decr:N \l_@@_replacement_csnames_int \else: - \__tl_build_one:n #1 + \@@_replacement_normal:n {#1} \fi: } % \end{macrocode} @@ -5084,6 +5243,39 @@ % % \subsubsection{Characters in replacement} % +% \begin{macro}[aux]{\@@_replacement_cat:NNN} +% Here, |#1| is a letter among |BEMTPUDSLOA| and |#2#3| denote the +% next character. Complain if we reach the end of the replacement or +% if the construction appears inside |\c{|\ldots{}|}| or +% |\u{|\ldots{}|}|, and detect the case of a parenthesis. In that +% case, store the current category in a sequence and switch to a new +% one. +% \begin{macrocode} +\cs_new_protected:Npn \@@_replacement_cat:NNN #1#2#3 + { + \token_if_eq_meaning:NNTF \prg_do_nothing: #3 + { \__msg_kernel_error:nn { regex } { replacement-catcode-end } } + { + \int_compare:nNnTF { \l_@@_replacement_csnames_int } > 0 + { + \__msg_kernel_error:nnnn + { regex } { replacement-catcode-in-cs } {#1} {#3} + #2 #3 + } + { + \str_if_eq:nnTF { #2 #3 } { \@@_replacement_normal:n ( } % ) + { + \seq_push:NV \l_@@_replacement_category_seq + \l_@@_replacement_category_tl + \tl_set:Nn \l_@@_replacement_category_tl {#1} + } + { \use:c { @@_replacement_c_#1:w } #2 #3 } + } + } + } +% \end{macrocode} +% \end{macro} +% % We will need to change the category code of the null character many % times, hence work in a group. The catcode-specific macros below are % defined in alphabetical order; if you are trying to understand the @@ -5099,16 +5291,13 @@ % wrapper for our purposes. The first argument is the null character % with various catcodes. The second and third arguments are grabbed % from the input stream: |#3| is the character whose character code to -% reproduce. +% reproduce. We could use \cs{char_generate:nn} but only for some +% catcodes (active characters and spaces are not supported). % \begin{macrocode} \cs_new_protected:Npn \@@_replacement_char:nNN #1#2#3 { - \if_meaning:w \prg_do_nothing: #3 - \__msg_kernel_error:nn { regex } { replacement-catcode-end } - \else: - \tex_lccode:D 0 = `#3 \scan_stop: - \tex_lowercase:D { \__tl_build_one:n {#1} } - \fi: + \tex_lccode:D 0 = `#3 \scan_stop: + \tex_lowercase:D { \__tl_build_one:n {#1} } } % \end{macrocode} % \end{macro} @@ -5234,15 +5423,11 @@ % \begin{macrocode} \cs_new_protected:Npn \@@_replacement_c_S:w #1#2 { - \if_meaning:w \prg_do_nothing: #2 - \__msg_kernel_error:nn { regex } { replacement-catcode-end } - \else: - \if_int_compare:w `#2 = 0 \exp_stop_f: - \__msg_kernel_error:nn { regex } { replacement-null-space } - \fi: - \tex_lccode:D `\ = `#2 \scan_stop: - \tex_lowercase:D { \__tl_build_one:n {~} } + \if_int_compare:w `#2 = 0 \exp_stop_f: + \__msg_kernel_error:nn { regex } { replacement-null-space } \fi: + \tex_lccode:D `\ = `#2 \scan_stop: + \tex_lowercase:D { \__tl_build_one:n {~} } } % \end{macrocode} % \end{macro} @@ -5433,23 +5618,32 @@ % \end{macrocode} % \end{variable} % -% \begin{variable}{\l_@@_submatch_int, \l_@@_zeroth_submatch_int} -% The end-points of each submatch are stored as main and stretch -% components of \tn{skip}\meta{submatch}, where \meta{submatch} ranges -% from \cs{l_@@_max_state_int} (inclusive) to +% \begin{variable}{\l_@@_min_submatch_int, \l_@@_submatch_int, \l_@@_zeroth_submatch_int} +% The end-points of each submatch are stored in two arrays whose index \meta{submatch} ranges +% from \cs{l_@@_min_submatch_int} (inclusive) to % \cs{l_@@_submatch_int} (exclusive). Each successful match comes % with a $0$-th submatch (the full match), and one match for each % capturing group: submatches corresponding to the last successful -% match are labelled starting at -% \texttt{zeroth_submatch}. Additionally, the shrink component of this -% $0$-th submatch is the position at which that match attempt started: -% this is used for splitting and replacements. +% match are labelled starting at \texttt{zeroth_submatch}. The entry +% \cs{l_@@_zeroth_submatch_int} in \cs{g_@@_submatch_prev_intarray} holds +% the position at which that match attempt started: this is used for +% splitting and replacements. % \begin{macrocode} +\int_new:N \l_@@_min_submatch_int \int_new:N \l_@@_submatch_int \int_new:N \l_@@_zeroth_submatch_int % \end{macrocode} % \end{variable} % +% \begin{variable}{\g_@@_submatch_prev_intarray, \g_@@_submatch_begin_intarray, \g_@@_submatch_end_intarray} +% Hold the place where the match attempt begun and the end-points of each submatch. +% \begin{macrocode} +\__intarray_new:Nn \g_@@_submatch_prev_intarray { 65536 } +\__intarray_new:Nn \g_@@_submatch_begin_intarray { 65536 } +\__intarray_new:Nn \g_@@_submatch_end_intarray { 65536 } +% \end{macrocode} +% \end{variable} +% % \begin{macro}[aux]{\@@_return:} % This function triggers either \cs{prg_return_false:} or % \cs{prg_return_true:} as appropriate to whether a match was found or @@ -5543,8 +5737,7 @@ % match, store the last part of the token list, which ranges from the % start of the match attempt to the end of the query. This step is % inhibited if the last match was empty and at the very end: decrement -% \cs{l_@@_submatch_int}, which controls which \tn{skip} registers -% will be used. +% \cs{l_@@_submatch_int}, which controls which matches will be used. % \begin{macrocode} \cs_new_protected:Npn \@@_split:nnN #1#2#3 { @@ -5553,16 +5746,30 @@ { \if_int_compare:w \l_@@_start_pos_int < \l_@@_success_pos_int \@@_extract: - \tex_skip:D \l_@@_zeroth_submatch_int - = \l_@@_start_pos_int sp - plus \tex_skip:D \l_@@_zeroth_submatch_int \scan_stop: + \__intarray_gset_fast:Nnn \g_@@_submatch_prev_intarray + { \l_@@_zeroth_submatch_int } { 0 } + \__intarray_gset_fast:Nnn \g_@@_submatch_end_intarray + { \l_@@_zeroth_submatch_int } + { + \__intarray_item_fast:Nn \g_@@_submatch_begin_intarray + { \l_@@_zeroth_submatch_int } + } + \__intarray_gset_fast:Nnn \g_@@_submatch_begin_intarray + { \l_@@_zeroth_submatch_int } + { \l_@@_start_pos_int } \fi: } #1 \@@_match:n {#2} %<assert>\assert_int:n { \l_@@_current_pos_int = \l_@@_max_pos_int } - \tex_skip:D \l_@@_submatch_int - = \l_@@_start_pos_int sp plus \l_@@_max_pos_int sp \scan_stop: + \__intarray_gset_fast:Nnn \g_@@_submatch_prev_intarray + { \l_@@_submatch_int } { 0 } + \__intarray_gset_fast:Nnn \g_@@_submatch_end_intarray + { \l_@@_submatch_int } + { \l_@@_max_pos_int } + \__intarray_gset_fast:Nnn \g_@@_submatch_begin_intarray + { \l_@@_submatch_int } + { \l_@@_start_pos_int } \int_incr:N \l_@@_submatch_int \if_meaning:w \c_true_bool \l_@@_empty_success_bool \if_int_compare:w \l_@@_start_pos_int = \l_@@_max_pos_int @@ -5575,8 +5782,8 @@ % \end{macro} % % \begin{macro}[aux]{\@@_group_end_extract_seq:N} -% The end-points of submatches are stored as the main and stretch -% components of \tn{skip} registers from \cs{l_@@_max_state_int} to +% The end-points of submatches are stored as entries of two arrays +% from \cs{l_@@_min_submatch_int} to % \cs{l_@@_submatch_int} (exclusive). Extract the relevant ranges % into \cs{l_@@_internal_a_tl}. We detect unbalanced results using % the two flags \texttt{@@_begin} and \texttt{@@_end}, raised @@ -5593,7 +5800,7 @@ { \s__seq \int_step_function:nnnN - { 2 * \l_@@_max_state_int } + { \l_@@_min_submatch_int } { 1 } { \l_@@_submatch_int - 1 } \@@_extract_seq_aux:n @@ -5649,14 +5856,14 @@ % {\@@_extract:, \@@_extract_b:wn, \@@_extract_e:wn} % Our task here is to extract from the property list % \cs{l_@@_success_submatches_prop} the list of end-points of -% submatches, and store them in \tn{skip} registers, from +% submatches, and store them in appropriate array entries, from % \cs{l_@@_zeroth_submatch_int} upwards. We begin by emptying those -% \tn{skip} registers. Then for each \meta{key}--\meta{value} pair in -% the property list update the appropriate \tn{skip} component. This +% entries. Then for each \meta{key}--\meta{value} pair in +% the property list update the appropriate entry. This % is somewhat a hack: the \meta{key} is a non-negative integer % followed by |<| or |>|, which we use in a comparison to $-1$. At the % end, store the information about the position at which the match -% attempt started, as a shrink component. +% attempt started, in \cs{g_@@_submatch_prev_intarray}. % \begin{macrocode} \cs_new_protected:Npn \@@_extract: { @@ -5664,7 +5871,12 @@ \int_set_eq:NN \l_@@_zeroth_submatch_int \l_@@_submatch_int \prg_replicate:nn \l_@@_capturing_group_int { - \tex_skip:D \l_@@_submatch_int 0 sp \scan_stop: + \__intarray_gset_fast:Nnn \g_@@_submatch_begin_intarray + { \l_@@_submatch_int } { 0 } + \__intarray_gset_fast:Nnn \g_@@_submatch_end_intarray + { \l_@@_submatch_int } { 0 } + \__intarray_gset_fast:Nnn \g_@@_submatch_prev_intarray + { \l_@@_submatch_int } { 0 } \int_incr:N \l_@@_submatch_int } \prop_map_inline:Nn \l_@@_success_submatches_prop @@ -5676,21 +5888,14 @@ \fi: \__int_eval:w \l_@@_zeroth_submatch_int + ##1 {##2} } - \tex_skip:D \l_@@_zeroth_submatch_int - = \tex_the:D \tex_skip:D \l_@@_zeroth_submatch_int - minus \l_@@_start_pos_int sp \scan_stop: + \__intarray_gset_fast:Nnn \g_@@_submatch_prev_intarray + { \l_@@_zeroth_submatch_int } { \l_@@_start_pos_int } \fi: } \cs_new_protected:Npn \@@_extract_b:wn #1 < #2 - { - \tex_skip:D #1 = #2 sp - plus \etex_gluestretch:D \tex_skip:D #1 \scan_stop: - } + { \__intarray_gset_fast:Nnn \g_@@_submatch_begin_intarray {#1} {#2} } \cs_new_protected:Npn \@@_extract_e:wn #1 > #2 - { - \tex_skip:D #1 - = 1 \tex_skip:D #1 plus #2 sp \scan_stop: - } + { \__intarray_gset_fast:Nnn \g_@@_submatch_end_intarray {#1} {#2} } % \end{macrocode} % \end{macro} % @@ -5730,7 +5935,10 @@ { \@@_replacement_do_one_match:n { \l_@@_zeroth_submatch_int } \@@_query_range:nn - { \etex_gluestretch:D \tex_skip:D \l_@@_zeroth_submatch_int } + { + \__intarray_item_fast:Nn \g_@@_submatch_end_intarray + { \l_@@_zeroth_submatch_int } + } { \l_@@_max_pos_int } } \@@_group_end_replace:N #3 @@ -5741,12 +5949,11 @@ % % \begin{macro}[aux]{\@@_replace_all:nnN} % Match multiple times, and for every match, extract submatches and -% additionally store the position at which the match attempt started -% (as the shrink component of a \tn{skip} register). The \tn{skip} -% registers from \cs{l_@@_max_state_int} to +% additionally store the position at which the match attempt started. +% The entries from \cs{l_@@_min_submatch_int} to % \cs{l_@@_submatch_int} hold information about submatches of every % match in order; each match corresponds to -% \cs{l_@@_capturing_group_int} consecutive \tn{skip} registers. +% \cs{l_@@_capturing_group_int} consecutive entries. % Compute the brace balance corresponding to doing all the % replacements: this is the sum of brace balances for replacing each % match. Join together the replacement texts for each match (including @@ -5763,7 +5970,7 @@ { 0 \int_step_function:nnnN - { 2 * \l_@@_max_state_int } + { \l_@@_min_submatch_int } \l_@@_capturing_group_int { \l_@@_submatch_int - 1 } \@@_replacement_balance_one_match:n @@ -5771,7 +5978,7 @@ \tl_set:Nx \l_@@_internal_a_tl { \int_step_function:nnnN - { 2 * \l_@@_max_state_int } + { \l_@@_min_submatch_int } \l_@@_capturing_group_int { \l_@@_submatch_int - 1 } \@@_replacement_do_one_match:n @@ -5825,23 +6032,23 @@ % Messages for the preparsing phase. % \begin{macrocode} \__msg_kernel_new:nnnn { regex } { trailing-backslash } - { Trailing~escape~character~`\iow_char:N\\'. } + { Trailing~escape~character~'\iow_char:N\\'. } { A~regular~expression~or~its~replacement~text~ends~with~ - the~escape~character~`\iow_char:N\\'.~It~will~be~ignored. + the~escape~character~'\iow_char:N\\'.~It~will~be~ignored. } \__msg_kernel_new:nnnn { regex } { x-missing-rbrace } - { Missing~closing~brace~in~`\iow_char:N\\x'~hexadecimal~sequence. } + { Missing~closing~brace~in~'\iow_char:N\\x'~hexadecimal~sequence. } { You~wrote~something~like~ - `\iow_char:N\\x\{...#1'.~ + '\iow_char:N\\x\{...#1'.~ The~closing~brace~is~missing. } \__msg_kernel_new:nnnn { regex } { x-overflow } - { Character~code~`#1'~too~large~in~`\iow_char:N\\x'~hexadecimal~sequence. } + { Character~code~'#1'~too~large~in~'\iow_char:N\\x'~hexadecimal~sequence. } { You~wrote~something~like~ - `\iow_char:N\\x\{\int_to_Hex:n{#1}\}'.~ + '\iow_char:N\\x\{\int_to_Hex:n{#1}\}'.~ The~character~code~#1~is~larger~than~ the~maximum~value~\int_use:N \c_max_char_int. } @@ -5850,11 +6057,11 @@ % Invalid quantifier. % \begin{macrocode} \__msg_kernel_new:nnnn { regex } { invalid-quantifier } - { Braced~quantifier~`#1'~may~not~be~followed~by~`#2'. } + { Braced~quantifier~'#1'~may~not~be~followed~by~'#2'. } { - The~character~`#2'~is~invalid~in~the~braced~quantifier~`#1'.~ - The~only~valid~quantifiers~are~`*',~`?',~`+',~`{<int>}',~ - `{<min>,}'~and~`{<min>,<max>}',~optionally~followed~by~`?'. + The~character~'#2'~is~invalid~in~the~braced~quantifier~'#1'.~ + The~only~valid~quantifiers~are~'*',~'?',~'+',~'{<int>}',~ + '{<min>,}'~and~'{<min>,<max>}',~optionally~followed~by~'?'. } % \end{macrocode} % @@ -5865,7 +6072,7 @@ { Missing~right~bracket~inserted~in~regular~expression. } { LaTeX~was~given~a~regular~expression~where~a~character~class~ - was~started~with~`[',~but~the~matching~`]'~is~missing. + was~started~with~'[',~but~the~matching~']'~is~missing. } \__msg_kernel_new:nnnn { regex } { missing-rparen } { @@ -5889,7 +6096,7 @@ % \begin{macrocode} \__msg_kernel_new:nnnn { regex } { bad-escape } { - Invalid~escape~`\iow_char:N\\#1'~ + Invalid~escape~'\iow_char:N\\#1'~ \@@_if_in_cs:TF { within~a~control~sequence. } { \@@_if_in_class:TF @@ -5898,16 +6105,16 @@ } } { - The~escape~sequence~`\iow_char:N\\#1'~may~not~appear~ + The~escape~sequence~'\iow_char:N\\#1'~may~not~appear~ \@@_if_in_cs:TF { within~a~control~sequence~test~introduced~by~ - `\iow_char:N\\c\iow_char:N\{'. + '\iow_char:N\\c\iow_char:N\{'. } { \@@_if_in_class:TF { within~a~character~class~ } - { following~a~category~test~such~as~`\iow_char:N\\cL'~ } + { following~a~category~test~such~as~'\iow_char:N\\cL'~ } because~it~does~not~match~exactly~one~character. } } @@ -5916,96 +6123,96 @@ % Range errors. % \begin{macrocode} \__msg_kernel_new:nnnn { regex } { range-missing-end } - { Invalid~end-point~for~range~`#1-#2'~in~character~class. } + { Invalid~end-point~for~range~'#1-#2'~in~character~class. } { - The~end-point~`#2'~of~the~range~`#1-#2'~may~not~serve~as~an~ + The~end-point~'#2'~of~the~range~'#1-#2'~may~not~serve~as~an~ end-point~for~a~range:~alphanumeric~characters~should~not~be~ escaped,~and~non-alphanumeric~characters~should~be~escaped. } \__msg_kernel_new:nnnn { regex } { range-backwards } - { Range~`[#1-#2]'~out~of~order~in~character~class. } + { Range~'[#1-#2]'~out~of~order~in~character~class. } { - In~ranges~of~characters~`[x-y]'~appearing~in~character~classes,~ + In~ranges~of~characters~'[x-y]'~appearing~in~character~classes,~ the~first~character~code~must~not~be~larger~than~the~second.~ - Here,~`#1'~has~character~code~\int_eval:n {`#1},~while~ - `#2'~has~character~code~\int_eval:n {`#2}. + Here,~'#1'~has~character~code~\int_eval:n {`#1},~while~ + '#2'~has~character~code~\int_eval:n {`#2}. } % \end{macrocode} % % Errors related to |\c| and |\u|. % \begin{macrocode} \__msg_kernel_new:nnnn { regex } { c-bad-mode } - { Invalid~nested~`\iow_char:N\\c'~escape~in~regular~expression. } + { Invalid~nested~'\iow_char:N\\c'~escape~in~regular~expression. } { - The~`\iow_char:N\\c'~escape~cannot~be~used~within~ - a~control~sequence~test~`\iow_char:N\\c{...}'.~ - To~combine~several~category~tests,~use~`\iow_char:N\\c[...]'. + The~'\iow_char:N\\c'~escape~cannot~be~used~within~ + a~control~sequence~test~'\iow_char:N\\c{...}'.~ + To~combine~several~category~tests,~use~'\iow_char:N\\c[...]'. } \__msg_kernel_new:nnnn { regex } { c-missing-rbrace } - { Missing~right~brace~inserted~for~`\iow_char:N\\c'~escape. } + { Missing~right~brace~inserted~for~'\iow_char:N\\c'~escape. } { LaTeX~was~given~a~regular~expression~where~a~ - `\iow_char:N\\c\iow_char:N\{...'~construction~was~not~ended~ - with~a~closing~brace~`\iow_char:N\}'. + '\iow_char:N\\c\iow_char:N\{...'~construction~was~not~ended~ + with~a~closing~brace~'\iow_char:N\}'. } \__msg_kernel_new:nnnn { regex } { c-missing-rbrack } - { Missing~right~bracket~inserted~for~`\iow_char:N\\c'~escape. } + { Missing~right~bracket~inserted~for~'\iow_char:N\\c'~escape. } { - A~construction~`\iow_char:N\\c[...'~appears~in~a~ - regular~expression,~but~the~closing~`]'~is~not~present. + A~construction~'\iow_char:N\\c[...'~appears~in~a~ + regular~expression,~but~the~closing~']'~is~not~present. } \__msg_kernel_new:nnnn { regex } { c-missing-category } - { Invalid~character~`#1'~following~`\iow_char:N\\c'~escape. } + { Invalid~character~'#1'~following~'\iow_char:N\\c'~escape. } { - In~regular~expressions,~the~`\iow_char:N\\c'~escape~sequence~ + In~regular~expressions,~the~'\iow_char:N\\c'~escape~sequence~ may~only~be~followed~by~a~left~brace,~a~left~bracket,~or~a~ capital~letter~representing~a~character~category,~namely~ - one~of~`ABCDELMOPSTU'. + one~of~'ABCDELMOPSTU'. } \__msg_kernel_new:nnnn { regex } { c-trailing } - { Trailing~category~code~escape~`\iow_char:N\\c'... } + { Trailing~category~code~escape~'\iow_char:N\\c'... } { - A~regular~expression~ends~with~`\iow_char:N\\c'~followed~ + A~regular~expression~ends~with~'\iow_char:N\\c'~followed~ by~a~letter.~It~will~be~ignored. } \__msg_kernel_new:nnnn { regex } { u-missing-lbrace } - { Missing~left~brace~following~`\iow_char:N\\u'~escape. } + { Missing~left~brace~following~'\iow_char:N\\u'~escape. } { - The~`\iow_char:N\\u'~escape~sequence~must~be~followed~by~ + The~'\iow_char:N\\u'~escape~sequence~must~be~followed~by~ a~brace~group~with~the~name~of~the~variable~to~use. } \__msg_kernel_new:nnnn { regex } { u-missing-rbrace } - { Missing~right~brace~inserted~for~`\iow_char:N\\u'~escape. } + { Missing~right~brace~inserted~for~'\iow_char:N\\u'~escape. } { LaTeX~ \str_if_eq_x:nnTF { } {#2} { reached~the~end~of~the~string~ } - { encountered~an~escaped~alphanumeric~character `\iow_char:N\\#2'~ } - when~parsing~the~argument~of~an~`\iow_char:N\\u\iow_char:N\{...\}'~escape. + { encountered~an~escaped~alphanumeric~character '\iow_char:N\\#2'~ } + when~parsing~the~argument~of~an~'\iow_char:N\\u\iow_char:N\{...\}'~escape. } % \end{macrocode} % % Errors when encountering the \textsc{posix} syntax |[:...:]|. % \begin{macrocode} \__msg_kernel_new:nnnn { regex } { posix-unsupported } - { POSIX~collating~element~`[#1 ~ #1]'~not~supported. } + { POSIX~collating~element~'[#1 ~ #1]'~not~supported. } { - The~`[.foo.]'~and~`[=bar=]'~syntaxes~have~a~special~meaning~ + The~'[.foo.]'~and~'[=bar=]'~syntaxes~have~a~special~meaning~ in~POSIX~regular~expressions.~This~is~not~supported~by~LaTeX.~ Maybe~you~forgot~to~escape~a~left~bracket~in~a~character~class? } \__msg_kernel_new:nnnn { regex } { posix-unknown } - { POSIX~class~`[:#1:]'~unknown. } + { POSIX~class~'[:#1:]'~unknown. } { - `[:#1:]'~is~not~among~the~known~POSIX~classes~ - `[:alnum:]',~`[:alpha:]',~`[:ascii:]',~`[:blank:]',~ - `[:cntrl:]',~`[:digit:]',~`[:graph:]',~`[:lower:]',~ - `[:print:]',~`[:punct:]',~`[:space:]',~`[:upper:]',~ - `[:word:]',~and~`[:xdigit:]'. + '[:#1:]'~is~not~among~the~known~POSIX~classes~ + '[:alnum:]',~'[:alpha:]',~'[:ascii:]',~'[:blank:]',~ + '[:cntrl:]',~'[:digit:]',~'[:graph:]',~'[:lower:]',~ + '[:print:]',~'[:punct:]',~'[:space:]',~'[:upper:]',~ + '[:word:]',~and~'[:xdigit:]'. } \__msg_kernel_new:nnnn { regex } { posix-missing-close } - { Missing~closing~`:]'~for~POSIX~class. } - { The~POSIX~syntax~`#1'~must~be~followed~by~`:]',~not~`#2'. } + { Missing~closing~':]'~for~POSIX~class. } + { The~POSIX~syntax~'#1'~must~be~followed~by~':]',~not~'#2'. } % \end{macrocode} % % In various cases, the result of a \pkg{l3regex} operation can leave us @@ -6025,62 +6232,72 @@ % Error message for unknown options. % \begin{macrocode} \__msg_kernel_new:nnnn { regex } { unknown-option } - { Unknown~option~`#1'~for~regular~expressions. } + { Unknown~option~'#1'~for~regular~expressions. } { - The~only~available~option~is~`case-insensitive',~toggled~by~ - `(?i)'~and~`(?-i)'. + The~only~available~option~is~'case-insensitive',~toggled~by~ + '(?i)'~and~'(?-i)'. } \__msg_kernel_new:nnnn { regex } { special-group-unknown } - { Unknown~special~group~`#1~...'~in~a~regular~expression. } + { Unknown~special~group~'#1~...'~in~a~regular~expression. } { - The~only~valid~constructions~starting~with~`(?'~are~ - `(?:~...~)',~`(?|~...~)',~`(?i)',~and~`(?-i)'. + The~only~valid~constructions~starting~with~'(?'~are~ + '(?:~...~)',~'(?|~...~)',~'(?i)',~and~'(?-i)'. } % \end{macrocode} % % Errors in the replacement text. % \begin{macrocode} \__msg_kernel_new:nnnn { regex } { replacement-c } - { Misused~`\iow_char:N\\c'~command~in~a~replacement~text. } + { Misused~'\iow_char:N\\c'~command~in~a~replacement~text. } { - In~a~replacement~text,~the~`\iow_char:N\\c'~escape~sequence~ - can~be~followed~by~one~of~the~letters~`ABCDELMOPSTU'~ - or~a~brace~group,~not~by~`#1'. + In~a~replacement~text,~the~'\iow_char:N\\c'~escape~sequence~ + can~be~followed~by~one~of~the~letters~'ABCDELMOPSTU'~ + or~a~brace~group,~not~by~'#1'. } \__msg_kernel_new:nnnn { regex } { replacement-u } - { Misused~`\iow_char:N\\u'~command~in~a~replacement~text. } + { Misused~'\iow_char:N\\u'~command~in~a~replacement~text. } { - In~a~replacement~text,~the~`\iow_char:N\\u'~escape~sequence~ + In~a~replacement~text,~the~'\iow_char:N\\u'~escape~sequence~ must~be~~followed~by~a~brace~group~holding~the~name~of~the~ variable~to~use. } \__msg_kernel_new:nnnn { regex } { replacement-g } { - Missing~brace~for~the~`\iow_char:N\\g'~construction~ + Missing~brace~for~the~'\iow_char:N\\g'~construction~ in~a~replacement~text. } { In~the~replacement~text~for~a~regular~expression~search,~ - submatches~are~represented~either~as~`\iow_char:N \\g{dd..d}',~ - or~`\\d',~where~`d'~are~single~digits.~Here,~a~brace~is~missing. + submatches~are~represented~either~as~'\iow_char:N \\g{dd..d}',~ + or~'\\d',~where~'d'~are~single~digits.~Here,~a~brace~is~missing. } \__msg_kernel_new:nnnn { regex } { replacement-catcode-end } { - Missing~character~for~the~`\iow_char:N\\c<category><character>'~ + Missing~character~for~the~'\iow_char:N\\c<category><character>'~ construction~in~a~replacement~text. } { - In~a~replacement~text,~the~`\iow_char:N\\c'~escape~sequence~ - can~be~followed~by~one~of~the~letters~`ABCDELMOPSTU'~representing~ + In~a~replacement~text,~the~'\iow_char:N\\c'~escape~sequence~ + can~be~followed~by~one~of~the~letters~'ABCDELMOPSTU'~representing~ the~character~category.~Then,~a~character~must~follow.~LaTeX~ reached~the~end~of~the~replacement~when~looking~for~that. } +\__msg_kernel_new:nnnn { regex } { replacement-catcode-in-cs } + { + Category~code~'\iow_char:N\\c#1#3'~ignored~inside~ + '\iow_char:N\\c\{...\}'~in~a~replacement~text. + } + { + In~a~replacement~text,~the~category~codes~of~the~argument~of~ + '\iow_char:N\\c\{...\}'~are~ignored~when~building~the~control~ + sequence~name. + } \__msg_kernel_new:nnnn { regex } { replacement-null-space } { TeX~cannot~build~a~space~token~with~character~code~0. } { You~asked~for~a~character~token~with~category~space,~ and~character~code~0,~for~instance~through~ - `\iow_char:N\\cS\iow_char:N\\x00'.~ + '\iow_char:N\\cS\iow_char:N\\x00'.~ This~specific~case~is~impossible~and~will~be~replaced~ by~a~normal~space. } @@ -6090,6 +6307,12 @@ There~ \int_compare:nTF { #1 = 1 } { was } { were } ~ #1~ missing~right~\int_compare:nTF { #1 = 1 } { brace } { braces } . } +\__msg_kernel_new:nnnn { regex } { replacement-missing-rparen } + { Missing~right~parenthesis~inserted~in~replacement~text. } + { + There~ \int_compare:nTF { #1 = 1 } { was } { were } ~ #1~ + missing~right~\int_compare:nTF { #1 = 1 } { parenthesis } { parentheses } . + } % \end{macrocode} % % \begin{macro}[aux]{\@@_msg_repeated:nnN} @@ -6136,7 +6359,7 @@ { \l_@@_max_state_int - 1 } { \trace:nnx { regex } { #1 } - { \iow_char:N \\toks ##1 = { \tex_the:D \tex_toks:D ##1 } } + { \iow_char:N \\toks ##1 = { \@@_toks_use:w ##1 } } } } %</trace> |