diff options
Diffstat (limited to 'Master/texmf-dist/source/latex/l3kernel/l3regex.dtx')
-rw-r--r-- | Master/texmf-dist/source/latex/l3kernel/l3regex.dtx | 851 |
1 files changed, 620 insertions, 231 deletions
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3regex.dtx b/Master/texmf-dist/source/latex/l3kernel/l3regex.dtx index ef7ab6becc5..0824cbcb974 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3regex.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3regex.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2021-02-18} +% \date{Released 2021-05-07} % % \maketitle % @@ -66,7 +66,7 @@ % \tl_set:Nn \l_my_tl { That~cat. } % \regex_replace_once:nnN { at } { is } \l_my_tl % \end{verbatim} -% the token list variable \cs{l_my_tl} holds the text +% the token list variable \cs[no-index]{l_my_tl} holds the text % \enquote{\texttt{This cat.}}, where the first % occurrence of \enquote{\texttt{at}} was replaced % by \enquote{\texttt{is}}. A more complicated example is @@ -87,8 +87,8 @@ % \begin{verbatim} % \regex_const:Nn \c_foo_regex { \c{begin} \cB. (\c[^BE].*) \cE. } % \end{verbatim} -% stores in \cs{c_foo_regex} a regular expression which matches the -% starting marker for an environment: \cs{begin}, followed by a +% stores in \cs[no-index]{c_foo_regex} a regular expression which matches the +% starting marker for an environment: \cs[no-index]{begin}, followed by a % begin-group token (|\cB.|), then any number of tokens which are % neither begin-group nor end-group character tokens (|\c[^BE].*|), % ending with an end-group token (|\cE.|). As explained in the next @@ -98,6 +98,8 @@ % % \section{Syntax of regular expressions} % +% \subsection{Regex examples} +% % We start with a few examples, and encourage the reader to apply % \cs{regex_show:n} to these regular expressions. % \begin{itemize} @@ -147,6 +149,8 @@ % other things all valid integer expressions (made only with explicit % integers). One should follow it with further testing. % +% \subsection{Characters in regular expressions} +% % Most characters match exactly themselves, % with an arbitrary category code. Some characters are % special and must be escaped with a backslash (\emph{e.g.}, |\*| @@ -191,6 +195,8 @@ % \item[\\t] Horizontal tab (hex 09). % \end{l3regex-syntax} % +% \subsection{Characters classes} +% % Character types. % \begin{l3regex-syntax} % \item[.] A single period matches any token. @@ -235,6 +241,21 @@ % except |p|, as well as control sequences (see below for a description % of |\c|). % +% In character classes, only |[|, |^|, |-|, |]|, |\| and spaces are +% special, and should be escaped. Other non-alphanumeric characters can +% still be escaped without harm. Any escape sequence which matches a +% single character (|\d|, |\D|, \emph{etc.}) is supported in character +% classes. If the first character is |^|, then +% the meaning of the character class is inverted; |^| appearing anywhere +% else in the range is not special. If the first character (possibly +% following a leading |^|) is |]| then it does not need to be escaped +% since ending the range there would make it empty. +% Ranges of characters +% can be expressed using |-|, for instance, |[\D 0-5]| and |[^6-9]| are +% equivalent. +% +% \subsection{Structure: alternatives, groups, repetitions} +% % Quantifiers (repetition). % \begin{l3regex-syntax} % \item[?] $0$ or $1$, greedy. @@ -250,24 +271,6 @@ % \item[\{$n,m$\}?] At least $n$, no more than $m$, lazy. % \end{l3regex-syntax} % -% Anchors and simple assertions. -% \begin{l3regex-syntax} -% \item[\\b] Word boundary: either the previous token is matched by -% |\w| and the next by |\W|, or the opposite. For this purpose, -% the ends of the token list are considered as |\W|. -% \item[\\B] Not a word boundary: between two |\w| tokens -% or two |\W| tokens (including the boundary). -% \item[\char`^ \textrm{or} \\A] -% Start of the subject token list. -% \item[\char`$\textrm{,} \\Z \textrm{or} \\z] ^^A $ -% End of the subject token list. -% \item[\\G] Start of the current match. This is only different from |^| -% in the case of multiple matches: for instance -% |\regex_count:nnN { \G a } { aaba } \l_tmpa_int| yields $2$, but -% replacing |\G| by |^| would result in \cs{l_tmpa_int} holding the -% value $1$. -% \end{l3regex-syntax} -% % Alternation and capturing groups. % \begin{l3regex-syntax} % \item[A\char`|B\char`|C] Either one of \texttt{A}, \texttt{B}, @@ -280,6 +283,31 @@ % group number. % \end{l3regex-syntax} % +% Capturing groups are a means of extracting information about the +% match. Parenthesized groups are labelled in the order of their +% opening parenthesis, starting at $1$. The contents of those groups +% corresponding to the \enquote{best} match (leftmost longest) +% can be extracted and stored in a sequence of token lists using for +% instance \cs{regex_extract_once:nnNTF}. +% +% The |\K| escape sequence resets the beginning of the match to the +% current position in the token list. This only affects what is reported +% as the full match. For instance, +% \begin{verbatim} +% \regex_extract_all:nnN { a \K . } { a123aaxyz } \l_foo_seq +% \end{verbatim} +% results in \cs[no-index]{l_foo_seq} containing the items |{1}| and |{a}|: the +% true matches are |{a1}| and |{aa}|, but they are trimmed by the use of +% |\K|. The |\K| command does not affect capturing groups: for instance, +% \begin{verbatim} +% \regex_extract_once:nnN { (. \K c)+ \d } { acbc3 } \l_foo_seq +% \end{verbatim} +% results in \cs[no-index]{l_foo_seq} containing the items |{c3}| and |{bc}|: the +% true match is |{acbc3}|, with first submatch |{bc}|, but |\K| resets +% the beginning of the match to the last position where it appears. +% +% \subsection{Matching exact tokens} +% % The |\c| escape sequence allows to test the category code of tokens, % and match control sequences. Each character category is represented % by a single uppercase letter: @@ -301,7 +329,7 @@ % \begin{l3regex-syntax} % \item[\\c\Arg{regex}] A control sequence whose csname matches the % \meta{regex}, anchored at the beginning and end, so that |\c{begin}| -% matches exactly \cs{begin}, and nothing else. +% matches exactly \cs[no-index]{begin}, and nothing else. % \item[\\cX] Applies to the next object, which can be a character, % character property, class, or group, and forces this object to % only match tokens with category |X| (any of |CBEMTPUDSLOA|. For @@ -328,11 +356,54 @@ % % The |\u| escape sequence allows to insert the contents of a token list % directly into a regular expression or a replacement, avoiding the need -% to escape special characters. Namely, |\u|\Arg{tl~var~name} matches -% the exact contents of the token list \meta{tl~var}. Within a |\c{...}| +% to escape special characters. Namely, |\u|\Arg{var~name} matches +% the exact contents (both character codes and category codes) of the +% variable \cs[no-index]{\meta{var~name}}, +% which are obtained by applying \cs{exp_not:v} \Arg{var~name} at the +% time the regular expression is compiled. Within a |\c{...}| % control sequence matching, the |\u| escape sequence only expands its -% argument once, in effect performing \cs{tl_to_str:v}. Quantifiers are -% not supported directly: use a group. +% argument once, in effect performing \cs{tl_to_str:v}. +% Quantifiers are supported. +% +% The |\ur| escape sequence allows to insert the contents of a |regex| +% variable into a larger regular expression. For instance, +% |A\ur{l_tmpa_regex}D| matches the tokens |A| and |D| separated by +% something that matches the regular expression +% \cs[no-index]{l_tmpa_regex}. This behaves as if a non-capturing group +% were surrounding \cs[no-index]{l_tmpa_regex}, and any group contained +% in \cs[no-index]{l_tmpa_regex} is converted to a non-capturing group. +% Quantifiers are supported. +% +% For instance, if \cs[no-index]{l_tmpa_regex} has value \verb"B|C", +% then |A\ur{l_tmpa_regex}D| is equivalent to \verb"A(?:B|C)D" (matching +% |ABD| or |ACD|) and not to \verb"AB|CD" (matching |AB| or |CD|). To +% get the latter effect, it is simplest to use \TeX{}'s expansion +% machinery directly: if \cs[no-index]{l_mymodule_BC_tl} contains +% \verb"B|C" then the following two lines show the same result: +% \begin{quote} +% \cs{regex_show:n} |{ A \u{l_mymodule_BC_tl} D }| \\ +% \cs{regex_show:n} \verb"{ A B | C D }" +% \end{quote} +% +% \subsection{Miscellaneous} +% +% Anchors and simple assertions. +% \begin{l3regex-syntax} +% \item[\\b] Word boundary: either the previous token is matched by +% |\w| and the next by |\W|, or the opposite. For this purpose, +% the ends of the token list are considered as |\W|. +% \item[\\B] Not a word boundary: between two |\w| tokens +% or two |\W| tokens (including the boundary). +% \item[\char`^ \textrm{or} \\A] +% Start of the subject token list. +% \item[\char`$\textrm{,} \\Z \textrm{or} \\z] ^^A $ +% End of the subject token list. +% \item[\\G] Start of the current match. This is only different from |^| +% in the case of multiple matches: for instance +% |\regex_count:nnN { \G a } { aaba } \l_tmpa_int| yields $2$, but +% replacing |\G| by |^| would result in \cs{l_tmpa_int} holding the +% value $1$. +% \end{l3regex-syntax} % % The option |(?i)| makes the match case insensitive (identifying % \texttt{A}--\texttt{Z} with \texttt{a}--\texttt{z}; no Unicode support @@ -346,42 +417,6 @@ % |i| option. % ^^A \] % -% In character classes, only |[|, |^|, |-|, |]|, |\| and spaces are -% special, and should be escaped. Other non-alphanumeric characters can -% still be escaped without harm. Any escape sequence which matches a -% single character (|\d|, |\D|, \emph{etc.}) is supported in character -% classes. If the first character is |^|, then -% the meaning of the character class is inverted; |^| appearing anywhere -% else in the range is not special. If the first character (possibly -% following a leading |^|) is |]| then it does not need to be escaped -% since ending the range there would make it empty. -% Ranges of characters -% can be expressed using |-|, for instance, |[\D 0-5]| and |[^6-9]| are -% equivalent. -% -% Capturing groups are a means of extracting information about the -% match. Parenthesized groups are labelled in the order of their -% opening parenthesis, starting at $1$. The contents of those groups -% corresponding to the \enquote{best} match (leftmost longest) -% can be extracted and stored in a sequence of token lists using for -% instance \cs{regex_extract_once:nnNTF}. -% -% The |\K| escape sequence resets the beginning of the match to the -% current position in the token list. This only affects what is reported -% as the full match. For instance, -% \begin{verbatim} -% \regex_extract_all:nnN { a \K . } { a123aaxyz } \l_foo_seq -% \end{verbatim} -% results in \cs{l_foo_seq} containing the items |{1}| and |{a}|: the -% true matches are |{a1}| and |{aa}|, but they are trimmed by the use of -% |\K|. The |\K| command does not affect capturing groups: for instance, -% \begin{verbatim} -% \regex_extract_once:nnN { (. \K c)+ \d } { acbc3 } \l_foo_seq -% \end{verbatim} -% results in \cs{l_foo_seq} containing the items |{c3}| and |{bc}|: the -% true match is |{acbc3}|, with first submatch |{bc}|, but |\K| resets -% the beginning of the match to the last position where it appears. -% % \section{Syntax of the replacement text} % % Most of the features described in regular expressions do not make @@ -412,7 +447,7 @@ % \tl_set:Nn \l_my_tl { Hello,~world! } % \regex_replace_all:nnN { ([er]?l|o) . } { (\0--\1) } \l_my_tl % \end{verbatim} -% results in \cs{l_my_tl} holding |H(ell--el)(o,--o) w(or--o)(ld--l)!| +% results in \cs[no-index]{l_my_tl} holding |H(ell--el)(o,--o) w(or--o)(ld--l)!| % % The submatches are numbered according to the order in which the % opening parenthesis of capturing groups appear in the regular @@ -423,10 +458,17 @@ % the last match is used in the replacement text. Submatches always keep % the same category codes as in the original token list. % -% The characters inserted by the replacement have category code $12$ -% (other) by default, with the exception of space characters. Spaces -% inserted through \verb*|\ | have category code $10$, while spaces -% inserted through |\x20| or |\x{20}| have category code $12$. +% By default, the category code of characters inserted by the +% replacement are determined by the prevailing category code regime at +% the time where the replacement is made, with two exceptions: +% \begin{itemize} +% \item space characters (with character code $32$) inserted with +% \verb*|\ | or |\x20| or |\x{20}| have category code~$10$ regardless +% of the prevailing category code regime; +% \item if the category code would be $0$~(escape), $5$~(newline), +% $9$~(ignore), $14$~(comment) or $15$~(invalid), it is replaced by +% $12$~(other) instead. +% \end{itemize} % The escape sequence |\c| allows to insert characters % with arbitrary category codes, as well as control sequences. % \begin{l3regex-syntax} @@ -441,8 +483,8 @@ % submatches |\0|, |\1|, and so on, as in the example for |\u| below. % \end{l3regex-syntax} % -% The escape sequence |\u|\Arg{tl~var~name} allows to insert the -% contents of the token list with name \meta{tl~var~name} directly into +% The escape sequence |\u|\Arg{var~name} allows to insert the +% contents of the variable with name \meta{var~name} directly into % the replacement, giving an easier control of category codes. When % nested in |\c{|\ldots{}|}| and |\u{|\ldots{}|}| constructions, the % |\u| and |\c|~escape sequences perform \cs{tl_to_str:v}, namely @@ -455,7 +497,16 @@ % \tl_set:Nn \l_my_tl { one , two , one , one } % \regex_replace_all:nnN { [^,]+ } { \u{l_my_\0_tl} } \l_my_tl % \end{verbatim} -% results in \cs{l_my_tl} holding |first,\emph{second},first,first|. +% results in \cs[no-index]{l_my_tl} holding |first,\emph{second},first,first|. +% +% Regex replacement is also a convenient way to produce token lists +% with arbitrary category codes. For instance +% \begin{verbatim} +% \tl_clear:N \l_tmpa_tl +% \regex_replace_all:nnN { } { \cU\% \cA\~ } \l_tmpa_tl +% \end{verbatim} +% results in \cs[no-index]{l_tmpa_tl} containing the percent character +% with category code~$7$ (superscript) and an active tilde character. % % \section{Pre-compiling regular expressions} % @@ -493,18 +544,21 @@ % which never change. % \end{function} % -% \begin{function}[added = 2017-05-26]{\regex_show:n, \regex_show:N} +% \begin{function}[added = 2021-04-26, updated = 2021-04-29] +% {\regex_show:N, \regex_show:n, \regex_log:N, \regex_log:n} % \begin{syntax} % \cs{regex_show:n} \Arg{regex} +% \cs{regex_log:n} \Arg{regex} % \end{syntax} -% Shows how \pkg{l3regex} interprets the \meta{regex}. For instance, +% Displays in the terminal or writes in the log file (respectively) +% how \pkg{l3regex} interprets the \meta{regex}. For instance, % \cs{regex_show:n} \verb+{\A X|Y}+ shows % \begin{verbatim} % +-branch % anchor at start (\A) -% char code 88 +% char code 88 (X) % +-branch -% char code 89 +% char code 89 (Y) % \end{verbatim} % indicating that the anchor |\A| only applies to the first branch: % the second branch is not anchored to the beginning of the match. @@ -515,7 +569,7 @@ % All regular expression functions are available in both |:n| and |:N| % variants. The former require a \enquote{standard} regular expression, % while the later require a compiled expression as generated by -% \cs{regex_(g)set:Nn}. +% \cs{regex_set:Nn}. % % \begin{function}[TF, added = 2017-05-26]{\regex_match:nn, \regex_match:Nn} % \begin{syntax} @@ -549,7 +603,7 @@ % \int_new:N \l_foo_int % \regex_count:nnN { (b+|c) } { abbababcbb } \l_foo_int % \end{verbatim} -% results in \cs{l_foo_int} taking the value $5$. +% results in \cs[no-index]{l_foo_int} taking the value $5$. % \end{function} % % \section{Submatch extraction} @@ -577,10 +631,10 @@ % Then the regular expression (anchored at the start with |\A| and % at the end with |\Z|) must match the whole token list. The first % capturing group, |(La)?|, matches |La|, and the second capturing -% group, |(!*)|, matches |!!!|. Thus, |\l_foo_seq| contains as a result +% group, |(!*)|, matches |!!!|. Thus, \cs[no-index]{l_foo_seq} contains as a result % the items |{LaTeX!!!}|, |{La}|, and |{!!!}|, and the \texttt{true} % branch is left in the input stream. -% Note that the $n$-th item of |\l_foo_seq|, as obtained using +% Note that the $n$-th item of \cs[no-index]{l_foo_seq}, as obtained using % \cs{seq_item:Nn}, correspond to the submatch numbered $(n-1)$ in % functions such as \cs{regex_replace_once:nnN}. % \end{function} @@ -695,7 +749,7 @@ % \item Add tracing information. % \item Detect attempts to use back-references and other % non-implemented syntax. -% \item Test for the maximum register \cs{c_max_register_int}. +% \item Test for the maximum register \cs[no-index]{c_max_register_int}. % \item Find out whether the fact that |\W| and friends match the % end-marker leads to bugs. Possibly update \cs[no-index]{__regex_item_reverse:n}. % \item The empty cs should be matched by |\c{}|, not by @@ -753,11 +807,6 @@ % \item Unicode properties: |\p{..}| and |\P{..}|; % |\X| which should match any \enquote{extended} Unicode sequence. % This requires to manipulate a lot of data, probably using tree-boxes. -% \item Provide a syntax such as |\ur{l_my_regex}| to use an -% already-compiled regex in a more complicated regex. This makes -% regexes more easily composable. -% \item Allowing |\u{l_my_tl}| in more places, for instance as the -% number of repetitions in a quantifier. % \end{itemize} % % The following features of \textsc{pcre} or Perl may or may not be @@ -1218,9 +1267,7 @@ \if_int_compare:w #1 = \l_@@_curr_char_int \exp_after:wN \@@_break_true:w \fi: - \if_int_compare:w \l_@@_case_changed_char_int = \c_max_int - \@@_compute_case_changed_char: - \fi: + \@@_maybe_compute_ccc: \if_int_compare:w #1 = \l_@@_case_changed_char_int \exp_after:wN \@@_break_true:w \fi: @@ -1232,9 +1279,7 @@ \exp_after:wN \exp_after:wN \exp_after:wN \@@_break_true:w \fi: \fi: - \if_int_compare:w \l_@@_case_changed_char_int = \c_max_int - \@@_compute_case_changed_char: - \fi: + \@@_maybe_compute_ccc: \reverse_if:N \if_int_compare:w #1 > \l_@@_case_changed_char_int \reverse_if:N \if_int_compare:w #2 < \l_@@_case_changed_char_int \exp_after:wN \exp_after:wN \exp_after:wN \@@_break_true:w @@ -1246,8 +1291,7 @@ % % \begin{macro}{\@@_compute_case_changed_char:} % This function is called when \cs{l_@@_case_changed_char_int} has -% not yet been computed (or rather, when it is set to the marker value -% \cs{c_max_int}). If the current character code is in the range +% not yet been computed. If the current character code is in the range % $[65,90]$ (upper-case), then add $32$, making it lowercase. If it is % in the lower-case letter range $[97,122]$, subtract $32$. % \begin{macrocode} @@ -1267,7 +1311,9 @@ { \c_@@_ascii_lower_int } \fi: \fi: + \cs_set_eq:NN \@@_maybe_compute_ccc: \prg_do_nothing: } +\cs_new_eq:NN \@@_maybe_compute_ccc: \@@_compute_case_changed_char: % \end{macrocode} % \end{macro} % @@ -1583,7 +1629,7 @@ \cs_new_eq:NN \@@_escape_break:w \prg_break: \cs_new:cpn { @@_escape_/break:w } { - \__kernel_msg_expandable_error:nn { kernel } { trailing-backslash } + \__kernel_msg_expandable_error:nn { regex } { trailing-backslash } \prg_break: } \cs_new:cpn { @@_escape_~:w } { } @@ -1620,7 +1666,7 @@ { \int_compare:nNnTF {#1} > \c_max_char_int { - \__kernel_msg_expandable_error:nnff { kernel } { x-overflow } + \__kernel_msg_expandable_error:nnff { regex } { x-overflow } {#1} { \int_to_Hex:n {#1} } } { @@ -1706,7 +1752,7 @@ } \cs_new:Npn \@@_escape_x_loop_error:n #1 { - \__kernel_msg_expandable_error:nnn { kernel } { x-missing-rbrace } {#1} + \__kernel_msg_expandable_error:nnn { regex } { x-missing-rbrace } {#1} \@@_escape_loop:N #1 } % \end{macrocode} @@ -2143,7 +2189,7 @@ \if_int_compare:w \l_@@_mode_int = \c_@@_class_mode_int \exp_after:wN \exp_after:wN \exp_after:wN \use:n \else: - \__kernel_msg_error:nn { kernel } { c-bad-mode } + \__kernel_msg_error:nn { regex } { c-bad-mode } \exp_after:wN \exp_after:wN \exp_after:wN \use_none:n \fi: \fi: @@ -2196,13 +2242,13 @@ { \@@_if_in_class:TF { - \__kernel_msg_error:nn { kernel } { missing-rbrack } + \__kernel_msg_error:nn { regex } { missing-rbrack } \use:c { @@_compile_]: } \prg_do_nothing: \prg_do_nothing: } { } \if_int_compare:w \l_@@_group_level_int > 0 \exp_stop_f: - \__kernel_msg_error:nnx { kernel } { missing-rparen } + \__kernel_msg_error:nnx { regex } { missing-rparen } { \int_use:N \l_@@_group_level_int } \prg_replicate:nn { \l_@@_group_level_int } @@ -2259,10 +2305,10 @@ \prg_do_nothing: \prg_do_nothing: \prg_do_nothing: \prg_do_nothing: \int_compare:nNnT \l_@@_mode_int = \c_@@_catcode_mode_int - { \__kernel_msg_error:nn { kernel } { c-trailing } } + { \__kernel_msg_error:nn { regex } { c-trailing } } \int_compare:nNnT \l_@@_mode_int < \c_@@_outer_mode_int { - \__kernel_msg_error:nn { kernel } { c-missing-rbrace } + \__kernel_msg_error:nn { regex } { c-missing-rbrace } \@@_compile_end_cs: \prg_do_nothing: \prg_do_nothing: \prg_do_nothing: \prg_do_nothing: @@ -2343,6 +2389,21 @@ % % \subsubsection{Quantifiers} % +% \begin{macro}{\@@_compile_if_quantifier:TFw} +% This looks ahead and checks whether there are any quantifier +% (special character equal to either of \texttt{?+*\{}). This is +% useful for the |\u| and |\ur| escape sequences. +% \begin{macrocode} +\cs_new_protected:Npn \@@_compile_if_quantifier:TFw #1#2#3#4 + { + \token_if_eq_meaning:NNTF #3 \@@_compile_special:N + { \cs_if_exist:cTF { @@_compile_quantifier_#4:w } } + { \use_ii:nn } + {#1} {#2} #3 #4 + } +% \end{macrocode} +% \end{macro} +% % \begin{macro}{\@@_compile_quantifier:w} % This looks ahead and finds any quantifier (special character equal % to either of \texttt{?+*\{}). @@ -2373,7 +2434,7 @@ \cs_new_protected:Npn \@@_compile_quantifier_abort:xNN #1#2#3 { \@@_compile_quantifier_none: - \__kernel_msg_warning:nnxx { kernel } { invalid-quantifier } {#1} {#3} + \__kernel_msg_warning:nnxx { regex } { invalid-quantifier } {#1} {#3} \@@_compile_abort_tokens:x {#1} #2 #3 } @@ -2489,7 +2550,7 @@ { \if_int_compare:w \l_@@_internal_a_int > \l_@@_internal_b_int - \__kernel_msg_error:nnxx { kernel } { backwards-quantifier } + \__kernel_msg_error:nnxx { regex } { backwards-quantifier } { \int_use:N \l_@@_internal_a_int } { \int_use:N \l_@@_internal_b_int } \int_zero:N \l_@@_internal_b_int @@ -2523,7 +2584,7 @@ % \begin{macrocode} \cs_new_protected:Npn \@@_compile_raw_error:N #1 { - \__kernel_msg_error:nnx { kernel } { bad-escape } {#1} + \__kernel_msg_error:nnx { regex } { bad-escape } {#1} \@@_compile_raw:N #1 } % \end{macrocode} @@ -2582,7 +2643,7 @@ \@@_if_end_range:NNTF #2 #3 { \if_int_compare:w `#1 > `#3 \exp_stop_f: - \__kernel_msg_error:nnxx { kernel } { range-backwards } {#1} {#3} + \__kernel_msg_error:nnxx { regex } { range-backwards } {#1} {#3} \else: \tl_build_put_right:Nx \l_@@_build_tl { @@ -2596,7 +2657,7 @@ \fi: } { - \__kernel_msg_warning:nnxx { kernel } { range-missing-end } + \__kernel_msg_warning:nnxx { regex } { range-missing-end } {#1} { \c_backslash_str #3 } \tl_build_put_right:Nx \l_@@_build_tl { @@ -2859,11 +2920,11 @@ { : { \@@_compile_class_posix:NNNNw } = { - \__kernel_msg_warning:nnx { kernel } + \__kernel_msg_warning:nnx { regex } { posix-unsupported } { = } } . { - \__kernel_msg_warning:nnx { kernel } + \__kernel_msg_warning:nnx { regex } { posix-unsupported } { . } } } @@ -2905,7 +2966,7 @@ } } { - \__kernel_msg_warning:nnx { kernel } { posix-unknown } + \__kernel_msg_warning:nnx { regex } { posix-unknown } { \l_@@_internal_a_tl } \@@_compile_abort_tokens:x { @@ -2915,7 +2976,7 @@ } } { - \__kernel_msg_error:nnxx { kernel } { posix-missing-close } + \__kernel_msg_error:nnxx { regex } { posix-missing-close } { [: \l_@@_internal_a_tl } { #2 #4 } \@@_compile_abort_tokens:x { [: \l_@@_internal_a_tl } #1 #2 #3 #4 @@ -2961,7 +3022,7 @@ \int_set_eq:NN \l_@@_catcodes_int \l_@@_default_catcodes_int \exp_after:wN \@@_compile_quantifier:w \else: - \__kernel_msg_warning:nn { kernel } { extra-rparen } + \__kernel_msg_warning:nn { regex } { extra-rparen } \exp_after:wN \@@_compile_raw:N \exp_after:wN ) \fi: } @@ -2980,7 +3041,7 @@ { \if_int_compare:w \l_@@_mode_int = \c_@@_catcode_in_class_mode_int - \__kernel_msg_error:nn { kernel } { c-lparen-in-class } + \__kernel_msg_error:nn { regex } { c-lparen-in-class } \exp_after:wN \@@_compile_raw:N \exp_after:wN ( \else: \exp_after:wN \@@_compile_lparen:w @@ -2994,7 +3055,7 @@ \cs_if_exist_use:cF { @@_compile_special_group_\token_to_str:N #4 :w } { - \__kernel_msg_warning:nnx { kernel } { special-group-unknown } + \__kernel_msg_warning:nnx { regex } { special-group-unknown } { (? #4 } \@@_compile_group_begin:N \@@_group:nnnN \@@_compile_raw:N ? #3 #4 @@ -3063,7 +3124,7 @@ { \@@_item_caseless_range:nn } } { - \__kernel_msg_warning:nnx { kernel } { unknown-option } { (?i #2 } + \__kernel_msg_warning:nnx { regex } { unknown-option } { (?i #2 } \@@_compile_raw:N ( \@@_compile_raw:N ? \@@_compile_raw:N i @@ -3082,7 +3143,7 @@ { \@@_item_caseful_range:nn } } { - \__kernel_msg_warning:nnx { kernel } { unknown-option } { (?-#2#4 } + \__kernel_msg_warning:nnx { regex } { unknown-option } { (?-#2#4 } \@@_compile_raw:N ( \@@_compile_raw:N ? \@@_compile_raw:N - @@ -3121,7 +3182,7 @@ } { \cs_if_exist_use:cF { @@_compile_c_#2:w } } { - \__kernel_msg_error:nnx { kernel } { c-missing-category } {#2} + \__kernel_msg_error:nnx { regex } { c-missing-category } {#2} #1 #2 } } @@ -3142,7 +3203,7 @@ { \token_if_eq_charcode:NNF #2 ( } % ) } { \use:n } - { \__kernel_msg_error:nnn { kernel } { c-C-invalid } {#2} } + { \__kernel_msg_error:nnn { regex } { c-C-invalid } {#2} } #1 #2 } % \end{macrocode} @@ -3195,7 +3256,7 @@ { \@@_compile_c_lbrack_end: } } { - \__kernel_msg_error:nnx { kernel } { c-missing-rbrack } {#2} + \__kernel_msg_error:nnx { regex } { c-missing-rbrack } {#2} \@@_compile_c_lbrack_end: #1 #2 } @@ -3237,6 +3298,21 @@ % \end{macrocode} % \end{macro} % +% \begin{macro}+\@@_compile_{:+ +% We forbid unescaped left braces inside a |\c{...}| escape because +% they otherwise lead to the confusing question of whether the first +% right brace in |\c{{}x}| should end |\c| or whether one should +% match braces. +% \begin{macrocode} +\cs_new_protected:cpn { @@_compile_ \c_left_brace_str : } + { + \@@_if_in_cs:TF + { \__kernel_msg_error:nnn { regex } { cu-lbrace } { c } } + { \exp_after:wN \@@_compile_raw:N \c_left_brace_str } + } +% \end{macrocode} +% \end{macro} +% % \begin{macro}+\@@_compile_}:+ % \begin{macro}{\@@_compile_end_cs:} % \begin{macro}[EXP]{\@@_compile_cs_aux:Nn, \@@_compile_cs_aux:NNnnnN} @@ -3320,36 +3396,58 @@ % \end{macro} % \end{macro} % -% \subsubsection{Raw token lists with \cs{u}} +% \subsubsection{Raw token lists with \cs[no-index]{u}} % % \begin{macro}{\@@_compile_/u:} -% \begin{macro}[EXP]{\@@_compile_u_loop:NN} % The |\u| escape is invalid in classes and directly following a -% catcode test. Otherwise, it must be followed by a left brace. We -% then collect the characters for the argument of |\u| within an -% \texttt{x}-expanding assignment. In principle we could just wait to -% encounter a right brace, but this is unsafe: if the right brace was -% missing, then we would reach the end-markers of the regex, and -% continue, leading to obscure fatal errors. Instead, we only allow -% raw and special characters, and stop when encountering a special -% right brace, any escaped character, or the end-marker. +% catcode test. Otherwise test for a following |r| (for |\ur|), and +% call an auxiliary responsible for finding the variable name. % \begin{macrocode} \cs_new_protected:cpn { @@_compile_/u: } #1#2 { \@@_if_in_class_or_catcode:TF { \@@_compile_raw_error:N u #1 #2 } { - \@@_two_if_eq:NNNNTF #1 #2 \@@_compile_special:N \c_left_brace_str - { - \__kernel_tl_set:Nx \l_@@_internal_a_tl { \if_false: } \fi: - \@@_compile_u_loop:NN - } - { - \__kernel_msg_error:nn { kernel } { u-missing-lbrace } - \@@_compile_raw:N u #1 #2 - } + \@@_two_if_eq:NNNNTF #1 #2 \@@_compile_raw:N r + { \@@_compile_u_brace:NNN \@@_compile_ur_end: } + { \@@_compile_u_brace:NNN \@@_compile_u_end: #1 #2 } + } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_compile_u_brace:NNN} +% This enforces the presence of a left brace, then starts a loop to +% find the variable name. +% \begin{macrocode} +\cs_new:Npn \@@_compile_u_brace:NNN #1#2#3 + { + \@@_two_if_eq:NNNNTF #2 #3 \@@_compile_special:N \c_left_brace_str + { + \tl_set:Nn \l_@@_internal_b_tl {#1} + \__kernel_tl_set:Nx \l_@@_internal_a_tl { \if_false: } \fi: + \@@_compile_u_loop:NN + } + { + \__kernel_msg_error:nn { regex } { u-missing-lbrace } + \token_if_eq_meaning:NNTF #1 \@@_compile_ur_end: + { \@@_compile_raw:N u \@@_compile_raw:N r } + { \@@_compile_raw:N u } + #2 #3 } } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[EXP]{\@@_compile_u_loop:NN} +% We collect the characters for the argument of |\u| within an +% \texttt{x}-expanding assignment. In principle we could just wait to +% encounter a right brace, but this is unsafe: if the right brace was +% missing, then we would reach the end-markers of the regex, and +% continue, leading to obscure fatal errors. Instead, we only allow +% raw and special characters, and stop when encountering a special +% right brace, any escaped character, or the end-marker. +% \begin{macrocode} \cs_new:Npn \@@_compile_u_loop:NN #1#2 { \token_if_eq_meaning:NNTF #1 \@@_compile_raw:N @@ -3358,29 +3456,92 @@ \token_if_eq_meaning:NNTF #1 \@@_compile_special:N { \exp_after:wN \token_if_eq_charcode:NNTF \c_right_brace_str #2 - { \if_false: { \fi: } \@@_compile_u_end: } - { #2 \@@_compile_u_loop:NN } + { \if_false: { \fi: } \l_@@_internal_b_tl } + { + \if_charcode:w \c_left_brace_str #2 + \__kernel_msg_expandable_error:nnn { regex } { cu-lbrace } { u } + \else: + #2 + \fi: + \@@_compile_u_loop:NN + } } { \if_false: { \fi: } - \__kernel_msg_error:nnx { kernel } { u-missing-rbrace } {#2} - \@@_compile_u_end: + \__kernel_msg_error:nnx { regex } { u-missing-rbrace } {#2} + \l_@@_internal_b_tl #1 #2 } } } % \end{macrocode} % \end{macro} +% +% \begin{macro}{\@@_compile_ur_end:, \@@_compile_ur:n} +% \begin{macro}[EXP]{\@@_compile_ur_aux:w} +% For the |\ur{...}| construction, once we have extracted the +% variable's name, we replace all groups by non-capturing groups in +% the compiled regex (passed as the +% argument of \cs{@@_compile_ur:n}). If that has a single branch +% (namely \cs{tl_if_empty:oTF} is false) and there is no quantifier, +% then simply insert the contents of this branch (obtained by +% \cs{use_ii:nn}, which is expanded later). In all other cases, +% insert a non-capturing group and look for quantifiers to determine +% the number of repetition etc. +% \begin{macrocode} +\cs_new_protected:Npn \@@_compile_ur_end: + { + \group_begin: + \cs_set:Npn \@@_group:nnnN { \@@_group_no_capture:nnnN } + \cs_set:Npn \@@_group_resetting:nnnN { \@@_group_no_capture:nnnN } + \exp_args:NNx + \group_end: + \@@_compile_ur:n { \use:c { \l_@@_internal_a_tl } } + } +\cs_new_protected:Npn \@@_compile_ur:n #1 + { + \tl_if_empty:oTF { \@@_compile_ur_aux:w #1 {} ? ? \q_@@_nil } + { \@@_compile_if_quantifier:TFw } + { \use_i:nn } + { + \tl_build_put_right:Nn \l_@@_build_tl + { \@@_group_no_capture:nnnN { \if_false: } \fi: #1 } + \@@_compile_quantifier:w + } + { \tl_build_put_right:Nn \l_@@_build_tl { \use_ii:nn #1 } } + } +\cs_new:Npn \@@_compile_ur_aux:w \@@_branch:n #1#2#3 \q_@@_nil {#2} +% \end{macrocode} +% \end{macro} % \end{macro} % -% \begin{macro}{\@@_compile_u_end:} -% Once we have extracted the variable's name, we store the contents of -% that variable in \cs{l_@@_internal_a_tl}. The behaviour of |\u| +% \begin{macro}{\@@_compile_u_end:, \@@_compile_u_payload:} +% Once we have extracted the variable's name, we check for +% quantifiers, in which case we set up a non-capturing group with a +% single branch. Inside this branch (we omit it and the group if +% there is no quantifier), \cs{@@_compile_u_payload:} puts +% the right tests corresponding to the contents of the variable, which +% we store in \cs{l_@@_internal_a_tl}. The behaviour of |\u| % then depends on whether we are within a |\c{...}| escape (in this % case, the variable is turned to a string), or not. % \begin{macrocode} \cs_new_protected:Npn \@@_compile_u_end: { + \@@_compile_if_quantifier:TFw + { + \tl_build_put_right:Nn \l_@@_build_tl + { + \@@_group_no_capture:nnnN { \if_false: } \fi: + \@@_branch:n { \if_false: } \fi: + } + \@@_compile_u_payload: + \tl_build_put_right:Nn \l_@@_build_tl { \if_false: { \fi: } } + \@@_compile_quantifier:w + } + { \@@_compile_u_payload: } + } +\cs_new_protected:Npn \@@_compile_u_payload: + { \tl_set:Nv \l_@@_internal_a_tl { \l_@@_internal_a_tl } \if_int_compare:w \l_@@_mode_int = \c_@@_outer_mode_int \@@_compile_u_not_cs: @@ -3465,6 +3626,160 @@ % % \subsubsection{Showing regexes} % +% \begin{macro}[rEXP] +% { +% \@@_clean_bool:n, \@@_clean_int:n, \@@_clean_int_aux:N, +% \@@_clean_regex:n, \@@_clean_regex_loop:w, \@@_clean_branch:n, +% \@@_clean_branch_loop:n, \@@_clean_assertion:Nn, +% \@@_clean_class:NnnnN, \@@_clean_group:nnnN, \@@_clean_class:n, +% \@@_clean_class_loop:nnn, \@@_clean_exact_cs:n, +% \@@_clean_exact_cs:w +% } +% Before showing a regex we check that it is \enquote{clean} in the +% sense that it has the correct internal structure. We do this (in +% the implementation of \cs{regex_show:N} and \cs{regex_log:N}) by +% comparing it with a cleaned-up version of the same regex. Along the +% way we also need similar functions for other types: all +% \cs[no-index]{@@_clean_\meta{type}:n} functions produce valid +% \meta{type} tokens (bool, explicit integer, etc.\@) from arbitrary +% input, and the output coincides with the input if that was valid. +% \begin{macrocode} +\cs_new:Npn \@@_clean_bool:n #1 + { + \tl_if_single:nTF {#1} + { \bool_if:NTF #1 \c_true_bool \c_false_bool } + { \c_true_bool } + } +\cs_new:Npn \@@_clean_int:n #1 + { + \tl_if_head_eq_meaning:nNTF {#1} - + { - \exp_args:No \@@_clean_int:n { \use_none:n #1 } } + { \int_eval:n { 0 \str_map_function:nN {#1} \@@_clean_int_aux:N } } + } +\cs_new:Npn \@@_clean_int_aux:N #1 + { + \if_int_compare:w 1 < 1 #1 ~ + #1 + \else: + \exp_after:wN \str_map_break: + \fi: + } +\cs_new:Npn \@@_clean_regex:n #1 + { + \@@_clean_regex_loop:w #1 + \@@_branch:n { \q_recursion_tail } \q_recursion_stop + } +\cs_new:Npn \@@_clean_regex_loop:w #1 \@@_branch:n #2 + { + \quark_if_recursion_tail_stop:n {#2} + \@@_branch:n { \@@_clean_branch:n {#2} } + \@@_clean_regex_loop:w + } +\cs_new:Npn \@@_clean_branch:n #1 + { + \@@_clean_branch_loop:n #1 + ? ? ? ? ? ? \prg_break_point: + } +\cs_new:Npn \@@_clean_branch_loop:n #1 + { + \tl_if_single:nF {#1} { \prg_break: } + \token_case_meaning:NnF #1 + { + \@@_command_K: { #1 \@@_clean_branch_loop:n } + \@@_assertion:Nn { #1 \@@_clean_assertion:Nn } + \@@_class:NnnnN { #1 \@@_clean_class:NnnnN } + \@@_group:nnnN { #1 \@@_clean_group:nnnN } + \@@_group_no_capture:nnnN { #1 \@@_clean_group:nnnN } + \@@_group_resetting:nnnN { #1 \@@_clean_group:nnnN } + } + { \prg_break: } + } +\cs_new:Npn \@@_clean_assertion:Nn #1#2 + { + \@@_clean_bool:n {#1} + \tl_if_single:nF {#2} { { \@@_A_test: } \prg_break: } + \token_case_meaning:NnTF #2 + { + \@@_A_test: { } + \@@_G_test: { } + \@@_Z_test: { } + \@@_b_test: { } + } + { {#2} } + { { \@@_A_test: } \prg_break: } + \@@_clean_branch_loop:n + } +\cs_new:Npn \@@_clean_class:NnnnN #1#2#3#4#5 + { + \@@_clean_bool:n {#1} + { \@@_clean_class:n {#2} } + { \int_max:nn { 0 } { \@@_clean_int:n {#3} } } + { \int_max:nn { -1 } { \@@_clean_int:n {#4} } } + \@@_clean_bool:n {#5} + \@@_clean_branch_loop:n + } +\cs_new:Npn \@@_clean_group:nnnN #1#2#3#4 + { + { \@@_clean_regex:n {#1} } + { \int_max:nn { 0 } { \@@_clean_int:n {#2} } } + { \int_max:nn { -1 } { \@@_clean_int:n {#3} } } + \@@_clean_bool:n {#4} + \@@_clean_branch_loop:n + } +\cs_new:Npn \@@_clean_class:n #1 + { \@@_clean_class_loop:nnn #1 ????? \prg_break_point: } +\cs_new:Npn \@@_clean_class_loop:nnn #1#2#3 + { + \tl_if_single:nF {#1} { \prg_break: } + \token_case_meaning:NnTF #1 + { + \@@_item_cs:n { #1 { \@@_clean_regex:n {#2} } } + \@@_item_exact_cs:n { #1 { \@@_clean_exact_cs:n {#2} } } + \@@_item_caseful_equal:n { #1 { \@@_clean_int:n {#2} } } + \@@_item_caseless_equal:n { #1 { \@@_clean_int:n {#2} } } + \@@_item_reverse:n { #1 { \@@_clean_class:n {#2} } } + } + { \@@_clean_class_loop:nnn {#3} } + { + \token_case_meaning:NnTF #1 + { + \@@_item_caseful_range:nn { } + \@@_item_caseless_range:nn { } + \@@_item_exact:nn { } + } + { #1 { \@@_clean_int:n {#2} } { \@@_clean_int:n {#3} } } + { + \token_case_meaning:NnTF #1 + { + \@@_item_catcode:nT { } + \@@_item_catcode_reverse:nT { } + } + { + #1 { \@@_clean_int:n {#2} } { \@@_clean_class:n {#3} } + \@@_clean_class_loop:nnn + } + { \prg_break: } + } + } + } +\cs_new:Npn \@@_clean_exact_cs:n #1 + { + \exp_last_unbraced:Nf \use_none:n + { + \@@_clean_exact_cs:w #1 + \scan_stop: \q_recursion_tail \scan_stop: + \q_recursion_stop + } + } +\cs_new:Npn \@@_clean_exact_cs:w #1 \scan_stop: + { + \quark_if_recursion_tail_stop:n {#1} + \scan_stop: \tl_to_str:n {#1} + \@@_clean_exact_cs:w + } +% \end{macrocode} +% \end{macro} +% % \begin{macro}{\@@_show:N} % Within a group and within \cs{tl_build_begin:N} \ldots{} \cs{tl_build_end:N} we % redefine all the function that can appear in a compiled regex, then @@ -3503,18 +3818,18 @@ \cs_set:Npn \@@_A_test: { anchor~at~start~(\iow_char:N\\A) } \cs_set:Npn \@@_G_test: { anchor~at~start~of~match~(\iow_char:N\\G) } \cs_set_protected:Npn \@@_item_caseful_equal:n ##1 - { \@@_show_one:n { char~code~\int_eval:n{##1} } } + { \@@_show_one:n { char~code~\@@_show_char:n{##1} } } \cs_set_protected:Npn \@@_item_caseful_range:nn ##1##2 { \@@_show_one:n - { range~[\int_eval:n{##1}, \int_eval:n{##2}] } + { range~[\@@_show_char:n{##1}, \@@_show_char:n{##2}] } } \cs_set_protected:Npn \@@_item_caseless_equal:n ##1 - { \@@_show_one:n { char~code~\int_eval:n{##1}~(caseless) } } + { \@@_show_one:n { char~code~\@@_show_char:n{##1}~(caseless) } } \cs_set_protected:Npn \@@_item_caseless_range:nn ##1##2 { \@@_show_one:n - { Range~[\int_eval:n{##1}, \int_eval:n{##2}]~(caseless) } + { Range~[\@@_show_char:n{##1}, \@@_show_char:n{##2}]~(caseless) } } \cs_set_protected:Npn \@@_item_catcode:nT { \@@_show_item_catcode:NnT \c_true_bool } @@ -3523,7 +3838,7 @@ \cs_set_protected:Npn \@@_item_reverse:n { \@@_show_scope:nn { Reversed~match } } \cs_set_protected:Npn \@@_item_exact:nn ##1##2 - { \@@_show_one:n { char~##2,~catcode~##1 } } + { \@@_show_one:n { char~\@@_show_char:n{##2},~catcode~##1 } } \cs_set_eq:NN \@@_item_exact_cs:n \@@_show_item_exact_cs:n \cs_set_protected:Npn \@@_item_cs:n { \@@_show_scope:nn { control~sequence } } @@ -3539,6 +3854,19 @@ % \end{macrocode} % \end{macro} % +% \begin{macro}[EXP]{\@@_show_char:n} +% Show a single character, together with its ascii representation if available. +% This could be extended to beyond ascii. It is not ideal for parentheses themselves. +% \begin{macrocode} +\cs_new:Npn \@@_show_char:n #1 + { + \int_eval:n {#1} + \int_compare:nT { 32 <= #1 <= 126 } + { ~ ( \char_generate:nn {#1} {12} ) } + } +% \end{macrocode} +% \end{macro} +% % \begin{macro}{\@@_show_one:n} % Every part of the final message go through this function, which adds % one line to the output, with the appropriate prefix. @@ -4796,7 +5124,7 @@ \int_add:Nn \l_@@_step_int { 2 } \int_incr:N \l_@@_curr_pos_int \int_set_eq:NN \l_@@_last_char_int \l_@@_curr_char_int - \int_set_eq:NN \l_@@_case_changed_char_int \c_max_int + \cs_set_eq:NN \@@_maybe_compute_ccc: \@@_compute_case_changed_char: \tl_set:Nn \l_@@_curr_token_tl {#1} \int_set:Nn \l_@@_curr_char_int {#2} \int_set:Nn \l_@@_curr_catcode_int { "#3 } @@ -5244,7 +5572,11 @@ \if_charcode:w \c_right_brace_str ##1 \@@_replacement_rbrace:N \else: - \@@_replacement_normal:n + \if_charcode:w \c_left_brace_str ##1 + \@@_replacement_lbrace:N + \else: + \@@_replacement_normal:n + \fi: \fi: ##1 } @@ -5253,14 +5585,14 @@ {#1} \prg_do_nothing: \prg_do_nothing: \if_int_compare:w \l_@@_replacement_csnames_int > 0 \exp_stop_f: - \__kernel_msg_error:nnx { kernel } { replacement-missing-rbrace } + \__kernel_msg_error:nnx { regex } { replacement-missing-rbrace } { \int_use:N \l_@@_replacement_csnames_int } \tl_build_put_right:Nx \l_@@_build_tl { \prg_replicate:nn \l_@@_replacement_csnames_int \cs_end: } \fi: \seq_if_empty:NF \l_@@_replacement_category_seq { - \__kernel_msg_error:nnx { kernel } { replacement-missing-rparen } + \__kernel_msg_error:nnx { regex } { replacement-missing-rparen } { \seq_count:N \l_@@_replacement_category_seq } \seq_clear:N \l_@@_replacement_category_seq } @@ -5303,7 +5635,7 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}{\@@_replacement_normal:n} +% \begin{macro}{\@@_replacement_normal:n, \@@_replacement_normal_aux:N} % Most characters are simply sent to the output by % \cs{tl_build_put_right:Nn}, unless a particular category code has been % requested: then \cs{@@_replacement_c_A:w} or a similar auxiliary is @@ -5312,13 +5644,16 @@ % sequence is non-empty there: it contains an empty entry % corresponding to the initial value of % \cs{l_@@_replacement_category_tl}. -% The argument |#1| can be a space, otherwise it is a single -% character. +% The argument |#1| is a single character (including the case of a catcode-other space). +% In case no specific catcode is requested, we taked into account the +% current catcode regime (at the time the replacement is performed) +% as much as reasonable, with all impossible catcodes (escape, +% newline, etc.) being mapped to \enquote{other}. % \begin{macrocode} \cs_new_protected:Npn \@@_replacement_normal:n #1 { \tl_if_empty:NTF \l_@@_replacement_category_tl - { \@@_replacement_put:n {#1} } + { \@@_replacement_normal_aux:N #1 } { % ( \token_if_eq_charcode:NNTF #1 ) { @@ -5326,15 +5661,37 @@ \l_@@_replacement_category_tl } { - \use:c - { - @@_replacement_c_ - \l_@@_replacement_category_tl :w - } - \@@_replacement_normal:n {#1} + \use:c { @@_replacement_c_ \l_@@_replacement_category_tl :w } + ? #1 } } } +\cs_new_protected:Npn \@@_replacement_normal_aux:N #1 + { + \token_if_eq_charcode:NNTF #1 \c_space_token + { \@@_replacement_c_S:w } + { + \exp_after:wN \exp_after:wN + \if_case:w \tex_catcode:D `#1 \exp_stop_f: + \@@_replacement_c_O:w + \or: \@@_replacement_c_B:w + \or: \@@_replacement_c_E:w + \or: \@@_replacement_c_M:w + \or: \@@_replacement_c_T:w + \or: \@@_replacement_c_O:w + \or: \@@_replacement_c_P:w + \or: \@@_replacement_c_U:w + \or: \@@_replacement_c_D:w + \or: \@@_replacement_c_O:w + \or: \@@_replacement_c_S:w + \or: \@@_replacement_c_L:w + \or: \@@_replacement_c_O:w + \or: \@@_replacement_c_A:w + \else: \@@_replacement_c_O:w + \fi: + } + ? #1 + } % \end{macrocode} % \end{macro} % @@ -5342,7 +5699,6 @@ % As in parsing a regular expression, we use an auxiliary built from % |#1| if defined. Otherwise, check for escaped digits (standing from % submatches from $0$ to $9$): anything else is a raw character. -% We use \cs{token_to_str:N} to give spaces the right category code. % \begin{macrocode} \cs_new_protected:Npn \@@_replacement_escaped:N #1 { @@ -5351,8 +5707,7 @@ \if_int_compare:w 1 < 1#1 \exp_stop_f: \@@_replacement_put_submatch:n {#1} \else: - \exp_args:No \@@_replacement_normal:n - { \token_to_str:N #1 } + \@@_replacement_normal:n {#1} \fi: } } @@ -5375,6 +5730,9 @@ { \if_int_compare:w #1 < \l_@@_capturing_group_int \@@_replacement_put_submatch_aux:n {#1} + \else: + \__kernel_msg_expandable_error:nnff { regex } { submatch-too-big } + {#1} { \int_eval:n { \l_@@_capturing_group_int - 1 } } \fi: } \cs_new_protected:Npn \@@_replacement_put_submatch_aux:n #1 @@ -5400,8 +5758,7 @@ % \begin{macrocode} \cs_new_protected:Npn \@@_replacement_g:w #1#2 { - \@@_two_if_eq:NNNNTF - #1 #2 \@@_replacement_normal:n \c_left_brace_str + \token_if_eq_meaning:NNTF #1 \@@_replacement_lbrace:N { \l_@@_internal_a_int = \@@_replacement_g_digits:NN } { \@@_replacement_error:NNN g #1 #2 } } @@ -5448,15 +5805,15 @@ { \token_if_eq_meaning:NNTF #1 \@@_replacement_normal:n { - \exp_after:wN \token_if_eq_charcode:NNTF \c_left_brace_str #2 + \cs_if_exist:cTF { @@_replacement_c_#2:w } + { \@@_replacement_cat:NNN #2 } + { \@@_replacement_error:NNN c #1#2 } + } + { + \token_if_eq_meaning:NNTF #1 \@@_replacement_lbrace:N { \@@_replacement_cu_aux:Nw \@@_replacement_exp_not:N } - { - \cs_if_exist:cTF { @@_replacement_c_#2:w } - { \@@_replacement_cat:NNN #2 } - { \@@_replacement_error:NNN c #1#2 } - } + { \@@_replacement_error:NNN c #1#2 } } - { \@@_replacement_error:NNN c #1#2 } } % \end{macrocode} % \end{macro} @@ -5490,8 +5847,7 @@ % \begin{macrocode} \cs_new_protected:Npn \@@_replacement_u:w #1#2 { - \@@_two_if_eq:NNNNTF - #1 #2 \@@_replacement_normal:n \c_left_brace_str + \token_if_eq_meaning:NNTF #1 \@@_replacement_lbrace:N { \@@_replacement_cu_aux:Nw \@@_replacement_exp_not:V } { \@@_replacement_error:NNN u #1#2 } } @@ -5515,6 +5871,21 @@ % \end{macrocode} % \end{macro} % +% \begin{macro}{\@@_replacement_lbrace:N} +% Within a |\c{...}| or |\u{...}| construction, this is +% forbidden. Otherwise, this is a raw left brace. +% \begin{macrocode} +\cs_new_protected:Npn \@@_replacement_lbrace:N #1 + { + \if_int_compare:w \l_@@_replacement_csnames_int > 0 \exp_stop_f: + \__kernel_msg_error:nnn { regex } { cu-lbrace } { u } + \else: + \@@_replacement_normal:n {#1} + \fi: + } +% \end{macrocode} +% \end{macro} +% % \subsubsection{Characters in replacement} % % \begin{macro}{\@@_replacement_cat:NNN} @@ -5528,12 +5899,12 @@ \cs_new_protected:Npn \@@_replacement_cat:NNN #1#2#3 { \token_if_eq_meaning:NNTF \prg_do_nothing: #3 - { \__kernel_msg_error:nn { kernel } { replacement-catcode-end } } + { \__kernel_msg_error:nn { regex } { replacement-catcode-end } } { \int_compare:nNnTF { \l_@@_replacement_csnames_int } > 0 { \__kernel_msg_error:nnnn - { kernel } { replacement-catcode-in-cs } {#1} {#3} + { regex } { replacement-catcode-in-cs } {#1} {#3} #2 #3 } { @@ -5549,7 +5920,7 @@ \@@_char_if_alphanumeric:NTF #3 { \__kernel_msg_error:nnnn - { kernel } { replacement-catcode-escaped } + { regex } { replacement-catcode-escaped } {#1} {#3} } { } @@ -5713,7 +6084,7 @@ \cs_new_protected:Npn \@@_replacement_c_S:w #1#2 { \if_int_compare:w `#2 = 0 \exp_stop_f: - \__kernel_msg_error:nn { kernel } { replacement-null-space } + \__kernel_msg_error:nn { regex } { replacement-null-space } \fi: \tex_lccode:D `\ = `#2 \scan_stop: \tex_lowercase:D { \@@_replacement_put:n {~} } @@ -5756,7 +6127,7 @@ % \begin{macrocode} \cs_new_protected:Npn \@@_replacement_error:NNN #1#2#3 { - \__kernel_msg_error:nnx { kernel } { replacement-#1 } {#3} + \__kernel_msg_error:nnx { regex } { replacement-#1 } {#3} #2 #3 } % \end{macrocode} @@ -5805,26 +6176,35 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}{\regex_show:N, \regex_show:n} +% \begin{macro} +% { +% \regex_show:n, \regex_log:n, \@@_show:Nn, +% \regex_show:N, \regex_log:N, \@@_show:NN +% } % User functions: the \texttt{n} variant requires compilation first. % Then show the variable with some appropriate text. The auxiliary -% is defined in a different section. +% \cs{@@_show:N} is defined in a different section. % \begin{macrocode} -\cs_new_protected:Npn \regex_show:n #1 +\cs_new_protected:Npn \regex_show:n { \@@_show:Nn \__kernel_msg_show:nnxxxx } +\cs_new_protected:Npn \regex_log:n { \@@_show:Nn \__kernel_msg_log:nnxxxx } +\cs_new_protected:Npn \@@_show:Nn #1#2 { - \@@_compile:n {#1} + \@@_compile:n {#2} \@@_show:N \l_@@_internal_regex - \msg_show:nnxxxx { LaTeX / kernel } { show-regex } - { \tl_to_str:n {#1} } { } + #1 { regex } { show } + { \tl_to_str:n {#2} } { } { \l_@@_internal_a_tl } { } } -\cs_new_protected:Npn \regex_show:N #1 +\cs_new_protected:Npn \regex_show:N { \@@_show:NN \__kernel_msg_show:nnxxxx } +\cs_new_protected:Npn \regex_log:N { \@@_show:NN \__kernel_msg_log:nnxxxx } +\cs_new_protected:Npn \@@_show:NN #1#2 { - \__kernel_chk_defined:NT #1 + \__kernel_chk_tl_type:NnnT #2 { regex } + { \exp_args:No \@@_clean_regex:n {#2} } { - \@@_show:N #1 - \msg_show:nnxxxx { LaTeX / kernel } { show-regex } - { } { \token_to_str:N #1 } + \@@_show:N #2 + #1 { regex } { show } + { } { \token_to_str:N #2 } { \l_@@_internal_a_tl } { } } } @@ -6148,7 +6528,7 @@ } = 0 { - \__kernel_msg_error:nnxxx { kernel } { result-unbalanced } + \__kernel_msg_error:nnxxx { regex } { result-unbalanced } { splitting~or~extracting~submatches } { \flag_height:n { @@_end } } { \flag_height:n { @@_begin } } @@ -6330,7 +6710,7 @@ { \if_int_compare:w \l_@@_balance_int = 0 \exp_stop_f: \else: - \__kernel_msg_error:nnxxx { kernel } { result-unbalanced } + \__kernel_msg_error:nnxxx { regex } { result-unbalanced } { replacing } { \int_max:nn { - \l_@@_balance_int } { 0 } } { \int_max:nn { \l_@@_balance_int } { 0 } } @@ -6731,14 +7111,14 @@ % \begin{macrocode} \use:x { - \__kernel_msg_new:nnn { kernel } { trailing-backslash } + \__kernel_msg_new:nnn { regex } { trailing-backslash } { Trailing~escape~char~'\iow_char:N\\'~in~regex~or~replacement. } - \__kernel_msg_new:nnn { kernel } { x-missing-rbrace } + \__kernel_msg_new:nnn { regex } { x-missing-rbrace } { Missing~brace~'\iow_char:N\}'~in~regex~ '...\iow_char:N\\x\iow_char:N\{...##1'. } - \__kernel_msg_new:nnn { kernel } { x-overflow } + \__kernel_msg_new:nnn { regex } { x-overflow } { Character~code~##1~too~large~in~ \iow_char:N\\x\iow_char:N\{##2\iow_char:N\}~regex. @@ -6748,7 +7128,7 @@ % % Invalid quantifier. % \begin{macrocode} -\__kernel_msg_new:nnnn { kernel } { invalid-quantifier } +\__kernel_msg_new:nnnn { regex } { invalid-quantifier } { Braced~quantifier~'#1'~may~not~be~followed~by~'#2'. } { The~character~'#2'~is~invalid~in~the~braced~quantifier~'#1'.~ @@ -6760,13 +7140,13 @@ % Messages for missing or extra closing brackets and parentheses, with % some fancy singular/plural handling for the case of parentheses. % \begin{macrocode} -\__kernel_msg_new:nnnn { kernel } { missing-rbrack } +\__kernel_msg_new:nnnn { regex } { missing-rbrack } { Missing~right~bracket~inserted~in~regular~expression. } { LaTeX~was~given~a~regular~expression~where~a~character~class~ was~started~with~'[',~but~the~matching~']'~is~missing. } -\__kernel_msg_new:nnnn { kernel } { missing-rparen } +\__kernel_msg_new:nnnn { regex } { missing-rparen } { Missing~right~ \int_compare:nTF { #1 = 1 } { parenthesis } { parentheses } ~ @@ -6776,7 +7156,7 @@ LaTeX~was~given~a~regular~expression~with~\int_eval:n {#1} ~ more~left~parentheses~than~right~parentheses. } -\__kernel_msg_new:nnnn { kernel } { extra-rparen } +\__kernel_msg_new:nnnn { regex } { extra-rparen } { Extra~right~parenthesis~ignored~in~regular~expression. } { LaTeX~came~across~a~closing~parenthesis~when~no~submatch~group~ @@ -6786,7 +7166,7 @@ % % Some escaped alphanumerics are not allowed everywhere. % \begin{macrocode} -\__kernel_msg_new:nnnn { kernel } { bad-escape } +\__kernel_msg_new:nnnn { regex } { bad-escape } { Invalid~escape~'\iow_char:N\\#1'~ \@@_if_in_cs:TF { within~a~control~sequence. } @@ -6814,14 +7194,14 @@ % % Range errors. % \begin{macrocode} -\__kernel_msg_new:nnnn { kernel } { range-missing-end } +\__kernel_msg_new:nnnn { regex } { range-missing-end } { Invalid~end-point~for~range~'#1-#2'~in~character~class. } { The~end-point~'#2'~of~the~range~'#1-#2'~may~not~serve~as~an~ end-point~for~a~range:~alphanumeric~characters~should~not~be~ escaped,~and~non-alphanumeric~characters~should~be~escaped. } -\__kernel_msg_new:nnnn { kernel } { range-backwards } +\__kernel_msg_new:nnnn { regex } { range-backwards } { Range~'[#1-#2]'~out~of~order~in~character~class. } { In~ranges~of~characters~'[x-y]'~appearing~in~character~classes,~ @@ -6833,7 +7213,7 @@ % % Errors related to |\c| and |\u|. % \begin{macrocode} -\__kernel_msg_new:nnnn { kernel } { c-bad-mode } +\__kernel_msg_new:nnnn { regex } { c-bad-mode } { Invalid~nested~'\iow_char:N\\c'~escape~in~regular~expression. } { The~'\iow_char:N\\c'~escape~cannot~be~used~within~ @@ -6841,33 +7221,40 @@ nor~another~category~test.~ To~combine~several~category~tests,~use~'\iow_char:N\\c[...]'. } -\__kernel_msg_new:nnnn { kernel } { c-C-invalid } +\__kernel_msg_new:nnnn { regex } { c-C-invalid } { '\iow_char:N\\cC'~should~be~followed~by~'.'~or~'(',~not~'#1'. } { The~'\iow_char:N\\cC'~construction~restricts~the~next~item~to~be~a~ control~sequence~or~the~next~group~to~be~made~of~control~sequences.~ It~only~makes~sense~to~follow~it~by~'.'~or~by~a~group. } -\__kernel_msg_new:nnnn { kernel } { c-lparen-in-class } +\__kernel_msg_new:nnnn { regex } { cu-lbrace } + { Left~braces~must~be~escaped~in~'\iow_char:N\\#1{...}'. } + { + Constructions~such~as~'\iow_char:N\\#1{...\iow_char:N\{...}'~are~ + not~allowed~and~should~be~replaced~by~ + '\iow_char:N\\#1{...\token_to_str:N\{...}'. + } +\__kernel_msg_new:nnnn { regex } { c-lparen-in-class } { Catcode~test~cannot~apply~to~group~in~character~class } { Construction~such~as~'\iow_char:N\\cL(abc)'~are~not~allowed~inside~a~ class~'[...]'~because~classes~do~not~match~multiple~characters~at~once. } -\__kernel_msg_new:nnnn { kernel } { c-missing-rbrace } +\__kernel_msg_new:nnnn { regex } { c-missing-rbrace } { Missing~right~brace~inserted~for~'\iow_char:N\\c'~escape. } { LaTeX~was~given~a~regular~expression~where~a~ '\iow_char:N\\c\iow_char:N\{...'~construction~was~not~ended~ with~a~closing~brace~'\iow_char:N\}'. } -\__kernel_msg_new:nnnn { kernel } { c-missing-rbrack } +\__kernel_msg_new:nnnn { regex } { c-missing-rbrack } { Missing~right~bracket~inserted~for~'\iow_char:N\\c'~escape. } { A~construction~'\iow_char:N\\c[...'~appears~in~a~ regular~expression,~but~the~closing~']'~is~not~present. } -\__kernel_msg_new:nnnn { kernel } { c-missing-category } +\__kernel_msg_new:nnnn { regex } { c-missing-category } { Invalid~character~'#1'~following~'\iow_char:N\\c'~escape. } { In~regular~expressions,~the~'\iow_char:N\\c'~escape~sequence~ @@ -6875,19 +7262,19 @@ capital~letter~representing~a~character~category,~namely~ one~of~'ABCDELMOPSTU'. } -\__kernel_msg_new:nnnn { kernel } { c-trailing } +\__kernel_msg_new:nnnn { regex } { c-trailing } { Trailing~category~code~escape~'\iow_char:N\\c'... } { A~regular~expression~ends~with~'\iow_char:N\\c'~followed~ by~a~letter.~It~will~be~ignored. } -\__kernel_msg_new:nnnn { kernel } { u-missing-lbrace } +\__kernel_msg_new:nnnn { regex } { u-missing-lbrace } { Missing~left~brace~following~'\iow_char:N\\u'~escape. } { The~'\iow_char:N\\u'~escape~sequence~must~be~followed~by~ a~brace~group~with~the~name~of~the~variable~to~use. } -\__kernel_msg_new:nnnn { kernel } { u-missing-rbrace } +\__kernel_msg_new:nnnn { regex } { u-missing-rbrace } { Missing~right~brace~inserted~for~'\iow_char:N\\u'~escape. } { LaTeX~ @@ -6901,14 +7288,14 @@ % % Errors when encountering the \textsc{posix} syntax |[:...:]|. % \begin{macrocode} -\__kernel_msg_new:nnnn { kernel } { posix-unsupported } +\__kernel_msg_new:nnnn { regex } { posix-unsupported } { POSIX~collating~element~'[#1 ~ #1]'~not~supported. } { The~'[.foo.]'~and~'[=bar=]'~syntaxes~have~a~special~meaning~ in~POSIX~regular~expressions.~This~is~not~supported~by~LaTeX.~ Maybe~you~forgot~to~escape~a~left~bracket~in~a~character~class? } -\__kernel_msg_new:nnnn { kernel } { posix-unknown } +\__kernel_msg_new:nnnn { regex } { posix-unknown } { POSIX~class~'[:#1:]'~unknown. } { '[:#1:]'~is~not~among~the~known~POSIX~classes~ @@ -6917,7 +7304,7 @@ '[:print:]',~'[:punct:]',~'[:space:]',~'[:upper:]',~ '[:word:]',~and~'[:xdigit:]'. } -\__kernel_msg_new:nnnn { kernel } { posix-missing-close } +\__kernel_msg_new:nnnn { regex } { posix-missing-close } { Missing~closing~':]'~for~POSIX~class. } { The~POSIX~syntax~'#1'~must~be~followed~by~':]',~not~'#2'. } % \end{macrocode} @@ -6926,7 +7313,7 @@ % with an unbalanced token list, which we must re-balance by adding % begin-group or end-group character tokens. % \begin{macrocode} -\__kernel_msg_new:nnnn { kernel } { result-unbalanced } +\__kernel_msg_new:nnnn { regex } { result-unbalanced } { Missing~brace~inserted~when~#1. } { LaTeX~was~asked~to~do~some~regular~expression~operation,~ @@ -6938,13 +7325,13 @@ % % Error message for unknown options. % \begin{macrocode} -\__kernel_msg_new:nnnn { kernel } { unknown-option } +\__kernel_msg_new:nnnn { regex } { unknown-option } { Unknown~option~'#1'~for~regular~expressions. } { The~only~available~option~is~'case-insensitive',~toggled~by~ '(?i)'~and~'(?-i)'. } -\__kernel_msg_new:nnnn { kernel } { special-group-unknown } +\__kernel_msg_new:nnnn { regex } { special-group-unknown } { Unknown~special~group~'#1~...'~in~a~regular~expression. } { The~only~valid~constructions~starting~with~'(?'~are~ @@ -6954,21 +7341,21 @@ % % Errors in the replacement text. % \begin{macrocode} -\__kernel_msg_new:nnnn { kernel } { replacement-c } +\__kernel_msg_new:nnnn { regex } { replacement-c } { Misused~'\iow_char:N\\c'~command~in~a~replacement~text. } { In~a~replacement~text,~the~'\iow_char:N\\c'~escape~sequence~ can~be~followed~by~one~of~the~letters~'ABCDELMOPSTU'~ or~a~brace~group,~not~by~'#1'. } -\__kernel_msg_new:nnnn { kernel } { replacement-u } +\__kernel_msg_new:nnnn { regex } { replacement-u } { Misused~'\iow_char:N\\u'~command~in~a~replacement~text. } { In~a~replacement~text,~the~'\iow_char:N\\u'~escape~sequence~ must~be~~followed~by~a~brace~group~holding~the~name~of~the~ variable~to~use. } -\__kernel_msg_new:nnnn { kernel } { replacement-g } +\__kernel_msg_new:nnnn { regex } { replacement-g } { Missing~brace~for~the~'\iow_char:N\\g'~construction~ in~a~replacement~text. @@ -6978,7 +7365,7 @@ submatches~are~represented~either~as~'\iow_char:N \\g{dd..d}',~ or~'\\d',~where~'d'~are~single~digits.~Here,~a~brace~is~missing. } -\__kernel_msg_new:nnnn { kernel } { replacement-catcode-end } +\__kernel_msg_new:nnnn { regex } { replacement-catcode-end } { Missing~character~for~the~'\iow_char:N\\c<category><character>'~ construction~in~a~replacement~text. @@ -6989,7 +7376,7 @@ the~character~category.~Then,~a~character~must~follow.~LaTeX~ reached~the~end~of~the~replacement~when~looking~for~that. } -\__kernel_msg_new:nnnn { kernel } { replacement-catcode-escaped } +\__kernel_msg_new:nnnn { regex } { replacement-catcode-escaped } { Escaped~letter~or~digit~after~category~code~in~replacement~text. } @@ -6999,7 +7386,7 @@ the~character~category.~Then,~a~character~must~follow,~not~ '\iow_char:N\\#2'. } -\__kernel_msg_new:nnnn { kernel } { replacement-catcode-in-cs } +\__kernel_msg_new:nnnn { regex } { replacement-catcode-in-cs } { Category~code~'\iow_char:N\\c#1#3'~ignored~inside~ '\iow_char:N\\c\{...\}'~in~a~replacement~text. @@ -7009,7 +7396,7 @@ '\iow_char:N\\c\{...\}'~are~ignored~when~building~the~control~ sequence~name. } -\__kernel_msg_new:nnnn { kernel } { replacement-null-space } +\__kernel_msg_new:nnnn { regex } { replacement-null-space } { TeX~cannot~build~a~space~token~with~character~code~0. } { You~asked~for~a~character~token~with~category~space,~ @@ -7018,31 +7405,33 @@ This~specific~case~is~impossible~and~will~be~replaced~ by~a~normal~space. } -\__kernel_msg_new:nnnn { kernel } { replacement-missing-rbrace } +\__kernel_msg_new:nnnn { regex } { replacement-missing-rbrace } { Missing~right~brace~inserted~in~replacement~text. } { There~ \int_compare:nTF { #1 = 1 } { was } { were } ~ #1~ missing~right~\int_compare:nTF { #1 = 1 } { brace } { braces } . } -\__kernel_msg_new:nnnn { kernel } { replacement-missing-rparen } +\__kernel_msg_new:nnnn { regex } { replacement-missing-rparen } { Missing~right~parenthesis~inserted~in~replacement~text. } { There~ \int_compare:nTF { #1 = 1 } { was } { were } ~ #1~ missing~right~ \int_compare:nTF { #1 = 1 } { parenthesis } { parentheses } . } +\__kernel_msg_new:nnn { regex } { submatch-too-big } + { Submatch~#1~used~but~regex~only~has~#2~group(s) } % \end{macrocode} % % Some escaped alphanumerics are not allowed everywhere. % \begin{macrocode} -\__kernel_msg_new:nnnn { kernel } { backwards-quantifier } +\__kernel_msg_new:nnnn { regex } { backwards-quantifier } { Quantifer~"{#1,#2}"~is~backwards. } { The~values~given~in~a~quantifier~must~be~in~order. } % \end{macrocode} % % Used when showing a regex. % \begin{macrocode} -\__kernel_msg_new:nnn { kernel } { show-regex } +\__kernel_msg_new:nnn { regex } { show } { >~Compiled~regex~ \tl_if_empty:nTF {#1} { variable~ #2 } { {#1} } : |