diff options
author | Karl Berry <karl@freefriends.org> | 2012-01-19 22:49:25 +0000 |
---|---|---|
committer | Karl Berry <karl@freefriends.org> | 2012-01-19 22:49:25 +0000 |
commit | 8764984f07460a77b8200111ddb513c8af6c23f3 (patch) | |
tree | 61782419c00b217fc7f14919b9e24bbae260274f /Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx | |
parent | 17d590a6bcf898806aeec2d7aa57d641a440ca10 (diff) |
l3experimental 3209 (19jan12)
git-svn-id: svn://tug.org/texlive/trunk@25158 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx')
-rw-r--r-- | Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx | 1361 |
1 files changed, 771 insertions, 590 deletions
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx index 9c276b6922d..d921b0b7e56 100644 --- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx @@ -1,6 +1,6 @@ % \iffalse meta-comment % -%% File: l3regex.dtx Copyright (C) 2011 The LaTeX3 Project +%% File: l3regex.dtx Copyright (C) 2011-2012 The LaTeX3 Project %% %% It may be distributed and/or modified under the conditions of the %% LaTeX Project Public License (LPPL), either version 1.3c of this @@ -35,7 +35,7 @@ % %<*driver|package> \RequirePackage{expl3} -\GetIdInfo$Id: l3regex.dtx 3073 2011-12-21 01:53:06Z bruno $ +\GetIdInfo$Id: l3regex.dtx 3200 2012-01-16 03:50:54Z bruno $ {L3 Experimental Regular Expressions} %</driver|package> %<*driver> @@ -104,16 +104,16 @@ % it can be compiled once, and stored in a token list % variable using \cs{regex_const:Nn}. For example, % \begin{verbatim} -% \regex_const:Nn \l_foo_regex_tl { \c{begin} \cB (\CB*?) \cE } +% \regex_const:Nn \c_foo_regex_tl { \c{begin} \cB. (\c[^BE].*) \cE. } % \end{verbatim} -% stores in \cs{l_foo_regex_tl} a regular expression which matches -% the starting marker for an environment: \cs{begin}, followed -% by a begin-group token (|\cB|), then tokens which are not -% begin-group characters (|\CB|), as few as possible (|*?|), -% followed by an end-group token (|\cE|). As explained in the -% next section, the parentheses \enquote{capture} the result of -% |\CB*?|, giving us access to the name of the environment -% when doing replacements. +% stores in \cs{c_foo_regex_tl} a regular expression which matches the +% starting marker for an environment: \cs{begin}, followed by a +% begin-group token (|\cB.|), then any number of tokens which are +% neither begin-group nor end-group character tokens (|\c[^BE].*|), +% ending with an end-group token (|\cE.|). As explained in the next +% section, the parentheses \enquote{capture} the result of |\c[^BE].*|, +% giving us access to the name of the environment when doing +% replacements. % % \subsection{Syntax of regular expressions} % @@ -191,7 +191,7 @@ % Matches any token other than the specified characters. % \item[{[x-y]}] Range (can be used with escaped characters). % \end{l3regex-syntax} -% For instance, |[a-oq-z\cC]| matches any lowercase latin letter +% For instance, |[a-oq-z\cC.]| matches any lowercase latin letter % except |p|, as well as control sequences (see below for a description % of |\c|). % @@ -240,29 +240,47 @@ % group number. % \end{l3regex-syntax} % -% Each catcode is represented by a single uppercase letter: -% |C| for control sequences; -% |B| for begin-group tokens; -% |E| for end-group tokens; -% |M| for math shift; -% |T| for alignment tab tokens; -% |P| for macro parameter tokens; -% |U| for superscript tokens (up); -% |D| for subscript tokens (down); -% |S| for spaces; -% |L| for letters; -% |O| for others; and -% |A| for active characters. +% The |\c| escape sequence allows to test the category code of tokens, +% and match control sequences. Each character category is represented +% by a single uppercase letter: +% \begin{itemize} +% \item |C| for control sequences; +% \item |B| for begin-group tokens; +% \item |E| for end-group tokens; +% \item |M| for math shift; +% \item |T| for alignment tab tokens; +% \item |P| for macro parameter tokens; +% \item |U| for superscript tokens (up); +% \item |D| for subscript tokens (down); +% \item |S| for spaces; +% \item |L| for letters; +% \item |O| for others; and +% \item |A| for active characters. +% \end{itemize} +% The |\c| escape sequence is used as follows. % \begin{l3regex-syntax} -% \item[\\cX] Any character token with catcode |X|, -% where |X| can be any of |CBEMTPUDSLOA|. -% \item[\\CX] Any token with catcode different from |X|, -% where |X| can be any of |CBEMTPUDSLOA|. -% \item[\\c\Arg{regex}] A control sequence whose csname matches -% the \meta{regex}, anchored at the beginning and end, -% so that |\c{begin}| matches exactly \cs{begin}, and nothing -% else. +% \item[\\c\Arg{regex}] A control sequence whose csname matches the +% \meta{regex}, anchored at the beginning and end, so that |\c{begin}| +% matches exactly \cs{begin}, and nothing else. +% \item[\\cX\meta{character or class}] Matches a token with category +% code |X| (any of |CBEMTPUDSLOA|) if it is also matched by the +% \meta{character or class}. For instance, |\cL[A-Z]| matches +% uppercase letters of category code letter, |\cC.| matches any +% control sequence, and |\cO\d| matches digits of category other. +% \item[{\\c[XYZ]\meta{character or class}}] Matches a token with +% category |X|, |Y|, or |Z| (each being any of |CBEMTPUDSLOA|), +% if it is also matched by the \meta{character or class}. For instance, +% |\c[LSO].| matches tokens of category letter, space, or other. +% \item[{\\c[\char`\^XYZ]\meta{character or class}}] Matches a token with +% category different from |X|, |Y|, or |Z| (each being any of +% |CBEMTPUDSLOA|), if it is also matched by the \meta{character or +% class}. For instance, |\c[LSO].| matches tokens of category +% letter, space, or other. % \end{l3regex-syntax} +% The category code tests can be used inside classes; for instance, +% |[\cO\d \c[LO][A-F]]| matches what \TeX{} considers as hexadecimal +% digits, namely digits with category other, or uppercase letters from +% |A| to |F| with category either letter or other. % % Options can be set with |(?|\meta{option}|)| and % unset with |(?-|\meta{option}|)|. Options are local @@ -526,6 +544,8 @@ % \begin{itemize} % \item The \texttt{\{\}} quantifiers are only partially implemented. % \item Clean up the use of messages. +% \item Triple check magic numbers. +% \item Test \cs{par}. % \item Add tests for every regex and replacement feature. % \item Make sure the documentation is correct, and the code comments % are as well. @@ -533,11 +553,12 @@ % % Code improvements to come. % \begin{itemize} +% \item \cs{regex_show:N} to show how a given regular expression is +% interpreted. % \item Test for the maximum register \cs{c_max_register_int}. % \item Use \tn{dimen} registers rather than \cs{l_regex_nesting_tl} % to build \cs{regex_nesting:n}. % \item Reduce the number of epsilon-transitions in alternatives. -% \item Merge the \texttt{build} and \texttt{class} functions where possible. % \item Improve nesting of |\c| in the replacement text. % \item Move the \enquote{reconstruction} part of \pkg{l3regex} % to \pkg{l3tl-analysis}. @@ -577,22 +598,24 @@ % the matching, because the regex code uses registers in an unsafe way; % \item Conditional subpatterns (other than with a look-ahead % or look-behind condition): this is non-regular, isn't it? +% \item Named subpatterns: \TeX{} programmers have lived so far without +% any need for named macro parameters. % \end{itemize} % % The following features of PCRE or perl will definitely not be implemented. % \begin{itemize} % \item |\cx|, similar to \TeX{}'s own |\^^x|; % \item Comments: \TeX{} already has its own system for comments. -% \item Named subpatterns: \TeX{} programmers have lived so far without -% any need for named macro parameters. % \item |\Q...\E| escaping: this would require to read the argument % verbatim, which is not in the scope of this module. % \item Atomic grouping, possessive quantifiers: those tools, mostly % meant to fix catastrophic backtracking, are unnecessary in a % non-backtracking algorithm, and difficult to implement. -% \item Subroutine calls: this syntactic sugar is difficult to -% include in a non-backtracking algorithm, in particular because -% the corresponding group should be treated as atomic. +% \item Subroutine calls: this syntactic sugar is difficult to include +% in a non-backtracking algorithm, in particular because the +% corresponding group should be treated as atomic. Also, we cannot +% afford to run user code within the regular expression matching, +% because of our \enquote{misuse} of registers. % \item Recursion: this is a non-regular feature. % \item Back-references: non-regular feature, this requires backtracking, % which is prohibitively slow. @@ -665,6 +688,15 @@ % % \subsubsection{Variables used while building} % +% \begin{variable}{\l_regex_build_mode_int} +% While building, ten modes are recognized, labelled $-63$, $-23$, +% $-6$, $-2$, $0$, $2$, $3$, $6$, $23$, $63$. See +% section~\ref{sec:regex-modes}. +% \begin{macrocode} +\int_new:N \l_regex_build_mode_int +% \end{macrocode} +% \end{variable} +% % \begin{variable}{\l_regex_max_state_int} % \begin{variable}{\l_regex_left_state_int, \l_regex_right_state_int} % The end-point states of the last group (which any quantifier @@ -723,16 +755,6 @@ % \end{variable} % \end{variable} % -% \begin{variable}{\l_regex_build_or_class_tl} -% The actions of various escape sequences and special characters -% are different within a class and outside a class. Rather than -% using a boolean, we store either \texttt{build} or \texttt{class} -% in this token list. -% \begin{macrocode} -\tl_new:N \l_regex_build_or_class_tl -% \end{macrocode} -% \end{variable} -% % \begin{variable}{\l_regex_one_or_group_tl} % When looking for quantifiers, this variable holds either % \enquote{one} or \enquote{group} depending on whether the @@ -743,11 +765,52 @@ % \end{macrocode} % \end{variable} % -% \begin{variable}{\l_regex_in_csname_bool} -% This boolean is true when building the regular expression -% for a given csname. +% \begin{variable}{\l_regex_catcodes_int, \l_regex_catcodes_default_int} +% \begin{variable} +% { +% \c_regex_catcode_C_int, \c_regex_catcode_B_int, \c_regex_catcode_E_int, +% \c_regex_catcode_M_int, \c_regex_catcode_T_int, \c_regex_catcode_P_int, +% \c_regex_catcode_U_int, \c_regex_catcode_D_int, \c_regex_catcode_S_int, +% \c_regex_catcode_L_int, \c_regex_catcode_O_int, \c_regex_catcode_A_int +% } +% \begin{variable}{\c_regex_catcodes_all_int} +% We wish to allow constructions such as |\c[^BE](..\cL[a-z]..)|, +% matching two tokens which are neither a begin-group nor an end-group +% token, followed by a token of category letter and character code in +% |[a-z]|, followed by two more tokens which are neither begin-group +% nor end-group tokens. For this to work, we need to keep track of +% lists of allowed category codes: \cs{l_regex_catcodes_int} and +% \cs{l_regex_catcodes_default_int} are bitmaps, sums of $4^c$, for +% all allowed catcodes $c$. The latter is local to each capturing +% group, and we reset \cs{l_regex_catcodes_int} to that value after +% each character or class, changing it only when encountering a |\c| +% escape. +% \begin{macrocode} +\int_new:N \l_regex_catcodes_int +\int_new:N \l_regex_catcodes_default_int +\int_const:Nn \c_regex_catcode_C_int { "1 } +\int_const:Nn \c_regex_catcode_B_int { "4 } +\int_const:Nn \c_regex_catcode_E_int { "10 } +\int_const:Nn \c_regex_catcode_M_int { "40 } +\int_const:Nn \c_regex_catcode_T_int { "100 } +\int_const:Nn \c_regex_catcode_P_int { "1000 } +\int_const:Nn \c_regex_catcode_U_int { "4000 } +\int_const:Nn \c_regex_catcode_D_int { "10000 } +\int_const:Nn \c_regex_catcode_S_int { "100000 } +\int_const:Nn \c_regex_catcode_L_int { "400000 } +\int_const:Nn \c_regex_catcode_O_int { "1000000 } +\int_const:Nn \c_regex_catcode_A_int { "4000000 } +\int_const:Nn \c_regex_catcodes_all_int { "5515155 } +% \end{macrocode} +% \end{variable} +% \end{variable} +% \end{variable} +% +% \begin{variable}{\l_regex_catcodes_bool} +% Controls whether the bitmap of category codes built should be +% inverted or not. % \begin{macrocode} -\bool_new:N \l_regex_in_csname_bool +\bool_new:N \l_regex_catcodes_bool % \end{macrocode} % \end{variable} % @@ -761,22 +824,22 @@ % % \subsubsection{Character classes} % -% \begin{macro}{\regex_build_tmp_class:n} % \begin{variable}{\l_regex_class_bool,\l_regex_class_tl} +% \begin{variable}{\l_regex_class_saved_bool,\l_regex_class_saved_tl} % \cs{l_regex_class_bool} is false for negative character classes. -% \cs{l_regex_class_tl} holds the tests which should be performed -% to decide whether the \cs{l_regex_current_char_int} matches that -% character class. Once the class is read completely, the full -% instructions are stored in \cs{regex_build_tmp_class:n}, whose -% argument is the target state if the test succeeds. It is also used -% for all other single-character situations (\emph{e.g.}, |\d|, or |.|). -% \begin{macrocode} -\cs_new_eq:NN \regex_build_tmp_class:n \use_none:n +% \cs{l_regex_class_tl} holds the tests which should be performed to +% decide whether the \cs{l_regex_current_char_int} matches that +% character class. In nested class, which can only occur as +% |[\c[...]]|, the two variables from the outer class must be saved +% while processing the inner class. +% \begin{macrocode} \bool_new:N \l_regex_class_bool \tl_new:N \l_regex_class_tl +\bool_new:N \l_regex_class_saved_bool +\tl_new:N \l_regex_class_saved_tl % \end{macrocode} % \end{variable} -% \end{macro} +% \end{variable} % % \begin{variable}{\c_regex_d_tl,\c_regex_D_tl} % \begin{variable}{\c_regex_h_tl,\c_regex_H_tl} @@ -1202,12 +1265,12 @@ % This is done \enquote{by hand}, in sequences whose items should % be run immediately. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_seq_pop_use:N #1 +\cs_new_protected:Npn \regex_seq_pop_use:N #1 { \seq_pop:NN #1 \l_regex_tmpa_tl \l_regex_tmpa_tl } -\cs_new_protected_nopar:Npn \regex_seq_get_use:N #1 +\cs_new_protected:Npn \regex_seq_get_use:N #1 { \seq_get:NN #1 \l_regex_tmpa_tl \l_regex_tmpa_tl @@ -1216,45 +1279,10 @@ % \end{macro} % \end{macro} % -% \begin{macro}[int, EXP]{\regex_seq_item:Nn} -% \begin{macro}[aux, EXP]{\regex_seq_item_aux:wNn} -% \begin{macrocode} -\cs_new:Npn \regex_seq_item:Nn #1#2 - { - \exp_after:wN \regex_seq_item_aux:wNn - \int_use:N \int_eval:w #2 \exp_after:wN ; - #1 { ? \prg_map_break: } { } - \prg_break_point:n { } - } -\cs_new:Npn \regex_seq_item_aux:wNn #1; #2#3 - { - \use_none:n #2 - \if_num:w #1 = \c_zero - \exp_after:wN \prg_map_break:n - \else: - \exp_after:wN \use_none:n - \fi: - { \exp_not:n {#3} } - \exp_after:wN \regex_seq_item_aux:wNn - \int_use:N \int_eval:w #1 - \c_one ; - } -% \end{macrocode} -% \end{macro} -% \end{macro} -% % \subsubsection{Testing characters} % -% \begin{macro}{\regex_item_dot:T} -% The dot meta-character matches any character, -% except the end marker. -% \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_item_dot:T - { \int_compare:nNnF \l_regex_current_char_int < \c_minus_one } -% \end{macrocode} -% \end{macro} -% % \begin{macro}[int]{\regex_break_point:TF} -% \begin{macro}[int]{\regex_break_true:w,\regex_break_false:w} +% \begin{macro}[int]{\regex_break_true:w} % When testing whether a character of the query token list matches % a given character class in the regular expression, we often % have to test it against several ranges of characters, checking @@ -1268,27 +1296,38 @@ % Otherwise, \cs{regex_break_point:TF} leaves the \meta{false code} % in the input stream. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_break_true:w +\cs_new_protected:Npn \regex_break_true:w #1 \regex_break_point:TF #2 #3 {#2} -\cs_new_protected_nopar:Npn \regex_break_false:w - #1 \regex_break_point:TF #2 #3 {#3} -\cs_new_protected_nopar:Npn \regex_break_point:TF #1 #2 { #2 } +\cs_new_protected:Npn \regex_break_point:TF #1 #2 { #2 } % \end{macrocode} % \end{macro} % \end{macro} % +% \begin{macro}{\regex_item_dot:} +% The dot meta-character matches any character, +% except the end marker. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \regex_item_dot: + { + \if_num:w \l_regex_current_char_int > - \c_two + \exp_after:wN \regex_break_true:w + \fi: + } +% \end{macrocode} +% \end{macro} +% % \begin{macro}[int]{\regex_item_caseful_equal:n} % \begin{macro}[int]{\regex_item_caseful_range:nn} % \begin{macro}[int]{\regex_item_caseful_geq:n} % Simple comparisons triggering \cs{regex_break_true:w} when true. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_item_caseful_equal:n #1 +\cs_new_protected:Npn \regex_item_caseful_equal:n #1 { \if_num:w #1 = \l_regex_current_char_int \exp_after:wN \regex_break_true:w \fi: } -\cs_new_protected_nopar:Npn \regex_item_caseful_range:nn #1 #2 +\cs_new_protected:Npn \regex_item_caseful_range:nn #1 #2 { \reverse_if:N \if_num:w #1 > \l_regex_current_char_int \reverse_if:N \if_num:w #2 < \l_regex_current_char_int @@ -1296,7 +1335,7 @@ \fi: \fi: } -\cs_new_protected_nopar:Npn \regex_item_caseful_geq:n #1 +\cs_new_protected:Npn \regex_item_caseful_geq:n #1 { \reverse_if:N \if_num:w #1 > \l_regex_current_char_int \exp_after:wN \regex_break_true:w @@ -1314,7 +1353,7 @@ % \cs{l_regex_current_char_int} and on % \cs{l_regex_case_changed_char_int}. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_item_caseless_equal:n #1 +\cs_new_protected:Npn \regex_item_caseless_equal:n #1 { \if_num:w #1 = \l_regex_current_char_int \exp_after:wN \regex_break_true:w @@ -1323,7 +1362,7 @@ \exp_after:wN \regex_break_true:w \fi: } -\cs_new_protected_nopar:Npn \regex_item_caseless_range:nn #1 #2 +\cs_new_protected:Npn \regex_item_caseless_range:nn #1 #2 { \reverse_if:N \if_num:w #1 > \l_regex_current_char_int \reverse_if:N \if_num:w #2 < \l_regex_current_char_int @@ -1336,7 +1375,7 @@ \fi: \fi: } -\cs_new_protected_nopar:Npn \regex_item_caseless_geq:n #1 +\cs_new_protected:Npn \regex_item_caseless_geq:n #1 { \reverse_if:N \if_num:w #1 > \l_regex_current_char_int \exp_after:wN \regex_break_true:w @@ -1387,26 +1426,30 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}[int]{\regex_item_catcode:n} -% \begin{macro}[int]{\regex_item_catcode_neq:n} -% Checking the category code is quite straightforward. +% \begin{macro}{\regex_item_catcode:nT} +% The argument is a sum of powers of $4$ with exponents given by the +% allowed category codes (between $0$ and $13$). Dividing by a given +% power of $4$ gives an odd result if and only if that category code +% is allowed. If the catcode does not match, then skip the character +% code tests which follow. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_item_catcode:n #1 - { - \if_num:w #1 = \l_regex_current_catcode_int - \exp_after:wN \regex_break_true:w - \fi: - } -\cs_new_protected_nopar:Npn \regex_item_catcode_neq:n #1 +\cs_new_protected:Npn \regex_item_catcode:nT #1 { - \if_num:w #1 = \l_regex_current_catcode_int + \if_int_odd:w \int_eval:w #1 / " + \if_case:w \l_regex_current_catcode_int + 1 \or: 4 \or: 10 \or: 40 + \or: 100 \or: \or: 1000 \or: 4000 + \or: 10000 \or: \or: 100000 \or: 400000 + \or: 1000000 \or: 4000000 \else: 1*\c_zero + \fi: + \int_eval_end: + \exp_after:wN \use:n \else: - \exp_after:wN \regex_break_true:w + \exp_after:wN \use_none:n \fi: } % \end{macrocode} % \end{macro} -% \end{macro} % % \begin{macro}[int]{\regex_item_cs:n} % Match a control sequence (the argument is a pre-compiled regex). @@ -1455,7 +1498,7 @@ { \if_false: } } \fi: \regex_get_digits_aux:NN } -\cs_new_nopar:Npn \regex_get_digits_aux:NN #1#2 +\cs_new:Npn \regex_get_digits_aux:NN #1#2 { \if_meaning:w \regex_build_raw:N #1 \if_charcode:w \c_space_token \exp_not:N #2 @@ -1471,7 +1514,7 @@ \fi: \regex_get_digits_aux:NN } -\cs_new_nopar:Npn \regex_get_digits_end:w #1 \fi: #2 \regex_get_digits_aux:NN +\cs_new:Npn \regex_get_digits_end:w #1 \fi: #2 \regex_get_digits_aux:NN { \fi: #2 \if_false: { { \fi: } } @@ -1568,6 +1611,66 @@ % % \subsection{Building} % +% \subsubsection{Build mode} +% \label{sec:regex-modes} +% +% When building the NFA corresponding to a given regex, we can be in +% ten distinct modes, which we label by some magic numbers: +% \begin{itemize} +% \item[-6] |[\c{...}]| control sequence in a class, +% \item[-2] |\c{...}| control sequence, +% \item[0] |...| outer, +% \item[2] |\c...| catcode test, +% \item[6] |[\c...]| catcode test in a class, +% \item[-63] |[\c{[...]}]| class inside mode $-6$, +% \item[-23] |\c{[...]}| class inside mode $-2$, +% \item[3] |[...]| class inside mode $-3$, +% \item[23] |\c[...]| class inside mode $2$, +% \item[63] |[\c[...]]| class inside mode $6$. +% \end{itemize} +% This list is exhaustive, because |\c| escape sequences cannot be +% nested, and character classes cannot be nested directly. The choice +% of numbers is such as to optimize the most useful tests, and make +% transitions from one mode to another as simple as possible. +% \begin{itemize} +% \item Even modes mean that we are not directly in a character class. +% In this case, a left bracket appends $3$ to the mode. In a character +% class, a right bracket changes the mode as $m\to (m-15)/13$, +% truncated. +% \item Grouping, assertion, and anchors are allowed in non-positive +% even modes ($0$, $-2$, $-6$), and do not change the mode. Otherwise, +% they trigger an error. +% \item A left bracket is special in even modes, appending $3$ to the +% mode; in those modes, quantifiers and the dot are recognized, and +% the right bracket is normal. In odd modes (within classes), the left +% bracket is normal, but the right bracket ends the class, changing +% the mode from $m$ to $(m-15)/13$, truncated; also, ranges are +% recognized. +% \item In non-negative modes, left and right braces are normal. In +% negative modes, however, left braces trigger a warning; right braces +% end the control sequence, going from $-2$ to $0$ or $-6$ to $3$, +% with error recovery for odd modes. +% \item Properties (such as the |\d| character class) can appear in any +% mode. +% \end{itemize} +% +% \begin{macro}[int, EXP]{\regex_build_if_in_class:TF} +% Test whether we are currently in a character class (at the +% inner-most level of nesting). There, many escape sequences are not +% recognized, and special characters are normal. Also, for every raw +% character, we must look ahead for a possible raw dash. +% \begin{macrocode} +\cs_new_nopar:Npn \regex_build_if_in_class:TF + { + \if_int_odd:w \l_regex_build_mode_int + \exp_after:wN \use_i:nn + \else: + \exp_after:wN \use_ii:nn + \fi: + } +% \end{macrocode} +% \end{macro} +% % \subsubsection{Helpers for building an NFA} % % \begin{macro}[int]{\regex_build_new_state:} @@ -1586,18 +1689,18 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_transition_aux:NN} -% \begin{macro}[aux]{\regex_build_transitions_aux:NNNN} +% \begin{macro}[aux]{\regex_build_transition:NN} +% \begin{macro}[aux]{\regex_build_transitions:NNNN} % These functions create a new state, and put one or two transitions % starting from the old current state. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_transition_aux:NN #1#2 +\cs_new_protected:Npn \regex_build_transition:NN #1#2 { \regex_build_new_state: \regex_toks_put_right:Nx \l_regex_left_state_int { #1 { \int_eval:n { #2 - \l_regex_left_state_int } } } } -\cs_new_protected_nopar:Npn \regex_build_transitions_aux:NNNN #1#2#3#4 +\cs_new_protected:Npn \regex_build_transitions:NNNN #1#2#3#4 { \regex_build_new_state: \regex_toks_put_right:Nx \l_regex_left_state_int @@ -1612,12 +1715,11 @@ % % \subsubsection{From regex to NFA: framework} % -% In order for the construction \verb"ab|cd" to work, we enclose -% the whole pattern within parentheses (in the code below, -% \cs{regex_build_open_aux:} and \cs{regex_build_close_aux:}). -% These have the added benefit -% to form a capturing group: hence we get the data of the whole match -% for free. +%^^A In order for the construction \verb"ab|cd" to work, we enclose the +%^^A whole pattern within parentheses (in the code below, +%^^A \cs{regex_build_open_aux:} and \cs{regex_build_close_aux:}). These +%^^A have the added benefit to form a capturing group: hence we get the +%^^A data of the whole match for free. % % \begin{macro}[int]{\regex_build:n} % First, reset a few variables. Then use the generic framework defined @@ -1633,11 +1735,11 @@ % \begin{macrocode} \cs_new_protected:Npn \regex_build:n #1 { - \bool_set_false:N \l_regex_in_csname_bool + \int_set_eq:NN \l_regex_build_mode_int \c_zero \regex_build:w \regex_build_new_state: \regex_toks_put_right:Nx \l_regex_left_state_int - { \regex_action_start_wildcard:nn {0} {1} } + { \regex_action_start_wildcard: } \regex_build_open_aux: \str_escape_use:NNNn \regex_build_i_unescaped:N @@ -1645,6 +1747,7 @@ \regex_build_i_raw:N { #1 } \prg_do_nothing: \prg_do_nothing: + \prg_do_nothing: \prg_do_nothing: \regex_build_end: } % \end{macrocode} @@ -1663,55 +1766,60 @@ % special meanings, while everything else should be treated as a raw % character. % \begin{macrocode} -\cs_new_nopar:Npn \regex_build_i_unescaped:N #1 +\cs_new:Npn \regex_build_i_unescaped:N #1 { \regex_aux_char_if_special:NTF #1 - { \exp_not:N \regex_build_control:N #1 } + { \exp_not:N \regex_build_special:N #1 } { \exp_not:N \regex_build_raw:N #1 } } -\cs_new_nopar:Npn \regex_build_i_escaped:N #1 +\cs_new:Npn \regex_build_i_escaped:N #1 { \regex_aux_char_if_alphanumeric:NTF #1 - { \exp_not:N \regex_build_control:N #1 } + { \exp_not:N \regex_build_escaped:N #1 } { \exp_not:N \regex_build_raw:N #1 } } -\cs_new_nopar:Npn \regex_build_i_raw:N #1 +\cs_new:Npn \regex_build_i_raw:N #1 { \exp_not:N \regex_build_raw:N #1 } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % -% \begin{macro}[aux]{\regex_build_control:N} -% If the control character has a particular meaning in regexes, -% the corresponding function is used. Otherwise, it is interpreted -% as a raw character. The \cs{regex_build/class_default_raw:N} -% functions are defined later. +% \begin{macro}[aux]{\regex_build_special:N} +% \begin{macro}[aux]{\regex_build_escaped:N} +% If the control character has a particular meaning in regexes, the +% corresponding function is used. Otherwise, it is interpreted as a +% raw character. We distinguish special characters from escaped +% alphanumeric characters because in character classes, unknown +% escaped alphanumeric characters raise an error, while special +% charcters are silently converted to raw characters. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_control:N #1 +\cs_new_protected:Npn \regex_build_special:N #1 { - \cs_if_exist_use:cF { regex_ \l_regex_build_or_class_tl _#1: } - { \use:c { regex_ \l_regex_build_or_class_tl _default_raw:N } #1 } + \cs_if_exist_use:cF { regex_build_#1: } + { \regex_build_raw:N #1 } + } +\cs_new_protected:Npn \regex_build_escaped:N #1 + { + \cs_if_exist_use:cF { regex_build_/#1: } + { \regex_build_raw:N #1 } } % \end{macrocode} % \end{macro} -% -%^^A todo: update -% This function is similar to \cs{regex_build_control:N}. If the control -% character has a meaning in character classes, call the corresponding -% function, otherwise, treat it as a raw character, with the -% \cs{regex_class_default_raw:N} function, defined later. +% \end{macro} % % \begin{macro}[int]{\regex_build:w} % Hopefully, we didn't forget to initialize anything here. % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_build:w { - \cs_set_eq:NN \regex_build_raw:N \regex_build_default_raw:N - \tl_set:Nn \l_regex_build_or_class_tl { build } - \int_set_eq:NN \l_regex_capturing_group_int \c_zero + \int_set_eq:NN \l_regex_catcodes_default_int \c_regex_catcodes_all_int + \int_set_eq:NN \l_regex_capturing_group_int \c_zero \int_zero:N \l_regex_max_state_int \regex_build_new_state: + \tl_clear:N \l_regex_class_tl + \bool_set_true:N \l_regex_class_bool + \int_set_eq:NN \l_regex_catcodes_int \l_regex_catcodes_default_int } % \end{macrocode} % \end{macro} @@ -1740,179 +1848,162 @@ % \end{macrocode} % \end{macro} % -% \subsubsection{Anchoring and simple assertions} -% -% \begin{macro}[int]{\regex_build_A:} -% \begin{macro}[int]+\regex_build_^:+ -% \begin{macro}[int]{\regex_build_G:} -% \begin{macro}[aux]{\regex_build_anchor_start:N} -% Anchoring at the start corresponds to checking that the current -% character is the first in the token list. Anchoring to the beginning -% of the match attempt uses \cs{l_regex_start_step_int} instead of -% \cs{c_zero}. +% \begin{macro}[int]{\regex_build_one:n, \regex_build_one:x} +% In a class, add the argument to the current class. Outside a class, +% this argument is the whole \enquote{class}, and we look for +% quantifiers. % \begin{macrocode} -\cs_new_protected_nopar:cpn { regex_build_^: } - { \regex_build_anchor_start:N \l_regex_min_step_int } -\cs_new_protected_nopar:Npn \regex_build_A: - { \regex_build_anchor_start:N \l_regex_min_step_int } -\cs_new_protected_nopar:Npn \regex_build_G: - { \regex_build_anchor_start:N \l_regex_start_step_int } -\cs_new_protected_nopar:Npn \regex_build_anchor_start:N #1 +\cs_new_protected:Npn \regex_build_one:n #1 + { \regex_build_one:x { \exp_not:n {#1} } } +\cs_new_protected:Npn \regex_build_one:x #1 { - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int + \tl_put_right:Nx \l_regex_class_tl { - \exp_not:N \int_compare:nNnT #1 = \l_regex_current_step_int - { - \regex_action_free:n - { - \int_eval:n - { \l_regex_right_state_int - \l_regex_left_state_int } - } - } + \if_num:w \l_regex_catcodes_int = \c_regex_catcodes_all_int + \exp_after:wN \use:n + \else: + \regex_item_catcode:nT { \int_use:N \l_regex_catcodes_int } + \fi: + {#1} } + \if_num:w \l_regex_build_mode_int = \c_two + \l_regex_build_mode_int = \c_zero + \else: + \if_num:w \l_regex_build_mode_int = \c_six + \l_regex_build_mode_int = \c_three + \fi: + \fi: + \if_int_odd:w \l_regex_build_mode_int \else: + \exp_after:wN \regex_build_one_quantifier: + \fi: } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}[aux]{\regex_build_Z:} -% \begin{macro}[aux]{\regex_build_z:} -% \begin{macro}[aux]+\regex_build_$:+ -% This matches the end of the token list, marked by a character code of $-2$. +% \begin{macro}{\regex_build_tmp_class:n} +% ^^A todo: rename, not tmp, and used for class & single. +% The argument is the target state if the test succeeds. % \begin{macrocode} -\cs_new_protected_nopar:cpn { regex_build_$: } % $ +\cs_new:Npn \regex_build_tmp_class:n #1 { - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int - { - \exp_not:N \int_compare:nNnT - \l_regex_current_char_int < \c_minus_one - { - \regex_action_free:n - { - \int_eval:n - { \l_regex_right_state_int - \l_regex_left_state_int } - } - } - } + \exp_not:o \l_regex_class_tl + \bool_if:NTF \l_regex_class_bool + { \regex_break_point:TF { \regex_action_cost:n {#1} } { } } + { \regex_break_point:TF { } { \regex_action_cost:n {#1} } } } -\cs_new_eq:Nc \regex_build_Z: { regex_build_$: } %$ -\cs_new_eq:Nc \regex_build_z: { regex_build_$: } %$ % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}[int]{\regex_build_b:} -% \begin{macro}[int]{\regex_build_B:} -% \begin{macro}[aux]{\regex_if_word_boundary:TF} -% Contrarily to |^| and |$|, which could be implemented without -% really knowing what precedes in the token list, this requires -% more information, namely, the knowledge of the last character -% code. Case sensitivity does not change word boundaries. +% \begin{macro}[int]{\regex_class_put:N} +% \begin{macro}[int]{\regex_class_put:NN} +% This function is used to rapidly put a character in a character +% class being built, avoiding some tests implied by \cs{regex_build_one:x}. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_b: +\cs_new_protected:Npn \regex_class_put:N #1 { - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int - { - \exp_not:N \regex_if_word_boundary:TF - { - \regex_action_free:n - { - \int_eval:n - { \l_regex_right_state_int - \l_regex_left_state_int } - } - } - { } - } + \tl_put_right:Nx \l_regex_class_tl + { \regex_item_equal:n { \int_value:w `#1 ~ } } } -\cs_new_protected_nopar:Npn \regex_build_B: - { - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int - { - \exp_not:N \regex_if_word_boundary:TF - { } - { - \regex_action_free:n - { - \int_eval:n - { \l_regex_right_state_int - \l_regex_left_state_int } - } - } - } - } -\cs_new_protected_nopar:Npn \regex_if_word_boundary:TF +\cs_new_protected:Npn \regex_class_put:NN #1#2 { - \group_begin: - \int_set_eq:NN \l_regex_current_char_int \l_regex_last_char_int - \c_regex_w_tl - \regex_break_point:TF - { \group_end: \c_regex_W_tl \regex_item_equal:n { -2 } } - { \group_end: \c_regex_w_tl } - \regex_break_point:TF + \tl_put_right:Nx \l_regex_class_tl + { \regex_item_range:nn { \int_value:w `#1 ~ } { \int_value:w `#2 ~ } } } % \end{macrocode} % \end{macro} % \end{macro} -% \end{macro} -% -% \subsubsection{Normal character, and simple character classes} % -% \begin{macro}[int]{\regex_build_one:n, \regex_build_one:x} +% \begin{macro}[int]{\regex_build_raw_alphanum_is_error:N} +% Within character classes, some special character and escape +% sequences do not have any meaning. Special characters are +% interpreted as their \enquote{raw} counterpart, but escape sequences +% without a meaning, \emph{e.g.}, |\A|, are most likely an error: +% this should be |A|. +% \begin{macrocode} +\cs_new_protected:Npn \regex_build_raw_alphanum_is_error:N #1 + { + \regex_aux_char_if_alphanumeric:NTF #1 + { \msg_kernel_error:nnx { regex } { class-bad-escape } { #1 } } + { \regex_build_raw:N #1 } + } +% \end{macrocode} +% \end{macro} % +% \begin{macro}[int]{\regex_build_raw:N} +% If we are in a character class and the next character is an +% unescaped dash, this denotes a range. Otherwise, the current +% character |#1| matches itself. % \begin{macrocode} -\cs_new_protected:Npn \regex_build_one:n #1 - { \regex_build_one:x { \exp_not:n {#1} } } -\cs_new_protected:Npn \regex_build_one:x #1 +\cs_new_protected:Npn \regex_build_raw:N #1#2#3 { - \cs_set_nopar:Npx \regex_build_tmp_class:n ##1 + \regex_build_if_in_class:TF + { \str_if_eq:nnTF {#2#3} { \regex_build_special:N - } } + { \use_ii:nn } + { \regex_class_range:Nw #1 } { - \exp_not:N \exp_not:n {#1} - \regex_break_point:TF { \regex_action_cost:n {##1} } { } + \regex_build_one:x { \regex_item_equal:n { \int_value:w `#1 ~ } } + #2 #3 } - \regex_build_one_quantifier: } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_default_raw:N} -% A normal alphanumeric or an escaped non-alphanumeric -% (actually, any unknown combination) will match itself -% and the thread will fail otherwise. We prepare -% \cs{regex_build_tmp_class:n} with the relevant test and -% commands. The state shift to be inserted in those -% commands will come as |##1|: we don't know -% yet what this will be before checking for quantifiers. +% \begin{macro}[aux]{\regex_class_range:Nw} +% We have just read a raw character followed by a dash; this should be +% followed by an end-point for the range. If the following character +% is an escaped alphanumeric, or if it is an unescaped right bracket, +% then we have an error, so we put the initial character and the dash +% back, as raw characters. Otherwise, build the range, checking that +% it is in the right order, and optimizing for equal end-points. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_default_raw:N #1 - { \regex_build_one:x { \regex_item_equal:n { \int_value:w `#1 ~ } } } +\cs_new_protected:Npn \regex_class_range:Nw #1#2#3 + { + \token_if_eq_meaning:NNTF #2 \regex_build_escaped:N % [ + { \use_i:nn } { \str_if_eq:nnTF { #2#3 } { \regex_build_special:N ] } } + { + \msg_kernel_warning:nnx { regex } { end-range-missing } % [ + { #1 - \if_meaning:w #3 ] \else: \c_backslash_str \fi: #3 } + \regex_class_put:N #1 + \regex_class_put:N - + #2#3 + } + { + \if_num:w `#1 > `#3 \exp_stop_f: + \msg_kernel_error:nnxx { regex } { backwards-range } {#1} {#3} + \else: + \if_num:w `#1 = `#3 \exp_stop_f: + \regex_class_put:N #3 + \else: + \regex_class_put:NN #1#3 + \fi: + \fi: + } + } % \end{macrocode} % \end{macro} % +% \subsubsection{Character properties} +% % \begin{macro}[aux]{\regex_build_.:} -% Similar to \cs{regex_build_default_raw:N} but accepts any character, -% and refuses $-1$, which marks the end of the token list. +% In a class, the dot has no special meaning. Outside, insert +% \cs{regex_item_dot:}, which matches any character or control +% sequence, and refuses $-2$, which marks the end of the token list. % \begin{macrocode} \cs_new_protected_nopar:cpn { regex_build_.: } { - \cs_set:Npn \regex_build_tmp_class:n ##1 - { \regex_item_dot:T { \regex_action_cost:n {##1} } } - \regex_build_one_quantifier: + \regex_build_if_in_class:TF + { \regex_build_raw:N . } + { \regex_build_one:x \regex_item_dot: } } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_d:,\regex_build_D:} -% \begin{macro}[aux]{\regex_build_h:,\regex_build_H:} -% \begin{macro}[aux]{\regex_build_s:,\regex_build_S:} -% \begin{macro}[aux]{\regex_build_v:,\regex_build_V:} -% \begin{macro}[aux]{\regex_build_w:,\regex_build_W:} -% \begin{macro}[aux]{\regex_build_N:} +% \begin{macro}[aux]{\regex_build_/d:,\regex_build_/D:} +% \begin{macro}[aux]{\regex_build_/h:,\regex_build_/H:} +% \begin{macro}[aux]{\regex_build_/s:,\regex_build_/S:} +% \begin{macro}[aux]{\regex_build_/v:,\regex_build_/V:} +% \begin{macro}[aux]{\regex_build_/w:,\regex_build_/W:} +% \begin{macro}[aux]{\regex_build_/N:} % The constants \cs{c_regex_d_tl}, \emph{etc.} hold % a list of tests which match the corresponding character % class, and jump to the \cs{regex_break_point:TF} marker. @@ -1920,7 +2011,7 @@ % \begin{macrocode} \tl_map_inline:nn { dDhHsSvVwWN } { - \cs_new_protected_nopar:cpx { regex_build_#1: } + \cs_new_protected_nopar:cpx { regex_build_/#1: } { \regex_build_one:n \exp_not:c { c_regex_#1_tl } } } % \end{macrocode} @@ -1931,263 +2022,338 @@ % \end{macro} % \end{macro} % -% \subsubsection{Character classes} +% \subsubsection{Anchoring and simple assertions} % -% \begin{macro}[aux]{\regex_class_one:n, \regex_class_one:x} -% This is the function used to add material -% to the class. +% \begin{macro}[aux]{\regex_build_simple_assertion:Nn} +% Assertions are not allowed within character classes: the raw +% character |#1| is inserted instead. Otherwise, use the test |#2|; +% if the assertion is successful, move from the \texttt{left} state to +% the \texttt{right} state. % \begin{macrocode} -\cs_new_protected:Npn \regex_class_one:n - { \tl_put_right:Nn \l_regex_class_tl } -\cs_new_protected:Npn \regex_class_one:x - { \tl_put_right:Nx \l_regex_class_tl } +\cs_new_protected:Npn \regex_build_simple_assertion:Nn #1#2 + { + \regex_build_if_in_class:TF + { \regex_build_raw_alphanum_is_error:N #1 } + { + \regex_build_new_state: + \regex_toks_put_right:Nx \l_regex_left_state_int + { + \exp_not:n {#2} + { + \regex_action_free:n + { + \int_eval:n + { \l_regex_right_state_int - \l_regex_left_state_int } + } + } + } + %^^A \regex_assertion_quantifier: + } + } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_[:} -% This starts a class. The code for the class is collected -% in \cs{l_regex_class_tl}. The first character is special. +% \begin{macro}[aux]+\regex_build_^:+ +% \begin{macro}[aux]{\regex_build_/A:} +% \begin{macro}[aux]{\regex_build_/G:} +% \begin{macro}[aux]+\regex_build_$:+ +% \begin{macro}[aux]{\regex_build_/Z:} +% \begin{macro}[aux]{\regex_build_/z:} +% Anchoring at the start corresponds to checking that the current +% character is the first in the token list. Anchoring to the beginning +% of the match attempt uses \cs{l_regex_start_step_int} instead of +% \cs{c_zero}. End anchors match the end of the token list, marked by +% a character code of $-2$. % \begin{macrocode} -\cs_new_protected_nopar:cpn { regex_build_[: } +\cs_new_protected_nopar:cpn { regex_build_^: } { - \tl_clear:N \l_regex_class_tl - \cs_set_eq:NN \regex_build_raw:N \regex_class_default_raw:N - \tl_set:Nn \l_regex_build_or_class_tl { class } - \regex_class_first:NN + \regex_build_simple_assertion:Nn ^ + { \int_compare:nNnT \l_regex_min_step_int = \l_regex_current_step_int } + } +\cs_new_protected_nopar:cpn { regex_build_/A: } + { + \regex_build_simple_assertion:Nn A + { \int_compare:nNnT \l_regex_min_step_int = \l_regex_current_step_int } + } +\cs_new_protected_nopar:cpn { regex_build_/G: } + { + \regex_build_simple_assertion:Nn G + { \int_compare:nNnT \l_regex_start_step_int = \l_regex_current_step_int } + } +\cs_new_protected_nopar:cpn { regex_build_$: } % $ + { + \regex_build_simple_assertion:Nn $ % $ + { \int_compare:nNnT \l_regex_current_char_int < \c_minus_one } + } +\cs_new_protected_nopar:cpn { regex_build_/Z: } + { + \regex_build_simple_assertion:Nn Z + { \int_compare:nNnT \l_regex_current_char_int < \c_minus_one } + } +\cs_new_protected_nopar:cpn { regex_build_/z: } + { + \regex_build_simple_assertion:Nn z + { \int_compare:nNnT \l_regex_current_char_int < \c_minus_one } } % \end{macrocode} % \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} % -% \begin{macro}[aux]{\regex_class_]:} -% If \texttt{]} appears as the first item of a class, then -% it doesn't end the class. Otherwise, it's the end, -% act just as for a single character, but with a more -% complicated test. And restore \cs{regex_build_raw:N}. +% \begin{macro}[int]{\regex_build_/b:} +% \begin{macro}[int]{\regex_build_/B:} +% \begin{macro}[aux]{\regex_if_word_boundary:TF} +% Contrarily to |^| and |$|, which could be implemented without +% really knowing what precedes in the token list, this requires +% more information, namely, the knowledge of the last character +% code. Case sensitivity does not change word boundaries. % \begin{macrocode} -\cs_new_protected_nopar:cpn { regex_class_]: } +\cs_new_protected_nopar:cpn { regex_build_/b: } { - \tl_if_empty:NTF \l_regex_class_tl %[ - { \regex_class_default_raw:N ] } - { - \cs_set_eq:NN \regex_build_raw:N \regex_build_default_raw:N - \tl_set:Nn \l_regex_build_or_class_tl { build } - \cs_set:Npn \regex_build_tmp_class:n ##1 - { - \exp_not:o \l_regex_class_tl - \bool_if:NTF \l_regex_class_bool - { \regex_break_point:TF { \regex_action_cost:n {##1} } { } } - { \regex_break_point:TF { } { \regex_action_cost:n {##1} } } - } - \regex_build_one_quantifier: - } + \regex_build_simple_assertion:Nn b + { \regex_if_word_boundary:TF \use:n \use_none:n } + } +\cs_new_protected_nopar:cpn { regex_build_/B: } + { + \regex_build_simple_assertion:Nn B + { \regex_if_word_boundary:TF { } } + } +\cs_new_protected_nopar:Npn \regex_if_word_boundary:TF + { + \group_begin: + \int_set_eq:NN \l_regex_current_char_int \l_regex_last_char_int + \c_regex_w_tl + \regex_break_point:TF + { \group_end: \c_regex_W_tl \regex_item_equal:n { -2 } } + { \group_end: \c_regex_w_tl } + \regex_break_point:TF } % \end{macrocode} % \end{macro} +% \end{macro} +% \end{macro} % -% \begin{macro}[aux]{\regex_class_first:NN} -% If the first non-space character is |^|, then the class is inverted. -% We keep track of this in \cs{l_regex_class_bool}. +% \subsubsection{Entering and exiting character classes} +% +% \begin{macro}[aux]{\regex_build_[:} +% In a class, left brackets mean nothing. Outside a class, this starts +% a class, whose first characters may have a special meaning. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_class_first:NN #1#2 +\cs_new_protected_nopar:cpn { regex_build_[: } { - \str_if_eq:nnTF {#1#2} { \regex_build_control:N ^ } - { \bool_set_false:N \l_regex_class_bool } - { - \bool_set_true:N \l_regex_class_bool - #1 #2 - } + \regex_build_if_in_class:TF + { \regex_build_raw:N [ } + { \regex_class_first:NNNN } } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_class_default_raw:N} -% \begin{macro}[aux]{\regex_class_single:N} -% Most characters are treated here. We look ahead for an unescaped dash. -% If there is none, then the character matches itself. +% \begin{macro}[aux]{\regex_build_]:} +% Outside a class, right brackets have no meaning. In a class, change +% the mode ($m\to (m-15)/13$, truncated) to reflect the fact that we +% are leaving the class. If we are still in a class after leaving one, +% then this is the case |[...\cL[...]...]|, and we insert the relevant +% closing material in \cs{l_regex_class_tl}. Otherwise look for +% quantifiers. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_class_default_raw:N #1#2#3 +\cs_new_protected:cpn { regex_build_]: } { - \str_if_eq:nnTF {#2#3} { \regex_build_control:N - } - { \regex_class_range:Nw #1 } + \regex_build_if_in_class:TF { - \regex_class_single:N #1 - #2 #3 + \if_num:w \l_regex_build_mode_int > \c_sixteen + \tl_set:Nx \l_regex_class_tl + { + \exp_not:o \l_regex_class_saved_tl + \if_num:w \l_regex_catcodes_int < \c_regex_catcodes_all_int + \regex_item_catcode:nT { \int_use:N \l_regex_catcodes_int } + \else: + \exp_after:wN \use:n + \fi: + { + \exp_not:o \l_regex_class_tl + \bool_if:NF \l_regex_class_bool + { \regex_break_point:TF { } { \regex_break_true:w } } + } + } + \bool_set_eq:NN \l_regex_class_bool \l_regex_class_saved_bool + \fi: + \tex_advance:D \l_regex_build_mode_int - \c_fifteen + \tex_divide:D \l_regex_build_mode_int \c_thirteen + \if_int_odd:w \l_regex_build_mode_int \else: + \exp_after:wN \regex_build_one_quantifier: + \fi: } + { \regex_build_raw:N ] } } -\cs_new_protected_nopar:Npn \regex_class_single:N #1 - { \regex_class_one:x { \regex_item_equal:n { \int_value:w `#1 } } } % \end{macrocode} % \end{macro} -% \end{macro} % -% \begin{macro}[aux]{\regex_class_range:Nw} -% \begin{macro}[aux]{\regex_class_range_put:NN} -% If the character is followed by a dash, we look for -% the end-point of the range. For \enquote{raw} characters, -% that's simply |#3|. Most \enquote{control} characters also -% have no meaning, and can serve as an end-point, but those -% with a meaning interrupt the range. -% In the case of a true range, check whether the end-points -% are in the right order, and optimize in the case of equal -% end-points. -% \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_class_range:Nw #1#2#3 - { - \token_if_eq_meaning:NNTF #2 \regex_build_control:N +%^^A todo: ignore spaces everywhere!! +% +% \begin{macro}[aux]{\regex_class_first:NNNN} +% This starts a class. Change the mode by appending $3$ to it, and +% reset the variables \cs{l_regex_class_tl} and +% \cs{l_regex_bool_tl}. In the special case of mode $63$ +% (|[\c[...]]|), we open a group, to avoid overriding the setting of +% \cs{l_regex_class_bool} and \cs{l_regex_class_tl}; the group ends at +% the matching right bracket. If the first character is |^|, then the +% class is inverted. We keep track of this in +% \cs{l_regex_class_bool}. If the next character is a right bracket, +% then it should be changed to a raw one (dirty hack here; the |F| +% argument of \cs{str_if_eq:nnTF} is the trailing |#3|). +% \begin{macrocode} +\cs_new_protected:Npn \regex_class_first:NNNN #1#2#3#4 + { + \l_regex_build_mode_int = \int_value:w \l_regex_build_mode_int 3 ~ % + \if_num:w \l_regex_build_mode_int > \c_sixteen + \tl_set_eq:NN \l_regex_class_saved_tl \l_regex_class_tl + \bool_set_eq:NN \l_regex_class_saved_bool \l_regex_class_bool + \fi: + \token_if_eq_meaning:NNTF #1 \regex_build_special:N { - \cs_if_exist:cTF { regex_class_#3: } + \token_if_eq_charcode:NNTF #2 ^ { - \regex_class_single:N #1 - \regex_class_single:N - - #2#3 + \bool_set_false:N \l_regex_class_bool % [ + \str_if_eq:nnTF {#3#4} { \regex_build_special:N ] } + { \regex_build_raw:N } + } + { % [ + \token_if_eq_charcode:NNTF #2 ] + { \regex_build_raw:N #2 } + { #1 #2 } } - { \regex_class_range_put:NN #1#3 } } - { \regex_class_range_put:NN #1#3 } - } -\cs_new_protected_nopar:Npn \regex_class_range_put:NN #1#2 - { - \if_num:w `#1 > `#2 \exp_stop_f: - \msg_kernel_error:nnxx { regex } { backwards-range } {#1} {#2} - \else: - \regex_class_one:x - { - \if_num:w `#1 = `#2 \exp_stop_f: - \regex_item_equal:n - \else: - \regex_item_range:nn { \int_value:w `#1 } - \fi: - { \int_value:w `#2 } - } - \fi: + { #1 #2 } + #3 #4 } % \end{macrocode} % \end{macro} -% \end{macro} % -% \begin{macro}[aux]{\regex_class_d:,\regex_class_D:} -% \begin{macro}[aux]{\regex_class_h:,\regex_class_H:} -% \begin{macro}[aux]{\regex_class_s:,\regex_class_S:} -% \begin{macro}[aux]{\regex_class_v:,\regex_class_V:} -% \begin{macro}[aux]{\regex_class_w:,\regex_class_W:} -% Similar to \cs{regex_class_single:N}, adding the appropriate -% ranges of characters to the class. The token lists are not -% expanded because it is more memory efficient, with a tiny -% overhead on execution. +% \subsubsection{Catcodes and csnames} +% +% \begin{macro}[aux]{\regex_build_/c:} +% The |\c| escape sequence is only allowed in modes $0$ and $3$, +% \emph{i.e.}, not within any other |\c| escape sequence. % \begin{macrocode} -\tl_map_inline:nn { dDhHsSvVwWN } +\cs_new_protected:cpn { regex_build_/c: } { - \cs_new_protected_nopar:cpx { regex_class_#1: } - { \regex_class_one:n \exp_not:c { c_regex_#1_tl } } + \if_case:w \l_regex_build_mode_int + \exp_after:wN \regex_build_c_aux:wNN + \or: \or: \or: \exp_after:wN \regex_build_c_aux:wNN + \fi: + \msg_kernel_error:nn { regex } { c-bad-mode } + \s_stop } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% -% \subsubsection{Catcodes and csnames} -% -% \begin{macro}[aux]{\regex_build_c:} -% \begin{macro}[aux]{\regex_build_C:} -% \begin{macro}[aux]{\regex_class_c:} -% \begin{macro}[aux]{\regex_class_C:} -% \begin{macro}[aux]{\regex_aux_c:nnNN} % +% \begin{macro}[aux]{\regex_build_c_aux:wNN} +% The |\c| escape sequence can be followed by a capital letter +% representing a character category, by a left bracket which starts a +% list of categories, or by a brace group holding a regular expression +% for a control sequence name. Otherwise, raise an error. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_c: - { \regex_aux_c:NNNN \regex_item_catcode:n \use_i:nn } -\cs_new_protected_nopar:Npn \regex_build_C: - { \regex_aux_c:NNNN \regex_item_catcode_neq:n \use_ii:nn } -\cs_new_eq:NN \regex_class_c: \regex_build_c: -\cs_new_eq:NN \regex_class_C: \regex_build_C: -\cs_new_protected:Npn \regex_aux_c:NNNN #1#2#3#4 +\cs_new_protected:Npn \regex_build_c_aux:wNN #1 \s_stop #2#3 { - \token_if_eq_meaning:NNTF #3 \regex_build_raw:N + \token_if_eq_meaning:NNTF #2 \regex_build_raw:N { - \regex_aux_c_test:NTF #4 + \cs_if_exist:cTF { c_regex_catcode_#3_int } { - \use:c { regex_ \l_regex_build_or_class_tl _one:x } - { #1 { \int_use:N \l_regex_current_catcode_int } } + \int_set_eq:Nc \l_regex_catcodes_int { c_regex_catcode_#3_int } + \l_regex_build_mode_int + = \if_case:w \l_regex_build_mode_int \c_two \else: \c_six \fi: } + } + { \cs_if_exist_use:cF { regex_build_c_#3: } } { - \msg_kernel_error:nnx { regex } { build-c } { #4 } - #3 #4 + \msg_kernel_error:nnxx { regex } { c-command } + { regular~expression } { #3 } + #2 #3 } - } - { - \if_charcode:w \c_lbrace_str \exp_not:N #4 - \exp_after:wN #2 - \else: - \exp_after:wN \use_ii:nn - \fi: - { - \group_begin: - \bool_set_true:N \l_regex_in_csname_bool - \regex_build:w - } - { - \msg_kernel_error:nnx { regex } { build-c } { #4 } - #3 #4 - } - } } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} +% +% \begin{macro}+\regex_build_c_{:+ ^^A } +% The case of a left brace is easy, based on what we have done so far: +% in a group, build the regular expression, after changing the mode to +% forbid nesting |\c|. +% \begin{macrocode} +\cs_new_protected:cpn { regex_build_c_ \c_lbrace_str : } + { + \group_begin: + \l_regex_build_mode_int + = - \if_case:w \l_regex_build_mode_int \c_two \else: \c_six \fi: + \regex_build:w + } +% \end{macrocode} % \end{macro} % -% The |\c| and |\C| escape sequences are used both in the -% regex and the replacement texts. They can be followed either -% by a capital letter representing a category code, or by a left -% brace (in the case of |\c| only). +% \begin{macro}[aux]{\regex_build_c_[:} ^^A] +% \begin{macro}[aux]{\regex_build_c_lbrack_loop:NN} +% \begin{macro}[aux]{\regex_build_c_lbrack_end:} +% \begin{macro}[aux]{\regex_build_add:N} % -% \begin{macro}{\regex_aux_c_test:NTF} -% If the first argument is a known catcode identifier -% (capital letters listed below), then -% \cs{l_regex_current_catcode_int} is set to the relevant -% catcode value (between $0$ for cs and $13$ for active chars), -% and the \texttt{true} branch is taken. -% Otherwise, the \texttt{false} branch is taken, -% and the current catcode is set to $-1$. % \begin{macrocode} -\cs_new_protected:Npn \regex_aux_c_test:NTF #1 +\cs_new_protected:cpn { regex_build_c_[: } #1#2 { - \int_set:Nn \l_regex_current_catcode_int + \int_zero:N \l_regex_catcodes_int + \str_if_eq:nnTF { #1 #2 } { \regex_build_special:N ^ } { - \if_case:w \int_eval:w `#1 - `A \int_eval_end: - \c_thirteen % Active - \or: \c_one % Begin-group - \or: \c_zero % Control sequence - \or: \c_eight % Down (subscript) - \or: \c_two % End-group - \else: - \if_case:w \int_eval:w `#1 - `L \int_eval_end: - \c_eleven % Letter - \or: \c_three % Math toggle - \or: \c_minus_one - \or: \c_twelve % Other - \or: \c_six % Parameter - \or: \c_minus_one - \or: \c_minus_one - \or: \c_ten % Space - \or: \c_four % Tab (alignment) - \or: \c_seven % Up (superscript) - \else: \c_minus_one - \fi: - \fi: + \bool_set_false:N \l_regex_catcodes_bool + \regex_build_c_lbrack_loop:NN } - \if_num:w \l_regex_current_catcode_int = \c_minus_one - \exp_after:wN \use_ii:nn + { + \bool_set_true:N \l_regex_catcodes_bool + \regex_build_c_lbrack_loop:NN + #1#2 + } + } +\cs_new_protected:Npn \regex_build_c_lbrack_loop:NN #1#2 + { + \token_if_eq_meaning:NNTF #1 \regex_build_raw:N + { + \cs_if_exist:cTF { c_regex_catcode_#2_int } + { + \exp_args:Nc \regex_build_c_lbrack_add:N + { c_regex_catcode_#2_int } + \regex_build_c_lbrack_loop:NN + } + } + { % [ + \token_if_eq_charcode:NNTF #2 ] + { \regex_build_c_lbrack_end: } + } + { + \msg_kernel_error:nnxx { regex } { c-command } + { regular-expression } { #2 } + \regex_build_c_lbrack_end: + } + } +\cs_new_protected_nopar:Npn \regex_build_c_lbrack_end: + { + \l_regex_build_mode_int + = \if_case:w \l_regex_build_mode_int \c_two \else: \c_six \fi: + \if_meaning:w \c_false_bool \l_regex_catcodes_bool + \int_set:Nn \l_regex_catcodes_int + { \c_regex_catcodes_all_int - \l_regex_catcodes_int } + \fi: + } +\cs_new_protected:Npn \regex_build_c_lbrack_add:N #1 + { + \if_int_odd:w \int_eval:w \l_regex_catcodes_int / #1 \int_eval_end: \else: - \exp_after:wN \use_i:nn + \tex_advance:D \l_regex_catcodes_int #1 \fi: } % \end{macrocode} % \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} % % ^^A{ % \begin{macro}+\regex_build}:+ @@ -2196,18 +2362,18 @@ % replace the brace with an escaped brace. % \begin{macrocode} \cs_new_protected:cpn { regex_build_ \c_rbrace_str : } - { - \bool_if:NTF \l_regex_in_csname_bool + { %^^A incorrect recovery for \c{...(...[...} with missing ) or ]. + \int_compare:nNnTF \l_regex_build_mode_int < \c_zero { - \regex_build_Z: + \use:c { regex_build_/Z: } \regex_toks_put_right:Nx \l_regex_right_state_int { \regex_action_success: } \int_incr:N \l_regex_capturing_group_int \use:x { \group_end: - \exp_not:n { \use:c { regex_ \l_regex_build_or_class_tl _one:n } } - { \regex_item_cs:n { \regex_set_aux:N ? } } + \regex_build_one:n + { \regex_item_cs:n { \regex_set_aux:N ? } } } } { \exp_after:wN \regex_build_raw:N \c_rbrace_str } @@ -2223,18 +2389,18 @@ % When all characters for the quantifier are found, the corresponding % function is called. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_quantifier:w #1#2 +\cs_new_protected:Npn \regex_build_quantifier:w #1#2 { - \token_if_eq_meaning:NNTF #1 \regex_build_control:N + \token_if_eq_meaning:NNTF #1 \regex_build_special:N { \cs_if_exist_use:cF { regex_build_quantifier_#2:w } { - \regex_build_quantifier_end:n { } + \regex_build_quantifier_end:nn { } { } #1 #2 } } { - \regex_build_quantifier_end:n { } + \regex_build_quantifier_end:nn { } { } #1 #2 } } @@ -2266,12 +2432,12 @@ % arguments that the final function needs in the case of braced % constructions, and is empty otherwise. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_quantifier_aux:nnNN #1#2#3#4 +\cs_new_protected:Npn \regex_build_quantifier_aux:nnNN #1#2#3#4 { - \str_if_eq:nnTF { #3 #4 } { \regex_build_control:N ? } - { \regex_build_quantifier_end:n { #2 #4 } #1 } + \str_if_eq:nnTF { #3 #4 } { \regex_build_special:N ? } + { \regex_build_quantifier_end:nn { #2 #4 } {#1} } { - \regex_build_quantifier_end:n { #2 } #1 + \regex_build_quantifier_end:nn { #2 } {#1} #3 #4 } } @@ -2287,44 +2453,44 @@ % \begin{macrocode} \cs_new_protected_nopar:cpn { regex_build_quantifier_ \c_lbrace_str :w } { \regex_get_digits:nw { \regex_build_quantifier_lbrace:n } } -\cs_new_protected_nopar:Npn \regex_build_quantifier_lbrace:n #1 +\cs_new_protected:Npn \regex_build_quantifier_lbrace:n #1 { \tl_if_empty:nTF {#1} { - \regex_build_quantifier_end:n { } + \regex_build_quantifier_end:nn { } { } \exp_after:wN \regex_build_raw:N \c_lbrace_str } { \regex_build_quantifier_lbrace:nw {#1} } } -\cs_new_protected_nopar:Npx \regex_build_quantifier_lbrace:nw #1#2#3 +\cs_new_protected:Npx \regex_build_quantifier_lbrace:nw #1#2#3 { \exp_not:N \prg_case_str:nnn { #2 #3 } { - { \exp_not:N \regex_build_control:N , } + { \exp_not:N \regex_build_special:N , } { \exp_not:N \regex_get_digits:nw { \exp_not:N \regex_build_quantifier_lbrace:nnw {#1} } } - { \exp_not:N \regex_build_control:N \c_rbrace_str } - { \exp_not:N \regex_build_quantifier_end:n {n} {#1} } + { \exp_not:N \regex_build_special:N \c_rbrace_str } + { \exp_not:N \regex_build_quantifier_end:nn {n} { {#1} } } } { - \exp_not:N \regex_build_quantifier_end:n { } + \exp_not:N \regex_build_quantifier_end:nn { } { } \exp_not:N \regex_build_raw:N \c_lbrace_str #1#2 } } -\cs_new_protected_nopar:Npn \regex_build_quantifier_lbrace:nnw #1#2#3#4 +\cs_new_protected:Npn \regex_build_quantifier_lbrace:nnw #1#2#3#4 { \str_if_eq:xxTF { \exp_not:n {#3#4} } - { \exp_not:N \regex_build_control:N \c_rbrace_str } + { \exp_not:N \regex_build_special:N \c_rbrace_str } { \tl_if_empty:nTF {#2} { \regex_build_quantifier_aux:nnNN { {#1} } { n* } } { \regex_build_quantifier_aux:nnNN { {#1} {#2} } { nn } } } %^^A todo: catch the case m>n { - \regex_build_quantifier_end:n { } + \regex_build_quantifier_end:nn { } { } \use:x { \exp_args:No \tl_map_function:nN @@ -2340,12 +2506,17 @@ % \end{macro} % \end{macro} % -% \begin{macro}[aux]{\regex_build_quantifier_end:n} +% \begin{macro}[aux]{\regex_build_quantifier_end:nn} % When all quantifiers are found, we will call the relevant % \cs{regex_build_one/group_\meta{quantifiers}:} function. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_quantifier_end:n #1 - { \use:c { regex_build_ \l_regex_one_or_group_tl _ #1 : } } +\cs_new_protected:Npn \regex_build_quantifier_end:nn #1#2 + { + \use:c { regex_build_ \l_regex_one_or_group_tl _ #1 : } #2 + \tl_clear:N \l_regex_class_tl + \bool_set_true:N \l_regex_class_bool + \int_set_eq:NN \l_regex_catcodes_int \l_regex_catcodes_default_int + } % \end{macrocode} % \end{macro} % @@ -2372,7 +2543,7 @@ % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_build_one_: { - \regex_build_transition_aux:NN + \regex_build_transition:NN \regex_build_tmp_class:n \l_regex_right_state_int } % \end{macrocode} @@ -2387,13 +2558,13 @@ % \begin{macrocode} \cs_new_protected_nopar:cpn { regex_build_one_?: } { - \regex_build_transitions_aux:NNNN + \regex_build_transitions:NNNN \regex_build_tmp_class:n \l_regex_right_state_int \regex_action_free:n \l_regex_right_state_int } \cs_new_protected_nopar:cpn { regex_build_one_??: } { - \regex_build_transitions_aux:NNNN + \regex_build_transitions:NNNN \regex_action_free:n \l_regex_right_state_int \regex_build_tmp_class:n \l_regex_right_state_int } @@ -2408,13 +2579,13 @@ % \begin{macrocode} \cs_new_protected_nopar:cpn { regex_build_one_*: } { - \regex_build_transitions_aux:NNNN + \regex_build_transitions:NNNN \regex_build_tmp_class:n \l_regex_left_state_int \regex_action_free:n \l_regex_right_state_int } \cs_new_protected_nopar:cpn { regex_build_one_*?: } { - \regex_build_transitions_aux:NNNN + \regex_build_transitions:NNNN \regex_action_free:n \l_regex_right_state_int \regex_build_tmp_class:n \l_regex_left_state_int } @@ -2433,7 +2604,7 @@ { \regex_build_one_: \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int - \regex_build_transitions_aux:NNNN + \regex_build_transitions:NNNN \regex_action_free:n \l_regex_tmpa_int \regex_action_free:n \l_regex_right_state_int } @@ -2441,7 +2612,7 @@ { \regex_build_one_: \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int - \regex_build_transitions_aux:NNNN + \regex_build_transitions:NNNN \regex_action_free:n \l_regex_right_state_int \regex_action_free:n \l_regex_tmpa_int } @@ -2456,7 +2627,7 @@ % are identical, since the number of repetitions is fixed. % Simply repeat |#1| times the effect of \cs{regex_build_one_:}. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_one_n: #1 +\cs_new_protected:Npn \regex_build_one_n: #1 { \prg_replicate:nn {#1} { \regex_build_one_: } } \cs_new_eq:cN { regex_build_one_n?: } \regex_build_one_n: % \end{macrocode} @@ -2468,12 +2639,12 @@ % This function is called in case the syntax is % \texttt{\{\meta{int},\}}. % \begin{macrocode} -\cs_new_protected_nopar:cpx { regex_build_one_n*: } #1 +\cs_new_protected:cpx { regex_build_one_n*: } #1 { \exp_not:N \prg_replicate:nn {#1} { \exp_not:N \regex_build_one_: } \exp_not:c { regex_build_one_*: } } -\cs_new_protected_nopar:cpx { regex_build_one_n*?: } #1 +\cs_new_protected:cpx { regex_build_one_n*?: } #1 { \exp_not:N \prg_replicate:nn {#1} { \exp_not:N \regex_build_one_: } \exp_not:c { regex_build_one_*?: } @@ -2488,7 +2659,7 @@ % This function is called when the syntax is % \texttt{\{\meta{int},\meta{int}\}}. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_one_nn_aux:Nnn #1#2#3 +\cs_new_protected:Npn \regex_build_one_nn_aux:Nnn #1#2#3 { \prg_replicate:nn {#2} { \regex_build_one_: } \prg_replicate:nn {#3-#2} {#1} @@ -2524,17 +2695,24 @@ % vertex to a new vertex. % \end{itemize} % \begin{macrocode} -\cs_new_protected_nopar:cpn { regex_build_(: } #1#2 +\cs_new_protected:cpn { regex_build_(: } #1#2 { - \str_if_eq:nnTF { #1 #2 } { \regex_build_control:N ? } - { \regex_build_special_group:NN } + \regex_build_if_in_class:TF { - \int_incr:N \l_regex_capturing_group_int - \regex_seq_push_int:NN - \l_regex_capturing_group_seq \l_regex_capturing_group_int - \regex_build_open_aux: + \regex_build_raw:N ( #1 #2 } + { + \str_if_eq:nnTF { #1 #2 } { \regex_build_special:N ? } + { \regex_build_special_group:NN } + { + \int_incr:N \l_regex_capturing_group_int + \regex_seq_push_int:NN + \l_regex_capturing_group_seq \l_regex_capturing_group_int + \regex_build_open_aux: + #1 #2 + } + } } \cs_new_protected_nopar:Npn \regex_build_open_aux: { @@ -2549,16 +2727,22 @@ } \cs_new_protected_nopar:cpn { regex_build_|: } { - \regex_build_end_alternation: - \regex_build_begin_alternation: + \regex_build_if_in_class:TF { \regex_build_raw:N | } + { + \regex_build_end_alternation: + \regex_build_begin_alternation: + } } \cs_new_protected_nopar:cpn { regex_build_): } { - \seq_if_empty:NTF \l_regex_capturing_group_seq - { \msg_kernel_error:nn { regex } { extra-rparen } } + \regex_build_if_in_class:TF { \regex_build_raw:N ) } { - \regex_build_close_aux: - \regex_build_group_quantifier: + \seq_if_empty:NTF \l_regex_capturing_group_seq + { \msg_kernel_error:nn { regex } { extra-rparen } } + { + \regex_build_close_aux: + \regex_build_group_quantifier: + } } } \cs_new_protected_nopar:Npn \regex_build_close_aux: @@ -2610,13 +2794,13 @@ % is known, then use that. Otherwise, treat the question mark % as if it had been escaped. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_special_group:NN #1#2 +\cs_new_protected:Npn \regex_build_special_group:NN #1#2 { \cs_if_exist_use:cF { regex_build_special_group_\token_to_str:N #2 : } { \msg_kernel_error:nnx { regex } { unsupported } { (? \token_to_str:N #2 } %) - \regex_build_control:N ( % ) + \regex_build_special:N ( % ) \regex_build_raw:N ? #1 #2 } @@ -2699,7 +2883,7 @@ { \regex_build_options:NNN - } -\cs_new_protected_nopar:Npn \regex_build_options:NNN #1#2#3 +\cs_new_protected:Npn \regex_build_options:NNN #1#2#3 { \token_if_eq_meaning:NNTF \regex_build_raw:N #2 { @@ -2752,7 +2936,7 @@ % Once the quantifier is found by \cs{regex_build_quantifier:w}, % we insert the code for tracking submatches. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_group_submatches:NN #1#2 +\cs_new_protected:Npn \regex_build_group_submatches:NN #1#2 { \seq_pop:NN \l_regex_capturing_group_seq \l_regex_tmpa_tl \int_compare:nNnF { \l_regex_tmpa_tl } < \c_zero @@ -2774,7 +2958,7 @@ { \regex_build_group_submatches:NN \l_regex_left_state_int \l_regex_right_state_int - \regex_build_transition_aux:NN + \regex_build_transition:NN \regex_action_free:n \l_regex_right_state_int } % \end{macrocode} @@ -2785,7 +2969,7 @@ % This is done by shifting the current contents of the \cs{tex_toks:D} % \cs{l_regex_tmpa_int} to a new state. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_group_shift:N #1 +\cs_new_protected:Npn \regex_build_group_shift:N #1 { \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int \regex_build_new_state: @@ -2827,11 +3011,11 @@ % operators, and higher priority (\texttt{put_left}) for % lazy operators. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_group_qs_aux:NN #1#2 +\cs_new_protected:Npn \regex_build_group_qs_aux:NN #1#2 { \regex_build_group_shift:N \regex_action_free:n \int_set_eq:NN \l_regex_right_state_int \l_regex_left_state_int - \regex_build_transition_aux:NN \regex_action_free:n #1 + \regex_build_transition:NN \regex_action_free:n #1 #2 \l_regex_tmpa_int { \regex_action_free:n @@ -2876,7 +3060,7 @@ \regex_build_group_submatches:NN \l_regex_left_state_int \l_regex_right_state_int \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int - \regex_build_transitions_aux:NNNN + \regex_build_transitions:NNNN \regex_action_free:n \l_regex_tmpa_int \regex_action_free:n \l_regex_right_state_int } @@ -2885,7 +3069,7 @@ \regex_build_group_submatches:NN \l_regex_left_state_int \l_regex_right_state_int \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int - \regex_build_transitions_aux:NNNN + \regex_build_transitions:NNNN \regex_action_free:n \l_regex_right_state_int \regex_action_free:n \l_regex_tmpa_int } @@ -2905,7 +3089,7 @@ % \cs{l_regex_max_state_int} is set appropriately to the largest % allocated \tn{toks} register. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_group_n_aux:n #1 +\cs_new_protected:Npn \regex_build_group_n_aux:n #1 { \regex_toks_put_right:Nx \l_regex_right_state_int { @@ -2939,10 +3123,10 @@ % are identical, since the number of repetitions is fixed. % We only record the submatch information at the last repetition. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_group_n: #1 +\cs_new_protected:Npn \regex_build_group_n: #1 { % ^^A todo: catch case #1 <= 0. \regex_build_group_n_aux:n {#1} - \regex_build_transition_aux:NN + \regex_build_transition:NN \regex_action_free:n \l_regex_right_state_int \regex_build_group_submatches:NN \l_regex_tmpa_int \l_regex_left_state_int @@ -2962,19 +3146,19 @@ % case no iteration was made, here, we know that the submatch % information is overwritten in any case. % \begin{macrocode} -\cs_new_protected_nopar:cpn { regex_build_group_n*: } #1 +\cs_new_protected:cpn { regex_build_group_n*: } #1 { % ^^A todo: catch case #1 <= 0. \regex_build_group_n_aux:n {#1} - \regex_build_transitions_aux:NNNN + \regex_build_transitions:NNNN \regex_action_free:n \l_regex_tmpa_int \regex_action_free:n \l_regex_right_state_int \regex_build_group_submatches:NN \l_regex_tmpa_int \l_regex_left_state_int } -\cs_new_protected_nopar:cpn { regex_build_group_n*?: } #1 +\cs_new_protected:cpn { regex_build_group_n*?: } #1 { % ^^A todo: catch case #1 <= 0. \regex_build_group_n_aux:n {#1} - \regex_build_transitions_aux:NNNN + \regex_build_transitions:NNNN \regex_action_free:n \l_regex_right_state_int \regex_action_free:n \l_regex_tmpa_int \regex_build_group_submatches:NN @@ -2989,12 +3173,12 @@ % These functions are called when the syntax is either % \texttt{\{\meta{int},\}} or \texttt{\{\meta{int},\meta{int}\}}. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_group_nn: #1#2 +\cs_new_protected:Npn \regex_build_group_nn: #1#2 { % ^^A Not Implemented Yet! \msg_expandable_error:n { Quantifier~{m,n}~not~implemented~yet } \use:c { regex_build_group_n*: } {#1} } -\cs_new_protected_nopar:cpn { regex_build_group_nn?: } #1#2 +\cs_new_protected:cpn { regex_build_group_nn?: } #1#2 { % ^^A Not Implemented Yet! \msg_expandable_error:n { Quantifier~{m,n}~not~implemented~yet } \use:c { regex_build_group_n*?: } {#1} @@ -3115,7 +3299,7 @@ { \regex_state_use_aux:n { } } \cs_new_protected_nopar:Npn \regex_state_use: { \regex_state_use_aux:n { \exp_after:wN \use_none_delimit_by_s_stop:w } } -\cs_new_protected_nopar:Npn \regex_state_use_aux:n #1 +\cs_new_protected:Npn \regex_state_use_aux:n #1 { \if_num:w \tex_dimen:D \l_regex_current_state_int < \l_regex_unique_id_int @@ -3227,7 +3411,7 @@ \int_set_eq:NN \l_regex_min_step_int \l_regex_current_step_int \exp_after:wN \regex_query_set_loop:ww \l_regex_tmpa_tl - \q_mark { ? \prg_map_break: } , + \s_tl { ? \prg_map_break: } \s_tl \prg_break_point:n { } \int_set_eq:NN \l_regex_max_step_int \l_regex_current_step_int \regex_query_set:nnn { } { -1 } { -2 } @@ -3235,7 +3419,7 @@ \regex_match_initial_setup: \regex_match_once: } -\cs_new:Npn \regex_query_set_loop:ww #1 \q_mark #2#3 , +\cs_new:Npn \regex_query_set_loop:ww #1 \s_tl #2#3 \s_tl { \use_none:n #2 \regex_query_set:nnn {#1} {"#2} {#3} @@ -3369,7 +3553,7 @@ \fi: \regex_match_loop: } -\cs_new_nopar:Npn \regex_match_one_index:w #1; +\cs_new:Npn \regex_match_one_index:w #1; { \if_num:w #1 > \l_regex_max_index_int \exp_after:wN \prg_map_break: @@ -3379,7 +3563,7 @@ \exp_after:wN \regex_match_one_index:w \int_use:N \int_eval:w #1 + \c_one ; } -\cs_new_protected_nopar:Npn \regex_match_one_index_aux:n #1 +\cs_new_protected:Npn \regex_match_one_index_aux:n #1 { \int_set:Nn \l_regex_current_state_int {#1} \prop_clear:N \l_regex_current_submatches_prop @@ -3420,7 +3604,7 @@ % % \subsubsection{Actions when matching} % -% \begin{macro}[aux]{\regex_action_start_wildcard:nn} +% \begin{macro}[aux]{\regex_action_start_wildcard:} %: the first state has a free transition to the second % state, where the regular expression really begins, and a costly % transition to itself, to try again at the next character. ^^A ?? @@ -3433,12 +3617,12 @@ % this character. For that, we use the boolean % \cs{l_regex_fresh_thread_bool}. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_action_start_wildcard:nn #1#2 +\cs_new_protected_nopar:Npn \regex_action_start_wildcard: { \bool_set_true:N \l_regex_fresh_thread_bool - \regex_action_free:n {#2} + \regex_action_free:n {1} \bool_set_false:N \l_regex_fresh_thread_bool - \regex_action_cost:n {#1} + \regex_action_cost:n {0} } % \end{macrocode} % \end{macro} @@ -3447,7 +3631,7 @@ % A transition which consumes the current character and moves % to state |#1|. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_action_cost:n #1 +\cs_new_protected:Npn \regex_action_cost:n #1 { \exp_args:Nf \regex_store_state:n { \int_eval:n { \l_regex_current_state_int + #1 } } @@ -3488,7 +3672,7 @@ % Then restore the old value of \cs{l_regex_current_state_int} % and of the current submatches. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_action_free:n #1 +\cs_new_protected:Npn \regex_action_free:n #1 { \cs_set_nopar:Npx \regex_tmp:w { @@ -3508,7 +3692,7 @@ % Update the current submatches with the information % from the current step. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_action_submatch:n #1 +\cs_new_protected:Npn \regex_action_submatch:n #1 { \prop_put:Nno \l_regex_current_submatches_prop {#1} { \int_use:N \l_regex_current_step_int } @@ -3552,7 +3736,7 @@ \regex_replacement_raw:N {#1} \prg_do_nothing: \prg_do_nothing: - \cs_set_nopar:Npx \regex_nesting:n ##1 + \cs_set:Npx \regex_nesting:n ##1 { + \int_use:N \l_regex_nesting_int \l_regex_nesting_tl @@ -3560,7 +3744,7 @@ } \use:x { - \exp_not:n { \cs_set_nopar:Npn \regex_replacement_tl:n ##1 } + \exp_not:n { \cs_set:Npn \regex_replacement_tl:n ##1 } { \str_aux_toks_range:nn \c_zero \l_regex_replacement_int } } % ^^A rename! Careful with \cP\#. @@ -3633,7 +3817,8 @@ } \cs_new_protected:Npn \regex_replacement_error:NNN #1#2#3 { - \msg_kernel_error:nn { regex } { replacement-#1 } + \msg_kernel_error:nnxx { regex } { #1-command } + { replacement~text } {#3} #2 #3 } \cs_new_protected:Npn \regex_replacement_g:w #1#2 @@ -3670,7 +3855,7 @@ { \regex_replacement_error:NNN g } #1 #2 } -\cs_new_protected_nopar:Npn \regex_replacement_c:w #1#2 +\cs_new_protected:Npn \regex_replacement_c:w #1#2 { \token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N { @@ -3685,7 +3870,9 @@ \regex_replacement_put:n { \exp_not:n { \exp_after:wN \regex_replacement_exp_not:N \cs:w } } } -\cs_new_nopar:Npn \regex_replacement_exp_not:N #1 { \exp_not:n {#1} } +\cs_new_eq:cc { regex_replacement_c_C:w } + { regex_replacement_c_ \c_lbrace_str :w } +\cs_new:Npn \regex_replacement_exp_not:N #1 { \exp_not:n {#1} } \group_begin: \char_set_catcode_math_superscript:N \^^@ @@ -3696,7 +3883,7 @@ \cs_new_protected_nopar:Npn \regex_replacement_c_T:w { \regex_replacement_char:nNN { ^^@ } } - \cs_new_protected_nopar:Npn \regex_replacement_c_S:w #1#2 + \cs_new_protected:Npn \regex_replacement_c_S:w #1#2 { \int_compare:nNnTF { `#2 } = \c_zero { \regex_replacement_error:NNN c #1#2 } @@ -3708,7 +3895,7 @@ \char_set_catcode_parameter:N \^^@ \cs_new_protected_nopar:Npn \regex_replacement_c_P:w - { \regex_replacement_char:nNN { ^^@^^@ } } + { \regex_replacement_char:nNN { ^^@^^@^^@^^@^^@^^@^^@^^@ } } \char_set_catcode_other:N \^^@ \cs_new_protected_nopar:Npn \regex_replacement_c_O:w @@ -3746,7 +3933,7 @@ \group_end: -\cs_new_protected_nopar:Npn \regex_replacement_char:nNN #1#2#3 +\cs_new_protected:Npn \regex_replacement_char:nNN #1#2#3 { \char_set_lccode:nn \c_zero { `#3 } \tl_to_lowercase:n @@ -3843,7 +4030,7 @@ \regex_set_aux:n \s_stop } -\cs_new_nopar:Npn \regex_set_aux:n #1 +\cs_new:Npn \regex_set_aux:n #1 { \tex_toks:D #1 { \tex_the:D \tex_toks:D #1 } } \cs_new:Npn \regex_nfa:Nw #1 { @@ -3870,13 +4057,13 @@ % token list variables. We do some more processing to start each % line with \cs{tex_toks:D} instead of a single very long line. % \begin{macrocode} -\cs_new_nopar:Npn \regex_to_str:N #1 +\cs_new:Npn \regex_to_str:N #1 { \exp_after:wN \regex_to_str_aux:Nw #1 \tex_toks:D \q_stop \prg_map_break: \tex_toks:D \prg_break_point:n { } } -\cs_new_nopar:Npn \regex_to_str_aux:Nw #1#2\tex_toks:D +\cs_new:Npn \regex_to_str_aux:Nw #1#2\tex_toks:D { \use_none_delimit_by_q_stop:w #2 \q_stop \tl_to_str:n {#1#2} \iow_newline: @@ -4027,12 +4214,12 @@ } \fi: } -\cs_new_protected_nopar:Npn \regex_extract_aux_b:wn #1 < #2 +\cs_new_protected:Npn \regex_extract_aux_b:wn #1 < #2 { \tex_skip:D #1 = #2 sp plus \etex_gluestretch:D \tex_skip:D #1 \scan_stop: } -\cs_new_protected_nopar:Npn \regex_extract_aux_e:wn #1 > #2 +\cs_new_protected:Npn \regex_extract_aux_e:wn #1 > #2 { \tex_skip:D #1 = 1 \tex_skip:D #1 plus #2 sp \scan_stop: @@ -4066,7 +4253,7 @@ \exp_args:NNNo \group_end: \tl_set:Nn #1 \l_regex_tmpa_tl } -\cs_new_nopar:Npn \regex_extract_seq_aux:n #1 +\cs_new:Npn \regex_extract_seq_aux:n #1 { \seq_item:n { @@ -4074,7 +4261,7 @@ \int_value:w \regex_submatch_nesting_aux:n {#1} ; #1; } } -\cs_new_nopar:Npn \regex_extract_seq_aux:ww #1; #2; +\cs_new:Npn \regex_extract_seq_aux:ww #1; #2; { %^^A todo: use flags \if_num:w #1 < \c_zero \flag_raise:N \l_regex_group_end_flag @@ -4293,7 +4480,7 @@ \group_end: \fi: } -\cs_new_protected_nopar:Npn \regex_replace_once_aux:w #1; +\cs_new_protected:Npn \regex_replace_once_aux:w #1; { } % \end{macrocode} @@ -4354,7 +4541,7 @@ \exp_args:NNNo \group_end: \tl_set:Nn #4 \l_regex_tmpa_tl } -\cs_new_nopar:Npn \regex_replace_all_aux:n #1 +\cs_new:Npn \regex_replace_all_aux:n #1 { \regex_query_substr:nn { \etex_glueshrink:D \tex_skip:D #1 } { \tex_skip:D #1 } @@ -4467,12 +4654,12 @@ % \end{macrocode} % % \begin{macrocode} -\msg_kernel_new:nnnn { regex } { build-c } - { Misused~\iow_char:N\\c~or\iow_char:N\\C~command~in~regular~expression. } +\msg_kernel_new:nnnn { regex } { c-command } + { Misused~\iow_char:N\\c~or\iow_char:N\\C~command~in~a~#1. } { - The~\iow_char:N\\C~escape~sequence~can~be~followed~by~ - one~of~the~letters~ABCDELMOPSTU.~The~\iow_char:N\\c~ - escape~sequence~can~also~be~followed~by~a~brace~group. + In~a~#1,~the~\iow_char:N\\C~escape~sequence~ + can~be~followed~by~one~of~the~letters~ABCDELMOPSTU~ + or~a~brace~group,~not~by~'#2'. } \msg_kernel_new:nnnn { regex } { unknown-option } { Unknown~option~`#1'~for~regular~expressions. } @@ -4486,19 +4673,13 @@ The~character~or~escape~sequence~`#1'~is~not~defined~ as~an~option~within~regular~expressions. } -\msg_kernel_new:nnnn { regex } { replacement-g } +\msg_kernel_new:nnnn { regex } { g-command } { Missing~brace~for~the~\iow_char:N\\g~construction~in~a~replacement~text. } { In~the~replacement~text~for~a~regular~expression~search,~ submatches~are~represented~either~as~\iow_char:N \\g{dd..d},~ or~\\d,~where~`d'~are~single~digits.~Here,~a~brace~is~missing. } -\msg_kernel_new:nnnn { regex } { replacement-c } - { Something~is~wrong~with~the~replacement~text. } - { - The~\iow_char:N\\c~escape~sequence~can~only~be~followed~by~ - ABCDELMN~or~by~a~brace. - } % \end{macrocode} % % \begin{macrocode} |