diff options
Diffstat (limited to 'Master/texmf-dist')
25 files changed, 4920 insertions, 2304 deletions
diff --git a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3flag.pdf b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3flag.pdf Binary files differnew file mode 100644 index 00000000000..681d939c173 --- /dev/null +++ b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3flag.pdf diff --git a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3regex.pdf b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3regex.pdf Binary files differindex 04338df8609..27ee4afc745 100644 --- a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3regex.pdf +++ b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3regex.pdf diff --git a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3str.pdf b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3str.pdf Binary files differnew file mode 100644 index 00000000000..d3ae4401372 --- /dev/null +++ b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3str.pdf diff --git a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3tl-analysis.pdf b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3tl-analysis.pdf Binary files differnew file mode 100644 index 00000000000..620d29b7f91 --- /dev/null +++ b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3tl-analysis.pdf diff --git a/Master/texmf-dist/doc/latex/l3experimental/xcoffins/xcoffins.pdf b/Master/texmf-dist/doc/latex/l3experimental/xcoffins/xcoffins.pdf Binary files differindex 7035f06a7e1..64244d3391b 100644 --- a/Master/texmf-dist/doc/latex/l3experimental/xcoffins/xcoffins.pdf +++ b/Master/texmf-dist/doc/latex/l3experimental/xcoffins/xcoffins.pdf diff --git a/Master/texmf-dist/doc/latex/l3experimental/xgalley/l3galley.pdf b/Master/texmf-dist/doc/latex/l3experimental/xgalley/l3galley.pdf Binary files differindex 7864fb8a764..ebfe08a82b2 100644 --- a/Master/texmf-dist/doc/latex/l3experimental/xgalley/l3galley.pdf +++ b/Master/texmf-dist/doc/latex/l3experimental/xgalley/l3galley.pdf diff --git a/Master/texmf-dist/doc/latex/l3experimental/xgalley/xgalley.pdf b/Master/texmf-dist/doc/latex/l3experimental/xgalley/xgalley.pdf Binary files differindex a2bd04b0349..426fe793fc2 100644 --- a/Master/texmf-dist/doc/latex/l3experimental/xgalley/xgalley.pdf +++ b/Master/texmf-dist/doc/latex/l3experimental/xgalley/xgalley.pdf diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx new file mode 100644 index 00000000000..eed657ad231 --- /dev/null +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx @@ -0,0 +1,236 @@ +% \iffalse meta-comment +% +%% File: l3flag.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +% +%<*driver|package> +\RequirePackage{expl3} +\GetIdInfo$Id: l3flag.dtx 3039 2011-12-08 09:22:35Z bruno $ + {L3 Experimental Flags} +%</driver|package> +%<*driver> +\documentclass[full]{l3doc} +\usepackage{amsmath} +\begin{document} + \DocInput{\jobname.dtx} +\end{document} +%</driver> +% \fi +% +% +% \title{^^A +% The \textsf{l3flag} package: expandable flags^^A +% \thanks{This file describes v\ExplFileVersion, +% last revised \ExplFileDate.}^^A +% } +% +% \author{^^A +% The \LaTeX3 Project\thanks +% {^^A +% E-mail: +% \href{mailto:latex-team@latex-project.org} +% {latex-team@latex-project.org}^^A +% }^^A +% } +% +% \date{Released \ExplFileDate} +% +% \maketitle +% +% \begin{documentation} +% +% Flags are the only data structure on which \TeX{} can perform +% assignments in expansion only contexts. This module is meant +% mostly for kernel use: in almost all cases, booleans should +% be preferred to flags, because they are faster. +% +% A flag can be in two states: \enquote{lowered} or \enquote{raised}. +% The status of a flag can be tested expandably (just like booleans). +% A flag can also be \emph{raised} expandably. However, a flag cannot +% be lowered expandably. +% +% Flag variables are always local. +% +% Testing for the existence of flags with \cs{cs_if_exist:NTF} will +% always result in a negative answer: from the point of view of +% \TeX{}, flags are either undefined control sequences, or let to +% \cs{scan_stop:}. This has a second consequence: flags don't need +% to be declared with \cs{flag_new:N}, although this has error-checking +% benefits. The ability of using flags without declaring them is used +% in \pkg{l3rand}. +% +% \begin{function}{\flag_new:N, \flag_new:c} +% \begin{syntax} +% \cs{flag_new:N} \meta{flag} +% \end{syntax} +% Creates a new \meta{flag} or raises an error if the +% name is already taken. The declaration is global, +% but \meta{flags} are always local variables. +% The \meta{flag} will initially be lowered. +% \end{function} +% +% \begin{function}[EXP,pTF]{\flag_test:N, \flag_test:c} +% \begin{syntax} +% \cs{flag_test:N} \meta{flag} +% \end{syntax} +% This function returns \texttt{true} if the \meta{flag} is raised, +% and \texttt{false} if the \meta{flag} is lowered. +% \end{function} +% +% \begin{function}[EXP]{\flag_raise:N} +% \begin{syntax} +% \cs{flag_raise:N} \meta{flag} +% \end{syntax} +% The \meta{flag} is raised. The assignment is local. +% This function is expandable, despite being an assignment. +% \end{function} +% +% \begin{function}{\flag_lower:N} +% \begin{syntax} +% \cs{flag_lower:N} \meta{flag} +% \end{syntax} +% The \meta{flag} is lowered. The assignment is local. +% \end{function} +% +% \end{documentation} +% +% \begin{implementation} +% +% \section{\pkg{l3str} implementation} +% +% \begin{macrocode} +%<*initex|package> +% \end{macrocode} +% +% \begin{macrocode} +\ProvidesExplPackage + {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription} +% \end{macrocode} +% +% \begin{variable}{\g_flag_list_tl} +% We keep track of the list of all defined flags in a token list. +% This is only used by \cs{flag_new:N} to check that the flag +% has not yet been defined. +% \begin{macrocode} +\tl_new:N \g_flag_list_tl +% \end{macrocode} +% \end{variable} +% +% \begin{macro}{\flag_new:N, \flag_new:c} +% If the control sequence is already defined, then we raise an error. +% Otherwise we check if it is already a flag, raising an appropriate +% error, finally lower the flag globally (this is the only global +% assignment to a flag\ldots{} perhaps an error?). +% The \texttt{c} variant can be created without worrying for this function. +% \begin{macrocode} +\cs_new_protected:Npn \flag_new:N #1 + { + \tl_if_in:NnTF \g_flag_list_tl {#1} + { + \msg_kernel_error:nnx + { flag } { already-defined } + { \token_to_str:N #1 } + } + { + \tl_gput_right:Nn \g_flag_list_tl {#1} + \cs_new_eq:NN #1 \c_undefined:D + } + } +\cs_generate_variant:Nn \flag_new:N { c } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[EXP, pTF]{\flag_test:N, \flag_test:c} +% Test if the control sequence is \texttt{undefined} or not. +% \begin{macrocode} +\prg_new_conditional:Npnn \flag_test:N #1 { p , T , F , TF } + { + \if_cs_exist:N #1 + \prg_return_true: + \else: + \prg_return_false: + \fi: + } +\prg_new_conditional:Npnn \flag_test:c #1 { p , T , F , TF } + { + \if_cs_exist:w #1 \cs_end: + \prg_return_true: + \else: + \prg_return_false: + \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[EXP]{\flag_raise:N, \flag_raise:c} +% Raising a flag expandably relies on the fact that \TeX{} +% automatically lets undefined control sequences to \tn{relax} +% when building an unknown csname. We build such a csname, then +% remove it with \cs{use_none:n}, essentially doing +% \cs{use_none:c}, but optimized slightly. When a flag is given +% as an \texttt{N}-type argument, a little more work is needed, +% converting to a csname before raising that. +% \begin{macrocode} +\cs_new:Npn \flag_raise:N #1 + { \exp_after:wN \use_none:n \cs:w \cs_to_str:N #1 \cs_end: } +\cs_new:Npn \flag_raise:c #1 + { \exp_after:wN \use_none:n \cs:w #1 \cs_end: } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\flag_lower:N, \flag_lower:c} +% Simply undefine the control sequence, locally. +% \begin{macrocode} +\cs_new_protected:Npn \flag_lower:N #1 + { \cs_set_eq:NN #1 \c_undefined:D } +\cs_generate_variant:Nn \flag_lower:N { c } +% \end{macrocode} +% \end{macro} +% +% \begin{macrocode} +\msg_kernel_new:nnnn { flag } { already-defined } + { The~control~sequence~#1~is~already~declared~as~a~flag. } + { + LaTeX~was~asked~to~define~the~flag~#1,~but~it~has~already~ + been~defined~as~a~flag.~The~flag~module~is~mostly~meant~ + for~kernel~use,~and~booleans~should~be~preferred. + } +% \end{macrocode} +% +% \begin{macrocode} +%</initex|package> +% \end{macrocode} +% +% \end{implementation} +% +% \PrintIndex diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx index 94bd054f0ce..9c276b6922d 100644 --- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx @@ -35,7 +35,7 @@ % %<*driver|package> \RequirePackage{expl3} -\GetIdInfo$Id: l3regex.dtx 3017 2011-11-28 00:10:02Z bruno $ +\GetIdInfo$Id: l3regex.dtx 3073 2011-12-21 01:53:06Z bruno $ {L3 Experimental Regular Expressions} %</driver|package> %<*driver> @@ -74,36 +74,53 @@ % \section{\pkg{l3regex} documentation} % % The \pkg{l3regex} package provides regular expression testing, -% extraction of submatches, splitting, and replacement, all acting on strings -% of characters. The syntax of regular expressions is mostly a subset -% of the PCRE syntax (and very close to POSIX). For performance -% reasons, only a limited set of features are implemented. Notably, -% back-references are not supported. +% extraction of submatches, splitting, and replacement, all acting +% on token lists. The syntax of regular expressions is mostly a subset +% of the PCRE syntax (and very close to POSIX), with some additions +% due to the fact that we act on lists of tokens rather than characters. +% For performance reasons, only a limited set of features are implemented. +% Notably, back-references are not supported. % % Let us give a few examples. After % \begin{verbatim} -% \str_set:Nn \l_my_str { That~cat. } -% \regex_replace_once:nnN { at } { is } \l_my_str +% \tl_set:Nn \l_my_tl { That~cat. } +% \regex_replace_once:nnN { at } { is } \l_my_tl % \end{verbatim} -% the string variable \cs{l_my_str} holds the text +% the token list variable \cs{l_my_tl} holds the text % \enquote{\texttt{This cat.}}, where the first % occurrence of \enquote{\texttt{at}} was replaced % by \enquote{\texttt{is}}. A more complicated example is % a pattern to add a comma at the end of each word: % \begin{verbatim} -% \regex_replace_all:nnN { \w+ } { \0 , } \l_my_str +% \regex_replace_all:nnN { \w+ } { \0 , } \l_my_tl % \end{verbatim} % The |\w| sequence represents any \enquote{word} character, % and |+| indicates that the |\w| sequence should be repeated % as many times as possible (at least once), hence matching a word in the -% input string. In the replacement text, |\0| denotes the full match +% input token list. In the replacement text, |\0| denotes the full match % (here, a word). % +% If a regular expression is to be used several times, +% it can be compiled once, and stored in a token list +% variable using \cs{regex_const:Nn}. For example, +% \begin{verbatim} +% \regex_const:Nn \l_foo_regex_tl { \c{begin} \cB (\CB*?) \cE } +% \end{verbatim} +% stores in \cs{l_foo_regex_tl} a regular expression which matches +% the starting marker for an environment: \cs{begin}, followed +% by a begin-group token (|\cB|), then tokens which are not +% begin-group characters (|\CB|), as few as possible (|*?|), +% followed by an end-group token (|\cE|). As explained in the +% next section, the parentheses \enquote{capture} the result of +% |\CB*?|, giving us access to the name of the environment +% when doing replacements. +% % \subsection{Syntax of regular expressions} % -% Most characters match exactly themselves. Some characters are +% Most characters match exactly themselves, +% with an arbitrary category code. Some characters are % special and must be escaped with a backslash (\emph{e.g.}, |\*| -% matches an explicit star character). Some escape sequences of +% matches a star character). Some escape sequences of % the form backslash--letter also have a special meaning % (for instance |\d| matches any digit). As a rule, % \begin{itemize} @@ -113,7 +130,7 @@ % |\A|, |\B|, \ldots{} have special meanings; % \item non-alphanumeric printable ascii characters can (and should) % always be escaped: many of them have special meanings (\emph{e.g.}, -% |(|, |)|, |?|, |.|); +% use |\(|, |\)|, |\?|, |\.|); % \item spaces should always be escaped (even in character % classes); % \item any other character may be escaped or not, without any @@ -121,12 +138,11 @@ % \end{itemize} % Note that these rules play nicely with the fact that many % non-alphanumeric characters are difficult to input into \TeX{} -% under normal category codes. For instance, |\$\%\^\\abc\#| -% matches the literal string |$%^\abc#|. -% \begin{texnote} -% When converting the regular expression to a string, -% the value of the escape character is set to be a backslash. -% \end{texnote} +% under normal category codes. For instance, |\\abc\%| +% matches the characters |\abc%| (with arbitrary category codes), +% but does not match the control sequence |\abc| followed by a +% percent character. Matching control sequences can be done +% using the |\c|\Arg{regex} syntax (see below). % % Any special character which appears at a place where its special % behaviour cannot apply matches itself instead (for instance, @@ -146,7 +162,7 @@ % % Character types. % \begin{l3regex-syntax} -% \item[.] A single period matches any character, including newlines. +% \item[.] A single period matches any token. % \item[\\d] Any decimal digit. % \item[\\h] Any horizontal space character, % equivalent to |[\ \^^I]|: space and tab. @@ -157,22 +173,27 @@ % but not a space, for compatibility with Perl. % \item[\\w] Any word character, \emph{i.e.}, % alpha-numerics and underscore, equivalent to |[A-Za-z0-9\_]|. -% \item[\\D] Any character not matched by |\d|. -% \item[\\H] Any character not matched by |\h|. -% \item[\\N] Any character other than the |\n| character (hex 0A). -% \item[\\S] Any character not matched by |\s|. -% \item[\\V] Any character not matched by |\v|. -% \item[\\W] Any character not matched by |\w|. +% \item[\\D] Any token not matched by |\d|. +% \item[\\H] Any token not matched by |\h|. +% \item[\\N] Any token other than the |\n| character (hex 0A). +% \item[\\S] Any token not matched by |\s|. +% \item[\\V] Any token not matched by |\v|. +% \item[\\W] Any token not matched by |\w|. % \end{l3regex-syntax} +% Of those, |.|, |\D|, |\H|, |\N|, |\S|, |\V|, and |\W| will match arbitrary +% control sequences. % % Character classes match exactly one character in the subject string. % \begin{l3regex-syntax} % \item[{[\ldots{}]}] Positive character class. -% Matches any of the specified characters. +% Matches any of the specified tokens. % \item[{[\char`\^\ldots{}]}] Negative character class. -% Matches any character other than the specified characters. +% Matches any token other than the specified characters. % \item[{[x-y]}] Range (can be used with escaped characters). % \end{l3regex-syntax} +% For instance, |[a-oq-z\cC]| matches any lowercase latin letter +% except |p|, as well as control sequences (see below for a description +% of |\c|). % % Quantifiers (repetition). % \begin{l3regex-syntax} @@ -191,12 +212,15 @@ % % Anchors and simple assertions. % \begin{l3regex-syntax} -% \item[\\b] Word boundary. -% \item[\\B] Not a word boundary. +% \item[\\b] Word boundary: either the previous token is matched by +% |\w| and the next by |\W|, or the opposite. For this purpose, +% the ends of the token list are considered as |\W|. +% \item[\\B] Not a word boundary: between two |\w| tokens +% or two |\W| tokens (including the boundary). % \item[\char`^ \textrm{or} \\A] -% Start of the subject string. +% Start of the subject token list. % \item[\char`$\textrm{,} \\Z \textrm{or} \\z] -% End of the subject string. +% End of the subject token list. % \item[\\G] Start of the current match. This is only different from |^| % in the case of multiple matches: for instance % |\regex_count:nnN { \G a } { aaba } \l_tmpa_int| yields $2$, but @@ -216,6 +240,30 @@ % group number. % \end{l3regex-syntax} % +% Each catcode is represented by a single uppercase letter: +% |C| for control sequences; +% |B| for begin-group tokens; +% |E| for end-group tokens; +% |M| for math shift; +% |T| for alignment tab tokens; +% |P| for macro parameter tokens; +% |U| for superscript tokens (up); +% |D| for subscript tokens (down); +% |S| for spaces; +% |L| for letters; +% |O| for others; and +% |A| for active characters. +% \begin{l3regex-syntax} +% \item[\\cX] Any character token with catcode |X|, +% where |X| can be any of |CBEMTPUDSLOA|. +% \item[\\CX] Any token with catcode different from |X|, +% where |X| can be any of |CBEMTPUDSLOA|. +% \item[\\c\Arg{regex}] A control sequence whose csname matches +% the \meta{regex}, anchored at the beginning and end, +% so that |\c{begin}| matches exactly \cs{begin}, and nothing +% else. +% \end{l3regex-syntax} +% % Options can be set with |(?|\meta{option}|)| and % unset with |(?-|\meta{option}|)|. Options are local % to the group in which they are set, and revert to their @@ -234,7 +282,7 @@ % In character classes, only |^|, |-|, |]|, |\| and spaces are special, % and should be escaped. Other non-alphanumeric characters can % still be escaped without harm. The escape sequences |\d|, -% |\D|, |\w|, |\W| are also supported in character classes. +% |\D|, etc. are also supported in character classes. % If the first character is |^|, then the meaning of the character % class is inverted. Ranges of characters can be expressed using % |-|, for instance, |[\D 0-5]| is equivalent to |[^6-9]|. @@ -256,10 +304,24 @@ % % For instance, % \begin{verbatim} -% \str_set:Nn \l_my_str { Hello,~world! } -% \regex_replace_all:nnN { ([er]?l|o) . } { \(\0\-\-\1\) } \l_my_str +% \tl_set:Nn \l_my_tl { Hello,~world! } +% \regex_replace_all:nnN { ([er]?l|o) . } { \(\0\-\-\1\) } \l_my_tl % \end{verbatim} -% results in \cs{l_my_str} holding |H(ell--el)(o,--o) w(or--o)(ld--l)!| +% results in \cs{l_my_tl} holding |H(ell--el)(o,--o) w(or--o)(ld--l)!| +% +% The characters inserted by the replacement have category code $12$ +% (other) by default. The escape sequence |\c| allows to insert characters +% with arbitrary category codes, as well as control sequences. +% \begin{l3regex-syntax} +% \item[\\cXY] Produces the character |Y| (which can be given as +% an escape sequence such as |\t| for tab) with category code |X|, +% which must be one of |CBEMTPUDSLOA|. +% \item[\\c\Arg{text}] Produces the control sequence with csname +% \meta{text}. The \meta{text} may contain references to the submatches +% |\0|, |\1| \emph{etc.} As an experimental feature, |\c{...}| can be +% nested, but the behaviour is not yet fully specified. Use at own risks. +%^^A todo: decide what the right behaviour should be. +% \end{l3regex-syntax} % % \subsection{Precompiling regular expressions} % @@ -285,14 +347,32 @@ % The assignment is local for \cs{regex_set:Nn} and global for % \cs{regex_gset:Nn}. Use \cs{regex_const:Nn} for precompiled expressions % which will never change. -% \begin{texnote} -% Precompiled regular expressions can be safely written to a file -% and read when the \LaTeX3 syntax is active (as triggered by -% \cs{ExplSyntaxOn}). -% \end{texnote} % \end{function} % -% \subsection{String matching} +% \begin{function}{\regex_to_str:N} +% \begin{syntax} +% \cs{regex_to_str:N} \meta{regex var} +% \end{syntax} +% Converts the \meta{regex var}, previously defined using +% \cs{regex_(g)set:Nn}, to a string, which can be safely +% read back when the \LaTeX3 syntax is active +% (as triggered by \cs{ExplSyntaxOn}). +% Line breaks are inserted at various places, to \emph{try} +% and keep the line length short. +% For instance, if \cs{l_my_stream} is an open stream for writing, +% and \cs{l_regex_tl} is a regex variable, you can save its definition +% using +% \begin{verbatim} +% \iow_now:Nx \l_my_stream +% { +% \tl_to_str:n { \tl_set:Nn \l_my_regex_tl } { \iow_newline: +% \regex_to_str:N \l_my_regex_tl +% } +% } +% \end{verbatim} +% \end{function} +% +% \subsection{Matching} % % All regular expression functions are available in both |:n| and |:N| % variants. The former require a \enquote{standard} regular expression, @@ -301,10 +381,10 @@ % % \begin{function}[TF]{\regex_match:nn, \regex_match:Nn} % \begin{syntax} -% \cs{regex_match:nnTF} \Arg{regex} \Arg{string} \Arg{true code} \Arg{false code} +% \cs{regex_match:nnTF} \Arg{regex} \Arg{token list} \Arg{true code} \Arg{false code} % \end{syntax} -% Tests whether the \meta{regular expression} matches any substring -% of \meta{string}. For instance, +% Tests whether the \meta{regular expression} matches any part +% of the \meta{token list}. For instance, % \begin{verbatim} % \regex_match:nnTF { b [cde]* } { abecdcx } { TRUE } { FALSE } % \regex_match:nnTF { [b-dq-w] } { example } { TRUE } { FALSE } @@ -314,15 +394,18 @@ % % \begin{function}{\regex_count:nnN, \regex_count:NnN} % \begin{syntax} -% \cs{regex_count:nnN} \Arg{regex} \Arg{string} \meta{int var} +% \cs{regex_count:nnN} \Arg{regex} \Arg{token list} \meta{int var} % \end{syntax} % Sets \meta{int var} within the current \TeX{} group level % equal to the number of times -% \meta{regular expression} appears in \meta{string}. +% \meta{regular expression} appears in \meta{token list}. % The search starts by finding the left-most longest match, % respecting greedy and ungreedy operators. Then the search % starts again from the character following the last character -% of the previous match, until reaching the end of the string. +% of the previous match, until reaching the end of the token list. +% Infinite loops are prevented in the case where the regular expression +% can match an empty string: then we count one match between each +% pair of characters. % For instance, % \begin{verbatim} % \int_new:N \l_foo_int @@ -335,25 +418,25 @@ % % \begin{function}[TF]{\regex_extract_once:nnN, \regex_extract_once:NnN} % \begin{syntax} -% \cs{regex_extract_once:nnN} \Arg{regex} \Arg{string} \meta{seq~var} -% \cs{regex_extract_once:nnNTF} \Arg{regex} \Arg{string} \meta{seq~var} \Arg{true code} \Arg{false code} +% \cs{regex_extract_once:nnN} \Arg{regex} \Arg{token list} \meta{seq~var} +% \cs{regex_extract_once:nnNTF} \Arg{regex} \Arg{token list} \meta{seq~var} \Arg{true code} \Arg{false code} % \end{syntax} % Finds the first match of the \meta{regular expression} -% in the \meta{string}. If it exists, the match is stored +% in the \meta{token list}. If it exists, the match is stored % as the zeroeth item of the \meta{seq~var}, and further % items are the contents of capturing groups, in the order % of their opening parenthesis. The \meta{seq~var} % is assigned locally. If there is no match, -% the \meta{seq~var} is not altered. +% the \meta{seq~var} is cleared. % The testing versions insert the \meta{true code} into the input % stream if a match was found, and the \meta{false code} otherwise. % For instance, assume that you type % \begin{verbatim} -% \regex_extract_once:nnNTF { ^(La)?TeX(!*)$ } { LaTeX!!! } -% \l_foo_seq { true } { false } +% \regex_extract_once:nnNTF { \A(La)?TeX(!*)\Z } { LaTeX!!! } \l_foo_seq +% { true } { false } % \end{verbatim} -% Then the regular expression (anchored at the start with |^| and -% at the end with |$|) will match the whole string. The first +% Then the regular expression (anchored at the start with |\A| and +% at the end with |\Z|) will match the whole token list. The first % capturing group, |(La)?|, matches |La|, and the second capturing % group, |(!*)|, matches |!!!|. Thus, |\l_foo_seq| will contain % the items |{LaTeX!!!}|, |{La}|, and |{!!!}|, and the \texttt{true} @@ -362,64 +445,79 @@ % % \begin{function}[TF]{\regex_extract_all:nnN, \regex_extract_all:NnN} % \begin{syntax} -% \cs{regex_extract_all:nnNTF} \Arg{regex} \Arg{string} \meta{seq~var} \Arg{true code} \Arg{false code} +% \cs{regex_extract_all:nnN} \Arg{regex} \Arg{token list} \meta{seq~var} +% \cs{regex_extract_all:nnNTF} \Arg{regex} \Arg{token list} \meta{seq~var} \Arg{true code} \Arg{false code} % \end{syntax} % Finds all matches of the \meta{regular expression} -% in the \meta{string}, and stores all the submatch information +% in the \meta{token list}, and stores all the submatch information % in a single sequence (concatenating the results of % multiple \cs{regex_extract_once:nnN} calls). +% The \meta{seq~var} is assigned locally. If there is no match, +% the \meta{seq~var} is cleared. +% The testing versions insert the \meta{true code} into the input +% stream if a match was found, and the \meta{false code} otherwise. % For instance, assume that you type % \begin{verbatim} -% \regex_extract_all:nnN { \w+ } { Hello,~world! } \l_foo_seq +% \regex_extract_all:nnNTF { \w+ } { Hello,~world! } \l_foo_seq +% { true } { false } % \end{verbatim} % Then the regular expression will match twice, and the resulting -% sequence contains the two items \texttt{Hello} and \texttt{world}. +% sequence contains the two items |{Hello}| and |{world}|, +% and the \texttt{true} branch is left in the input stream. % \end{function} % -% \begin{function}{\regex_split:nnN, \regex_split:NnN} +% \begin{function}[TF]{\regex_split:nnN, \regex_split:NnN} % \begin{syntax} -% \cs{regex_split:nnN} \Arg{regular expression} \meta{string} \meta{seq~var} +% \cs{regex_split:nnN} \Arg{regular expression} \Arg{token list} \meta{seq~var} +% \cs{regex_split:nnNTF} \Arg{regular expression} \Arg{token list} \meta{seq~var} \Arg{true code} \Arg{false code} % \end{syntax} -% Searches the \meta{string} into a sequence of substrings, delimited by +% Splits the \meta{token list} into a sequence of parts, delimited by % matches of the \meta{regular expression}. If the \meta{regular expression} -% has capturing groups, then the substrings that they match are stored as +% has capturing groups, then the token lists that they match are stored as % items of the sequence as well. The assignment to \meta{seq~var} is local. -% If no match is found the resulting \meta{seq~var} has the \meta{string} as -% its sole item. If the \meta{regular expression} matches the empty string, -% then the \meta{string} is split into single character substrings. +% If no match is found the resulting \meta{seq~var} has the +% \meta{token list} as its sole item. If the \meta{regular expression} +% matches the empty token list, then the \meta{token list} is split +% into single tokens. +% The testing versions insert the \meta{true code} into the input +% stream if a match was found, and the \meta{false code} otherwise. % For example, after % \begin{verbatim} % \seq_new:N \l_path_seq -% \regex_split:nnN { / } { the/path/for/this/file.tex } \l_path_seq +% \regex_split:nnNTF { / } { the/path/for/this/file.tex } \l_path_seq +% { true } { false } % \end{verbatim} % the sequence |\l_path_seq| contains the items |{the}|, |{path}|, -% |{for}|, |{this}|, and |{file.tex}|. +% |{for}|, |{this}|, and |{file.tex}|, and the \texttt{true} branch +% is left in the input stream. % \end{function} % -% \subsection{String replacement} +% \subsection{Replacement} % % \begin{function}[TF]{\regex_replace_once:nnN,\regex_replace_once:NnN} % \begin{syntax} -% \cs{regex_replace_once:nnN} \Arg{regular expression} \Arg{replacement} \meta{str~var} +% \cs{regex_replace_once:nnN} \Arg{regular expression} \Arg{replacement} \meta{tl~var} +% \cs{regex_replace_once:nnNTF} \Arg{regular expression} \Arg{replacement} \meta{tl~var} \Arg{true code} \Arg{false code} % \end{syntax} -% Searches for the \meta{regular expression} in the \meta{string} -% and replaces the matching part with the \meta{replacement}. The result -% is assigned locally to \meta{str~var}. In the \meta{replacement}, +% Searches for the \meta{regular expression} in the \meta{token list} +% and replaces the first match with the \meta{replacement}. The result +% is assigned locally to \meta{tl~var}. In the \meta{replacement}, % |\0| represents the full match, |\1| represent the contents % of the first capturing group, |\2| of the second, \emph{etc.} % \end{function} % -% \begin{function}{\regex_replace_all:nnN, \regex_replace_all:NnN} +% \begin{function}[TF]{\regex_replace_all:nnN, \regex_replace_all:NnN} % \begin{syntax} -% \cs{regex_replace_all:nnN} \Arg{regular expression} \Arg{replacement} \meta{str~var} +% \cs{regex_replace_all:nnN} \Arg{regular expression} \Arg{replacement} \meta{tl~var} +% \cs{regex_replace_all:nnNTF} \Arg{regular expression} \Arg{replacement} \meta{tl~var} \Arg{true code} \Arg{false code} % \end{syntax} % Replaces all occurrences of the \cs{regular expression} -% in the \meta{string} by the \meta{replacement}, where +% in the \meta{token list} by the \meta{replacement}, where % |\0| represents the full match, |\1| % represent the contents of the first capturing group, % |\2| of the second, \emph{etc.} Every match % is treated independently, and matches cannot overlap. -% The result is assigned locally to \meta{str~var}. +% The result is assigned locally to \meta{tl~var}. % \end{function} % % \subsection{Bugs, misfeatures, future work, and other possibilities} @@ -427,11 +525,27 @@ % The following need to be done now. % \begin{itemize} % \item The \texttt{\{\}} quantifiers are only partially implemented. -% \item Check the catcode of spaces everywhere. -% \item Newline conventions are not done. -% In particular, we should have an option for |.| not to match newlines. -% Also, |\A| should differ from |^|, and |\Z|, |\z| and |$| should -% differ. +% \item Clean up the use of messages. +% \item Add tests for every regex and replacement feature. +% \item Make sure the documentation is correct, and the code comments +% are as well. +% \end{itemize} +% +% Code improvements to come. +% \begin{itemize} +% \item Test for the maximum register \cs{c_max_register_int}. +% \item Use \tn{dimen} registers rather than \cs{l_regex_nesting_tl} +% to build \cs{regex_nesting:n}. +% \item Reduce the number of epsilon-transitions in alternatives. +% \item Merge the \texttt{build} and \texttt{class} functions where possible. +% \item Improve nesting of |\c| in the replacement text. +% \item Move the \enquote{reconstruction} part of \pkg{l3regex} +% to \pkg{l3tl-analysis}. +% \item Optimize regexes for csnames when the regex is a simple string. +% \item Optimize simple strings: use less states +% (|abcade| should give two states, for |abc| and |ade|). +% \item Optimize groups with no alternative. +% \item Optimize the use of \cs{prg_stepwise_...} functions. % \end{itemize} % % The following features are likely to be implemented at some point @@ -441,30 +555,33 @@ % \item Regex matching on external files. % \item Conditional subpatterns with look ahead/behind: \enquote{if % what follows is [\ldots{}], then [\ldots{}]}. -% \item |(*..)| and |(?..)| sequences to set some options. -% \item Reduce the number of epsilon-transitions in alternatives. -% \item Optimize simple strings: use less states -% (|abcade| should give two states, for |abc| and |ade|). -% \item Optimize groups with no alternative. -% \item Optimize the use of \cs{prg_stepwise_...} functions. +% \item |(*..)| and |(?..)| sequences to set some options +% (partially implemented). % \item |\K| for resetting the beginning of the match. % \item UTF-8 mode for pdf\TeX{}. +% \item Newline conventions are not done. +% In particular, we should have an option for |.| not to match newlines. +% Also, |\A| should differ from |^|, and |\Z|, |\z| and |$| should +% differ. +% \item Unicode properties: |\p{..}| and |\P{..}|; +% |\X| which should match any \enquote{extended} Unicode sequence. +% This requires to manipulate a lot of data, hence a lot of optimization +% ahead. % \end{itemize} % % The following features of PCRE or Perl will probably not be implemented. % \begin{itemize} -% \item |\cx|, similar to \TeX{}'s own |\^^x|; % \item |\ddd|, matching the character with code \texttt{ddd} in octal; -% \item POSIX character classes |[:alpha:]| \emph{etc.}; -% \item Unicode properties: |\p{..}| and |\P{..}|; -% \item |\X| which should match any \enquote{extended} Unicode sequence; -% \item Callout with |(?C...)|; +% \item POSIX character classes |[:alpha:]| \emph{etc.}, this is redundant; +% \item Callout with |(?C...)|, we cannot run arbitrary user code during +% the matching, because the regex code uses registers in an unsafe way; % \item Conditional subpatterns (other than with a look-ahead % or look-behind condition): this is non-regular, isn't it? % \end{itemize} % % The following features of PCRE or perl will definitely not be implemented. % \begin{itemize} +% \item |\cx|, similar to \TeX{}'s own |\^^x|; % \item Comments: \TeX{} already has its own system for comments. % \item Named subpatterns: \TeX{} programmers have lived so far without % any need for named macro parameters. @@ -495,7 +612,7 @@ % \begin{macrocode} \ProvidesExplPackage {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription} -\RequirePackage{l3str} +\RequirePackage{l3str, l3tl-analysis, l3flag} % \end{macrocode} % % Most regex engines use backtracking. This allows to provide very @@ -506,8 +623,8 @@ % % Given a regular expression of $n$ characters, we build a % non-deterministic finite automaton (NFA) with roughly $n$ states, -% which accepts precisely those strings matching that regular expression. -% We then run the string through the NFA, and check the return value. +% which accepts precisely those token lists matching that regular expression. +% We then run the token list through the NFA, and check the return value. % % The code is structured as follows. Various helper functions are % introduced in the next subsection, to limit the clutter in later @@ -520,14 +637,14 @@ % \subsection{Constants and variables} % % \begin{macro}{\regex_tmp:w} -% \begin{variable}{\g_regex_tmpa_tl, \l_regex_tmpa_tl, \l_regex_tmpb_tl} +% \begin{variable}{\l_regex_tmpa_tl, \l_regex_tmpb_tl, \l_regex_tmpc_tl} % \begin{variable}{\l_regex_tmpa_int, \l_regex_tmpb_int} % Temporary variables. % \begin{macrocode} \cs_new:Npn \regex_tmp:w { } -\tl_new:N \g_regex_tmpa_tl \tl_new:N \l_regex_tmpa_tl \tl_new:N \l_regex_tmpb_tl +\tl_new:N \l_regex_tmpc_tl \int_new:N \l_regex_tmpa_int \int_new:N \l_regex_tmpb_int % \end{macrocode} @@ -535,14 +652,27 @@ % \end{variable} % \end{macro} % +% \begin{variable}{\l_regex_group_begin_flag} +% \begin{variable}{\l_regex_group_end_flag} +% Those flags are raised to indicate extra begin-group +% or end-group tokens when extracting submatches. +% \begin{macrocode} +\flag_new:N \l_regex_group_begin_flag +\flag_new:N \l_regex_group_end_flag +% \end{macrocode} +% \end{variable} +% \end{variable} +% % \subsubsection{Variables used while building} % % \begin{variable}{\l_regex_max_state_int} % \begin{variable}{\l_regex_left_state_int, \l_regex_right_state_int} -% The last state that was allocated is stored in \cs{l_regex_max_state_int}, -% and \cs{l_regex_left/right_state_int} point to both end-points of the -% last group (which any quantifier would repeat). For simple strings of -% characters, the left and right pointers only differ by one. +% The end-point states of the last group (which any quantifier +% would repeat) are stored as \cs{l_regex_left/right_state_int}. +% For simple strings of characters, the left and right pointers +% only differ by one. +% The last state that was allocated is $\cs{l_regex_max_state_int}-1$, +% so that \cs{l_regex_max_state_int} always points to a free state. % \begin{macrocode} \int_new:N \l_regex_max_state_int \int_new:N \l_regex_left_state_int @@ -593,6 +723,16 @@ % \end{variable} % \end{variable} % +% \begin{variable}{\l_regex_build_or_class_tl} +% The actions of various escape sequences and special characters +% are different within a class and outside a class. Rather than +% using a boolean, we store either \texttt{build} or \texttt{class} +% in this token list. +% \begin{macrocode} +\tl_new:N \l_regex_build_or_class_tl +% \end{macrocode} +% \end{variable} +% % \begin{variable}{\l_regex_one_or_group_tl} % When looking for quantifiers, this variable holds either % \enquote{one} or \enquote{group} depending on whether the @@ -603,6 +743,22 @@ % \end{macrocode} % \end{variable} % +% \begin{variable}{\l_regex_in_csname_bool} +% This boolean is true when building the regular expression +% for a given csname. +% \begin{macrocode} +\bool_new:N \l_regex_in_csname_bool +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_regex_tmpa_regex_tl} +% This holds a temporary pre-compiled regular expression +% when matching a control sequence name. +% \begin{macrocode} +\tl_new:N \l_regex_tmpa_regex_tl +% \end{macrocode} +% \end{variable} +% % \subsubsection{Character classes} % % \begin{macro}{\regex_build_tmp_class:n} @@ -710,19 +866,37 @@ % % \subsubsection{Variables used when matching} % -% \begin{variable}{\l_regex_query_other_str} -% The string that is being matched. -% In that string, spaces have category code \enquote{other}. +% \begin{variable}{\l_regex_nesting_int} +% This integer is used to keep track of begin and end-group tokens. +% \begin{macrocode} +\int_new:N \l_regex_nesting_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_regex_min_step_int} +% \begin{variable}{\l_regex_max_step_int} +% This integer holds the value of the step corresponding to +% the left end of the token list, \emph{i.e.}, when the token +% list is stored in \tn{toks} registers, the \tn{toks} register +% \cs{l_regex_min_step_int} holds the first token in the token +% list. In fact, this number is always one more than +% \cs{l_regex_max_state_int}, but it is more practical to give +% that a name. % \begin{macrocode} -\tl_new:N \l_regex_query_other_str +\int_new:N \l_regex_min_step_int +\int_new:N \l_regex_max_step_int % \end{macrocode} % \end{variable} +% \end{variable} % % \begin{variable}{\l_regex_current_step_int} % \begin{variable}{\l_regex_start_step_int} % \begin{variable}{\l_regex_success_step_int} -% While reading through the query string, \cs{l_regex_current_step_int} -% is the position in the string. Each match begins at the position +% While reading through the query token list, +% \cs{l_regex_current_step_int} is the position in the +% token list, starting at \cs{l_regex_min_step_int} for +% the left-most token. +% Each match begins at the position % given by \cs{l_regex_start_step_int}. Whenever an execution thread % succeeds, the corresponding step is stored into % \cs{l_regex_success_step_int}, which will be the next starting step @@ -736,24 +910,26 @@ % \end{variable} % \end{variable} % -% \begin{variable}{\l_regex_unique_step_int} +% \begin{variable}{\l_regex_unique_id_int} % In the case of repeated matches, \cs{l_regex_current_step_int} % is reset to the end-position of the previous match. In contrast, -% \cs{l_regex_unique_step_int} is simply incremented to provide +% \cs{l_regex_unique_id_int} is simply incremented to provide % a unique number for each iteration of the matching loop. This % is handy to attach each set of submatch information to a given % iteration (and automatically discard it when it corresponds to % a past iteration). % \begin{macrocode} -\int_new:N \l_regex_unique_step_int +\int_new:N \l_regex_unique_id_int % \end{macrocode} % \end{variable} % % \begin{variable}{\l_regex_current_char_int} +% \begin{variable}{\l_regex_current_catcode_int} +% \begin{variable}{\l_regex_current_token_tl} % \begin{variable}{\l_regex_last_char_int} % \begin{variable}{\l_regex_case_changed_char_int} % The character codes of the character at the current position -% in the string, and at the previous position, and the current +% in the token list, and at the previous position, and the current % character with its case changed (|A-Z|$\leftrightarrow$|a-z|). % The \cs{l_regex_last_char_int} is used to test for word boundaries % (|\b| and |\B|). The \cs{l_regex_case_changed_char_int} is @@ -761,12 +937,16 @@ % is used in the regex. % \begin{macrocode} \int_new:N \l_regex_current_char_int +\int_new:N \l_regex_current_catcode_int +\tl_new:N \l_regex_current_token_tl \int_new:N \l_regex_last_char_int \int_new:N \l_regex_case_changed_char_int % \end{macrocode} % \end{variable} % \end{variable} % \end{variable} +% \end{variable} +% \end{variable} % % \begin{variable}{\l_regex_caseless_bool} % True if caseless matching is used within the regular expression. @@ -777,7 +957,7 @@ % \end{variable} % % \begin{variable}{\l_regex_current_state_int} -% For every character in the string, each of the active states is +% For every character in the token list, each of the active states is % considered in turn. % The variable \cs{l_regex_current_state_int} holds the state % of the NFA which is currently considered: transitions are then @@ -823,26 +1003,38 @@ % \begin{macro}[int]{\l_regex_every_match_tl} % Every time a match is found, this token list is used. % For single matching, the token list is set to removing -% the remainder of the query string. For multiple matching, +% the remainder of the query token list. For multiple matching, % the token list is set to repeat the matching. % \begin{macrocode} \tl_new:N \l_regex_every_match_tl % \end{macrocode} % \end{macro} % -% \begin{variable}{\l_regex_success_bool} -% The boolean \cs{l_regex_success_bool} is true if the current match -% attempt was successful. +% \begin{variable}{\g_regex_success_bool} +% \begin{variable}{\l_regex_saved_success_bool} +% \begin{variable}{\l_regex_success_match_bool} +% The boolean \cs{g_regex_success_bool} is true if there was +% at least one match, and \cs{l_regex_success_match_bool} is +% true if the current match attempt was successful. +% The variable \cs{g_regex_success_bool} is the only global +% variable in this whole module. When nesting \cs{regex} +% functions internally, the value of \cs{g_regex_success_bool} +% is saved into \cs{l_regex_saved_success_bool}, which is local, +% hence not affected by the changes due to inner regex functions. % \begin{macrocode} -\bool_new:N \l_regex_success_bool +\bool_new:N \g_regex_success_bool +\bool_new:N \l_regex_saved_success_bool +\bool_new:N \l_regex_success_match_bool % \end{macrocode} % \end{variable} +% \end{variable} +% \end{variable} % % \begin{macro}{\regex_last_match_empty:F} % \begin{macro}[aux]{\regex_last_match_empty_no:F} % \begin{macro}[aux]{\regex_last_match_empty_yes:F} % When doing multiple matches, we need to avoid infinite loops where -% each iteration matches the same empty string. When we detect such +% each iteration matches the same empty token list. When we detect such % a situation, the next match attempt is shifted by one character. % Namely, an empty match is discarded if it follows an empty match % at the same position. If the previous match was non-empty, @@ -876,86 +1068,61 @@ % \end{variable} % \end{variable} % -% \subsubsection{Variables used for user functions} +% \subsubsection{Variables used when building the replacement} % -% \begin{variable}{\g_regex_submatches_seq} -% This holds temporarily a sequence of submatches, -% global so that it exits the group. +% \begin{variable}{\l_regex_replacement_int} % \begin{macrocode} -\seq_new:N \g_regex_submatches_seq +\int_new:N \l_regex_replacement_int % \end{macrocode} % \end{variable} % -% \begin{variable}{\g_regex_match_count_int} -% The number of matches found so far is stored -% in \cs{g_regex_match_count_int}. This is only used -% in the \cs{regex_count:nnN} functions. +% \begin{variable}{\l_regex_replacement_csnames_int} % \begin{macrocode} -\int_new:N \g_regex_match_count_int +\int_new:N \l_regex_replacement_csnames_int % \end{macrocode} % \end{variable} % -% \begin{variable}{\g_regex_extract_all_seq} -% The \cs{regex_extract_all:nnN} function stores its result -% in that sequence variable before assigning it to the -% variable provided by the user. +% \begin{variable}{\l_regex_submatch_int} +% \begin{variable}{\l_regex_submatch_start_int} % \begin{macrocode} -\seq_new:N \g_regex_extract_all_seq +\int_new:N \l_regex_submatch_int +\int_new:N \l_regex_submatch_start_int % \end{macrocode} % \end{variable} -% -% \begin{variable}{\g_regex_split_seq} -% The \cs{regex_split:nnN} function stores its result -% in that sequence variable before assigning it to the -% variable provided by the user. -% \begin{macrocode} -\seq_new:N \g_regex_split_seq -% \end{macrocode} % \end{variable} % -% \begin{variable}{\l_regex_replacement_tl} -% \begin{variable}{\g_regex_replaced_str} -% The replacement code stores a processed version -% of the user's argument in \cs{l_regex_replacement_tl}. -% The result of the replacement is stored in \cs{g_regex_replaced_str}, -% global to exit the group. +% \subsubsection{Variables used for user functions} +% +% \begin{variable}{\l_regex_match_count_int} +% The number of matches found so far is stored +% in \cs{l_regex_match_count_int}. This is only used +% in the \cs{regex_count:nnN} functions. % \begin{macrocode} -\tl_new:N \l_regex_replacement_tl -\tl_new:N \g_regex_replaced_str +\int_new:N \l_regex_match_count_int % \end{macrocode} % \end{variable} -% \end{variable} % % \subsection{Helpers} % -% \subsubsection{Scan markers} -% -% \begin{macro}[int]{\s_regex_stop} -% Unexpandable markers, which does nothing if it reaches \TeX{}'s stomach. -% When performing the matching, the \tn{toks} registers hold submatch -% information, followed by the instruction for a given state of the NFA. -% The two parts are separated by \cs{s_regex_stop}. -% See \cs{regex_nfa:Nw} for another use. -% \begin{macrocode} -\cs_new_eq:NN \s_regex_stop \scan_stop: -% \end{macrocode} -% \end{macro} -% % \subsubsection{Toks} % +% When performing the matching, the \tn{toks} registers hold submatch +% information, followed by the instruction for a given state of the NFA. +% The two parts are separated by \cs{s_stop}. +% % \begin{macro}[int]{\regex_toks_put_left:Nx} % \begin{macro}[int]{\regex_toks_put_right:Nx} % During the building phase, every \tn{toks} register starts with -% \cs{s_regex_stop}, and we wish to add \texttt{x}-expanded material +% \cs{s_stop}, and we wish to add \texttt{x}-expanded material % to those registers. The expansion is done \enquote{by hand} for % optimization (these operations are used quite a lot). When adding % material to the left, we define \cs{regex_tmp:w} to remove the -% \cs{s_regex_stop} marker and put it back to the left of the new +% \cs{s_stop} marker and put it back to the left of the new % material. % \begin{macrocode} \cs_new_protected:Npn \regex_toks_put_left:Nx #1#2 { - \cs_set_nopar:Npx \regex_tmp:w \s_regex_stop { \s_regex_stop #2 } + \cs_set_nopar:Npx \regex_tmp:w \s_stop { \s_stop #2 } \tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN { \exp_after:wN \regex_tmp:w \tex_the:D \tex_toks:D #1 } } @@ -969,30 +1136,37 @@ % \end{macro} % \end{macro} % -% \subsubsection{Unsafe substring extraction} +% \subsubsection{Extracting parts of the query token list} % -% \begin{macro}[int]{\regex_query_substr:NN} -% \begin{macro}[aux]{\regex_query_substr_aux:nnn} -% Extracting submatches from the query string is done -% in a way similar to \cs{str_substr:Nnn}, but omitting -% most checks, hence faster. +% \begin{macro}[int]{\regex_query_substr:nn} +% \begin{macro}[int]{\regex_query_submatch:nn} +% \begin{macro}[int]{\regex_query_submatch:w} +% The token list is stored in \tn{toks} registers: +% the first is \cs{l_regex_min_step_int}, the last is (we don't care yet). % \begin{macrocode} -\cs_new_nopar:Npn \regex_query_substr:NN #1#2 +\cs_new:Npn \regex_query_substr:nn #1#2 { - \exp_after:wN \regex_query_substr_aux:NN - \exp_after:wN #1 - \exp_after:wN #2 - \l_regex_query_other_str - \q_stop + \if_num:w \int_eval:w #1 < \int_eval:w #2 \int_eval_end: + \str_aux_toks_range:nn {#1} {#2} + \fi: + } +\cs_new:Npn \regex_query_submatch:nn #1#2 + { + \if_num:w #1 < \l_regex_capturing_group_int + \exp_after:wN \regex_query_submatch:w + \int_use:N \int_eval:w #1 + #2 ; + \fi: } -\cs_new:Npn \regex_query_substr_aux:NN #1#2 +\cs_new:Npn \regex_query_submatch:w #1 ; { - \str_skip_do:nn {#1} - { \str_collect_do:nn { #2 - #1 } { \use_i_delimit_by_q_stop:nw } } + \regex_query_substr:nn + { \tex_skip:D #1 } + { \etex_gluestretch:D \tex_skip:D #1 } } % \end{macrocode} % \end{macro} % \end{macro} +% \end{macro} % % \subsubsection{Sequences} % @@ -1042,20 +1216,46 @@ % \end{macro} % \end{macro} % +% \begin{macro}[int, EXP]{\regex_seq_item:Nn} +% \begin{macro}[aux, EXP]{\regex_seq_item_aux:wNn} +% \begin{macrocode} +\cs_new:Npn \regex_seq_item:Nn #1#2 + { + \exp_after:wN \regex_seq_item_aux:wNn + \int_use:N \int_eval:w #2 \exp_after:wN ; + #1 { ? \prg_map_break: } { } + \prg_break_point:n { } + } +\cs_new:Npn \regex_seq_item_aux:wNn #1; #2#3 + { + \use_none:n #2 + \if_num:w #1 = \c_zero + \exp_after:wN \prg_map_break:n + \else: + \exp_after:wN \use_none:n + \fi: + { \exp_not:n {#3} } + \exp_after:wN \regex_seq_item_aux:wNn + \int_use:N \int_eval:w #1 - \c_one ; + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% % \subsubsection{Testing characters} % % \begin{macro}{\regex_item_dot:T} % The dot meta-character matches any character, -% except the end-of-string marker. +% except the end marker. % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_item_dot:T - { \int_compare:nNnF \l_regex_current_char_int = \c_minus_one } + { \int_compare:nNnF \l_regex_current_char_int < \c_minus_one } % \end{macrocode} % \end{macro} % % \begin{macro}[int]{\regex_break_point:TF} % \begin{macro}[int]{\regex_break_true:w,\regex_break_false:w} -% When testing whether a character of the query string matches +% When testing whether a character of the query token list matches % a given character class in the regular expression, we often % have to test it against several ranges of characters, checking % if any one of those matches. This is done with a structure like @@ -1187,6 +1387,56 @@ % \end{macrocode} % \end{macro} % +% \begin{macro}[int]{\regex_item_catcode:n} +% \begin{macro}[int]{\regex_item_catcode_neq:n} +% Checking the category code is quite straightforward. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \regex_item_catcode:n #1 + { + \if_num:w #1 = \l_regex_current_catcode_int + \exp_after:wN \regex_break_true:w + \fi: + } +\cs_new_protected_nopar:Npn \regex_item_catcode_neq:n #1 + { + \if_num:w #1 = \l_regex_current_catcode_int + \else: + \exp_after:wN \regex_break_true:w + \fi: + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int]{\regex_item_cs:n} +% Match a control sequence (the argument is a pre-compiled regex). +% First test the catcode of the current token to be zero. +% Then perform the matching test, and break if the csname +% indeed matches. The three \cs{exp_after:wN} expand the contents +% of \cs{l_regex_current_token_tl} (of the form \cs{exp_not:n} +% \Arg{control sequence}) to \meta{control sequence}. +% \begin{macrocode} +\cs_new_protected:Npn \regex_item_cs:n #1 + { + \int_compare:nNnT \l_regex_current_catcode_int = \c_zero + { + \tl_set:Nn \l_regex_tmpa_regex_tl {#1} + \bool_set_eq:NN \l_regex_saved_success_bool \g_regex_success_bool + \exp_args:NNx \regex_match:NnTF \l_regex_tmpa_regex_tl + { + \exp_after:wN \exp_after:wN + \exp_after:wN \cs_to_str:N \l_regex_current_token_tl + } + { + \bool_gset_eq:NN \g_regex_success_bool \l_regex_saved_success_bool + \regex_break_true:w + } + { \bool_gset_eq:NN \g_regex_success_bool \l_regex_saved_success_bool } + } + } +% \end{macrocode} +% \end{macro} +% % \subsubsection{Grabbing digits} % % \begin{macro}[int]{\regex_get_digits:nw} @@ -1323,19 +1573,15 @@ % \begin{macro}[int]{\regex_build_new_state:} % Here, we add a new state to the NFA. At the end of the building % phase, we want every \tn{toks} register to start with -% \cs{s_regex_stop}, hence initialize the new register appropriately. +% \cs{s_stop}, hence initialize the new register appropriately. % Then set \cs{l_regex_left/right_state_int} to their new values. % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_build_new_state: { - \int_compare:nNnTF \l_regex_max_state_int > { 32766 } - { \msg_error:nn { regex } { -997 } } - { - \int_incr:N \l_regex_max_state_int - \tex_toks:D \l_regex_max_state_int { \s_regex_stop } - } + \tex_toks:D \l_regex_max_state_int { \s_stop } \int_set_eq:NN \l_regex_left_state_int \l_regex_right_state_int \int_set_eq:NN \l_regex_right_state_int \l_regex_max_state_int + \int_incr:N \l_regex_max_state_int } % \end{macrocode} % \end{macro} @@ -1377,7 +1623,9 @@ % First, reset a few variables. Then use the generic framework defined % in \pkg{l3str} to parse the regular expression once, recognizing % which characters are raw characters, and which have special meanings. -% The result is stored in \cs{g_str_result_tl}, and can be run directly. +% The search is not anchored: to achieve that, we insert state(s) +% responsible for repeating the match attempt on every token +% in the token list. % The trailing \cs{prg_do_nothing:} ensure that the look-ahead done by % some of the operations is harmless. % Finally, \cs{regex_build_end:} adds the finishing code @@ -1385,16 +1633,18 @@ % \begin{macrocode} \cs_new_protected:Npn \regex_build:n #1 { - \regex_build_setup: - \str_aux_escape:NNNn - \regex_build_i_unescaped:N - \regex_build_i_escaped:N - \regex_build_i_raw:N - { #1 } + \bool_set_false:N \l_regex_in_csname_bool + \regex_build:w + \regex_build_new_state: + \regex_toks_put_right:Nx \l_regex_left_state_int + { \regex_action_start_wildcard:nn {0} {1} } \regex_build_open_aux: - \g_str_result_tl \prg_do_nothing: \prg_do_nothing: - \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_zero - \regex_build_close_aux: \regex_build_group_: + \str_escape_use:NNNn + \regex_build_i_unescaped:N + \regex_build_i_escaped:N + \regex_build_i_raw:N + { #1 } + \prg_do_nothing: \prg_do_nothing: \regex_build_end: } % \end{macrocode} @@ -1403,7 +1653,7 @@ % \begin{macro}[aux]{\regex_build_i_unescaped:N} % \begin{macro}[aux]{\regex_build_i_escaped:N} % \begin{macro}[aux]{\regex_build_i_raw:N} -% The \pkg{l3str} function \cs{str_aux_escape:NNNn} goes through +% The \pkg{l3str} function \cs{str_escape_use:NNNn} goes through % the regular expression and finds the |\a|, |\e|, |\f|, |\n|, |\r|, % |\t|, and |\x| escape sequences, then distinguishes three cases: % non-escaped characters, escaped characters, and \enquote{raw} @@ -1432,35 +1682,36 @@ % \end{macro} % \end{macro} % -% \begin{macro}[aux]{\regex_build_default_control:N} -% If the control character has a particular meaning in regular expressions, -% the corresponding function is used. Otherwise, it is interpreted as a raw -% character. The \cs{regex_build_default_raw:N} function is defined later. +% \begin{macro}[aux]{\regex_build_control:N} +% If the control character has a particular meaning in regexes, +% the corresponding function is used. Otherwise, it is interpreted +% as a raw character. The \cs{regex_build/class_default_raw:N} +% functions are defined later. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_default_control:N #1 +\cs_new_protected_nopar:Npn \regex_build_control:N #1 { - \cs_if_exist_use:cF { regex_build_#1: } - { \regex_build_default_raw:N #1 } + \cs_if_exist_use:cF { regex_ \l_regex_build_or_class_tl _#1: } + { \use:c { regex_ \l_regex_build_or_class_tl _default_raw:N } #1 } } % \end{macrocode} % \end{macro} % -% \begin{macro}[int]{\regex_build_setup:} +%^^A todo: update +% This function is similar to \cs{regex_build_control:N}. If the control +% character has a meaning in character classes, call the corresponding +% function, otherwise, treat it as a raw character, with the +% \cs{regex_class_default_raw:N} function, defined later. +% +% \begin{macro}[int]{\regex_build:w} % Hopefully, we didn't forget to initialize anything here. -% The search is not anchored: to achieve that, we insert state(s) -% responsible for repeating the match attempt on every character -% of the string. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_setup: +\cs_new_protected_nopar:Npn \regex_build:w { - \cs_set_eq:NN \regex_build_control:N \regex_build_default_control:N \cs_set_eq:NN \regex_build_raw:N \regex_build_default_raw:N + \tl_set:Nn \l_regex_build_or_class_tl { build } \int_set_eq:NN \l_regex_capturing_group_int \c_zero \int_zero:N \l_regex_max_state_int \regex_build_new_state: - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int - { \regex_action_start_wildcard:nn {0} {1} } } % \end{macrocode} % \end{macro} @@ -1472,15 +1723,19 @@ % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_build_end: { + \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_zero + \regex_build_close_aux: \regex_build_group_: \seq_if_empty:NF \l_regex_capturing_group_seq { - \msg_error:nn { regex } {22} + \msg_kernel_error:nnx { regex } { missing-rparen } + { \seq_length:N \l_regex_capturing_group_seq } \prg_replicate:nn { \seq_length:N \l_regex_capturing_group_seq } { \regex_build_close_aux: \regex_build_group_: } } \regex_toks_put_right:Nx \l_regex_right_state_int { \regex_action_success: } + \int_incr:N \l_regex_capturing_group_int } % \end{macrocode} % \end{macro} @@ -1492,14 +1747,14 @@ % \begin{macro}[int]{\regex_build_G:} % \begin{macro}[aux]{\regex_build_anchor_start:N} % Anchoring at the start corresponds to checking that the current -% character is the first in the string. Anchoring to the beginning +% character is the first in the token list. Anchoring to the beginning % of the match attempt uses \cs{l_regex_start_step_int} instead of % \cs{c_zero}. % \begin{macrocode} \cs_new_protected_nopar:cpn { regex_build_^: } - { \regex_build_anchor_start:N \c_zero } + { \regex_build_anchor_start:N \l_regex_min_step_int } \cs_new_protected_nopar:Npn \regex_build_A: - { \regex_build_anchor_start:N \c_zero } + { \regex_build_anchor_start:N \l_regex_min_step_int } \cs_new_protected_nopar:Npn \regex_build_G: { \regex_build_anchor_start:N \l_regex_start_step_int } \cs_new_protected_nopar:Npn \regex_build_anchor_start:N #1 @@ -1507,7 +1762,7 @@ \regex_build_new_state: \regex_toks_put_right:Nx \l_regex_left_state_int { - \exp_not:N \int_compare:nNnT {#1} = \l_regex_current_step_int + \exp_not:N \int_compare:nNnT #1 = \l_regex_current_step_int { \regex_action_free:n { @@ -1526,7 +1781,7 @@ % \begin{macro}[aux]{\regex_build_Z:} % \begin{macro}[aux]{\regex_build_z:} % \begin{macro}[aux]+\regex_build_$:+ -% This matches the end of the string, marked by a character code of $-1$. +% This matches the end of the token list, marked by a character code of $-2$. % \begin{macrocode} \cs_new_protected_nopar:cpn { regex_build_$: } % $ { @@ -1534,7 +1789,7 @@ \regex_toks_put_right:Nx \l_regex_left_state_int { \exp_not:N \int_compare:nNnT - \c_minus_one = \l_regex_current_char_int + \l_regex_current_char_int < \c_minus_one { \regex_action_free:n { @@ -1555,7 +1810,7 @@ % \begin{macro}[int]{\regex_build_B:} % \begin{macro}[aux]{\regex_if_word_boundary:TF} % Contrarily to |^| and |$|, which could be implemented without -% really knowing what precedes in the string, this requires +% really knowing what precedes in the token list, this requires % more information, namely, the knowledge of the last character % code. Case sensitivity does not change word boundaries. % \begin{macrocode} @@ -1597,7 +1852,7 @@ \int_set_eq:NN \l_regex_current_char_int \l_regex_last_char_int \c_regex_w_tl \regex_break_point:TF - { \group_end: \c_regex_W_tl \regex_item_equal:n { -1 } } + { \group_end: \c_regex_W_tl \regex_item_equal:n { -2 } } { \group_end: \c_regex_w_tl } \regex_break_point:TF } @@ -1608,6 +1863,23 @@ % % \subsubsection{Normal character, and simple character classes} % +% \begin{macro}[int]{\regex_build_one:n, \regex_build_one:x} +% +% \begin{macrocode} +\cs_new_protected:Npn \regex_build_one:n #1 + { \regex_build_one:x { \exp_not:n {#1} } } +\cs_new_protected:Npn \regex_build_one:x #1 + { + \cs_set_nopar:Npx \regex_build_tmp_class:n ##1 + { + \exp_not:N \exp_not:n {#1} + \regex_break_point:TF { \regex_action_cost:n {##1} } { } + } + \regex_build_one_quantifier: + } +% \end{macrocode} +% \end{macro} +% % \begin{macro}[aux]{\regex_build_default_raw:N} % A normal alphanumeric or an escaped non-alphanumeric % (actually, any unknown combination) will match itself @@ -1618,20 +1890,13 @@ % yet what this will be before checking for quantifiers. % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_build_default_raw:N #1 - { - \cs_set:Npx \regex_build_tmp_class:n ##1 - { - \regex_item_equal:n { \int_value:w `#1 ~ } - \regex_break_point:TF { \regex_action_cost:n { ##1 } } { } - } - \regex_build_one_quantifier: - } + { \regex_build_one:x { \regex_item_equal:n { \int_value:w `#1 ~ } } } % \end{macrocode} % \end{macro} % % \begin{macro}[aux]{\regex_build_.:} % Similar to \cs{regex_build_default_raw:N} but accepts any character, -% and refuses $-1$, which marks the end of the string. +% and refuses $-1$, which marks the end of the token list. % \begin{macrocode} \cs_new_protected_nopar:cpn { regex_build_.: } { @@ -1642,34 +1907,21 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_d:,\regex_build_D} -% \begin{macro}[aux]{\regex_build_h:,\regex_build_H} -% \begin{macro}[aux]{\regex_build_s:,\regex_build_S} -% \begin{macro}[aux]{\regex_build_v:,\regex_build_V} -% \begin{macro}[aux]{\regex_build_w:,\regex_build_W} +% \begin{macro}[aux]{\regex_build_d:,\regex_build_D:} +% \begin{macro}[aux]{\regex_build_h:,\regex_build_H:} +% \begin{macro}[aux]{\regex_build_s:,\regex_build_S:} +% \begin{macro}[aux]{\regex_build_v:,\regex_build_V:} +% \begin{macro}[aux]{\regex_build_w:,\regex_build_W:} % \begin{macro}[aux]{\regex_build_N:} -% \begin{macro}[aux]{\regex_build_char_type:N} % The constants \cs{c_regex_d_tl}, \emph{etc.} hold % a list of tests which match the corresponding character % class, and jump to the \cs{regex_break_point:TF} marker. % As for a normal character, we check for quantifiers. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_char_type:N #1 - { - \cs_set:Npn \regex_build_tmp_class:n ##1 - { - \exp_not:N #1 - \regex_break_point:TF { \regex_action_cost:n {##1} } { } - } - \regex_build_one_quantifier: - } \tl_map_inline:nn { dDhHsSvVwWN } { \cs_new_protected_nopar:cpx { regex_build_#1: } - { - \exp_not:N \regex_build_char_type:N - \exp_not:c { c_regex_#1_tl } - } + { \regex_build_one:n \exp_not:c { c_regex_#1_tl } } } % \end{macrocode} % \end{macro} @@ -1678,10 +1930,20 @@ % \end{macro} % \end{macro} % \end{macro} -% \end{macro} % % \subsubsection{Character classes} % +% \begin{macro}[aux]{\regex_class_one:n, \regex_class_one:x} +% This is the function used to add material +% to the class. +% \begin{macrocode} +\cs_new_protected:Npn \regex_class_one:n + { \tl_put_right:Nn \l_regex_class_tl } +\cs_new_protected:Npn \regex_class_one:x + { \tl_put_right:Nx \l_regex_class_tl } +% \end{macrocode} +% \end{macro} +% % \begin{macro}[aux]{\regex_build_[:} % This starts a class. The code for the class is collected % in \cs{l_regex_class_tl}. The first character is special. @@ -1689,41 +1951,26 @@ \cs_new_protected_nopar:cpn { regex_build_[: } { \tl_clear:N \l_regex_class_tl - \cs_set_eq:NN \regex_build_control:N \regex_class_control:N - \cs_set_eq:NN \regex_build_raw:N \regex_class_raw:N + \cs_set_eq:NN \regex_build_raw:N \regex_class_default_raw:N + \tl_set:Nn \l_regex_build_or_class_tl { class } \regex_class_first:NN } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_class_control:N} -% This function is similar to \cs{regex_build_control:N}. If the control -% character has a meaning in character classes, call the corresponding -% function, otherwise, treat it as a raw character, with the -% \cs{regex_class_raw:N} function, defined later. -% \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_class_control:N #1 - { - \cs_if_exist_use:cF { regex_class_#1: } - { \regex_class_raw:N #1 } - } -% \end{macrocode} -% \end{macro} -% % \begin{macro}[aux]{\regex_class_]:} % If \texttt{]} appears as the first item of a class, then % it doesn't end the class. Otherwise, it's the end, % act just as for a single character, but with a more -% complicated test. And restore \cs{regex_build_control:N} -% and \cs{regex_build_raw:N}. +% complicated test. And restore \cs{regex_build_raw:N}. % \begin{macrocode} \cs_new_protected_nopar:cpn { regex_class_]: } { \tl_if_empty:NTF \l_regex_class_tl %[ - { \regex_class_raw:N ] } + { \regex_class_default_raw:N ] } { - \cs_set_eq:NN \regex_build_control:N \regex_build_default_control:N \cs_set_eq:NN \regex_build_raw:N \regex_build_default_raw:N + \tl_set:Nn \l_regex_build_or_class_tl { build } \cs_set:Npn \regex_build_tmp_class:n ##1 { \exp_not:o \l_regex_class_tl @@ -1753,12 +2000,12 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_class_raw:N} +% \begin{macro}[aux]{\regex_class_default_raw:N} % \begin{macro}[aux]{\regex_class_single:N} % Most characters are treated here. We look ahead for an unescaped dash. % If there is none, then the character matches itself. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_class_raw:N #1#2#3 +\cs_new_protected_nopar:Npn \regex_class_default_raw:N #1#2#3 { \str_if_eq:nnTF {#2#3} { \regex_build_control:N - } { \regex_class_range:Nw #1 } @@ -1768,10 +2015,7 @@ } } \cs_new_protected_nopar:Npn \regex_class_single:N #1 - { - \tl_put_right:Nx \l_regex_class_tl - { \regex_item_equal:n { \int_value:w `#1 } } - } + { \regex_class_one:x { \regex_item_equal:n { \int_value:w `#1 } } } % \end{macrocode} % \end{macro} % \end{macro} @@ -1804,9 +2048,9 @@ \cs_new_protected_nopar:Npn \regex_class_range_put:NN #1#2 { \if_num:w `#1 > `#2 \exp_stop_f: - \msg_error:nn { regex } {8} + \msg_kernel_error:nnxx { regex } { backwards-range } {#1} {#2} \else: - \tl_put_right:Nx \l_regex_class_tl + \regex_class_one:x { \if_num:w `#1 = `#2 \exp_stop_f: \regex_item_equal:n @@ -1834,9 +2078,59 @@ \tl_map_inline:nn { dDhHsSvVwWN } { \cs_new_protected_nopar:cpx { regex_class_#1: } + { \regex_class_one:n \exp_not:c { c_regex_#1_tl } } + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \subsubsection{Catcodes and csnames} +% +% \begin{macro}[aux]{\regex_build_c:} +% \begin{macro}[aux]{\regex_build_C:} +% \begin{macro}[aux]{\regex_class_c:} +% \begin{macro}[aux]{\regex_class_C:} +% \begin{macro}[aux]{\regex_aux_c:nnNN} +% +% \begin{macrocode} +\cs_new_protected_nopar:Npn \regex_build_c: + { \regex_aux_c:NNNN \regex_item_catcode:n \use_i:nn } +\cs_new_protected_nopar:Npn \regex_build_C: + { \regex_aux_c:NNNN \regex_item_catcode_neq:n \use_ii:nn } +\cs_new_eq:NN \regex_class_c: \regex_build_c: +\cs_new_eq:NN \regex_class_C: \regex_build_C: +\cs_new_protected:Npn \regex_aux_c:NNNN #1#2#3#4 + { + \token_if_eq_meaning:NNTF #3 \regex_build_raw:N + { + \regex_aux_c_test:NTF #4 + { + \use:c { regex_ \l_regex_build_or_class_tl _one:x } + { #1 { \int_use:N \l_regex_current_catcode_int } } + } + { + \msg_kernel_error:nnx { regex } { build-c } { #4 } + #3 #4 + } + } { - \tl_put_right:Nn \exp_not:N \l_regex_class_tl - { \exp_not:c { c_regex_#1_tl } } + \if_charcode:w \c_lbrace_str \exp_not:N #4 + \exp_after:wN #2 + \else: + \exp_after:wN \use_ii:nn + \fi: + { + \group_begin: + \bool_set_true:N \l_regex_in_csname_bool + \regex_build:w + } + { + \msg_kernel_error:nnx { regex } { build-c } { #4 } + #3 #4 + } } } % \end{macrocode} @@ -1846,6 +2140,81 @@ % \end{macro} % \end{macro} % +% The |\c| and |\C| escape sequences are used both in the +% regex and the replacement texts. They can be followed either +% by a capital letter representing a category code, or by a left +% brace (in the case of |\c| only). +% +% \begin{macro}{\regex_aux_c_test:NTF} +% If the first argument is a known catcode identifier +% (capital letters listed below), then +% \cs{l_regex_current_catcode_int} is set to the relevant +% catcode value (between $0$ for cs and $13$ for active chars), +% and the \texttt{true} branch is taken. +% Otherwise, the \texttt{false} branch is taken, +% and the current catcode is set to $-1$. +% \begin{macrocode} +\cs_new_protected:Npn \regex_aux_c_test:NTF #1 + { + \int_set:Nn \l_regex_current_catcode_int + { + \if_case:w \int_eval:w `#1 - `A \int_eval_end: + \c_thirteen % Active + \or: \c_one % Begin-group + \or: \c_zero % Control sequence + \or: \c_eight % Down (subscript) + \or: \c_two % End-group + \else: + \if_case:w \int_eval:w `#1 - `L \int_eval_end: + \c_eleven % Letter + \or: \c_three % Math toggle + \or: \c_minus_one + \or: \c_twelve % Other + \or: \c_six % Parameter + \or: \c_minus_one + \or: \c_minus_one + \or: \c_ten % Space + \or: \c_four % Tab (alignment) + \or: \c_seven % Up (superscript) + \else: \c_minus_one + \fi: + \fi: + } + \if_num:w \l_regex_current_catcode_int = \c_minus_one + \exp_after:wN \use_ii:nn + \else: + \exp_after:wN \use_i:nn + \fi: + } +% \end{macrocode} +% \end{macro} +% +% ^^A{ +% \begin{macro}+\regex_build}:+ +% Non-escaped right braces are only special if they appear +% when building the regular expression for a csname. Otherwise, +% replace the brace with an escaped brace. +% \begin{macrocode} +\cs_new_protected:cpn { regex_build_ \c_rbrace_str : } + { + \bool_if:NTF \l_regex_in_csname_bool + { + \regex_build_Z: + \regex_toks_put_right:Nx \l_regex_right_state_int + { \regex_action_success: } + \int_incr:N \l_regex_capturing_group_int + \use:x + { + \group_end: + \exp_not:n { \use:c { regex_ \l_regex_build_or_class_tl _one:n } } + { \regex_item_cs:n { \regex_set_aux:N ? } } + } + } + { \exp_after:wN \regex_build_raw:N \c_rbrace_str } + } +% \end{macrocode} +% \end{macro} +% % \subsubsection{Quantifiers} % % \begin{macro}[int]{\regex_build_quantifier:w} @@ -1952,12 +2321,8 @@ { \tl_if_empty:nTF {#2} { \regex_build_quantifier_aux:nnNN { {#1} } { n* } } - { - \int_compare:nNnT {#1} > {#2} - { \msg_error:nn { regex } {4} } - \regex_build_quantifier_aux:nnNN { {#1} {#2} } { nn } - } - } + { \regex_build_quantifier_aux:nnNN { {#1} {#2} } { nn } } + } %^^A todo: catch the case m>n { \regex_build_quantifier_end:n { } \use:x @@ -2190,7 +2555,7 @@ \cs_new_protected_nopar:cpn { regex_build_): } { \seq_if_empty:NTF \l_regex_capturing_group_seq - { \msg_error:nn { regex } { 22 } } + { \msg_kernel_error:nn { regex } { extra-rparen } } { \regex_build_close_aux: \regex_build_group_quantifier: @@ -2249,7 +2614,8 @@ { \cs_if_exist_use:cF { regex_build_special_group_\token_to_str:N #2 : } { - \msg_error:nn { regex } { -998 } + \msg_kernel_error:nnx { regex } { unsupported } + { (? \token_to_str:N #2 } %) \regex_build_control:N ( % ) \regex_build_raw:N ? #1 #2 @@ -2433,7 +2799,7 @@ { \tex_toks:D \l_regex_tmpa_int { - \s_regex_stop + \s_stop #1 { \int_eval:n { \l_regex_right_state_int - \l_regex_tmpa_int } } } } @@ -2544,16 +2910,19 @@ \regex_toks_put_right:Nx \l_regex_right_state_int { \regex_action_free:n - { \int_eval:n { \l_regex_max_state_int - \l_regex_left_state_int } } + { + \int_eval:n %^^A todo: document why that value. + { \l_regex_max_state_int - \c_one - \l_regex_left_state_int } + } } \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int - \int_set:Nn \l_regex_tmpb_int { \l_regex_max_state_int + \c_one } + \int_set_eq:NN \l_regex_tmpb_int \l_regex_max_state_int \int_set:Nn \l_regex_max_state_int { - \l_regex_left_state_int - \c_one - + #1 * ( \l_regex_max_state_int - \l_regex_left_state_int + \c_one ) + \l_regex_left_state_int + + #1 * ( \l_regex_max_state_int - \l_regex_left_state_int ) } - \int_until_do:nNnn \l_regex_tmpb_int > \l_regex_max_state_int + \int_while_do:nNnn \l_regex_tmpb_int < \l_regex_max_state_int { \tex_toks:D \l_regex_tmpb_int = \tex_toks:D \l_regex_tmpa_int \int_incr:N \l_regex_tmpa_int @@ -2640,12 +3009,12 @@ % % The first step in matching a regular expression is to build % the corresponding NFA and store its states in the \tn{toks} -% registers. Then loop through the query string one character +% registers. Then loop through the query token list one character % (one \enquote{step}) at a time, exploring in parallel every % possible path through the NFA. We keep track of an array of % the states currently \enquote{active}. More precisely, % \tn{skip} registers hold the state numbers to be considered -% when the next character of the string is read. +% when the next token is read. % % At every step, we unpack that array of active states and % empty it. Then loop over all active states, and perform the @@ -2670,7 +3039,7 @@ % the \enquote{best} match. Execution paths through the NFA % are ordered by precedence: for instance, the regular % expression \texttt{a?} creates two paths, matching either -% an empty string or a single \texttt{a}; the path matching +% an empty token list or a single \texttt{a}; the path matching % an \texttt{a} has higher precedence. When two paths collide, % the path with the highest precedence is kept, and the other % one is discarded. The submatch information for a given path @@ -2704,6 +3073,7 @@ % % The current approach means that stretch and shrink components % of \tn{skip} registers, +%^^A todo: update: submatches' end-points are muskips. % as well as all \tn{muskip} registers are unused. It could seem that % \tn{count} registers are also free for use, but we still want to be % able to safely use integers, which are implemented as \tn{count} @@ -2737,21 +3107,20 @@ % but not by \cs{regex_state_use_with_submatches:}. % A state is free if it is not marker as taken, namely % if the corresponding \tn{dimen} register is not -% \cs{l_regex_unique_step_int} in \texttt{sp}. +% \cs{l_regex_unique_id_int} in \texttt{sp}. % The primitive conditional is ended before unpacking % the \tn{toks} register. % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_state_use_with_submatches: { \regex_state_use_aux:n { } } \cs_new_protected_nopar:Npn \regex_state_use: - { \regex_state_use_aux:n { \exp_after:wN \regex_state_use_aux_ii:w } } -\cs_new_nopar:Npn \regex_state_use_aux_ii:w #1 \s_regex_stop { } + { \regex_state_use_aux:n { \exp_after:wN \use_none_delimit_by_s_stop:w } } \cs_new_protected_nopar:Npn \regex_state_use_aux:n #1 { \if_num:w \tex_dimen:D \l_regex_current_state_int - < \l_regex_unique_step_int + < \l_regex_unique_id_int \tex_dimen:D \l_regex_current_state_int - = \l_regex_unique_step_int sp \scan_stop: + = \l_regex_unique_id_int sp \scan_stop: #1 \tex_the:D \tex_toks:D \exp_after:wN \l_regex_current_state_int \fi: \scan_stop: @@ -2799,26 +3168,26 @@ \tex_the:D \tex_toks:D #1 } } -\cs_new_protected:Npn \regex_store_submatches_aux:w #1 \s_regex_stop +\cs_new_protected:Npn \regex_store_submatches_aux:w #1 \s_stop { \regex_store_submatches_aux_ii:Nnnw #1 \regex_state_submatches:nn \c_minus_one \q_prop - \s_regex_stop + \s_stop } \cs_new_protected:Npn \regex_store_submatches_aux_ii:Nnnw - \regex_state_submatches:nn #1 #2 #3 \s_regex_stop + \regex_state_submatches:nn #1 #2 #3 \s_stop { \exp_after:wN \c_zero \exp_after:wN \regex_state_submatches:nn \exp_after:wN { \int_value:w \int_eval:w - \l_regex_unique_step_int + \c_one + \l_regex_unique_id_int + \c_one \exp_after:wN } \exp_after:wN { \l_regex_current_submatches_prop } \regex_state_submatches:nn {#1} {#2} - \s_regex_stop + \s_stop } % \end{macrocode} % \end{macro} @@ -2832,7 +3201,7 @@ % \begin{macrocode} \cs_new_protected:Npn \regex_state_submatches:nn #1#2 { - \if_num:w #1 = \l_regex_unique_step_int + \if_num:w #1 = \l_regex_unique_id_int \tl_set:Nn \l_regex_current_submatches_prop { #2 } \fi: } @@ -2842,7 +3211,7 @@ % \subsubsection{Matching: framework} % % \begin{macro}[int]{\regex_match:n} -% Store the query string in \cs{l_regex_query_other_str}. +%^^A todo: update doc. % Then reset a few variables which should be set only once, % before the first match, even in the case of multiple matches. % Then run the NFA (\cs{regex_match_once:} matches multiple times @@ -2850,70 +3219,95 @@ % \begin{macrocode} \cs_new_protected:Npn \regex_match:n #1 { - \tl_set:Nx \l_regex_query_other_str { \tl_to_other_str:n {#1} } + \tl_set_analysis:Nn \l_regex_tmpa_tl {#1} + + \int_zero:N \l_regex_nesting_int + \int_set_eq:NN \l_regex_current_step_int \l_regex_max_state_int + \regex_query_set:nnn { } { -1 } { -2 } + \int_set_eq:NN \l_regex_min_step_int \l_regex_current_step_int + \exp_after:wN \regex_query_set_loop:ww + \l_regex_tmpa_tl + \q_mark { ? \prg_map_break: } , + \prg_break_point:n { } + \int_set_eq:NN \l_regex_max_step_int \l_regex_current_step_int + \regex_query_set:nnn { } { -1 } { -2 } + \regex_match_initial_setup: \regex_match_once: } +\cs_new:Npn \regex_query_set_loop:ww #1 \q_mark #2#3 , + { + \use_none:n #2 + \regex_query_set:nnn {#1} {"#2} {#3} + \if_case:w "#2 \exp_stop_f: + \or: \int_incr:N \l_regex_nesting_int + \or: \int_decr:N \l_regex_nesting_int + \fi: + \regex_query_set_loop:ww + } +\cs_new_protected:Npn \regex_query_set:nnn #1#2#3 + { + \tex_muskip:D \l_regex_current_step_int + = \etex_gluetomu:D + #3 sp + plus #2 sp + minus \l_regex_nesting_int sp + \scan_stop: + \tex_toks:D \l_regex_current_step_int {#1} + \int_incr:N \l_regex_current_step_int + } +\cs_new_protected_nopar:Npn \regex_query_get: + { + \tl_set:Nx \l_regex_current_token_tl + { \tex_the:D \tex_toks:D \l_regex_current_step_int } + \l_regex_current_char_int + = \etex_mutoglue:D \tex_muskip:D \l_regex_current_step_int + \l_regex_current_catcode_int = \etex_gluestretch:D + \etex_mutoglue:D \tex_muskip:D \l_regex_current_step_int + } % \end{macrocode} % \end{macro} % % \begin{macro}[int]{\regex_match_once:} -% \begin{macro}[aux]{\regex_match_once_aux:} -% \begin{macro}[aux]{\regex_match_once_aux_ii:N} -% After setting up more variables in \cs{regex_match_setup:}, -% skip the \cs{l_regex_start_step_int} first characters of the -% query string, and loop over it. +% Set up more variables in \cs{regex_match_setup:}. % If there was a match, use the token list \cs{l_regex_every_match_tl}, -% which may call \cs{regex_match_once:} to achieve multiple matches. +% which may call \cs{regex_match_once:} again to achieve multiple matches. % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_match_once: { \regex_match_setup: - \exp_after:wN \regex_match_once_aux: \l_regex_query_other_str - \q_recursion_tail \q_recursion_stop - \bool_if:NT \l_regex_success_bool { \l_regex_every_match_tl } - } -\cs_new_protected_nopar:Npn \regex_match_once_aux: - { - \int_compare:nNnTF \l_regex_start_step_int = \c_zero - { - \int_set_eq:NN \l_regex_current_char_int \c_minus_one - \regex_match_loop:N - } + \regex_query_get: + \regex_match_loop: + \prg_break_point:n { } + \bool_if:NT \l_regex_success_match_bool { - \str_skip_do:nn - { \l_regex_start_step_int - \c_one } - { \regex_match_once_aux_ii:N } + \bool_gset_true:N \g_regex_success_bool + \l_regex_every_match_tl } } -\cs_new_protected_nopar:Npn \regex_match_once_aux_ii:N #1 - { - \int_set:Nn \l_regex_current_char_int { `#1 } - \regex_match_loop:N - } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} % % \begin{macro}[aux]{\regex_match_initial_setup:} % This function holds the setup that should be done -% only once for one given pattern matching on a given -% string. It is called only once for the whole string. +% only once for one given pattern matching. +% It is called only once for the whole token list. % On the other hand, \cs{regex_match_setup:} -% is called for every match in the string in case of -% repeated matches, and \cs{regex_match_loop_setup:N} -% is called at every step. +% is called for every match in the token list in case of +% repeated matches. % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_match_initial_setup: { - \prg_stepwise_inline:nnnn {1} {1} { \l_regex_max_state_int } + \prg_stepwise_inline:nnnn {0} {1} { \l_regex_max_state_int - \c_one } { \tex_dimen:D ##1 \c_minus_one sp \scan_stop: } - \int_set_eq:NN \l_regex_unique_step_int \c_minus_one - \int_set_eq:NN \l_regex_start_step_int \c_minus_one - \int_set_eq:NN \l_regex_current_step_int \c_zero - \int_set_eq:NN \l_regex_success_step_int \c_zero + \int_set_eq:NN \l_regex_unique_id_int \c_minus_one + \int_set:Nn \l_regex_start_step_int + { \l_regex_min_step_int - \c_one } + \int_set_eq:NN \l_regex_current_step_int \l_regex_min_step_int + \int_set_eq:NN \l_regex_success_step_int \l_regex_min_step_int + \int_set_eq:NN \l_regex_submatch_int \l_regex_max_state_int \bool_set_false:N \l_regex_success_empty_bool + \bool_gset_false:N \g_regex_success_bool } % \end{macrocode} % \end{macro} @@ -2929,47 +3323,61 @@ { \cs_set_eq:NN \regex_last_match_empty:F \regex_last_match_empty_yes:F } { \cs_set_eq:NN \regex_last_match_empty:F \regex_last_match_empty_no:F } \int_set_eq:NN \l_regex_start_step_int \l_regex_success_step_int - \int_set_eq:NN \l_regex_current_step_int \l_regex_start_step_int - \int_decr:N \l_regex_current_step_int - \bool_set_false:N \l_regex_success_bool + \int_set:Nn \l_regex_current_step_int + { \l_regex_start_step_int - \c_one } + \bool_set_false:N \l_regex_success_match_bool \int_zero:N \l_regex_max_index_int - \regex_store_state:n {1} + \regex_store_state:n {0} %^^A _state_int! } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_match_loop:N} +% \begin{macro}[aux]{\regex_match_loop:} % \begin{macro}[aux]{\regex_match_one_index:n} % \begin{macro}[aux]{\regex_match_one_index_aux:n} % Setup what needs to be reset at every character, % then set \cs{l_regex_current_char_int} to the -% character code of the character that is read -% (and $-1$ for the end of the string), and loop +% character code of the token that is read +% (and $-1$ for the end of the token list), and loop % over the elements of the \tn{skip} array. Then repeat. -% There are a couple of tests to stop reading the string +% There are a couple of tests to stop reading the token list % when no active state is left, or when the end is reached. +% At every step in reading the token list, we store the character +% code of the current character in \cs{l_regex_current_char_int}, +% unless the end was reached: then we store $-1$. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_match_loop:N #1 +\cs_new_protected_nopar:Npn \regex_match_loop: { - \regex_match_loop_setup:N #1 + \int_incr:N \l_regex_current_step_int + \int_incr:N \l_regex_unique_id_int + \bool_set_false:N \l_regex_fresh_thread_bool + \int_set_eq:NN \l_regex_last_char_int \l_regex_current_char_int + \regex_query_get: + \regex_match_loop_case_hook: \cs_set_nopar:Npx \regex_tmp:w { \int_zero:N \l_regex_max_index_int - \prg_stepwise_function:nnnN - {1} {1} { \l_regex_max_index_int } - \regex_match_one_index:n + \regex_match_one_index:w 1 ; \prg_break_point:n { } + \exp_not:N \prg_break_point:n { } } - \regex_tmp:w \prg_break_point:n { } + \regex_tmp:w + \if_num:w \l_regex_current_char_int < \c_minus_one + \exp_after:wN \prg_map_break: + \fi: \if_num:w \l_regex_max_index_int = \c_zero - \exp_after:wN \use_none_delimit_by_q_recursion_stop:w + \exp_after:wN \prg_map_break: \fi: - \quark_if_recursion_tail_stop:N #1 - \regex_match_loop:N + \regex_match_loop: } -\cs_new_nopar:Npn \regex_match_one_index:n #1 +\cs_new_nopar:Npn \regex_match_one_index:w #1; { + \if_num:w #1 > \l_regex_max_index_int + \exp_after:wN \prg_map_break: + \fi: \regex_match_one_index_aux:n { \int_value:w \tex_skip:D #1 } + \exp_after:wN \regex_match_one_index:w + \int_use:N \int_eval:w #1 + \c_one ; } \cs_new_protected_nopar:Npn \regex_match_one_index_aux:n #1 { @@ -2982,28 +3390,6 @@ % \end{macro} % \end{macro} % -% \begin{macro}[aux]{\regex_match_loop_setup:N} -% This is called for every character in the string. -% At every step in reading the query string, we store the character -% code of the current character in \cs{l_regex_current_char_int}, -% unless the end was reached: then we store $-1$. -% \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_match_loop_setup:N #1 - { - \int_incr:N \l_regex_current_step_int - \int_incr:N \l_regex_unique_step_int - \bool_set_false:N \l_regex_fresh_thread_bool - \int_set_eq:NN \l_regex_last_char_int \l_regex_current_char_int - \if_meaning:w #1 \q_recursion_tail - \int_set_eq:NN \l_regex_current_char_int \c_minus_one - \else: - \int_set:Nn \l_regex_current_char_int {`#1} - \fi: - \regex_match_loop_case_hook: - } -% \end{macrocode} -% \end{macro} -% % \begin{macro}{\regex_match_loop_case_hook:} % \begin{macro}{\regex_match_loop_caseless_hook:} % In the case where the regular expression contains caseless matching, @@ -3041,7 +3427,7 @@ % The search is made unanchored at the start by putting % a free transition to the real start of the NFA, and a % costly transition to the same state, waiting for the -% next character in the query string. This combination +% next token in the query. This combination % could be reused (with some changes). We sometimes need % to know that the match for a given thread starts at % this character. For that, we use the boolean @@ -3063,7 +3449,7 @@ % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_action_cost:n #1 { - \exp_args:Nf \regex_store_state:n %^^A optimize! + \exp_args:Nf \regex_store_state:n { \int_eval:n { \l_regex_current_state_int + #1 } } } % \end{macrocode} @@ -3082,11 +3468,10 @@ { \regex_last_match_empty:F { - \bool_set_true:N \l_regex_success_bool + \bool_set_true:N \l_regex_success_match_bool \bool_set_eq:NN \l_regex_success_empty_bool \l_regex_fresh_thread_bool - \int_set_eq:NN \l_regex_success_step_int - \l_regex_current_step_int + \int_set_eq:NN \l_regex_success_step_int \l_regex_current_step_int \prop_set_eq:NN \l_regex_success_submatches_prop \l_regex_current_submatches_prop \prg_map_break: @@ -3131,46 +3516,252 @@ % \end{macrocode} % \end{macro} % -% \subsection{Submatches, once the correct match is found} +% \subsection{Replacement} % -% \begin{macro}[int]{\regex_extract:} -% \begin{macro}[aux]{\regex_extract_aux:nTF} +% \begin{macro}[rEXP]{\regex_submatch_nesting_aux:n} % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_extract: +\cs_new_protected:Npn \regex_submatch_nesting_aux:n #1 + { + + \etex_glueshrink:D \etex_mutoglue:D \etex_muexpr:D + \tex_muskip:D \etex_gluestretch:D \tex_skip:D #1 + - \tex_muskip:D \tex_skip:D #1 + \scan_stop: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int]{\regex_replacement:n} +% Our goal here is to analyse the replacement text. First take care +% of detecting escaped and non-escaped characters using +% \cs{str_escape_use:NNNn} with three protected arguments. +% This inserts in the input stream a token list of the form +% \meta{fn 1} \meta{char 1} \ldots{} \meta{fn $N$} \meta{char $N$}, +% where \meta{fn $i$} is one of the three functions, and \meta{char $i$} +% a character in the string |#1|. +% \begin{macrocode} +\cs_new:Npn \regex_nesting:n #1 { } %^^A move. Rename? +\tl_new:N \l_regex_nesting_tl +\cs_new_protected:Npn \regex_replacement:n #1 { - \seq_gclear:N \g_regex_submatches_seq - \prg_stepwise_inline:nnnn - {0} {1} { \l_regex_capturing_group_int } + \int_zero:N \l_regex_replacement_int + \int_zero:N \l_regex_nesting_int + \tl_clear:N \l_regex_nesting_tl + \str_escape_use:NNNn + \regex_replacement_unescaped:N + \regex_replacement_escaped:N + \regex_replacement_raw:N + {#1} + \prg_do_nothing: \prg_do_nothing: + \cs_set_nopar:Npx \regex_nesting:n ##1 { - \regex_extract_aux:nTF { ##1 } - { - \seq_gput_right:Nx \g_regex_submatches_seq - { \regex_query_substr:NN \l_regex_tmpa_tl \l_regex_tmpb_tl } - } - { \seq_gput_right:Nn \g_regex_submatches_seq { } } + + \int_use:N \l_regex_nesting_int + \l_regex_nesting_tl + - \regex_submatch_nesting_aux:n {##1} + } + \use:x + { + \exp_not:n { \cs_set_nopar:Npn \regex_replacement_tl:n ##1 } + { \str_aux_toks_range:nn \c_zero \l_regex_replacement_int } + } + % ^^A rename! Careful with \cP\#. + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_replacement_raw:N} +% \begin{macrocode} +\cs_new_protected:Npn \regex_replacement_raw:N #1 + { \regex_replacement_put:n {#1} } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_replacement_put:n} +% Raw characters are stored in a toks register. +% \begin{macrocode} +\cs_new_protected:Npn \regex_replacement_put:n #1 + { + \tex_toks:D \l_regex_replacement_int {#1} + \int_incr:N \l_regex_replacement_int + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_replacement_escaped:N} +% \begin{macro}[aux]{\regex_replacement_submatch:w} +% \begin{macro}[aux]{\regex_replacement_submatch_aux:nN} +% \begin{macrocode} +\cs_new_protected:Npn \regex_replacement_unescaped:N #1 + { + \if_charcode:w \c_rbrace_str #1 + \if_num:w \l_regex_replacement_csnames_int > \c_zero + \regex_replacement_put:n \cs_end: + \else: + \regex_replacement_put:n #1 + \fi: + \else: + \regex_replacement_put:n #1 + \fi: + } +\cs_new_protected:Npn \regex_replacement_escaped:N #1 + { + \if_charcode:w c #1 + \exp_after:wN \regex_replacement_c:w + \else: + \if_charcode:w g #1 + \exp_after:wN \exp_after:wN \exp_after:wN \regex_replacement_g:w + \else: + \if_num:w \c_one < 1#1 \exp_stop_f: + \regex_replacement_put_submatch:n {#1} + \else: + \regex_replacement_put:n #1 + \fi: + \fi: + \fi: + } +\cs_new_protected:Npn \regex_replacement_put_submatch:n #1 + { + \regex_replacement_put:n + { \regex_query_submatch:nn {#1} {##1} } + \if_num:w \l_regex_replacement_csnames_int = \c_zero + \tl_put_right:Nn \l_regex_nesting_tl + { + \exp_not:N \if_num:w #1 < \l_regex_capturing_group_int + \regex_submatch_nesting_aux:n { \int_eval:w #1+##1 \int_eval_end: } + \exp_not:N \fi: + } + \fi: + } +\cs_new_protected:Npn \regex_replacement_error:NNN #1#2#3 + { + \msg_kernel_error:nn { regex } { replacement-#1 } + #2 #3 + } +\cs_new_protected:Npn \regex_replacement_g:w #1#2 + { + \str_if_eq:xxTF + { \exp_not:n { #1#2 } } + { \regex_replacement_unescaped:N \c_lbrace_str } + { + \int_zero:N \l_regex_tmpa_int + \regex_replacement_g_digits:NN } + { \regex_replacement_error:NNN g #1 #2 } } -\cs_new_protected_nopar:Npn \regex_extract_aux:nTF #1#2#3 +\cs_new_protected:Npn \regex_replacement_g_digits:NN #1#2 { - \prop_get:NnNTF \l_regex_success_submatches_prop - { #1 < } \l_regex_tmpa_tl + \token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N { - \prop_get:NnNTF \l_regex_success_submatches_prop - { #1 > } \l_regex_tmpb_tl - {#2} - {#3} + \if_num:w \c_one < 1#2 \exp_stop_f: + \int_set:Nn \l_regex_tmpa_int { \c_ten * \l_regex_tmpa_int + #2 } + \exp_after:wN \use_i:nnn + \exp_after:wN \regex_replacement_g_digits:NN + \else: + \if_charcode:w \c_rbrace_str #2 + \exp_args:No \regex_replacement_put_submatch:n + { \int_use:N \l_regex_tmpa_int } + \exp_after:wN \exp_after:wN \exp_after:wN \use_none:nn + \else: + \exp_after:wN \exp_after:wN + \exp_after:wN \regex_replacement_error:NNN + \exp_after:wN \exp_after:wN \exp_after:wN g + \fi: + \fi: } - {#3} + { \regex_replacement_error:NNN g } + #1 #2 + } +\cs_new_protected_nopar:Npn \regex_replacement_c:w #1#2 + { + \token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N + { + \cs_if_exist_use:cF { regex_replacement_c_#2:w } + { \regex_replacement_error:NNN c #1#2 } + } + { \regex_replacement_error:NNN c #1#2 } + } +\cs_new_protected_nopar:cpn { regex_replacement_c_ \c_lbrace_str :w } + { + \int_incr:N \l_regex_replacement_csnames_int + \regex_replacement_put:n + { \exp_not:n { \exp_after:wN \regex_replacement_exp_not:N \cs:w } } + } +\cs_new_nopar:Npn \regex_replacement_exp_not:N #1 { \exp_not:n {#1} } +\group_begin: + + \char_set_catcode_math_superscript:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_U:w + { \regex_replacement_char:nNN { ^^@ } } + + \char_set_catcode_alignment:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_T:w + { \regex_replacement_char:nNN { ^^@ } } + + \cs_new_protected_nopar:Npn \regex_replacement_c_S:w #1#2 + { + \int_compare:nNnTF { `#2 } = \c_zero + { \regex_replacement_error:NNN c #1#2 } + { + \char_set_lccode:nn {32} { `#2 } + \tl_to_lowercase:n { \regex_replacement_put:n {~} } + } + } + + \char_set_catcode_parameter:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_P:w + { \regex_replacement_char:nNN { ^^@^^@ } } + + \char_set_catcode_other:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_O:w + { \regex_replacement_char:nNN { ^^@ } } + + \char_set_catcode_math_toggle:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_M:w + { \regex_replacement_char:nNN { ^^@ } } + + \char_set_catcode_letter:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_L:w + { \regex_replacement_char:nNN { ^^@ } } + + \char_set_catcode_group_end:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_E:w + { + \int_decr:N \l_regex_nesting_int + \regex_replacement_char:nNN { \if_false: { \fi: ^^@ } + } + + \char_set_catcode_math_subscript:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_D:w + { \regex_replacement_char:nNN { ^^@ } } + + \char_set_catcode_group_begin:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_B:w + { + \int_incr:N \l_regex_nesting_int + \regex_replacement_char:nNN { \exp_after:wN ^^@ \if_false: } \fi: } + } + + \char_set_catcode_active:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_A:w + { \regex_replacement_char:nNN { \exp_not:N ^^@ } } + +\group_end: + +\cs_new_protected_nopar:Npn \regex_replacement_char:nNN #1#2#3 + { + \char_set_lccode:nn \c_zero { `#3 } + \tl_to_lowercase:n + { \regex_replacement_put:n { \exp_not:n {#1} } } } % \end{macrocode} % \end{macro} % \end{macro} +% \end{macro} % % \subsection{User commands} % % \subsubsection{Precompiled pattern} % -% A given pattern is often reused to match many different strings. +% A given pattern is often reused to match many different query token lists. % We thus give a means of storing the NFA corresponding to a given % pattern in a token list variable of the form % \begin{quote} @@ -3179,7 +3770,7 @@ % \cs{tex_toks:D} 0 \{ \meta{instruction0} \} \\ % \ldots{} \\ % \cs{tex_toks:D} $n$ \{ \meta{instruction$\sb{n}$} \} \\ -% \cs{s_regex_stop} +% \cs{s_stop} % \end{quote} % where $n$ is the number of states in the NFA, % and the various \meta{instruction$\sb{i}$} control @@ -3191,672 +3782,678 @@ % contents of \meta{nfa var} may overwrite data which is used % elsewhere, unless everything is done carefully in a group. % -% \begin{macro}{\regex_set:Nn} % \begin{macro}{\regex_gset:Nn} +% \begin{macro}{\regex_const:Nn} +% \begin{macro}{\regex_set:Nn} % \begin{macro}[aux]{\regex_set_aux:NNn} -% \begin{macro}[aux]{\regex_set_aux:n} -% \begin{macro}[aux]{\regex_nfa:Nw} -% Within a group, build the NFA corresponding to the given regular -% expression, with submatch tracking. Then save the contents of all -% relevant \tn{toks} registers into \cs{g_regex_tmpa_tl}, then -% transferred to the user's tl variable. -% The auxiliary \cs{regex_nfa:Nw} is not protected: this ensures that -% the NFA will properly be replaced by an error message in expansion -% contexts. +% The three user functions only differ with which function is used +% to assign the pre-compiled regular expression to the user's variable. +% Internally, they all first build the NFA corresponding to the regex, +% then store the contents of all the necessary \tn{toks} registers +% in the user's variable. % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_set:Nn - { \regex_set_aux:NNn \tl_set_eq:NN } + { \regex_set_aux:NNn \tl_set:Nn } \cs_new_protected_nopar:Npn \regex_gset:Nn - { \regex_set_aux:NNn \tl_gset_eq:NN } + { \regex_set_aux:NNn \tl_gset:Nn } +\cs_new_protected_nopar:Npn \regex_const:Nn + { \regex_set_aux:NNn \tl_const:Nn } \cs_new_protected:Npn \regex_set_aux:NNn #1#2#3 { \group_begin: \regex_build:n {#3} - \tl_gset:Nx \g_regex_tmpa_tl - { - \exp_not:N \regex_nfa:Nw \exp_not:N #2 - \l_regex_max_state_int - = \int_use:N \l_regex_max_state_int - \l_regex_capturing_group_int - = \int_use:N \l_regex_capturing_group_int - \token_if_eq_meaning:NNT - \regex_match_loop_case_hook: - \regex_match_loop_caseless_hook: - { - \cs_set_eq:NN \regex_match_loop_case_hook: - \regex_match_loop_caseless_hook: - } - \prg_stepwise_function:nnnN - {1} {1} {\l_regex_max_state_int} - \regex_set_aux:n - \s_regex_stop - } + \cs_set_nopar:Npx \regex_tmp:w + { #1 \exp_not:N #2 { \regex_set_aux:N #2 } } + \exp_after:wN \group_end: - #1 #2 \g_regex_tmpa_tl + \regex_tmp:w } -\cs_new_nopar:Npn \regex_set_aux:n #1 - { \tex_toks:D #1 { \tex_the:D \tex_toks:D #1 } } -\cs_new:Npn \regex_nfa:Nw #1 #2 \s_regex_stop - { \msg_expandable_kernel_error:nnn { regex } { nfa-misused } {#1} } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} -% \end{macro} % -% \begin{macro}{\regex_const:Nn} -% The same idea as for setting, but using the new function. -% \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_const:Nn - { \regex_set_aux:NNn \cs_new_eq:NN } -% \end{macrocode} -% \end{macro} -% -% \begin{macro}[int,TF]{\regex_check_nfa:N} -% If a token list variable starts with \cs{regex_nfa:Nw}, -% then it most likely holds the data for a precompiled pattern. +% \begin{macro}[int, rEXP]{\regex_set_aux:N} +% \begin{macro}[aux, rEXP]{\regex_set_aux:n} +% \begin{macro}[aux, EXP]{\regex_nfa:Nw} +% Within a group, build the NFA corresponding to the given regular +% expression, with submatch tracking. Then save the contents of all +% relevant \tn{toks} registers into the variable outside the group. +% The auxiliary \cs{regex_nfa:Nw} is not protected: this ensures that +% the NFA will properly be replaced by an error message in expansion +% contexts. % \begin{macrocode} -\prg_new_protected_conditional:Npnn \regex_check_nfa:N #1 { T , TF } - { \exp_after:wN \regex_check_nfa_aux:Nw #1 \q_stop } -\cs_new:Npn \regex_check_nfa_aux:Nw #1 #2 \q_stop +\cs_new:Npn \regex_set_aux:N #1 + { + \exp_not:n { \regex_nfa:Nw #1 } + \l_regex_max_state_int + = \int_use:N \l_regex_max_state_int + \l_regex_capturing_group_int + = \int_use:N \l_regex_capturing_group_int + \token_if_eq_meaning:NNT + \regex_match_loop_case_hook: + \regex_match_loop_caseless_hook: + { + \cs_set_eq:NN \regex_match_loop_case_hook: + \regex_match_loop_caseless_hook: + } + \prg_stepwise_function:nnnN + {0} {1} {\l_regex_max_state_int - \c_one } + \regex_set_aux:n + \s_stop + } +\cs_new_nopar:Npn \regex_set_aux:n #1 + { \tex_toks:D #1 { \tex_the:D \tex_toks:D #1 } } +\cs_new:Npn \regex_nfa:Nw #1 { - \if_meaning:w \regex_nfa:Nw #1 - \prg_return_true: - \else: - \msg_error:nnx { regex } { not-nfa } { \token_to_str:N #1 } - \prg_return_false: - \fi: + \msg_expandable_kernel_error:nnn { regex } { nfa-misused } {#1} + \use_none_delimit_by_s_stop:w } % \end{macrocode} % \end{macro} +% \end{macro} +% \end{macro} % % \begin{macro}[int]{\regex_use:N} % No error-checking. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_use:N #1 - { \exp_after:wN \use_none:nn #1 } +\cs_new_protected_nopar:Npn \regex_use:N + { \exp_last_unbraced:No \use_none:nn } % \end{macrocode} % \end{macro} % -% \subsubsection{Generic auxiliary functions} -% -% \begin{macro}[aux]{\regex_user_aux:n} -% This is an auxiliary used by most user functions. -% The first and second arguments control whether we should track -% submatches, and whether we track one or multiple submatches. -% Everything is done within a group, so that |#1| can perform -% \enquote{unsafe} assignments. Most user functions return a -% result using \cs{group_insert_after:N}. +% \begin{macro}[int, rEXP]{\regex_to_str:N} +% \begin{macro}[int, rEXP]{\regex_to_str_aux:w} +% Of course, we could simply set \cs{regex_to_str:N} equal +% to \cs{tl_to_str:N}. After all, regex variables are in particular +% token list variables. We do some more processing to start each +% line with \cs{tex_toks:D} instead of a single very long line. % \begin{macrocode} -\cs_new_protected:Npn \regex_user_aux:n #1 +\cs_new_nopar:Npn \regex_to_str:N #1 { - \group_begin: - \tl_clear:N \l_regex_every_match_tl - #1 - \group_end: + \exp_after:wN \regex_to_str_aux:Nw #1 + \tex_toks:D \q_stop \prg_map_break: \tex_toks:D + \prg_break_point:n { } + } +\cs_new_nopar:Npn \regex_to_str_aux:Nw #1#2\tex_toks:D + { + \use_none_delimit_by_q_stop:w #2 \q_stop + \tl_to_str:n {#1#2} \iow_newline: + \regex_to_str_aux:Nw \tex_toks:D } % \end{macrocode} % \end{macro} +% \end{macro} +% +% \subsubsection{Generic auxiliary functions} +% +% Most of \pkg{l3regex}'s work is done within a group. % -% \begin{macro}[aux]{\regex_return_after_group:} -% Most of \pkg{l3regex}'s work is done within a group. +% \begin{macro}[aux]{\regex_aux_return:} % This function triggers either \cs{prg_return_false:} % or \cs{prg_return_true:} as appropriate to whether a % match was found or not. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_return_after_group: +\cs_new_protected_nopar:Npn \regex_aux_return: { - \if_meaning:w \c_true_bool \l_regex_success_bool - \group_insert_after:N \prg_return_true: + \if_meaning:w \c_true_bool \g_regex_success_bool + \prg_return_true: \else: - \group_insert_after:N \prg_return_false: - \fi: - } -% \end{macrocode} -% \end{macro} -% -% \begin{macro}[aux]{\regex_extract_after_group:N} -% Extract submatches, and store them in the user-given variable -% after the group has ended. -% \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_extract_after_group:N #1 - { - \if_meaning:w \c_true_bool \l_regex_success_bool - \regex_extract: - \group_insert_after:N \seq_set_eq:NN - \group_insert_after:N #1 - \group_insert_after:N \g_regex_submatches_seq + \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_count_after_group:N} -% Same procedure as \cs{regex_extract_after_group:N}, but simpler -% since the match counting has already taken place. +% \begin{macro}{\regex_aux_build_match:nn} +% This auxiliary is used by user functions whose \meta{regex} argument +% is given as an explicit regular expression within braces. In that +% case, we need to build the automaton corresponding to that regular +% expression, then perform the matching on the given token list |#2|. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_count_after_group:N #1 +\cs_new_protected:Npn \regex_aux_build_match:nn #1#2 { - \group_insert_after:N \int_set_eq:NN - \group_insert_after:N #1 - \group_insert_after:N \g_regex_match_count_int + \regex_build:n {#1} + \regex_match:n {#2} } % \end{macrocode} % \end{macro} % -% \subsubsection{Matching} -% -% \begin{macro}[TF]{\regex_match:nn} -% \begin{macro}[TF]{\regex_match:Nn} -% We don't track submatches. Then either build the NFA corresponding -% to the regular expression, or use a precompiled pattern. Then match, -% using the internal \cs{regex_match:n}. Finally return the result -% after closing the group. +% \begin{macro}{\regex_aux_use_match:Nn} +% This auxiliary is used by user functions whose \meta{regex} argument +% is given as a pre-compiled regex variable. We make sure that +% the token list variable indeed is an automaton (by testing the +% first token). If not, the match is deemed unsuccessful, after +% raising an error. If we have an automaton, \enquote{use} it, +% then perform the matching on the given token list |#2|. % \begin{macrocode} -\prg_new_protected_conditional:Npnn \regex_match:nn #1#2 { T , F , TF } +\cs_new_protected:Npn \regex_aux_use_match:Nn #1#2 { - \regex_user_aux:n + \exp_args:No \tl_if_head_eq_meaning:nNTF {#1} \regex_nfa:Nw { - \regex_disable_submatches: - \regex_build:n {#1} + \regex_use:N #1 \regex_match:n {#2} - \regex_return_after_group: } - } -\prg_new_protected_conditional:Npnn \regex_match:Nn #1#2 { T , F , TF } - { - \regex_check_nfa:NTF #1 { - \regex_user_aux:n - { - \regex_disable_submatches: - \regex_use:N #1 - \regex_match:n {#2} - \regex_return_after_group: - } + \msg_kernel_error:nnx { regex } { not-nfa } { \token_to_str:N #1 } + \bool_gset_false:N \g_regex_success_bool } - { \prg_return_false: } } % \end{macrocode} % \end{macro} -% \end{macro} % -% \begin{macro}{\regex_count:nnN} -% \begin{macro}{\regex_count:NnN} -% Instead of aborting once the first \enquote{best match} is found, -% we repeat the search. The code is such that the search will not -% start on the same character, hence avoiding infinite loops. +% \begin{macro} +% { +% \regex_extract_once:nnN, \regex_extract_once:NnN, +% \regex_extract_all:nnN, \regex_extract_all:NnN, +% \regex_replace_once:nnN, \regex_replace_once:NnN, +% \regex_replace_all:nnN, \regex_replace_all:NnN, +% \regex_split:nnN, \regex_split:NnN +% } +% \begin{macro}[TF] +% { +% \regex_extract_once:nnN, \regex_extract_once:NnN, +% \regex_extract_all:nnN, \regex_extract_all:NnN, +% \regex_replace_once:nnN, \regex_replace_once:NnN, +% \regex_replace_all:nnN, \regex_replace_all:NnN, +% \regex_split:nnN, \regex_split:NnN +% } +% We define here $40$ user functions, following a common pattern +% in terms of an auxiliary such as \cs{regex_extract_once_aux:NnnN} +% (those auxiliaries are defined in the coming sections). +% The arguments handed to the auxiliary are +% \cs{regex_aux_build_match:nn} or \cs{regex_aux_use_match:Nn}, +% followed by the three arguments of the user function. +% The conditionals call \cs{regex_aux_return:} to return +% either \texttt{true} or \texttt{false} once matching +% has been performed. % \begin{macrocode} -\cs_new_protected:Npn \regex_count:nnN #1#2#3 +\cs_set_protected:Npn \regex_tmp:w #1#2#3 { - \regex_user_aux:n + \cs_new_protected_nopar:Npn #1 + { #3 \regex_aux_build_match:nn } + \cs_new_protected_nopar:Npn #2 + { #3 \regex_aux_use_match:Nn } + \prg_new_protected_conditional:Npnn #1 ##1##2##3 { T , F , TF } { - \regex_disable_submatches: - \int_gzero:N \g_regex_match_count_int - \tl_set:Nn \l_regex_every_match_tl - { - \int_gincr:N \g_regex_match_count_int - \regex_match_once: - } - \regex_build:n {#1} - \regex_match:n {#2} - \regex_count_after_group:N #3 + #3 \regex_aux_build_match:nn {##1} {##2} ##3 + \regex_aux_return: } - } -\cs_new_protected:Npn \regex_count:NnN #1#2#3 - { - \regex_check_nfa:NT #1 + \prg_new_protected_conditional:Npnn #2 ##1##2##3 { T , F , TF } { - \regex_user_aux:n - { - \regex_disable_submatches: - \int_gzero:N \g_regex_match_count_int - \tl_set:Nn \l_regex_every_match_tl - { - \int_gincr:N \g_regex_match_count_int - \regex_match_once: - } - \regex_use:N #1 - \regex_match:n {#2} - \regex_count_after_group:N #3 - } + #3 \regex_aux_use_match:Nn {##1} {##2} ##3 + \regex_aux_return: } } +\tl_map_inline:nn + { + { extract_once } { extract_all } + { replace_once } { replace_all } + { split } + } + { + \exp_args:Nccc \regex_tmp:w + { regex_#1:nnN } { regex_#1:NnN } { regex_#1_aux:NnnN } + } % \end{macrocode} % \end{macro} % \end{macro} % -% \subsubsection{Submatch extraction} +% \subsubsection{Submatches, once the correct match is found} +% +% \begin{macro}[int]{\regex_extract:} +% \begin{macro}[aux]{\regex_extract_aux_b:wn} +% \begin{macro}[aux]{\regex_extract_aux_e:wn} % -% \begin{macro}{\regex_extract_once:nnN} -% \begin{macro}{\regex_extract_once:NnN} -% \begin{macro}[TF]{\regex_extract_once:nnN} -% \begin{macro}[TF]{\regex_extract_once:NnN} % \begin{macrocode} -\cs_new_protected:Npn \regex_extract_once:nnN #1#2#3 +\cs_new_protected_nopar:Npn \regex_extract: { - \regex_user_aux:n - { - \regex_build:n {#1} - \regex_match:n {#2} - \regex_extract_after_group:N #3 - } + \int_set_eq:NN \l_regex_submatch_start_int \l_regex_submatch_int + \if_meaning:w \c_true_bool \g_regex_success_bool + \prg_replicate:nn \l_regex_capturing_group_int + { + \tex_skip:D \l_regex_submatch_int \c_zero sp \scan_stop: + \int_incr:N \l_regex_submatch_int + } + \prop_map_inline:Nn \l_regex_success_submatches_prop + { + \if_num:w ##1 \c_max_int + \exp_after:wN \regex_extract_aux_b:wn \int_use:N + \else: + \exp_after:wN \regex_extract_aux_e:wn \int_use:N + \fi: + \int_eval:w \l_regex_submatch_start_int + ##1 {##2} + } + \fi: } -\prg_new_protected_conditional:Npnn \regex_extract_once:nnN #1#2#3 - { T , F , TF } +\cs_new_protected_nopar:Npn \regex_extract_aux_b:wn #1 < #2 { - \regex_user_aux:n - { - \regex_build:n {#1} - \regex_match:n {#2} - \regex_extract_after_group:N #3 - \regex_return_after_group: - } + \tex_skip:D #1 = #2 sp + plus \etex_gluestretch:D \tex_skip:D #1 \scan_stop: } -\cs_new_protected:Npn \regex_extract_once:NnN #1#2#3 +\cs_new_protected_nopar:Npn \regex_extract_aux_e:wn #1 > #2 { - \regex_check_nfa:NT #1 - { - \regex_user_aux:n - { - \regex_use:N #1 - \regex_match:n {#2} - \regex_extract_after_group:N #3 - } - } - } -\prg_new_protected_conditional:Npnn \regex_extract_once:NnN #1#2#3 - { T , F , TF } - { - \regex_check_nfa:NTF #1 - { - \regex_user_aux:n - { - \regex_use:N #1 - \regex_match:n {#2} - \regex_extract_after_group:N #3 - \regex_return_after_group: - } - } - { \prg_return_false: } + \tex_skip:D #1 + = 1 \tex_skip:D #1 plus #2 sp \scan_stop: } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} -% \end{macro} % -% \begin{macro}{\regex_extract_all:nnN} -% \begin{macro}{\regex_extract_all:NnN} -% \begin{macro}[aux]{\regex_extract_all_aux:} -% \begin{macro}[aux]{\regex_extract_all_after_group:N} -% Similarly to \cs{regex_count:nnN} functions, -% recurse through matches of the pattern. Then we do -% something slightly different, extracting submatches. -% Submatches are not extracted if the pattern matched -% an empty string at the start of the match attempt -% (to avoid adding spurious empty items to the resulting -% sequence). +% \begin{macro}{\regex_group_end_extract_seq:N} +% \begin{macro}[aux]{\regex_extract_seq_aux:n} +% \begin{macro}[aux]{\regex_extract_seq_aux:ww} % \begin{macrocode} -\cs_new_protected:Npn \regex_extract_all:nnN #1#2#3 +\cs_new_protected:Npn \regex_group_end_extract_seq:N #1 { - \regex_user_aux:n - { - \seq_gclear:N \g_regex_extract_all_seq - \tl_set:Nn \l_regex_every_match_tl { \regex_extract_all_aux: } - \regex_build:n {#1} - \regex_match:n {#2} - \regex_extract_all_after_group:N #3 - } + \cs_set_eq:NN \seq_item:n \scan_stop: + \flag_lower:N \l_regex_group_begin_flag + \flag_lower:N \l_regex_group_end_flag + \tl_set:Nx \l_regex_tmpa_tl + { + \prg_stepwise_function:nnnN + \l_regex_max_state_int + \c_one + { \l_regex_submatch_int - \c_one } + \regex_extract_seq_aux:n + } + \flag_test:NF \l_regex_group_begin_flag + { \flag_test:NF \l_regex_group_end_flag { \use_none:nnn } } + \msg_kernel_error:nn { regex } { sequence-unbalanced } + \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl } + \exp_args:NNNo \group_end: + \tl_set:Nn #1 \l_regex_tmpa_tl } -\cs_new_protected:Npn \regex_extract_all:NnN #1#2#3 +\cs_new_nopar:Npn \regex_extract_seq_aux:n #1 { - \regex_check_nfa:NT #1 + \seq_item:n { - \regex_user_aux:n - { - \seq_gclear:N \g_regex_extract_all_seq - \tl_set:Nn \l_regex_every_match_tl { \regex_extract_all_aux: } - \regex_use:N #1 - \regex_match:n {#2} - \regex_extract_all_after_group:N #3 - } + \exp_after:wN \regex_extract_seq_aux:ww + \int_value:w \regex_submatch_nesting_aux:n {#1} ; #1; } } -\cs_new_protected_nopar:Npn \regex_extract_all_aux: - { - \regex_extract: - \seq_gconcat:NNN \g_regex_extract_all_seq - \g_regex_extract_all_seq \g_regex_submatches_seq - \regex_match_once: - } -\cs_new_protected_nopar:Npn \regex_extract_all_after_group:N #1 - { - \seq_if_empty:NF \g_regex_extract_all_seq - { - \regex_extract: - \group_insert_after:N \seq_set_eq:NN - \group_insert_after:N #1 - \group_insert_after:N \g_regex_extract_all_seq - } +\cs_new_nopar:Npn \regex_extract_seq_aux:ww #1; #2; + { %^^A todo: use flags + \if_num:w #1 < \c_zero + \flag_raise:N \l_regex_group_end_flag + \prg_replicate:nn {-#1} { \exp_not:n { { \if_false: } \fi: } } + \fi: + \regex_query_submatch:w #2; + \if_num:w #1 > \c_zero + \flag_raise:N \l_regex_group_begin_flag + \prg_replicate:nn {#1} { \exp_not:n { \if_false: { \fi: } } } + \fi: } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} -% \end{macro} % -% \subsubsection{Splitting a string by matches of a regex} +% \subsubsection{Matching} % -% \begin{macro}{\regex_split:nnN} -% \begin{macro}{\regex_split:NnN} -% \begin{macro}[aux]{\regex_split_aux:} -% \begin{macro}[aux]{\regex_split_after_group:N} -% Similarly to \cs{regex_count:nnN} functions, -% recurse through matches of the pattern. Then we do -% something slightly different, extracting submatches. -% Submatches are not extracted if the pattern matched -% an empty string at the start of the match attempt -% (to avoid adding spurious empty items to the resulting -% sequence). +% \begin{macro}[TF]{\regex_match:nn} +% \begin{macro}[TF]{\regex_match:Nn} +% \begin{macro}{\regex_match_aux:n} +% We don't track submatches. Then either build the NFA corresponding +% to the regular expression, or use a precompiled pattern. Then match, +% using the internal \cs{regex_match:n}. Finally return the result +% after closing the group. % \begin{macrocode} -\cs_new_protected:Npn \regex_split:nnN #1#2#3 +\prg_new_protected_conditional:Npnn \regex_match:nn #1#2 { T , F , TF } { - \regex_user_aux:n - { - \seq_gclear:N \g_regex_split_seq - \tl_set:Nn \l_regex_every_match_tl { \regex_split_aux: } - \regex_build:n {#1} - \regex_match:n {#2} - \regex_split_after_group:N #3 - } + \regex_match_aux:n + { \regex_aux_build_match:nn {#1} {#2} } } -\cs_new_protected:Npn \regex_split:NnN #1#2#3 +\prg_new_protected_conditional:Npnn \regex_match:Nn #1#2 { T , F , TF } { - \regex_check_nfa:NT #1 - { - \regex_user_aux:n - { - \seq_gclear:N \g_regex_split_seq - \tl_set:Nn \l_regex_every_match_tl { \regex_split_aux: } - \regex_use:N #1 - \regex_match:n {#2} - \regex_split_after_group:N #3 - } - } + \regex_match_aux:n + { \regex_aux_use_match:Nn #1 {#2} } } -\cs_new_protected_nopar:Npn \regex_split_aux: +\cs_new_protected:Npn \regex_match_aux:n #1 { - \int_compare:nNnF \l_regex_start_step_int = \l_regex_success_step_int - { - \regex_extract: - \seq_pop:NN \g_regex_submatches_seq \l_regex_tmpa_tl - \regex_extract_aux:nTF {0} - { - \seq_gput_left:Nx \g_regex_submatches_seq - { - \regex_query_substr:NN - \l_regex_start_step_int \l_regex_tmpa_tl - } - } - { \msg_error:nn { regex } { internal } } - \seq_gconcat:NNN \g_regex_split_seq - \g_regex_split_seq \g_regex_submatches_seq - } - \regex_match_once: + \group_begin: + \tl_clear:N \l_regex_every_match_tl + \regex_disable_submatches: + #1 + \group_end: + \regex_aux_return: } -\cs_new_protected_nopar:Npn \regex_split_after_group:N #1 +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \begin{macro}{\regex_count:nnN} +% \begin{macro}{\regex_count:NnN} +% \begin{macro}{\regex_count_aux:n} +% Instead of aborting once the first \enquote{longest match} is found, +% we repeat the search. The code is such that the search will not +% start on the same character, hence avoiding infinite loops. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \regex_count:nnN + { \regex_count_aux:NnnN \regex_aux_build_match:nn } +\cs_new_protected_nopar:Npn \regex_count:NnN + { \regex_count_aux:NnnN \regex_aux_use_match:Nn } +\cs_new_protected:Npn \regex_count_aux:NnnN #1#2#3#4 { - \int_compare:nNnTF \l_regex_start_step_int = \l_regex_current_step_int - { - \bool_if:NF \l_regex_success_empty_bool - { \seq_gput_right:Nn \g_regex_split_seq { } } - } - { - \seq_gput_right:Nx \g_regex_split_seq - { - \regex_query_substr:NN - \l_regex_start_step_int \l_regex_current_step_int - } - } - \group_insert_after:N \seq_set_eq:NN - \group_insert_after:N #1 - \group_insert_after:N \g_regex_split_seq + \group_begin: + \regex_disable_submatches: + \int_zero:N \l_regex_match_count_int + \tl_set:Nn \l_regex_every_match_tl + { + \int_incr:N \l_regex_match_count_int + \regex_match_once: + } + #1 {#2} {#3} + \exp_args:NNNo + \group_end: + \int_set:Nn #4 { \int_use:N \l_regex_match_count_int } } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} -% \end{macro} % -% \subsubsection{String replacement} +% \subsubsection{Submatch extraction} % -% \begin{macro}[int]{\regex_replacement:n} -% \begin{macro}[aux]{\regex_replacement_escaped:N} -% \begin{macro}[aux]{\regex_replacement_submatch:w} -% \begin{macro}[aux]{\regex_replacement_submatch_aux:nN} +% \begin{macro}[aux]{\regex_extract_once_aux:NnnN} +% As announced, here comes the auxiliary for extracting one match. +% Since we only want one match, \cs{l_regex_every_match_tl} +% is empty, and does not trigger the matching code again. +% After matching, \cs{regex_extract:} extracts submatches +% into various \tn{skip} registers, and those are then concatenated +% into a sequence by \cs{regex_group_end_extract_seq:N}. That function +% is also responsible for closing the group. % \begin{macrocode} -\cs_new_protected:Npn \regex_replacement:n #1 - { - \str_aux_escape:NNNn - \prg_do_nothing: - \regex_replacement_escaped:N - \prg_do_nothing: - {#1} - \tl_set_eq:NN \l_regex_replacement_tl \g_str_result_tl - \tl_set:Nx \l_regex_replacement_tl - { \l_regex_replacement_tl \prg_do_nothing: } - } -\cs_new_nopar:Npn \regex_replacement_escaped:N #1 +\cs_new_protected:Npn \regex_extract_once_aux:NnnN #1#2#3#4 { - \regex_token_if_other_digit:NTF #1 - { - \exp_not:n - { \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } } {#1} - } - { - \token_if_eq_charcode:NNTF g #1 - { \exp_not:N \regex_replacement_submatch:w } - { #1 } - } + \group_begin: + \tl_set:Nn \l_regex_every_match_tl { \regex_extract: } + #1 {#2} {#3} + \regex_group_end_extract_seq:N #4 } -\cs_new_nopar:Npn \regex_replacement_submatch:w #1 +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\regex_extract_all_aux:NnnN} +%^^A todo: update doc +% The set of submatches will be built progressively +% in \cs{l_regex_result_seq}. For each match, extract +% the submatches, and concatenate that to the right of +% the result sequence, then start matching again. +% Finally, copy the result in the user's sequence variable. +% \begin{macrocode} +\cs_new_protected:Npn \regex_extract_all_aux:NnnN #1#2#3#4 { - \exp_after:wN \token_if_eq_meaning:NNTF \c_lbrace_str #1 - { \regex_replacement_submatch_aux:nN { } } - { - \regex_token_if_other_digit:NTF #1 - { \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } } - { \msg_expandable_kernel_error:nn { regex } { g-misused } } - #1 - } - } -\cs_new_nopar:Npn \regex_replacement_submatch_aux:nN #1 #2 + \group_begin: + \tl_set:Nn \l_regex_every_match_tl + { \regex_extract: \regex_match_once: } + #1 {#2} {#3} + \regex_group_end_extract_seq:N #4 + } +% \end{macrocode} +% \end{macro} +% +% \subsubsection{Splitting a token list by matches of a regex} +% +% \begin{macro}[aux]{\regex_split_aux:NnnN} +% Recurse through the matches, and for each, do the following. +% Extract the submatches into various \tn{skip} registers, +% then replace the match |\0|, which should not be kept in the +% final result, and replace it by the part of the token list +% before the match. +% This process must be inhibited to avoid creating empty items +% if the regex matched an empty token list at the place where +% the match attempt started. +% After the last successful match, we need to add to the result +% the part of the token list after the last match, unless the +% last match was empty and at the very end. +% Finally, \cs{regex_group_end_extract_seq:N} builds a sequence +% from all the \tn{skip} registers, and assigns it to |#4| +% after closing the group. +% \begin{macrocode} +\cs_new_protected:Npn \regex_split_aux:NnnN #1#2#3#4 { - \regex_token_if_other_digit:NTF #2 - { \regex_replacement_submatch_aux:nN {#1#2} } - { - \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } {#1} - \exp_after:wN \token_if_eq_meaning:NNF \c_rbrace_str #2 - { - \msg_expandable_kernel_error:nn { regex } { g-misused } - #2 - } - } + \group_begin: + \tl_set:Nn \l_regex_every_match_tl + { + \if_num:w \l_regex_start_step_int < \l_regex_success_step_int + \regex_extract: + \tex_skip:D \l_regex_submatch_start_int + = \l_regex_start_step_int sp + plus \tex_skip:D \l_regex_submatch_start_int \scan_stop: + \fi: + \regex_match_once: + } + #1 {#2} {#3} + \tex_skip:D \l_regex_submatch_int + = \l_regex_start_step_int sp + plus \l_regex_current_step_int sp \scan_stop: + \int_incr:N \l_regex_submatch_int + \if_num:w \l_regex_start_step_int = \l_regex_current_step_int + \if_meaning:w \c_true_bool \l_regex_success_empty_bool + \int_decr:N \l_regex_submatch_int + \fi: + \fi: + \regex_group_end_extract_seq:N #4 } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}[aux]{\regex_replace_after_group:N} -% I wonder why I implemented things with \tn{aftergroup} -% rather than simply placing that code after the explicit -% \cs{group_end:} that I have. (General remark.) +% \subsubsection{Replacement} +% +% \begin{macro}[aux]{\regex_replace_once_aux:NnnN} +% The replacement text is analysed by \cs{regex_replacement:n}, +% which defines \cs{regex_replacement_tl:n} to expand to the +% replaced token list, assuming that submatches are stored in +% various \tn{skip} registers, as done by \cs{regex_extract:}. +% If there is a match, we grab +% the parts before and after it, and get the result +% by \texttt{x}-expanding twice. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_replace_after_group:N #1 +\cs_new_protected:Npn \regex_replace_once_aux:NnnN #1#2#3#4 + { + \group_begin: + \tl_clear:N \l_regex_every_match_tl + \regex_replacement:n {#3} + \exp_args:Nno #1 {#2} #4 + \if_meaning:w \c_true_bool \g_regex_success_bool + \regex_extract: + \int_set:Nn \l_regex_tmpa_int + { \regex_nesting:n { \l_regex_submatch_start_int } } + \if_num:w \l_regex_tmpa_int = \c_zero + \else: + \msg_kernel_error:nnx { regex } { replace-unbalanced } + { \l_regex_tmpa_int } + \fi: + \tl_set:Nx \l_regex_tmpa_tl + { + \if_num:w \l_regex_tmpa_int < \c_zero + \prg_replicate:nn { - \l_regex_tmpa_int } + { \exp_not:n { { \if_false: } \fi: } } + \fi: + \regex_query_substr:nn + { \l_regex_min_step_int } + { \tex_skip:D \l_regex_submatch_start_int } + \regex_replacement_tl:n { \l_regex_submatch_start_int } + \regex_query_substr:nn + { \etex_gluestretch:D \tex_skip:D \l_regex_submatch_start_int } + { \l_regex_max_step_int } + \if_num:w \l_regex_tmpa_int > \c_zero + \prg_replicate:nn { \l_regex_tmpa_int } + { \exp_not:n { \if_false: { \fi: } } } + \fi: + } + \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl } + \exp_args:NNNo \group_end: + \tl_set:Nn #4 \l_regex_tmpa_tl + \else: + \group_end: + \fi: + } +\cs_new_protected_nopar:Npn \regex_replace_once_aux:w #1; { - \group_insert_after:N \tl_set:Nx - \group_insert_after:N #1 - \group_insert_after:N { - \group_insert_after:N \tl_to_str:N - \group_insert_after:N \g_regex_replaced_str - \group_insert_after:N } } % \end{macrocode} % \end{macro} % -% \begin{macro}{\regex_replace_once:nnN} -% \begin{macro}{\regex_replace_once:NnN} -% \begin{macro}[TF]{\regex_replace_once:nnN} -% \begin{macro}[TF]{\regex_replace_once:NnN} -% \begin{macro}[aux]{\regex_replace_once_aux:Nn} +% \begin{macro}[aux]{\regex_replace_all_aux:NnnN} +% For every match, extract submatches, and add the part before +% the beginning of the match, as well as the replacement, +% to the result. After the last match, extract the end +% of the token list, and add it to the replaced token list. % \begin{macrocode} -\cs_new_protected:Npn \regex_replace_once_aux:Nn #1#2 +\cs_new_protected:Npn \regex_replace_all_aux:NnnN #1#2#3#4 { \group_begin: - \regex_replace_after_group:N #1 - \tl_clear:N \l_regex_every_match_tl - #2 - \exp_args:No \regex_match:n {#1} - \regex_extract: - \regex_extract_aux:nTF {0} + \tl_set:Nn \l_regex_every_match_tl { - \cs_set_nopar:Npn \regex_tmp:w - { \str_skip_do:nn { \l_regex_tmpb_tl } { } } - \tl_gset:Nx \g_regex_replaced_str - { - \regex_query_substr:NN \c_zero \l_regex_tmpa_tl - \l_regex_replacement_tl - \exp_after:wN \regex_tmp:w \l_regex_query_other_str - } + \regex_extract: + \tex_skip:D \l_regex_submatch_start_int + = \tex_the:D \tex_skip:D \l_regex_submatch_start_int + minus \l_regex_start_step_int sp \scan_stop: + \regex_match_once: } - { \tl_gset_eq:NN \g_regex_replaced_str \l_regex_query_other_str } - \group_end: + \regex_replacement:n {#3} + \exp_args:Nno #1 {#2} #4 + \int_set:Nn \l_regex_tmpa_int + { + 0 + \prg_stepwise_function:nnnN + \l_regex_max_state_int + \l_regex_capturing_group_int + { \l_regex_submatch_int - \c_one } + \regex_nesting:n + } + \if_num:w \l_regex_tmpa_int = \c_zero + \else: + \msg_kernel_error:nnx { regex } { replace-unbalanced } + { \l_regex_tmpa_int } + \fi: + \tl_set:Nx \l_regex_tmpa_tl + { + \if_num:w \l_regex_tmpa_int < \c_zero + \prg_replicate:nn { - \l_regex_tmpa_int } + { \exp_not:n { { \if_false: } \fi: } } + \fi: + \prg_stepwise_function:nnnN + \l_regex_max_state_int + \l_regex_capturing_group_int + { \l_regex_submatch_int - \c_one } + \regex_replace_all_aux:n + \regex_query_substr:nn + \l_regex_start_step_int \l_regex_max_step_int + \if_num:w \l_regex_tmpa_int > \c_zero + \prg_replicate:nn { \l_regex_tmpa_int } + { \exp_not:n { \if_false: { \fi: } } } + \fi: + } + \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl } + \exp_args:NNNo \group_end: + \tl_set:Nn #4 \l_regex_tmpa_tl } -\cs_new_protected:Npn \regex_replace_once:nnN #1#2#3 +\cs_new_nopar:Npn \regex_replace_all_aux:n #1 { - \regex_replace_once_aux:Nn #3 - { - \regex_build:n {#1} - \regex_replacement:n {#2} - } + \regex_query_substr:nn + { \etex_glueshrink:D \tex_skip:D #1 } { \tex_skip:D #1 } + \regex_replacement_tl:n {#1} } -\cs_new_protected:Npn \regex_replace_once:NnN #1#2#3 +% \end{macrocode} +% \end{macro} +% +% \subsection{Messages} +% +% \begin{macrocode} +\msg_kernel_new:nnnn { regex } { sequence-unbalanced } { - \regex_replace_once_aux:Nn #3 + Missing~ + \flag_test:NTF \l_regex_group_end_flag { - \regex_use:N #1 - \regex_replacement:n {#2} + left~ + \flag_test:NTF \l_regex_group_begin_flag + { and~right~braces } { brace } } + { right~brace } + \ inserted~in~extracted~match. } -\prg_new_protected_conditional:Npnn \regex_replace_once:nnN #1#2#3 {T,F,TF} { - \regex_replace_once_aux:Nn #3 - { - \regex_build:n {#1} - \regex_replacement:n {#2} - \regex_return_after_group: - } + LaTeX~was~asked~to~extract~submatches~or~split~a~token~list~ + according~to~a~given~regular~expression,~but~some~of~the~resulting~ + items~were~not~balanced. } -\prg_new_protected_conditional:Npnn \regex_replace_once:NnN #1#2#3 {T,F,TF} +\msg_kernel_new:nnnn { regex } { replace-unbalanced } + { The~result~of~a~replacement~does~not~have~balanced~braces. } { - \regex_replace_once_aux:Nn #3 + LaTeX~was~asked~to~do~some~regular~expression~replacement,~ + and~the~resulting~token~list~would~not~have~the~same~number~ + of~begin-group~and~end-group~tokens. \\ \\ + \ \ \ \ + \prg_case_int:nnn {#1} { - \regex_use:N #1 - \regex_replacement:n {#2} - \regex_return_after_group: + { -1 } { A~left~brace~was } + { 1 } { A~right~brace~was } } + { + \int_abs:n {#1} ~ + \int_compare:nNnTF {#1} < \c_zero { left } { right } ~ + braces ~ were + } + \ inserted. } % \end{macrocode} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}{\regex_replace_all:nnN} -% \begin{macro}{\regex_replace_all:NnN} -% \begin{macro}[aux]{\regex_replace_all_aux:Nn} -% \begin{macro}[aux]{\regex_replace_all_aux:} % \begin{macrocode} -\cs_new_protected:Npn \regex_replace_all_aux:Nn #1#2 +\msg_kernel_new:nnnn { regex } { missing-rparen } + { Missing~right~parenthesis~added~in~regular~expression. } { - \group_begin: - \regex_replace_after_group:N #1 - \tl_set:Nn \l_regex_every_match_tl { \regex_replace_all_aux: } - \tl_gclear:N \g_regex_replaced_str - #2 - \exp_args:No \regex_match:n {#1} - \cs_set_nopar:Npn \regex_tmp:w - { \str_skip_do:nn { \l_regex_start_step_int } { } } - \tl_gput_right:Nx \g_regex_replaced_str - { \exp_after:wN \regex_tmp:w \l_regex_query_other_str } - \group_end: + LaTeX~was~given~a~regular~expression~with~\int_eval:n{#1}~ + more~left~parenthes\int_compare:nTF{#1=1}{i}{e}s~than~right~ + parenthes\int_compare:nTF{#1=1}{i}{e}s. } -\cs_new_protected_nopar:Npn \regex_replace_all_aux: +\msg_kernel_new:nnnn { regex } { extra-rparen } + { Extra~right~parenthesis~ignored~in~regular~expression. } { - \regex_extract: - \regex_extract_aux:nTF {0} - { - \tl_gput_right:Nx \g_regex_replaced_str - { - \regex_query_substr:NN - \l_regex_start_step_int \l_regex_tmpa_tl - \l_regex_replacement_tl - } - } - { \msg_error:nn { regex } { internal } } - \regex_match_once: + LaTeX~came~across~a~closing~parenthesis~when~no~submatch~group~ + was~open.~The~parenthesis~will~be~ignored. } -\cs_new_protected:Npn \regex_replace_all:nnN #1#2#3 +\msg_kernel_new:nnnn { regex } { backwards-range } + { Range~[#1-#2]~out~of~order~in~character~class. } { - \regex_replace_all_aux:Nn #3 - { - \regex_build:n {#1} - \regex_replacement:n {#2} - } + In~ranges~of~characters~[x-y]~appearing~in~character~classes,~ + the~first~character~code~must~not~be~larger~than~the~second.~ + Here,~#1~has~character~code~\int_eval:n {`#1},~while~#2~has~ + character~code~\int_eval:n {`#2}. } -\cs_new_protected:Npn \regex_replace_all:NnN #1#2#3 +\msg_kernel_new:nnnn { regex } { unsupported } + { Unsupported~construction~`#1'. } { - \regex_replace_all_aux:Nn #3 - { - \regex_use:N #1 - \regex_replacement:n {#2} - } + The~construction~`#1'~is~not~supported~by~the~LaTeX~ + regular~expression~module.~Perhaps~some~character~should~ + have~been~escaped? + } +\msg_kernel_new:nnnn { regex } { not-nfa } + { This~is~not~a~regular~expression~variable. } + { + LaTeX~was~expecting~`#1'~to~be~a~regular~expression~variable.\\ + This~control~sequence~is~not~a~regex~variable.~It's~current~meaning~ + is~\\\\ + \ \ \ \ \token_to_str:N #1 = \token_to_meaning:N #1 . } % \end{macrocode} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} % -% \subsection{Messages} -% -% Negative codes are specific to the \LaTeX3 implementation. -% Other error codes match with the PCRE codes (not all of -% the PCRE errors can occur, since many constructions are -% not supported). % \begin{macrocode} -% \msg_new:nnn { regex } { -999 } { File~not~found } -\msg_new:nnn { regex } { -998 } { Unsupported~construct } -\msg_new:nnn { regex } { -997 } - { The~regular~expression~is~too~large~(32768~states). } -\msg_new:nnn { regex } { 1 } { \iow_char:N\\~at~end~of~pattern } +% \msg_new:nnn { regex } { 1 } { \iow_char:N\\~at~end~of~pattern } % \msg_new:nnn { regex } { 2 } { \iow_char:N\\c~at~end~of~pattern } -\msg_new:nnn { regex } { 4 } - { Numbers~out~of~order~in~\iow_char:N\{\iow_char\}~quantifier. } -\msg_new:nnn { regex } { 6 } - { Missing~terminating~\iow_char:N\]~for~character~class } -\msg_new:nnn { regex } { 7 } - { Invalid~escape~sequence~in~character~class } -\msg_new:nnn { regex } { 8 } - { Range~out~of~order~in~character~class } -\msg_new:nnn { regex } { 22 } { Mismatched~parentheses } -\msg_new:nnn { regex } { 34 } - { Character~value~in~\iow_char:N\\x{...}~sequence~is~too~large } +% \msg_new:nnn { regex } { 4 } +% { Numbers~out~of~order~in~\iow_char:N\{\iow_char\}~quantifier. } +% \msg_new:nnn { regex } { 6 } +% { Missing~terminating~\iow_char:N\]~for~character~class } +% \msg_new:nnn { regex } { 7 } +% { Invalid~escape~sequence~in~character~class } +% \msg_new:nnn { regex } { 34 } +% { Character~value~in~\iow_char:N\\x{...}~sequence~is~too~large } % \msg_new:nnn { regex } { 44 } { Invalid~UTF-8~string } % \msg_new:nnn { regex } { 46 } % { Malformed~\iow_char:N\\P~or\iow_char:N\\p~sequence } @@ -3867,18 +4464,46 @@ % \end{macrocode} % % \begin{macrocode} -\msg_new:nnn { regex } { not-nfa } +% \end{macrocode} +% +% \begin{macrocode} +\msg_kernel_new:nnnn { regex } { build-c } + { Misused~\iow_char:N\\c~or\iow_char:N\\C~command~in~regular~expression. } + { + The~\iow_char:N\\C~escape~sequence~can~be~followed~by~ + one~of~the~letters~ABCDELMOPSTU.~The~\iow_char:N\\c~ + escape~sequence~can~also~be~followed~by~a~brace~group. + } +\msg_kernel_new:nnnn { regex } { unknown-option } + { Unknown~option~`#1'~for~regular~expressions. } + { + LaTeX~came~across~something~like~`(?#1)'~in~a~regular~expression,~ + but~the~option~`#1'~is~not~known.~It~will~be~ignored. + } +\msg_kernel_new:nnnn { regex } { invalid-in-option } + { Invalid~character~in~option~of~a~regular~expression. } + { + The~character~or~escape~sequence~`#1'~is~not~defined~ + as~an~option~within~regular~expressions. + } +\msg_kernel_new:nnnn { regex } { replacement-g } + { Missing~brace~for~the~\iow_char:N\\g~construction~in~a~replacement~text. } + { + In~the~replacement~text~for~a~regular~expression~search,~ + submatches~are~represented~either~as~\iow_char:N \\g{dd..d},~ + or~\\d,~where~`d'~are~single~digits.~Here,~a~brace~is~missing. + } +\msg_kernel_new:nnnn { regex } { replacement-c } + { Something~is~wrong~with~the~replacement~text. } { - LaTeX~was~expecting~a~regular~expression~variable.\\ - Instead,~LaTeX~found~'#1'. + The~\iow_char:N\\c~escape~sequence~can~only~be~followed~by~ + ABCDELMN~or~by~a~brace. } % \end{macrocode} % % \begin{macrocode} \msg_kernel_new:nnn { regex } { nfa-misused } { Automaton~#1 used~incorrectly. } -\msg_kernel_new:nnn { regex } { g-misused } - { \g misused~in~replacement~text. } % \end{macrocode} % % \begin{macrocode} diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx index db73291a70b..a33d4909a1a 100644 --- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx @@ -35,7 +35,7 @@ % %<*driver|package> \RequirePackage{expl3} -\GetIdInfo$Id: l3str.dtx 3035 2011-12-07 11:48:46Z bruno $ +\GetIdInfo$Id: l3str.dtx 3088 2011-12-23 02:00:14Z bruno $ {L3 Experimental Strings} %</driver|package> %<*driver> @@ -76,25 +76,25 @@ % String variables are simply specialised token lists, but by convention % should be named with the suffix \texttt{\ldots str}. Such variables should % contain characters with category code $12$ (other), except spaces, which -% have category code $10$ (blank space). All the \enquote{safe} functions in +% have category code $10$ (blank space). All the functions in % this module first convert their argument to a string for internal processing, % and will not treat a token list or the corresponding string representation % differently. % % Most functions in this module come in three flavours: % \begin{itemize} -% \item \cs{str_...:N...}, which expect a token list +% \item \cs{str_...:N...}, which expect a token list or string % variable as their argument; % \item \cs{str_...:n...}, taking any token list (or string) as an argument; % \item \cs{str_..._ignore_spaces:n...}, which ignores any space encountered % during the operation: these functions are faster than those % which take care of escaping spaces appropriately; % \end{itemize} -% When performance is important, the internal \cs{str_..._unsafe:n...} -% functions, which expect a \enquote{safe} string in which spaces +% When performance is critical, the internal \cs{str_..._unsafe:n...} +% functions, which expect an \enquote{other string} in which spaces % have category code $12$ instead of $10$, might be useful. % -% \section{Conversion and input of strings} +% \section{Building strings} % % \begin{variable} % { @@ -106,6 +106,7 @@ % \c_percent_str % } % Constant strings, containing a single character, with category code $12$. +% Other characters can be accessed as \cs{iow_char:N} |\|\meta{character}. % \end{variable} % % \begin{function}[EXP]{\tl_to_str:N, \tl_to_str:n} @@ -124,77 +125,49 @@ % Creates a new constant \meta{str~var} or raises an error % if the name is already taken. The value of the % \meta{str~var} will be set globally to the -% \meta{token list}, after being converted to a string. -% \end{function} -% -% \begin{function}{\str_set:Nn, \str_set:Nx, \str_set:cn, \str_set:cx} -% \begin{syntax} -% \cs{str_set:Nn} \meta{str var} \Arg{token list} -% \end{syntax} -% Converts the \meta{token list} to a \meta{string}, -% and saves the result in \meta{str var}. -% \end{function} -% -% \begin{function}{\str_gset:Nn, \str_gset:Nx, \str_gset:cn, \str_gset:cx} -% \begin{syntax} -% \cs{str_gset:Nn} \meta{str~var} \Arg{token list} -% \end{syntax} -% Converts the \meta{token list} to a \meta{string}, -% and saves the result in \meta{str~var} globally. +% \meta{token list}, converted to a string. % \end{function} % % \begin{function} % { -% \str_put_left:Nn, \str_put_left:Nx, -% \str_put_left:cn, \str_put_left:cx +% \str_set:Nn, \str_set:Nx, \str_set:cn, \str_set:cx, +% \str_gset:Nn, \str_gset:Nx, \str_gset:cn, \str_gset:cx % } % \begin{syntax} -% \cs{str_put_left:Nn} \meta{str var} \Arg{token list} +% \cs{str_set:Nn} \meta{str var} \Arg{token list} % \end{syntax} % Converts the \meta{token list} to a \meta{string}, -% and prepends the result to \meta{str var}. -% The current contents of the \meta{str var} are not -% automatically converted to a string. +% and stores the result in \meta{str var}. % \end{function} % % \begin{function} % { +% \str_put_left:Nn, \str_put_left:Nx, +% \str_put_left:cn, \str_put_left:cx, % \str_gput_left:Nn, \str_gput_left:Nx, % \str_gput_left:cn, \str_gput_left:cx % } % \begin{syntax} -% \cs{str_gput_left:Nn} \meta{str var} \Arg{token list} -% \end{syntax} -% Converts the \meta{token list} to a \meta{string}, -% and prepends the result to \meta{str var}, globally. -% The current contents of the \meta{str var} are not -% automatically converted to a string. -% \end{function} -% -% \begin{function} -% { -% \str_put_right:Nn, \str_put_right:Nx, -% \str_put_right:cn, \str_put_right:cx -% } -% \begin{syntax} -% \cs{str_put_right:Nn} \meta{str var} \Arg{token list} +% \cs{str_put_left:Nn} \meta{str var} \Arg{token list} % \end{syntax} % Converts the \meta{token list} to a \meta{string}, -% and appends the result to \meta{str var}. +% and prepends the result to \meta{str var}. % The current contents of the \meta{str var} are not % automatically converted to a string. % \end{function} % % \begin{function} % { +% \str_put_right:Nn, \str_put_right:Nx, +% \str_put_right:cn, \str_put_right:cx, % \str_gput_right:Nn, \str_gput_right:Nx, % \str_gput_right:cn, \str_gput_right:cx % } % \begin{syntax} -% \cs{str_gput_right:Nn} \meta{str var} \Arg{token list} +% \cs{str_put_right:Nn} \meta{str var} \Arg{token list} % \end{syntax} % Converts the \meta{token list} to a \meta{string}, -% and appends the result to \meta{str var}, globally. +% and appends the result to \meta{str var}. % The current contents of the \meta{str var} are not % automatically converted to a string. % \end{function} @@ -204,9 +177,7 @@ % \cs{str_input:Nn} \meta{str var} \Arg{token list} % \end{syntax} % Converts the \meta{token list} into a \meta{string}, and stores -% it in the \meta{str var}, within the current \TeX{} group level -% for the \texttt{input} variant and globally for the \texttt{ginput} -% version. +% it in the \meta{str var}. % Special characters can be input by % escaping them with a backslash. % \begin{itemize} @@ -218,7 +189,7 @@ % are read to give a number between $0$ and $255$. % \item |\x{hh...}| produces the character with code \texttt{hh...} % (an arbitrary number of hexadecimal digits are read): -% this is mostly useful for \LuaTeX{} and \XeTeX{}. +% this is mostly useful for Unicode-aware engines. % \item |\a|, |\e|, |\f|, |\n|, |\r|, |\t| stand for specific characters: % \begin{center} % \begin{tabular}{cccc} @@ -292,7 +263,7 @@ % The functions differ in the case where the first character % is a space: \cs{str_tail:N} and \cs{str_tail:n} will trim % only that space, while \cs{str_tail_ignore_spaces:n} trims -% the first non-space character. +% the first non-space character and any space before it. % \end{function} % % \begin{function}[EXP]{\str_item:Nn, \str_item:nn, \str_item_ignore_spaces:nn} @@ -321,8 +292,17 @@ % and leaves in the input stream the characters % between \meta{start index} (inclusive) and \meta{end index} (exclusive). % If either of \meta{start index} or \meta{end index} is negative, -% then it is incremented by the length of the list. Both \meta{start index} -% and \meta{end index} count from $0$ for the first (left most) character. +% then it is incremented by the length of the list. +% If either of \meta{start index} or \meta{end index} is empty, +% it is replaced by the corresponding end-point of the string. +% Both \meta{start index} and \meta{end index} count from $0$ +% for the first (left most) character. +% For instance, +% \begin{verbatim} +% \iow_term:x { \str_substr:nnn { abcdef } { 1 } { 4 } } +% \iow_term:x { \str_substr:nnn { abcdef } { -4 } { } } +% \end{verbatim} +% will print \texttt{bcd} and \texttt{cdef}. % \end{function} % % \section{String conditionals} @@ -348,15 +328,6 @@ % expansion). % \end{function} % -% \begin{function}[EXP, pTF]{\str_if_bytes:N} -% \begin{syntax} -% \cs{str_if_bytes:NTF} \meta{str var} \Arg{true code} \Arg{false code} -% \end{syntax} -% Tests whether the \meta{str var} only contains characters -% in the range $0$ to $255$. This is automatically true when -% the pdf\TeX{} engine is in use. -% \end{function} -% % \section{Encoding functions} % % Traditionally, string encodings only specify how strings of characters @@ -370,12 +341,13 @@ % \enquote{encoding}. This can be \textsc{utf-16}, % \textsc{iso 8859-1}, \emph{etc.} % See Table~\ref{tab:encodings} for a list -% of encodings supported. +% of encodings supported.\footnote{Encodings and escapings +% will be added as they are requested.} % \item Bytes are translated to \TeX{} tokens through % a given \enquote{escaping}. Those are defined for % the most part by the \texttt{pdf} file format. % See Table~\ref{tab:escapings} for a list -% of escaping methods supported. +% of escaping methods supported.\footnotemark % \end{itemize} % % \begin{table}\centering @@ -408,7 +380,7 @@ % \texttt{iso885916}, \texttt{latin10} & \textsc{iso 8859-16} \\ % \midrule % \texttt{native} & Native Unicode string. \\ -% \texttt{internal} & Comma-list of integers. \\ +% \texttt{internal} & For curious programmers. \\ % \bottomrule % \end{tabular} % \end{table} @@ -427,7 +399,7 @@ % & see \tn{pdfescapename} \\ % \texttt{string} % & see \tn{pdfescapestring} \\ -% \texttt{url}, \texttt{percent} +% \texttt{url} % & encoding used in \textsc{url}s \\ % \bottomrule % \end{tabular} @@ -471,89 +443,101 @@ % an error. % Characters which cannot be encoded in the \meta{encoding 2} % are silently ignored. -%^^A todo: add doc and examples about each encoding and escaping method. % \end{function} % % \section{Internal string functions} % -% \begin{function}[EXP]{\tl_to_other_str:N, \tl_to_other_str:n} +% \begin{function}{\str_escape_use:NNNn} % \begin{syntax} -% \cs{tl_to_other_str:n} \Arg{token list} +% \cs{str_escape_use:NNNn} \meta{fn1} \meta{fn2} \meta{fn3} \Arg{token list} % \end{syntax} -% Converts the \meta{token list} to an \meta{other string}, where -% spaces have category code \enquote{other}. These functions create -% \enquote{safe} strings. -% \begin{texnote} -% These functions can be \texttt{f}-expanded without fear of losing -% a leading space, since spaces do not have category code $10$ in -% their result. -% \end{texnote} +% The \meta{token list} is converted to a string, then read from +% left to right, interpreting backslashes as escaping the next character. +% Unescaped characters are fed to the function \meta{fn1}, +% and escaped characters are fed to the function \meta{fn2} +% within an \texttt{x}-expansion context (typically those functions +% perform some tests on their argument to decide how to output them). +% The escape sequences |\a|, |\e|, |\f|, |\n|, |\r|, |\t| and |\x| +% are recognized as described for \cs{str_input:Nn}, and those +% are replaced by the corresponding character, then fed to +% \meta{fn3}. The result is then left in the input stream. % \end{function} % -% \begin{function}[EXP]{\str_sanitize_args:Nn, \str_sanitize_args:Nnn} +% \begin{function}[rEXP]{\str_aux_toks_range:nn} % \begin{syntax} -% \cs{str_sanitize_args:Nnn} \meta{function} -% ~~\Arg{token list1} \Arg{token list2} +% \cs{str_aux_toks_range:nn} \meta{start} \meta{end} % \end{syntax} -% Converts the \meta{token lists} to \enquote{safe} strings -% (where spaces have category code \enquote{other}), -% and hands-in the result as arguments to \meta{function}. +% Expands to the contents of the \tn{toks} registers numbered +% from \meta{start} (inclusive) to \meta{end} (exclusive). +% This function is used in \pkg{l3regex}. % \end{function} % -% \begin{function}{\str_aux_gset_other:Nn} +% \begin{function}{\str_aux_hexadecimal_use:NTF} % \begin{syntax} -% \cs{str_aux_gset_other:Nn} \meta{tl~var} \Arg{token list} +% \cs{str_aux_hexadecimal_use:NTF} \meta{token} +% ~~\Arg{true code} \Arg{false code} % \end{syntax} -% Converts the \meta{token list} to an \meta{other string}, where -% spaces have category code \enquote{other}, and assigns the result -% to the \meta{tl~var}, globally. +% If the \meta{token} is a hexdecimal digit (upper case or lower case), +% its upper-case version is left in the input strem, \emph{followed} +% by the \meta{true code}. Otherwise, the \meta{false code} is left +% in the input stream. +% \begin{texnote} +% This function will fail if the escape character is a hexadecimal +% digit, or has a character code in the range $[0,5]$. We are thus +% careful to set the escape character to a known value before using it. +% \end{texnote} % \end{function} % -% \begin{function}[rEXP]{\str_map_tokens:Nn, \str_map_tokens:nn} +% \begin{function}[EXP]{\tl_to_other_str:n} % \begin{syntax} -% \cs{str_map_tokens:Nn} \meta{str var} \meta{tokens} +% \cs{tl_to_other_str:n} \Arg{token list} % \end{syntax} -% Maps the \meta{tokens} over every character in the \meta{str var}. +% Converts the \meta{token list} to a \meta{other string}, where +% spaces have category code \enquote{other}. +% \begin{texnote} +% These functions can be \texttt{f}-expanded without fear of losing +% a leading space, since spaces do not have category code $10$ in +% their result. +% \end{texnote} % \end{function} % -% \begin{function}[EXP, pTF] -% {\str_if_contains_char:NN, \str_if_contains_char:nN} +% \begin{function}{\str_gset_other:Nn} % \begin{syntax} -% \cs{str_if_contains_char:nN} \Arg{token list} \meta{char} +% \cs{str_gset_other:Nn} \meta{tl~var} \Arg{token list} % \end{syntax} -% Converts the \meta{token list} to a \meta{string} -% and tests whether the \meta{char} is present in the \meta{string}. -% The \meta{char} can be given either directly, or as a one -% letter control sequence. +% Converts the \meta{token list} to an \meta{other string}, where +% spaces have category code \enquote{other}, and assigns the result +% to the \meta{tl~var}, globally. % \end{function} % -% \begin{function}{\str_aux_escape:NNNn} +% \begin{function}[EXP] +% { +% \str_if_contains_char:NNT, +% \str_if_contains_char:NNTF, +% \str_if_contains_char:nNTF +% } % \begin{syntax} -% \cs{str_aux_escape:NNNn} \meta{fn1} \meta{fn2} \meta{fn3} \Arg{token list} +% \cs{str_if_contains_char:nNTF} \Arg{token list} \meta{char} % \end{syntax} -% The \meta{token list} is converted to a string, then read from -% left to right, interpreting backslashes as escaping the next character. -% Unescaped characters are fed to the function \meta{fn1}, -% and escaped characters are fed to the function \meta{fn2} -% within an \texttt{x}-expansion context (typically those functions -% perform some tests on their argument to decide how to output them). -% The escape sequences |\a|, |\e|, |\f|, |\n|, |\r|, |\t| and |\x| -% are recognized as described for \cs{str_input:Nn}, and those -% are replaced by the corresponding character, then fed to -% \meta{fn3}. The result is assigned globally to \cs{g_str_result_tl}. +% Converts the \meta{token list} to a \meta{string} +% and tests whether the \meta{char} is present in the \meta{string}. +% Spaces are ignored. % \end{function} % -% \section{Possibilities} +% \section{Possibilities, and things to do} % % \begin{itemize} -% \item More encodings (see Heiko's \pkg{stringenc}). -% In particular, what is needed for pdf: \textsc{utf-16}? -% \item CESU -% \item \cs{str_if_head_eq:nN} alias of \cs{tl_if_head_eq_charcode:nN} +% \item Use flags rather than expandable errors. +% \item Add documentation about each encoding and escaping method, +% and add examples. +% \item Perhaps use the pdf\TeX{} primitives for escapings when available. +% But this changes error detection between engines. +% \item More encodings (see Heiko's \pkg{stringenc}). CESU. +% \item More escapings: shell escapes, lua escapes, etc? +% \item \cs{str_if_head_eq:nN} % \item \cs{str_if_numeric/decimal/integer:n}, perhaps in \pkg{l3fp}? % \item Should \cs{str_item:Nn} be \cs{str_char:Nn}? % \end{itemize} -% Some functionalities of \pkg{stringstrings} and \pkg{xstring} as well. % % \end{documentation} % @@ -582,52 +566,9 @@ % \item \cs{str_length_loop:NNNNNNNNN} (unchanged) % \end{itemize} % -% \subsection{General functions} -% -% \begin{macro}[EXP,aux]{\use_i:nnnnnnnn} -% A function which may already be defined elsewhere. -% \begin{macrocode} -\cs_if_exist:NF \use_i:nnnnnnnn - { \cs_new:Npn \use_i:nnnnnnnn #1#2#3#4#5#6#7#8 {#1} } -% \end{macrocode} -% \end{macro} +% \subsection{Helpers} % -% \begin{variable} -% { -% \c_forty_eight, \c_fifty_eight, \c_sixty_five, \c_ninety_one, -% \c_ninety_seven, \c_one_hundred_twenty_three, -% \c_one_hundred_twenty_seven -% } -% We declare here some integer values which delimit ranges of -% ASCII characters of various types. This is mostly used in -% \pkg{l3regex}. -% \begin{macrocode} -\int_const:Nn \c_forty_eight { 48 } -\int_const:Nn \c_fifty_eight { 58 } -\int_const:Nn \c_sixty_five { 65 } -\int_const:Nn \c_ninety_one { 91 } -\int_const:Nn \c_ninety_seven { 97 } -\int_const:Nn \c_one_hundred_twenty_three { 123 } -\int_const:Nn \c_one_hundred_twenty_seven { 127 } -% \end{macrocode} -% \end{variable} -% -% \begin{variable}{\c_max_char_int} -% The maximum character code depends on the engine. -% \begin{macrocode} -\pdftex_if_engine:TF - { \int_const:Nn \c_max_char_int { 255 } } - { \int_const:Nn \c_max_char_int { 1114111 } } -% \end{macrocode} -% \end{macro} -% -% \begin{variable}{\c_str_replacement_char_int} -% When converting, invalid bytes are replaced by the Unicode -% replacement character \textsc{u-fffd}. -% \begin{macrocode} -\int_const:Nn \c_str_replacement_char_int { "FFFD } -% \end{macrocode} -% \end{variable} +% \subsubsection{Assigning strings} % % \begin{macro} % { @@ -666,7 +607,7 @@ % \end{macrocode} % \end{macro} % -% \subsection{Variables and constants} +% \subsubsection{Variables and constants} % % \begin{macro}{\str_tmp:w} % \begin{variable}{\l_str_tmpa_tl} @@ -674,6 +615,7 @@ % \begin{macrocode} \cs_new_protected_nopar:Npn \str_tmp:w { } \tl_new:N \l_str_tmpa_tl +\int_new:N \l_str_tmpa_int % \end{macrocode} % \end{variable} % \end{macro} @@ -699,6 +641,42 @@ % % \begin{variable} % { +% \c_forty_eight, \c_fifty_eight, \c_sixty_five, \c_ninety_one, +% \c_ninety_seven, \c_one_hundred_twenty_three, +% \c_one_hundred_twenty_seven +% } +% We declare here some integer values which delimit ranges of +% ASCII characters of various types. This is mostly used in +% \pkg{l3regex}. +% \begin{macrocode} +\int_const:Nn \c_forty_eight { 48 } +\int_const:Nn \c_fifty_eight { 58 } +\int_const:Nn \c_sixty_five { 65 } +\int_const:Nn \c_ninety_one { 91 } +\int_const:Nn \c_ninety_seven { 97 } +\int_const:Nn \c_one_hundred_twenty_three { 123 } +\int_const:Nn \c_one_hundred_twenty_seven { 127 } +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\c_max_char_int} +% The maximum character code depends on the engine. +% \begin{macrocode} +\int_const:Nn \c_max_char_int + { \pdftex_if_engine:TF { 255 } { 1114111 } } +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\c_str_replacement_char_int} +% When converting, invalid bytes are replaced by the Unicode +% replacement character \textsc{u-fffd}. +% \begin{macrocode} +\int_const:Nn \c_str_replacement_char_int { "FFFD } +% \end{macrocode} +% \end{variable} +% +% \begin{variable} +% { % \c_backslash_str, % \c_lbrace_str, % \c_rbrace_str, @@ -734,18 +712,17 @@ \prop_gput:Nnn \g_str_aliases_prop { latin9 } { iso885915 } \prop_gput:Nnn \g_str_aliases_prop { latin10 } { iso885916 } \prop_gput:Nnn \g_str_aliases_prop { hexadecimal } { hex } -\prop_gput:Nnn \g_str_aliases_prop { percent } { url } % \end{macrocode} % \end{variable} % -% \subsection{Internal functions} -% % \subsubsection{Escaping spaces} % -% \begin{macro}[EXP]{\tl_to_other_str:N, \tl_to_other_str:n} +% \begin{macro}[EXP]{\tl_to_other_str:n} % \begin{macro}[EXP,aux]{\tl_to_other_str_loop:w, \tl_to_other_str_end:w} % Replaces all spaces by \enquote{other} spaces, after converting % the token list to a string via \cs{tl_to_str:n}. +% This function is \texttt{f}-expandable, storing the part of the string +% with escaped spaces between the \cs{q_mark} and \cs{q_stop} markers. % \begin{macrocode} \group_begin: \char_set_lccode:nn { `\* } { `\ } @@ -770,29 +747,19 @@ \cs_new_nopar:Npn \tl_to_other_str_end:w \fi: #1 \q_mark #2 * A #3 \q_stop { \fi: #2 } } -\cs_new_nopar:Npn \tl_to_other_str:N { \exp_args:No \tl_to_other_str:n } % \end{macrocode} % \end{macro} % \end{macro} % -% \begin{macro}[EXP]{\str_sanitize_args:Nn, \str_sanitize_args:Nnn} -% Here, \texttt{f}-expansion does not lose leading spaces, -% since they have catcode \enquote{other} after \cs{str_sanitize:n}. -% \begin{macrocode} -\cs_new:Npn \str_sanitize_args:Nn #1#2 - { \exp_args:Nf #1 { \tl_to_other_str:n {#2} } } -\cs_new:Npn \str_sanitize_args:Nnn #1#2#3 - { - \exp_args:Nff #1 - { \tl_to_other_str:n {#2} } - { \tl_to_other_str:n {#3} } - } -% \end{macrocode} -% \end{macro} -% -% \begin{macro}[int]{\str_aux_gset_other:Nn} -% \begin{macro}[aux,EXP]{\str_aux_gset_other_loop:w} -% \begin{macro}[aux,EXP]{\str_aux_gset_other_end:w} +% \begin{macro}[int]{\str_gset_other:Nn} +% \begin{macro}[aux,EXP]{\str_gset_other_loop:w} +% \begin{macro}[aux,EXP]{\str_gset_other_end:w} +% This function could be done by using \cs{tl_to_other_str:n} +% within an \texttt{x}-expansion, but that would take a time +% quadratic in the size of the string. Instead, we can +% \enquote{leave the result behind us} in the input stream, +% to be captured into the expanding assignment. This gives us +% a linear time. % \begin{macrocode} \group_begin: \char_set_lccode:nn { `\* } { `\ } @@ -800,24 +767,24 @@ \tl_to_lowercase:n { \group_end: - \cs_new_protected:Npn \str_aux_gset_other:Nn #1#2 + \cs_new_protected:Npn \str_gset_other:Nn #1#2 { \tl_gset:Nx #1 { - \exp_after:wN \str_aux_gset_other_loop:w \tl_to_str:n {#2} ~ % + \exp_after:wN \str_gset_other_loop:w \tl_to_str:n {#2} ~ % A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ \q_mark \q_stop } } - \cs_new_nopar:Npn \str_aux_gset_other_loop:w + \cs_new_nopar:Npn \str_gset_other_loop:w #1 ~ #2 ~ #3 ~ #4 ~ #5 ~ #6 ~ #7 ~ #8 ~ #9 ~ { \if_meaning:w A #9 - \str_aux_gset_other_end:w + \str_gset_other_end:w \fi: #1 * #2 * #3 * #4 * #5 * #6 * #7 * #8 * #9 - \str_aux_gset_other_loop:w * + \str_gset_other_loop:w * } - \cs_new_nopar:Npn \str_aux_gset_other_end:w \fi: #1 * A #2 \q_stop + \cs_new_nopar:Npn \str_gset_other_end:w \fi: #1 * A #2 \q_stop { \fi: #1 } } % \end{macrocode} @@ -825,54 +792,46 @@ % \end{macro} % \end{macro} % -% \subsubsection{Mapping} +% \subsubsection{Functions unrelated to strings} % -% \begin{macro}[rEXP,int]{\str_map_tokens:nn} -% \begin{macro}[rEXP,int]{\str_map_tokens:Nn} -% \begin{macro}[rEXP,aux]{\str_map_tokens_aux:nn} -% \begin{macro}[rEXP,aux]{\str_map_tokens_loop:nw} -% \begin{macro}[rEXP,aux]{\str_map_tokens_loop:nN} -% \begin{macro}[rEXP,int]{\str_map_break_do:n} -% We simply need to be careful with spaces. -% Spaces are fed to the mapped tokens with -% category code $12$, not $10$. +% \begin{macro}[EXP,aux]{\use_i:nnnnnnnn} +% A function which may already be defined elsewhere. % \begin{macrocode} -\cs_new_nopar:Npn \str_map_tokens:Nn - { \exp_args:No \str_map_tokens:nn } -\cs_new:Npn \str_map_tokens:nn #1 - { \exp_args:No \str_map_tokens_aux:nn { \tl_to_str:n {#1} } } -\cs_new:Npn \str_map_tokens_aux:nn #1#2 +\cs_if_exist:NF \use_i:nnnnnnnn + { \cs_new:Npn \use_i:nnnnnnnn #1#2#3#4#5#6#7#8 {#1} } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int, rEXP]{\str_aux_toks_range:nn} +% \begin{macro}[aux, rEXP]{\str_aux_toks_range:ww} +% Some non-expandable functions (in this module, +% \cs{str_aux_escape:NNNn}, which is used by \cs{str_(g)input:Nn}) +% store pieces of their result in \tn{toks} registers. +% Those pieces are concatenated using \cs{str_aux_toks_range:nn}, +% which expects two integer registers, \meta{start} and \meta{end}, +% and expands to the contents of the \tn{toks} registers from +% \meta{start} (inclusive) to \meta{end} (exclusive). +% This function is also used in the \pkg{l3regex} module. +% \begin{macrocode} +\cs_new:Npn \str_aux_toks_range:nn #1#2 { - \str_map_tokens_loop:nw {#2} #1 - { ? \use_none_delimit_by_q_recursion_stop:w } - ?~ ?~ ?~ ?~ ?~ ?~ ?~ ?~ \q_recursion_stop + \exp_after:wN \str_aux_toks_range:ww + \int_use:N \int_eval:w #1 \exp_after:wN ; + \int_use:N \int_eval:w #2 ; + \prg_break_point:n { } } -\group_begin: - \char_set_lccode:nn { `* } { `\ } - \tl_to_lowercase:n - { - \group_end: - \cs_new:Npn \str_map_tokens_loop:nw #1 #2~ #3~ #4~ #5~ #6~ #7~ #8~ #9~ - { - \str_map_tokens_loop:nN {#1} #2* #3* #4* #5* #6* #7* #8* #9* - { ? \use_none_delimit_by_q_stop:w } \q_stop - \str_map_tokens_loop:nw {#1} - } - } -\cs_new:Npn \str_map_tokens_loop:nN #1#2 +\cs_new:Npn \str_aux_toks_range:ww #1 ; #2 ; { - \use_none:n #2 - #1 #2 - \str_map_tokens_loop:nN {#1} + \if_num:w #1 = #2 \exp_stop_f: + \exp_after:wN \prg_map_break: + \fi: + \tex_the:D \tex_toks:D #1 \exp_stop_f: + \exp_after:wN \str_aux_toks_range:ww + \int_use:N \int_eval:w #1 + \c_one ; #2 ; } -\cs_new_eq:NN \str_map_break_do:n \use_i_delimit_by_q_recursion_stop:nw % \end{macrocode} % \end{macro} % \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} % % \subsection{Characters given by their position} % @@ -889,7 +848,8 @@ % by \cs{use_none_delimit_by_q_stop:w}. % \begin{macrocode} \cs_new_nopar:Npn \str_length:N { \exp_args:No \str_length:n } -\cs_new:Npn \str_length:n { \str_sanitize_args:Nn \str_length_unsafe:n } +\cs_new:Npn \str_length:n #1 + { \exp_args:Nf \str_length_unsafe:n { \tl_to_other_str:n {#1} } } \cs_new_nopar:Npn \str_length_unsafe:n #1 { \str_length_aux:n { \str_length_loop:NNNNNNNNN #1 } } \cs_new:Npn \str_length_ignore_spaces:n #1 @@ -1110,15 +1070,15 @@ % the parameters given to \cs{str_skip_do:nn} are necessarily % within the bounds of the length of the string. % The \cs{str_item_ignore_spaces:nn} function cheats a little bit -% in that it doesn't hand to \cs{str_item_unsafe:nn} a truly -% \enquote{safe} string. This is alright, as everything else +% in that it doesn't hand to \cs{str_item_unsafe:nn} an +% \enquote{other string}. This is alright, as everything else % is done with undelimited arguments. % \begin{macrocode} \cs_new_nopar:Npn \str_item:Nn { \exp_args:No \str_item:nn } \cs_new:Npn \str_item:nn #1#2 { \exp_args:Nf \tl_to_str:n - { \str_sanitize_args:Nn \str_item_unsafe:nn {#1} {#2} } + { \exp_args:Nf \str_item_unsafe:nn { \tl_to_other_str:n {#1} } {#2} } } \cs_new:Npn \str_item_ignore_spaces:nn #1 { \exp_args:No \str_item_unsafe:nn { \tl_to_str:n {#1} } } @@ -1161,13 +1121,21 @@ % \begin{macro}[EXP]{\str_substr:nnn} % \begin{macro}[EXP]{\str_substr_ignore_spaces:nnn} % \begin{macro}[EXP]{\str_substr_unsafe:nnn} -% \begin{macro}[EXP,aux]{\str_substr_aux:nnnw} -% \begin{macro}[EXP,aux]{\str_substr_aux_ii:nnw} -% \begin{macro}[EXP,aux]{\str_aux_eval_args:Nnnn} +% \begin{macro}[EXP,aux]{\str_substr_aux:www} +% \begin{macro}[EXP,aux]{\str_substr_aux:nnw} % \begin{macro}[EXP,aux]{\str_aux_normalize_range:nn} -% Sanitize the string, then limit the second and third arguments -% to be at most the length of the string (this avoids needing -% to check for the end of the string when grabbing characters). +% Sanitize the string. Then evaluate the arguments, replacing +% them by \cs{c_zero} or \cs{c_max_int} if they are empty. +% This is done by using the construction +% \begin{verbatim} +% \int_use:N \int_eval:w #2 \c_zero \c_zero ; +% \end{verbatim} +% which expands to the value of |#2|, followed by |\c_zero \c_zero ;| +% if |#2| is an expression, and expands to |0\c_zero ;| otherwise. +% The same is done to the end-point of the range. +% Then limit the range to be at most the length of the string +% (this avoids needing to check for the end of the string when +% grabbing characters). % Afterwards, skip characters, then keep some more, and finally % drop the end of the string. % \begin{macrocode} @@ -1175,26 +1143,31 @@ \cs_new:Npn \str_substr:nnn #1#2#3 { \exp_args:Nf \tl_to_str:n - { \str_sanitize_args:Nn \str_substr_unsafe:nnn {#1} {#2} {#3} } + { + \exp_args:Nf \str_substr_unsafe:nnn + { \tl_to_other_str:n {#1} } {#2} {#3} + } } \cs_new:Npn \str_substr_ignore_spaces:nnn #1 { \exp_args:No \str_substr_unsafe:nnn { \tl_to_str:n {#1} } } \cs_new:Npn \str_substr_unsafe:nnn #1#2#3 { - \str_aux_eval_args:Nnnn \str_substr_aux:nnnw - { \str_length_unsafe:n {#1} } - {#2} - {#3} + \exp_after:wN \str_substr_aux:www + \int_value:w \str_length_unsafe:n {#1} \exp_after:wN ; + \int_use:N \int_eval:w #2 + \c_zero \exp_after:wN ; + \int_use:N \int_eval:w \exp_args:Nf \str_substr_unsafe:nN {#3} \c_max_int ; #1 \q_stop } -\cs_new:Npn \str_substr_aux:nnnw #1#2#3 +\cs_new:Npn \str_substr_unsafe:nN #1 #2 + { \tl_if_empty:nTF {#1} {#2} {#1} } +\cs_new:Npn \str_substr_aux:www #1; #2; #3; { - \exp_args:Nf \str_substr_aux_ii:nnw + \exp_args:Nf \str_substr_aux:nnw { \str_aux_normalize_range:nn {#2} {#1} } { \str_aux_normalize_range:nn {#3} {#1} } } -\cs_new:Npn \str_substr_aux_ii:nnw #1#2 +\cs_new:Npn \str_substr_aux:nnw #1#2 { \str_skip_do:nn {#1} { @@ -1203,13 +1176,6 @@ { \use_i_delimit_by_q_stop:nw } } } -\cs_new:Npn \str_aux_eval_args:Nnnn #1#2#3#4 - { - \exp_after:wN #1 - \exp_after:wN { \int_value:w \int_eval:w #2 \exp_after:wN } - \exp_after:wN { \int_value:w \int_eval:w #3 \exp_after:wN } - \exp_after:wN { \int_value:w \int_eval:w #4 } - } \cs_new:Npn \str_aux_normalize_range:nn #1#2 { \int_eval:n @@ -1237,14 +1203,15 @@ % \end{macro} % \end{macro} % \end{macro} -% \end{macro} % % \subsection{String conditionals} % % \begin{macro}[EXP,pTF]{\str_if_eq:NN} % \begin{macro}[EXP,pTF]{\str_if_eq:nn,\str_if_eq:xx} % The \texttt{nn} and \texttt{xx} variants are already -% defined in \pkg{l3basics}. +% defined in \pkg{l3basics}. Note that \cs{str_if_eq:NN} +% is different from \cs{tl_if_eq:NN} because it needs to +% ignore category codes. % \begin{macrocode} \prg_new_conditional:Npnn \str_if_eq:NN #1#2 { p , TF , T , F } { @@ -1255,112 +1222,45 @@ % \end{macro} % \end{macro} % -%^^A treat spaces separately, or make that internal and faster. -% \begin{macro}[EXP,pTF]{\str_if_contains_char:NN} -% \begin{macro}[EXP,pTF]{\str_if_contains_char:nN} +% \begin{macro}[EXP]{\str_if_contains_char:NNT, \str_if_contains_char:NNTF} +% \begin{macro}[EXP]{\str_if_contains_char:nNTF} % \begin{macro}[EXP,aux]{\str_if_contains_char_aux:NN} -% \begin{macro}[EXP,aux]{\str_if_contains_char_end:w} +% \begin{macro}[EXP,aux]{\str_if_contains_char_end:} % Loop over the characters of the string, comparing character codes. % We allow |#2| to be a single-character control sequence, hence the % use of \cs{int_compare:nNnT} rather than \cs{token_if_eq_charcode:NNT}. % The loop is broken if character codes match. Otherwise we return % \enquote{false}. % \begin{macrocode} -\prg_new_conditional:Npnn \str_if_contains_char:NN #1#2 { p , T , F , TF } +\prg_new_conditional:Npnn \str_if_contains_char:NN #1#2 { T , TF } { - \str_map_tokens:Nn #1 { \str_if_contains_char_aux:NN #2 } + \exp_after:wN \str_if_contains_char_aux:NN \exp_after:wN #2 + #1 { \prg_map_break:n { ? \fi: } } + \prg_break_point:n { } \prg_return_false: } -\prg_new_conditional:Npnn \str_if_contains_char:nN #1#2 { p , T , F , TF } +\prg_new_conditional:Npnn \str_if_contains_char:nN #1#2 { TF } { - \str_map_tokens:nn {#1} { \str_if_contains_char_aux:NN #2 } + \exp_after:wN \str_if_contains_char_aux:NN \exp_after:wN #2 + \tl_to_str:n {#1} { \prg_map_break:n { ? \fi: } } + \prg_break_point:n { } \prg_return_false: } \cs_new_nopar:Npn \str_if_contains_char_aux:NN #1#2 { - \if_num:w `#1 = `#2 \exp_stop_f: - \str_if_contains_char_end:w + \if_charcode:w #1 #2 + \exp_after:wN \str_if_contains_char_end: \fi: + \str_if_contains_char_aux:NN #1 } -\cs_new_nopar:Npn \str_if_contains_char_end:w \fi: #1 \prg_return_false: - { \fi: \prg_return_true: } +\cs_new_nopar:Npn \str_if_contains_char_end: + { \prg_map_break:n { \prg_return_true: \use_none:n } } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % -% \begin{macro}[EXP,pTF]{\str_if_bytes:N} -% \begin{macro}[EXP,aux]{\str_if_bytes_aux:N} -% \begin{macro}[EXP,aux]{\str_if_bytes_aux:} -% In the pdf\TeX{} engine, every character is a byte, -% so there is nothing to check. In other engines, we -% loop over the string, checking if every character -% code is less than $256$. -% \begin{macrocode} -\pdftex_if_engine:TF - { - \cs_new_eq:NN \str_if_bytes_p:N \c_true_bool - \cs_new_eq:NN \str_if_bytes:NT \use_ii:nn - \cs_new_eq:NN \str_if_bytes:NF \use_none:nn - \cs_new_eq:NN \str_if_bytes:NTF \use_ii:nnn - } - { - \prg_new_conditional:Npnn \str_if_bytes:N #1 { p , T , F , TF } - { - \exp_args:Nf \tl_map_function:nN - { \tl_to_str:N #1 } - \str_if_bytes_aux:N - \prg_return_true: - } - \cs_new_nopar:Npn \str_if_bytes_aux:N #1 - { - \if_num:w `#1 > \c_two_hundred_fifty_five - \exp_after:wN \str_if_bytes_aux: - \fi: - } - \cs_new_nopar:Npn \str_if_bytes_aux: - { \tl_map_break:n { \prg_return_false: \use_none:n } } - } -% \end{macrocode} -% \end{macro} -% \end{macro} -% \end{macro} -% -% \subsection{Conversions} -% -% \subsubsection{Internal conditionals} -% -% \begin{macro}[aux]{\str_aux_hexadecimal_test:NTF} -% This test is used when reading hexadecimal digits, for the |\x| escape -% sequence. It returns \meta{true} if the -% token is a hexadecimal digit, and \meta{false} otherwise. It has the -% additional side-effect of updating the value of -% \cs{l_str_char_int} (number formed in base $16$ from the digits -% read so far). -%^^A todo: speed tweaks? -% \begin{macrocode} -\prg_new_protected_conditional:Npnn \str_aux_hexadecimal_test:N #1 { TF } - { - \tl_if_in:onTF { \tl_to_str:n { abcdef } } {#1} - { - \int_set:Nn \l_str_char_int - { \c_sixteen * \l_str_char_int + `#1 - 87 } - \prg_return_true: - } - { - \if_num:w \c_fifteen < "1 \exp_not:N #1 \exp_stop_f: - \int_set:Nn \l_str_char_int - { \c_sixteen * \l_str_char_int + "#1 } - \prg_return_true: - \else: - \prg_return_false: - \fi: - } - } -% \end{macrocode} -% \end{macro} -% % \begin{macro}[aux, rEXP]{\str_aux_octal_use:NTF} % \TeX{} dutifully detects octal digits for us: if |#1| % is an octal digit, then the right-hand side of the @@ -1370,7 +1270,7 @@ % \begin{macrocode} \prg_new_conditional:Npnn \str_aux_octal_use:N #1 { TF } { - \if_num:w \c_one < '1 #1 \exp_stop_f: + \if_num:w \c_one < '1 \token_to_str:N #1 \exp_stop_f: #1 \prg_return_true: \else: \prg_return_false: @@ -1387,27 +1287,35 @@ % \begin{macrocode} \prg_new_conditional:Npnn \str_aux_hexadecimal_use:N #1 { TF } { - \if_num:w \c_two < "1 #1 \exp_stop_f: + \if_num:w \c_two < "1 \token_to_str:N #1 \exp_stop_f: #1 \prg_return_true: \else: - \if_case:w \int_eval:w `#1 - `a \int_eval_end: - A \prg_return_true: - \or: B \prg_return_true: - \or: C \prg_return_true: - \or: D \prg_return_true: - \or: E \prg_return_true: - \or: F \prg_return_true: - \else: \prg_return_false: + \if_case:w \int_eval:w + \exp_after:wN ` \token_to_str:N #1 - `a + \int_eval_end: + A + \or: B + \or: C + \or: D + \or: E + \or: F + \else: + \prg_return_false: + \exp_after:wN \use_none:n \fi: + \prg_return_true: \fi: } % \end{macrocode} % \end{macro} % +% \subsection{Conversions} +% % \subsubsection{Producing one byte or character} % % \begin{variable}{\c_str_byte_0_tl} % \begin{variable}{\c_str_byte_-1_tl} +% \begin{variable}{\c_str_positive_bytes_tl} % For each integer $N$ in the range $[0,255]$, we create % a constant token list which holds three character tokens % with category code other: the character with character @@ -1415,8 +1323,11 @@ % two hexadecimal digits. % The value $-1$ is given a default token list which ensures % that later functions give an empty result for the input $-1$. +% Simultaneously, we build a list of all bytes (from which we +% remove the null byte) in \cs{c_str_positive_bytes_tl}. % \begin{macrocode} \group_begin: + \tl_clear:N \l_str_tmpa_tl \char_set_catcode_other:n { \c_zero } \tl_gset:Nx \g_str_result_tl { \tl_to_str:n { 0123456789ABCDEF } } \tl_map_inline:Nn \g_str_result_tl @@ -1425,20 +1336,24 @@ { \tl_map_inline:Nn \g_str_result_tl { - \char_set_lccode:nn { \c_zero } {"##1#1} + \char_set_lccode:nn { \c_zero } {"#1##1} \tl_to_lowercase:n { \tl_const:cx - { c_str_byte_ \int_eval:n {"##1#1} _tl } - { ^^@ ##1 #1 } + { c_str_byte_ \int_eval:n {"#1##1} _tl } + { ^^@ #1 ##1 } + \tl_put_right:Nn \l_str_tmpa_tl { ^^@ } } } } + \tl_const:Nx \c_str_positive_bytes_tl + { \exp_after:wN \use_none:n \l_str_tmpa_tl } \group_end: \tl_const:cn { c_str_byte_-1_tl } { { } \use_none:n { } } % \end{macrocode} % \end{variable} % \end{variable} +% \end{variable} % % \begin{macro}[int, EXP]{\str_output_byte:n} % \begin{macro}[aux, EXP]{\str_output_byte:w} @@ -1476,11 +1391,13 @@ % \end{macro} % \end{macro} % -% \subsubsection{}%^^A here +% \subsubsection{Mapping functions for encoding conversion} % % \begin{macro}{\str_aux_gmap_result:N} % \begin{macro}[aux, rEXP]{\str_aux_gmap_result_loop:NN} -% +% This maps the function |#1| over all characters +% in \cs{g_str_result_tl}, which should be a byte +% string in most cases, sometimes a native string. % \begin{macrocode} \cs_new_protected_nopar:Npn \str_aux_gmap_result:N #1 { @@ -1504,23 +1421,25 @@ % % \begin{macro}{\str_aux_gmap_internal_result:N} % \begin{macro}[aux, rEXP]{\str_aux_gmap_internal_result_loop:Nw} -% +% This maps the function |#1| over all character codes +% in \cs{g_str_result_tl}, which must be in the internal +% representation. % \begin{macrocode} \cs_new_protected_nopar:Npn \str_aux_gmap_internal_result:N #1 { \tl_gset:Nx \g_str_result_tl { - \exp_after:wN \str_aux_gmap_internal_result_loop:Nw + \exp_after:wN \str_aux_gmap_internal_result_loop:Nww \exp_after:wN #1 - \g_str_result_tl \q_stop \prg_map_break: , + \g_str_result_tl \q_mark \q_stop \prg_map_break: , \prg_break_point:n { } } } -\cs_new_nopar:Npn \str_aux_gmap_internal_result_loop:Nw #1#2, +\cs_new_nopar:Npn \str_aux_gmap_internal_result_loop:Nww #1#2\q_mark #3, { - \use_none_delimit_by_q_stop:w #2 \q_stop - #1 {#2} - \str_aux_gmap_internal_result_loop:Nw #1 + \use_none_delimit_by_q_stop:w #3 \q_stop + #1 {#3} + \str_aux_gmap_internal_result_loop:Nww #1 } % \end{macrocode} % \end{macro} @@ -1566,19 +1485,28 @@ % \end{macrocode} % \end{macro} % +% \begin{macro}[int]{\str_escape_use:NNNn} +% Use the internal \cs{str_aux_escape:NNNn}, then leave the result +% in the input stream. This is used in \pkg{l3regex}. +% \begin{macrocode} +\cs_new_protected:Npn \str_escape_use:NNNn #1#2#3#4 + { + \str_aux_escape:NNNn #1#2#3 {#4} + \g_str_result_tl + } +% \end{macrocode} +% \end{macro} +% % \begin{macro}[int]{\str_aux_escape:NNNn} -% \begin{macro}[aux]{\str_aux_escape_unescaped:N} -% \begin{macro}[aux]{\str_aux_escape_escaped:N} -% \begin{macro}[aux]{\str_aux_escape_raw:N} % \begin{macro}[aux]{\str_aux_escape_loop:N} % \begin{macro}[aux]+\str_aux_escape_\:w+ -% Treat the argument as an \meta{escaped string}, -% and store the corresponding \meta{string} globally -% in \cs{g_str_result_tl}. +% Most of the work is done within an \texttt{x}-expanding assignment, +% but the |\x| escape sequence cannot be done in that way. Therefore, +% we interrupt the assignment at each |\x| escape sequence, store the +% partial result in a \tn{toks} register, and at the end unpack all +% of the \tn{toks} registers to the left of the last chunk of +% \cs{g_str_result_tl}. % \begin{macrocode} -\cs_new_eq:NN \str_aux_escape_unescaped:N \use:n -\cs_new_eq:NN \str_aux_escape_escaped:N \use:n -\cs_new_eq:NN \str_aux_escape_raw:N \use:n \cs_new_protected:Npn \str_aux_escape:NNNn #1#2#3#4 { \group_begin: @@ -1586,12 +1514,15 @@ \cs_set_nopar:Npn \str_aux_escape_escaped:N { #2 } \cs_set_nopar:Npn \str_aux_escape_raw:N { #3 } \int_set:Nn \tex_escapechar:D { 92 } - \str_aux_gset_other:Nn \g_str_result_tl {#4} + \str_gset_other:Nn \g_str_result_tl {#4} + \int_zero:N \l_str_tmpa_int \tl_gset:Nx \g_str_result_tl { \exp_after:wN \str_aux_escape_loop:N \g_str_result_tl \q_recursion_tail \q_recursion_stop } + \tl_gput_left:Nx \g_str_result_tl + { \str_aux_toks_range:nn \c_zero \l_str_tmpa_int } \group_end: } \cs_new_nopar:Npn \str_aux_escape_loop:N #1 @@ -1611,6 +1542,17 @@ % \end{macro} % \end{macro} % \end{macro} +% +% \begin{macro}[aux]{\str_aux_escape_unescaped:N} +% \begin{macro}[aux]{\str_aux_escape_escaped:N} +% \begin{macro}[aux]{\str_aux_escape_raw:N} +% Those functions are never called before being given a new meaning, +% so their definitions here don't matter. +% \begin{macrocode} +\cs_new_eq:NN \str_aux_escape_unescaped:N \prg_do_nothing: +\cs_new_eq:NN \str_aux_escape_escaped:N \prg_do_nothing: +\cs_new_eq:NN \str_aux_escape_raw:N \prg_do_nothing: +% \end{macrocode} % \end{macro} % \end{macro} % \end{macro} @@ -1662,18 +1604,20 @@ % When |\x| is encountered, interrupt the assignment, % and distinguish the cases of a braced or unbraced syntax. % In the braced case, collect arbitrarily many hexadecimal digits, -% building the number in \cs{l_str_char_int} (this is a side effect -% of \cs{str_aux_hexadecimal_test:NTF}), and check that -% the run of digits was interrupted by a closing brace. +% building the number in \cs{l_str_char_int} (using +% \cs{str_aux_hexadecimal_use:NTF}), and check that +% the run of digits was stopped by a closing brace. % In the unbraced case, collect up to two hexadecimal digits, % possibly less, building the character number in \cs{l_str_char_int}. % In both cases, once all digits have been collected, use % the \TeX{} primitive \tn{lowercase} to produce that character, -% and use a \cs{if_false:} trick to restart the assignment. +% and use an \cs{if_false:} trick to restart the assignment. % \begin{macrocode} \cs_new_nopar:cpn { str_aux_escape_/x:w } \str_aux_escape_loop:N { \if_false: { \fi: } + \tex_toks:D \l_str_tmpa_int \exp_after:wN { \g_str_result_tl } + \int_incr:N \l_str_tmpa_int \int_zero:N \l_str_char_int \str_aux_escape_x_test:N } @@ -1682,35 +1626,36 @@ \exp_not:N \token_if_eq_charcode:NNTF \c_space_token #1 { \exp_not:N \str_aux_escape_x_test:N } { - \exp_not:N \token_if_eq_charcode:NNTF \c_lbrace_str #1 - { \exp_not:N \str_aux_escape_x_braced_loop:N } - { \exp_not:N \str_aux_escape_x_unbraced_i:N #1 } + \l_str_char_int = "0 + \exp_not:N \token_if_eq_charcode:NNTF \c_lbrace_str #1 + { \exp_not:N \str_aux_escape_x_braced_loop:N } + { \exp_not:N \str_aux_escape_x_unbraced_i:N #1 } } } -\cs_new_protected_nopar:Npn \str_aux_escape_x_unbraced_i:N #1 +\cs_new_nopar:Npn \str_aux_escape_x_unbraced_i:N #1 { - \str_aux_hexadecimal_test:NTF #1 + \str_aux_hexadecimal_use:NTF #1 { \str_aux_escape_x_unbraced_ii:N } - { \str_aux_escape_x_end: #1 } + { \exp_stop_f: \str_aux_escape_x_end: #1 } } -\cs_new_protected_nopar:Npn \str_aux_escape_x_unbraced_ii:N #1 +\cs_new_nopar:Npn \str_aux_escape_x_unbraced_ii:N #1 { \token_if_eq_charcode:NNTF \c_space_token #1 { \str_aux_escape_x_unbraced_ii:N } { - \str_aux_hexadecimal_test:NTF #1 - { \str_aux_escape_x_end: } - { \str_aux_escape_x_end: #1 } + \str_aux_hexadecimal_use:NTF #1 + { \exp_stop_f: \str_aux_escape_x_end: } + { \exp_stop_f: \str_aux_escape_x_end: #1 } } } -\cs_new_protected_nopar:Npn \str_aux_escape_x_braced_loop:N #1 +\cs_new_nopar:Npn \str_aux_escape_x_braced_loop:N #1 { \token_if_eq_charcode:NNTF \c_space_token #1 { \str_aux_escape_x_braced_loop:N } { - \str_aux_hexadecimal_test:NTF #1 + \str_aux_hexadecimal_use:NTF #1 { \str_aux_escape_x_braced_loop:N } - { \str_aux_escape_x_braced_end:N #1 } + { \exp_stop_f: \str_aux_escape_x_braced_end:N #1 } } } \cs_new_protected_nopar:Npx \str_aux_escape_x_braced_end:N #1 @@ -1729,7 +1674,7 @@ \tex_lccode:D \c_zero \l_str_char_int \tl_to_lowercase:n { - \tl_gput_right:Nx \g_str_result_tl + \tl_gset:Nx \g_str_result_tl { \if_false: } \fi: \str_aux_escape_raw:N ^^@ \str_aux_escape_loop:N @@ -1803,13 +1748,13 @@ % must be empty. Otherwise, the escaping |#2| is performed. % \begin{macrocode} \cs_new_protected:Npn \str_set_convert:Nnnn - { \str_convert_aux_i:NNnnn \str_set:Nx } + { \str_convert_aux_i:NNnnn \tl_set_eq:NN } \cs_new_protected:Npn \str_gset_convert:Nnnn - { \str_convert_aux_i:NNnnn \str_gset:Nx } + { \str_convert_aux_i:NNnnn \tl_gset_eq:NN } \cs_new_protected:Npn \str_convert_aux_i:NNnnn #1#2#3#4#5 { \group_begin: - \str_aux_gset_other:Nn \g_str_result_tl {#5} + \str_gset_other:Nn \g_str_result_tl {#5} \exp_after:wN \str_convert_aux_ii:wwwnn \tl_to_str:n {#3} /// \q_stop @@ -2002,11 +1947,6 @@ % \end{macrocode} % \end{macro} % -%^^A todo: document, move. -% \begin{macrocode} -\int_new:N \l_str_encoding_int -% \end{macrocode} -% % \begin{macro}[int]{\str_convert_from_eight_bit:n} % \begin{macro}[aux]{\str_convert_from_eight_bit_load:nn} % \begin{macro}[aux]{\str_convert_from_eight_bit_load_missing:n} @@ -2016,7 +1956,7 @@ \cs_new_protected:Npn \str_convert_from_eight_bit:n #1 { \group_begin: - \int_zero:N \l_str_encoding_int + \int_zero:N \l_str_tmpa_int \exp_last_unbraced:Nx \str_convert_from_eight_bit_load:nn { \tl_use:c { c_str_encoding_#1_tl } } { \q_stop \prg_map_break: } { } @@ -2031,24 +1971,25 @@ \cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load:nn #1#2 { \use_none_delimit_by_q_stop:w #1 \q_stop - \tex_dimen:D "#1 = \l_str_encoding_int sp \scan_stop: - \tex_skip:D \l_str_encoding_int = "#1 sp \scan_stop: - \tex_toks:D \l_str_encoding_int \exp_after:wN { \int_value:w "#2 } - \tex_advance:D \l_str_encoding_int \c_one + \tex_dimen:D "#1 = \l_str_tmpa_int sp \scan_stop: + \tex_skip:D \l_str_tmpa_int = "#1 sp \scan_stop: + \tex_toks:D \l_str_tmpa_int \exp_after:wN { \int_value:w "#2 } + \tex_advance:D \l_str_tmpa_int \c_one \str_convert_from_eight_bit_load:nn } \cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load_missing:n #1 { \use_none_delimit_by_q_stop:w #1 \q_stop - \tex_dimen:D "#1 = \l_str_encoding_int sp \scan_stop: - \tex_skip:D \l_str_encoding_int = "#1 sp \scan_stop: - \tex_toks:D \l_str_encoding_int \exp_after:wN { \int_value:w "FFFD } - \tex_advance:D \l_str_encoding_int \c_one + \tex_dimen:D "#1 = \l_str_tmpa_int sp \scan_stop: + \tex_skip:D \l_str_tmpa_int = "#1 sp \scan_stop: + \tex_toks:D \l_str_tmpa_int \exp_after:wN { \int_value:w "FFFD } + \tex_advance:D \l_str_tmpa_int \c_one \str_convert_from_eight_bit_load_missing:n } \cs_new:Npn \str_convert_from_eight_bit_aux:N #1 { - \if_num:w \tex_dimen:D `#1 < \l_str_encoding_int + #1 \exp_not:N \q_mark + \if_num:w \tex_dimen:D `#1 < \l_str_tmpa_int \if_num:w \tex_skip:D \tex_dimen:D `#1 = `#1 \exp_stop_f: \tex_the:D \tex_toks:D \tex_dimen:D \fi: @@ -2070,7 +2011,7 @@ \cs_new_protected:Npn \str_convert_to_eight_bit:n #1 { \group_begin: - \int_zero:N \l_str_encoding_int + \int_zero:N \l_str_tmpa_int \exp_last_unbraced:Nx \str_convert_to_eight_bit_load:nn { \tl_use:c { c_str_encoding_#1_tl } } { \q_stop \prg_map_break: } { } @@ -2081,17 +2022,17 @@ \cs_new_protected_nopar:Npn \str_convert_to_eight_bit_load:nn #1#2 { \use_none_delimit_by_q_stop:w #1 \q_stop - \tex_dimen:D "#2 = \l_str_encoding_int sp \scan_stop: - \tex_skip:D \l_str_encoding_int = "#2 sp \scan_stop: - \exp_args:NNf \tex_toks:D \l_str_encoding_int + \tex_dimen:D "#2 = \l_str_tmpa_int sp \scan_stop: + \tex_skip:D \l_str_tmpa_int = "#2 sp \scan_stop: + \exp_args:NNf \tex_toks:D \l_str_tmpa_int { \str_output_byte:n { "#1 } } - \tex_advance:D \l_str_encoding_int \c_one + \tex_advance:D \l_str_tmpa_int \c_one \str_convert_to_eight_bit_load:nn } \cs_new:Npn \str_convert_to_eight_bit_aux:n #1 { \if_num:w #1 < "8000 \exp_stop_f: - \if_num:w \tex_dimen:D #1 < \l_str_encoding_int + \if_num:w \tex_dimen:D #1 < \l_str_tmpa_int \if_num:w \tex_skip:D \tex_dimen:D #1 = #1 \exp_stop_f: \tex_the:D \tex_toks:D \tex_dimen:D #1 \exp_stop_f: \exp_after:wN \exp_after:wN \exp_after:wN \use_none:nn @@ -2100,7 +2041,7 @@ \str_convert_to_eight_bit_aux_ii:n {#1} \else: \msg_expandable_kernel_error:nnn - { str } { eight_bit-to-byte-overflow } {#1} + { str } { eight-bit-to-byte-overflow } {#1} \fi: } \cs_new:Npn \str_convert_to_eight_bit_aux_ii:n #1 @@ -2204,52 +2145,107 @@ \cs_new_protected_nopar:Npn \str_convert_from_native: { \str_aux_gmap_result:N \str_convert_from_native_aux:N } \cs_new:Npn \str_convert_from_native_aux:N #1 - { \int_value:w `#1 , } + { #1 \exp_not:N \q_mark \int_value:w `#1 , } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}[int]{\str_convert_to_native:} % \begin{macro}[aux]{\str_convert_to_native_aux:w} -% Convert the string from the given encoding -% to the internal format. We grab integers -% from the internal string using the low-level -% method to assign integers. -% Except at the end of the loop, the function -% \cs{str_convert_to_native_aux:w} is followed by a -% comma, hence its argument is empty. -% We are making heavy use of \TeX{} primitives here, -% because the function can become very slow for large -% strings. +% This function is based on the internal functions +% \cs{str_aux_convert_to_native:n} and +% \cs{str_aux_convert_to_native_step:n}. +% Loop through the comma-list, and call +% \cs{str_aux_convert_to_native_step:n} +% with the relevant integer argument. +% We need to store \cs{g_str_result_tl} in a temporary +% token list, because that global \enquote{result} is +% cleared silently by \cs{str_aux_convert_to_native:n}. +%^^A todo: update % \begin{macrocode} \cs_new_protected_nopar:Npn \str_convert_to_native: { \tl_set_eq:NN \l_str_tmpa_tl \g_str_result_tl + \str_aux_convert_to_native:n + { + \exp_after:wN \str_convert_to_native_aux:w + \l_str_tmpa_tl \q_mark { \q_stop \prg_map_break: } , + } + } +\cs_new_protected_nopar:Npn \str_convert_to_native_aux:w #1\q_mark #2, + { + \use_none_delimit_by_q_stop:w #2 \q_stop + \str_aux_convert_to_native_step:n {#2} + \str_convert_to_native_aux:w + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int]{\str_aux_convert_to_native:n} +% \begin{macro}[int]{\str_aux_convert_to_native_step:n} +% \begin{macro}[aux]{\str_aux_convert_to_native_flush:} +% \begin{macro}[aux, rEXP]{\str_aux_convert_to_native_filter:N} +%^^A todo: document. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \str_aux_convert_to_native:n #1 + { + \tex_lccode:D \c_zero \c_zero + \int_zero:N \l_str_tmpa_int \tl_gclear:N \g_str_result_tl - \tex_afterassignment:D \str_convert_to_native_aux:w - \l_str_char_int = \l_str_tmpa_tl \c_zero \prg_map_break: , - \prg_break_point:n { } + #1 + \prg_break_point:n { \str_aux_convert_to_native_flush: } + } +\cs_new_protected_nopar:Npn \str_aux_convert_to_native_step:n #1 + { + \if_num:w \l_str_tmpa_int = \c_two_hundred_fifty_five + \str_aux_convert_to_native_flush: + \l_str_tmpa_int \c_zero + \fi: + \tex_advance:D \l_str_tmpa_int \c_one + \if_int_compare:w #1 > \c_max_char_int + \msg_kernel_error:nnx { str } { overflow } {#1} + \tex_lccode:D \l_str_tmpa_int \c_str_replacement_char_int + \else: + \tex_lccode:D \l_str_tmpa_int \int_eval:w #1 \int_eval_end: + \fi: + } +\cs_new_protected_nopar:Npn \str_aux_convert_to_native_flush: + { + \use:x + { + \tl_to_lowercase:n + { + \tl_gput_right:Nx \exp_not:N \g_str_result_tl + { + \exp_after:wN \str_aux_convert_to_native_filter:N + \c_str_positive_bytes_tl + { ? = \c_zero \fi: \prg_map_break: } + \prg_break_point:n { } + } + } + } } \group_begin: - \char_set_catcode_other:n { `\^^@ } - \cs_new_protected_nopar:Npn \str_convert_to_native_aux:w #1 , + \char_set_catcode_other:n { 0 } + \cs_new:Npn \str_aux_convert_to_native_filter:N #1 { - #1 - \if_int_compare:w \l_str_char_int > \c_max_char_int - \msg_kernel_error:nnx { str } { overflow } - { \int_use:N \l_str_char_int } + \if_num:w `#1 > \l_str_tmpa_int + \exp_after:wN \prg_map_break: + \fi: + \if_num:w \tex_lccode:D `#1 = \c_zero + ^^@ \else: - \tex_lccode:D \c_zero \l_str_char_int - \tl_to_lowercase:n - { \tex_xdef:D \g_str_result_tl { \g_str_result_tl ^^@ } } + #1 \fi: - \tex_afterassignment:D \str_convert_to_native_aux:w - \l_str_char_int = + \str_aux_convert_to_native_filter:N } \group_end: % \end{macrocode} % \end{macro} % \end{macro} +% \end{macro} +% \end{macro} % % \subsection{Messages} % @@ -2331,10 +2327,8 @@ % as per the pdf\TeX{} primitive \tn{pdfescapehex} % \item \texttt{name}, as per the pdf\TeX{} primitive \tn{pdfescapename} % \item \texttt{string}, as per the pdf\TeX{} primitive \tn{pdfescapestring} -% \item \texttt{url} or \texttt{percent}, as per the percent encoding of urls. +% \item \texttt{url}, as per the percent encoding of urls. % \end{itemize} -%^^A todo: in pdfTeX, use the \tn{pdfescape...} -%^^A todo: add various shell escapes, lua escapes etc? % % \subsubsection{Input methods} % @@ -2353,13 +2347,16 @@ %<*hex> \cs_new_protected_nopar:Npn \str_convert_input_hex: { - \tl_gset:Nx \g_str_result_tl - { - \str_output_byte:w " - \exp_after:wN \str_convert_input_hex_aux:N - \g_str_result_tl 0 { ? 0 - \c_one \prg_map_break: } - \prg_break_point:n { \str_output_end: } - } + \group_begin: + \int_set:Nn \tex_escapechar:D { 92 } + \tl_gset:Nx \g_str_result_tl + { + \str_output_byte:w " + \exp_after:wN \str_convert_input_hex_aux:N + \g_str_result_tl 0 { ? 0 - \c_one \prg_map_break: } + \prg_break_point:n { \str_output_end: } + } + \group_end: } \cs_new_nopar:Npn \str_convert_input_hex_aux:N #1 { @@ -2388,7 +2385,6 @@ % \begin{macro}[int]{\str_convert_input_name:} % \begin{macro}[aux, rEXP]{\str_convert_input_name_aux:wNN} % \begin{macro}[int]{\str_convert_input_url:} -% \begin{macro}[int]{\str_convert_input_percent:} % \begin{macro}[aux, rEXP]{\str_convert_input_url_aux:wNN} % The \cs{str_convert_input_name:} function replaces each % occurrence of |#| followed by two hexadecimal digits @@ -2417,12 +2413,15 @@ { \cs_new_protected_nopar:Npn #2 { - \tl_gset:Nx \g_str_result_tl - { - \exp_after:wN #3 \g_str_result_tl - #1 ? { ? \prg_map_break: } - \prg_break_point:n { } - } + \group_begin: + \int_set:Nn \tex_escapechar:D { 92 } + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN #3 \g_str_result_tl + #1 ? { ? \prg_map_break: } + \prg_break_point:n { } + } + \group_end: } \cs_new_nopar:Npn #3 ##1#1##2##3 { @@ -2450,7 +2449,6 @@ % \end{macro} % \end{macro} % \end{macro} -% \end{macro} % % \begin{macro}[int]{\str_convert_input_string:} % \begin{macro}[aux, rEXP]{\str_convert_input_string_aux:wN} @@ -2488,18 +2486,21 @@ { \cs_new_protected_nopar:Npn \str_convert_input_string: { - \tl_gset:Nx \g_str_result_tl - { - \exp_after:wN \str_convert_input_string_aux:wN - \g_str_result_tl \prg_map_break: ^^M ? - \prg_break_point:n { } - } - \tl_gset:Nx \g_str_result_tl - { - \exp_after:wN \str_convert_input_string_aux:wNNN - \g_str_result_tl * ?? { ? \prg_map_break: } - \prg_break_point:n { } - } + \group_begin: + \int_set:Nn \tex_escapechar:D { 92 } + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN \str_convert_input_string_aux:wN + \g_str_result_tl \prg_map_break: ^^M ? + \prg_break_point:n { } + } + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN \str_convert_input_string_aux:wNNN + \g_str_result_tl * ?? { ? \prg_map_break: } + \prg_break_point:n { } + } + \group_end: } \cs_new_nopar:Npn \str_convert_input_string_aux:wNNN #1 *#2#3#4 } @@ -2670,7 +2671,6 @@ % \end{macro} % % \begin{macro}[int]{\str_convert_output_url:} -% \begin{macro}[int]{\str_convert_output_percent:} % \begin{macro}[aux, rEXP]{\str_convert_output_url_aux:N} % \begin{macro}[aux, rEXP]{\str_convert_output_url_aux:NTF} %^^A todo: Figure out what bytes should be percent encoded! Safe could be @@ -2705,7 +2705,6 @@ % \end{macro} % \end{macro} % \end{macro} -% \end{macro} % % \subsection{Encoding definition files} % @@ -2790,32 +2789,33 @@ \tl_gset:Nx \g_str_result_tl { \exp_after:wN \str_aux_from_utf_viii:N \g_str_result_tl - { ? \prg_map_break: \q_stop \str_aux_from_utf_viii_error: } + { \prg_map_break: \str_aux_from_utf_viii_error: } \prg_break_point:n { } } } \cs_new_nopar:Npn \str_aux_from_utf_viii:N #1 { - \use_none:n #1 - \int_use:N \int_eval:w - \if_num:w `#1 < "C0 \exp_stop_f: - \if_num:w `#1 < "80 \exp_stop_f: - `#1 - \else: - \msg_expandable_kernel_error:nnn - { str } { utf8-extra-conti } {#1} - \c_str_replacement_char_int - \fi: + #1 + \if_num:w `#1 < "C0 \exp_stop_f: + \exp_not:N \q_mark + \if_num:w `#1 < "80 \exp_stop_f: + \int_value:w `#1 \else: - \if_num:w `#1 < "F5 \exp_stop_f: - \exp_after:wN \str_aux_from_utf_viii:wwNN - \int_use:N \int_eval:w `#1 - "C0 - \else: - \msg_expandable_kernel_error:nnn - { str } { utf8-invalid-byte } {#1} - \c_str_replacement_char_int - \fi: + \msg_expandable_kernel_error:nnn + { str } { utf8-extra-conti } {#1} + \int_use:N \c_str_replacement_char_int \fi: + \else: + \if_num:w `#1 < "F5 \exp_stop_f: + \exp_after:wN \str_aux_from_utf_viii:wwNN + \int_use:N \int_eval:w `#1 - "C0 + \else: + \exp_not:N \q_mark + \msg_expandable_kernel_error:nnn + { str } { utf8-invalid-byte } {#1} + \int_use:N \c_str_replacement_char_int + \fi: + \fi: , \use_none_delimit_by_q_stop:w "80 , "800 , "10000 , "10FFFF , @@ -2824,7 +2824,7 @@ } \cs_new_nopar:Npn \str_aux_from_utf_viii:wwNN #1 , #2 \q_stop #3#4 { - \use_none_delimit_by_q_stop:w #4 \q_stop + \use_none:n #4 \exp_after:wN \str_aux_from_utf_viii:wNNww \int_use:N \int_eval:w #1 * "40 + `#4 - "80 , #4 @@ -2837,8 +2837,9 @@ \if_num:w #1 < #4 \exp_stop_f: \str_aux_from_utf_viii_error:w #1 , \else: + #2 \if_num:w #1 < #5 \exp_stop_f: - #1 + \exp_not:N \q_mark #1 \else: \exp_after:wN \str_aux_from_utf_viii:wwNN \int_use:N \int_eval:w #1 - #5 @@ -2852,12 +2853,14 @@ } \cs_new_nopar:Npn \str_aux_from_utf_viii_error:w #1 , { + \exp_not:N \q_mark \msg_expandable_kernel_error:nnn { str } { utf8-overlong } {#1} \c_str_replacement_char_int } \cs_new_nopar:Npn \str_aux_from_utf_viii_error:NwN #1 #2 \q_stop #3 { + \exp_not:N \q_mark \msg_expandable_kernel_error:nnn { str } { utf8-missing-conti } {#1} \c_str_replacement_char_int , @@ -2865,6 +2868,7 @@ } \cs_new_nopar:Npn \str_aux_from_utf_viii_error: { + \exp_not:N \q_mark \msg_expandable_kernel_error:nn { str } { utf8-premature-end } \c_str_replacement_char_int , @@ -3033,13 +3037,13 @@ \if_meaning:w \scan_stop: #2 \str_aux_from_utf_xvi_end:Nw #1 \fi: - \int_use:N \int_eval:w - \if_case:w - \int_eval:w ( \str_tmp:w #1#2 - "D6 ) / \c_four \int_eval_end: - \or: \exp_after:wN \str_aux_from_utf_xvi:wNNN - \or: \exp_after:wN \str_aux_from_utf_xvi_error:wNNw - \fi: - "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 , + \if_case:w + \int_eval:w ( \str_tmp:w #1#2 - "D6 ) / \c_four \int_eval_end: + \or: #1#2 \exp_after:wN \str_aux_from_utf_xvi:wNNN + \or: \exp_after:wN \str_aux_from_utf_xvi_error:wNNw + \else: #1#2 \exp_not:N \q_mark \int_use:N \int_eval:w + \fi: + "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 , \str_aux_from_utf_xvi:NN } \cs_new:Npn \str_aux_from_utf_xvi:wNNN #1, #2#3#4 @@ -3048,8 +3052,10 @@ \str_aux_from_utf_xvi_end:wNNww #1 , #3 \fi: \if_num:w \int_eval:w ( \str_tmp:w #3#4 - "DA ) / \c_four = \c_one - ( #1 - "D800 ) * "400 - + \str_tmp:w #3#4 * "100 + \str_tmp:w #4#3 - "DC00 , + #3 #4 + \int_use:N \int_eval:w + ( #1 - "D800 ) * "400 + + \str_tmp:w #3#4 * "100 + \str_tmp:w #4#3 - "DC00 , \exp_after:wN \use_i:nnn \else: \str_aux_from_utf_xvi_error:wNNw #1 , @@ -3058,6 +3064,7 @@ } \cs_new:Npn \str_aux_from_utf_xvi_error:wNNw #1 \str_tmp:w #2#3 #4, { + \exp_not:N \q_mark \msg_expandable_kernel_error:nnn { str } { utf16-surrogate } { #2#3 } \c_str_replacement_char_int , @@ -3067,15 +3074,17 @@ \fi: \if_meaning:w \scan_stop: #1 \else: + #1 \exp_not:N \q_mark \msg_expandable_kernel_error:nnn { str } { utf16-odd } {#1} - \c_str_replacement_char_int , + \int_use:N \c_str_replacement_char_int , \fi: \prg_map_break: } \cs_new:Npn \str_aux_from_utf_xvi_end:wNNww #1 `#2#3 #4, #5 \fi: { \fi: + \exp_not:N \q_mark \if_meaning:w \scan_stop: #5 \msg_expandable_kernel_error:nnn { str } { utf16-surrogate } { #2#3 } @@ -3083,7 +3092,7 @@ \msg_expandable_kernel_error:nnn { str } { utf16-odd } { #2#3#5 } \fi: - \c_str_replacement_char_int , + \int_use:N \c_str_replacement_char_int , \prg_map_break: } % \end{macrocode} @@ -3215,31 +3224,33 @@ \cs_new:Npn \str_aux_from_utf_xxxii:NNNN #1#2#3#4 { \if_meaning:w \scan_stop: #4 - \str_aux_from_utf_xxxii_end:ww #1#2#3#4 + \exp_after:wN \str_aux_from_utf_xxxii_end:w + \fi: + #1#2#3#4 \exp_not:N \q_mark + \if_num:w \int_eval:w + \str_tmp:w #1#4 * \c_two_hundred_fifty_six + + \str_tmp:w #2#3 + > \c_sixteen + \msg_expandable_kernel_error:nnn + { str } { utf32-overflow } { #1#2#3#4 } + \int_use:N \c_str_replacement_char_int + \else: + \int_use:N \int_eval:w + \str_tmp:w #2#3*"10000 + + \str_tmp:w #3#2 * \c_two_hundred_fifty_six + + \str_tmp:w #4#1 \fi: - \int_use:N \int_eval:w - \if_num:w \int_eval:w - \str_tmp:w #1#4 * \c_two_hundred_fifty_six - + \str_tmp:w #2#3 - > \c_sixteen - \msg_expandable_kernel_error:nnn - { str } { utf32-overflow } { #1#2#3#4 } - \c_str_replacement_char_int - \else: - \str_tmp:w #2#3*"10000 - + \str_tmp:w #3#2 * \c_two_hundred_fifty_six - + \str_tmp:w #4#1 - \fi: , \str_aux_from_utf_xxxii:NNNN } - \cs_new:Npn \str_aux_from_utf_xxxii_end:ww #1 \scan_stop: #2 \fi: + \cs_new:Npn \str_aux_from_utf_xxxii_end:w #1 \scan_stop: { - \fi: \tl_if_empty:nF {#1} { \msg_expandable_kernel_error:nnn { str } { utf32-truncated } { #1 } + #1 \exp_not:N \q_mark + \int_use:N \c_str_replacement_char_int , } \prg_map_break: } diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins index cb3e893d52c..55b4066a306 100644 --- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins @@ -35,8 +35,10 @@ Do not distribute a modified version of this file. \keepsilent +\generate{\file{l3flag.sty} {\from{l3flag.dtx} {package}}} \generate{\file{l3str.sty} {\from{l3str.dtx} {package}}} \generate{\file{l3regex.sty} {\from{l3regex.dtx} {package}}} +\generate{\file{l3tl-analysis.sty} {\from{l3tl-analysis.dtx} {package}}} %\generate{\file{l3regex-trace.sty} {\from{l3regex.dtx} {package,trace}}} % Escapings. diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx new file mode 100644 index 00000000000..c3310545ac0 --- /dev/null +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx @@ -0,0 +1,901 @@ +% \iffalse meta-comment +% +%% File: l3tl-analysis.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +% +%<*driver|package> +\RequirePackage{expl3} +\GetIdInfo$Id: l3tl-analysis.dtx 3039 2011-12-08 09:22:35Z bruno $ + {L3 Experimental Analysis of Token lists} +%</driver|package> +%<*driver> +\documentclass[full]{l3doc} +\usepackage{amsmath} +\begin{document} + \DocInput{\jobname.dtx} +\end{document} +%</driver> +% \fi +% +% +% \title{^^A +% The \textsf{l3tl-analysis} package: analysing token lists^^A +% \thanks{This file describes v\ExplFileVersion, +% last revised \ExplFileDate.}^^A +% } +% +% \author{^^A +% The \LaTeX3 Project\thanks +% {^^A +% E-mail: +% \href{mailto:latex-team@latex-project.org} +% {latex-team@latex-project.org}^^A +% }^^A +% } +% +% \date{Released \ExplFileDate} +% +% \maketitle +% +% \begin{documentation} +% +% \section{\pkg{l3tl-analysis} documentation} +% +% This module moslty provides internal functions for use +% in the \pkg{l3regex} module. However, it provides as a +% side-effect a user debugging function, very similar to +% the \cs{ShowTokens} macro from the \pkg{ted} package. +% +% \begin{function}{\tl_show_analysis:N, \tl_show_analysis:n} +% \begin{syntax} +% \cs{tl_show_analysis:n} \Arg{token list} +% \end{syntax} +% Displays to the terminal the detailed decomposition of +% the \meta{token list} into tokens, +% showing the category code of each character token, +% and the meaning of control sequences and active characters. +% \end{function} +% +% \subsection{Internal} +% +% \begin{function}{\tl_set_analysis:Nn} +% \begin{syntax} +% \cs{tl_analysis:Nn} \meta{tl var} \Arg{token list} +% \end{syntax} +% This function analysiss each token in the \meta{token list}, +% converting each \meta{token} to +% \begin{quote} +% \Arg{tokens} \cs{q_mark} \meta{catcode} \meta{char code} |,| +% \end{quote} +% where \meta{tokens} expands in one step to the \meta{token}, +% and \meta{catcode} and \meta{char code} are the category code +% and character code of the token if it is a character token, +% and otherwise are $0$ and $-1$ respectively. +% The result of the conversion is then stored in \meta{tl var}. +% \end{function} +% +% \end{documentation} +% +% \begin{implementation} +% +% \section{\pkg{l3tl-analysis} implementation} +% +% \begin{macrocode} +%<*initex|package> +% \end{macrocode} +% +% \begin{macrocode} +\ProvidesExplPackage + {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription} +\RequirePackage{l3str} +% \end{macrocode} +% +% \subsection{Variables and helper functions} +% +% \begin{variable}{\l_tl_analysis_normal_int} +% The number of normal (\texttt{N}-type argument) tokens +% since the last special token. +% \begin{macrocode} +\int_new:N \l_tl_analysis_normal_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_tl_analysis_index_int} +% During the first pass, this is the index in the array being built. +% During the second pass, it is equal to the maximum index in the +% array from the first pass. +% \begin{macrocode} +\int_new:N \l_tl_analysis_index_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_tl_analysis_nesting_int} +% Nesting depth of explicit begin-group and end-group characters +% during the first pass. This lets us detect the end of the token +% list without a reserved end-marker. +% \begin{macrocode} +\int_new:N \l_tl_analysis_nesting_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_tl_analysis_char_int} +% Holds the character code of the token currently considered, +% in cases where it is a character. +% \begin{macrocode} +\int_new:N \l_tl_analysis_char_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_tl_analysis_type_int} +% When encountering special characters, we record their \enquote{type} +% in this integer. +% \begin{macrocode} +\int_new:N \l_tl_analysis_type_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\g_tl_analysis_result_tl} +% The result of the conversion is stored in this token list, +% with a succession of items of the form +% \begin{quote} +% \meta{tokens} \cs{q_mark} \meta{catcode} \meta{char code} |,| +% \end{quote} +% \begin{macrocode} +\tl_new:N \g_tl_analysis_result_tl +% \end{macrocode} +% \end{variable} +% +% \begin{macro}[int]{\tl_analysis_extract_charcode:} +% \begin{macro}[aux]{\tl_analysis_extract_charcode_aux:w} +% Extracting the character code from the meaning of \cs{l_tl_analysis_token}. +% This has no error checking, and should only be assumed to work +% for begin-group and end-group tokens. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \tl_analysis_extract_charcode: + { + \exp_after:wN \tl_analysis_extract_charcode_aux:w + \token_to_meaning:N \l_tl_analysis_token + } +\cs_new_protected_nopar:Npn \tl_analysis_extract_charcode_aux:w #1 ~ #2 ~ { ` } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int]{\tl_analysis_cs_space_count:NN} +% \begin{macro}[aux]{\tl_analysis_cs_space_count:w} +% \begin{macro}[aux]{\tl_analysis_cs_space_count_end:w} +% Counts the number of spaces in the string representation of its +% second argument, as well as the number of characters following +% the last space in that representation, and feeds the two numbers +% as semicolon-delimited arguments to the first argument. +% \begin{macrocode} +\cs_new:Npn \tl_analysis_cs_space_count:NN #1 #2 + { + \exp_after:wN #1 + \int_value:w \int_eval:w \c_zero + \exp_after:wN \tl_analysis_cs_space_count:w + \token_to_str:N #2 + \fi: \tl_analysis_cs_space_count_end:w ; ~ ! + } +\cs_new:Npn \tl_analysis_cs_space_count:w #1 ~ + { + \if_false: #1 #1 \fi: + + \c_one + \tl_analysis_cs_space_count:w + } +\cs_new:Npn \tl_analysis_cs_space_count_end:w ; #1 \fi: #2 ! + { \exp_after:wN ; \int_value:w \str_length_ignore_spaces:n {#1} ; } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \subsection{Plan of attack} +% +% Our goal is to produce a token list of the form roughly +% \begin{quote} +% \Arg{token 1} \cs{q_mark} \meta{catcode 1} \meta{char code 1} |,| \\ +% \Arg{token 2} \cs{q_mark} \meta{catcode 2} \meta{char code 2} |,| \\ +% \ldots{} +% \Arg{token N} \cs{q_mark} \meta{catcode N} \meta{char code N} |,| +% \end{quote} +% Most but not all tokens can be grabbed as an undelimited +% (\texttt{N}-type) argument by \TeX{}. +% The plan is to have a two pass system. In the first pass, +% locate special tokens, and store them in various \tn{toks} registers. +% In the second pass, which is done within an \texttt{x}-expanding +% assignment, normal tokens are taken in as \texttt{N}-type +% arguments, and special tokens are retrieved from the \tn{toks} +% registers, and removed from the input stream by some means. +% +% To ease the difficult first pass, we first do some setup +% with \cs{tl_analysis_i_setup:n}. +% Active characters set equal to non-active characters +% cause trouble, so we disable all active characters by +% setting them equal to \texttt{undefined}. +% We also set there the escape character to be printable +% (backslash, but this later changes to a forward slash): +% this makes it possible to distinguish characters from +% control sequences. +% +% A token has two characteristics: its \tn{meaning}, +% and what it looks like for \TeX{} when it is in +% scanning mode (\emph{e.g.}, when capturing parameters +% for a macro). +% For our purposes, the token's meaning can be +% \begin{itemize} +% \item a begin-group token (category code $1$), +% either space (character code $32$), or non-space; +% \item an end-group token (category code $2$), +% either space (character code $32$), or non-space; +% \item a space token (category code $10$, character code $32$); +%^^A \item a character with category code $3$, $4$, $6$, $7$, $8$, $11$, +%^^A or $12$, or with category code $10$ but character code $\neq 32$; +%^^A \item a non-character (primitive, macro, register, font, +%^^A undefined, \emph{etc.}). +% \item anything else (then the token is always an \texttt{N}-type argument). +% \end{itemize} +% The token itself can \enquote{look like} one of the following +% \begin{itemize} +% \item a non-active character, in which case its meaning +% is automatically that associated to its character code +% and category code, we call it \enquote{true} character; +% \item an active character; +% \item a control sequence. +% \end{itemize} +% The only tokens which are not valid \texttt{N}-type arguments are +% true begin-group characters, true end-group characters, and true spaces. +% We will detect those characters by scanning ahead with \tn{futurelet}, +% then distinguishing true characters from tokens set equal to them +% using the \tn{string} representation. +% +% The second pass is a simple exercise in expandable loops. +% +% \begin{macro}[int]{\tl_analysis:n} +% \begin{macro}[int]{\tl_set_analysis:Nn} +% Everything is done within a group, and all definitions will be local. +% We use \cs{group_align_safe_begin/end:} to avoid problems in case +% \cs{tl_analysis:n} is used within an alignment and its argument +% contains alignment tab tokens. +% \begin{macrocode} +\cs_new_protected:Npn \tl_analysis:n #1 + { + \group_begin: + \group_align_safe_begin: + \tl_analysis_setup:n {#1} + \tl_analysis_i:n {#1} + \tl_analysis_ii:n {#1} + \group_align_safe_end: + \group_end: + } +\cs_new_protected:Npn \tl_set_analysis:Nn #1#2 + { + \tl_analysis:n {#2} + \tl_set_eq:NN #1 \g_tl_analysis_result_tl + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \subsection{Setup} +% +% \begin{macro}[int]{\tl_analysis_setup:n} +% \begin{macro}[aux]{\tl_analysis_disable_loop:N} +% Active characters can cause problems later on in the processing, +% so the first step is to disable them, by setting them to +% \texttt{undefined}. Since Unicode contains too many characters +% to loop over all of them, we instead loop over the input token +% list as a string: any active character in the token list +% must appear in its string representation. +% ^^A todo: ask LuaTeX list for an \ifx\undefined <active char> +% ^^A which does not add the <active char> in memory. +% \begin{macrocode} +\cs_new_protected:Npn \tl_analysis_setup:n #1 + { + \int_set:Nn \tex_escapechar:D { 92 } + \exp_after:wN \tl_analysis_disable_loop:N + \tl_to_str:n {#1} { ~ } { ? ~ \prg_map_break: } + \prg_break_point:n { } + } +\group_begin: + \char_set_catcode_active:N \^^@ + \cs_new_protected_nopar:Npn \tl_analysis_disable_loop:N #1 + { + \tex_lccode:D \c_zero `#1 ~ + \tl_to_lowercase:n { \tex_let:D ^^@ } \c_undefined:D + \tl_analysis_disable_loop:N + } +\group_end: +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \subsection{First pass} +% +% The goal of this pass is to detect special (non-\texttt{N}-type) tokens, +% and count how many \texttt{N}-type tokens lie between special tokens. +% Also, we wish to store some representation of each special token +% in a \tn{toks} register. +% +% After the setup step, we have $11$ types of tokens: +% \begin{itemize} +% \item[1] a true non-space begin-group character; +% \item[2] a true space begin-group character; +% \item[3] a true non-space end-group character; +% \item[4] a true space end-group character; +% \item[5] a true space blank space character; +% \item[6] an undefined active character; +% \item[7] any other true character; +% \item[8] a control sequence equal to a begin-group token (category code $1$); +% \item[9] a control sequence equal to an end-group token (category code $2$); +% \item[10] a control sequence equal to a space token +% (character code $32$, category code $10$); +% \item[11] any other control sequence. +% \end{itemize} +% Our first tool is \tn{futurelet}. This cannot distinguish +% cases $1$ and $2$ from $8$, nor cases $3$ and $4$ from $9$, +% nor case $5$ from case $10$. Those cases will be distinguished +% by applying the \tn{string} primitive to the following token, +% after possibly changing the escape character to ensure that +% a control sequence's string representation cannot be mistaken +% for the true character. +% +% In cases $6$, $7$, and $11$, the following token is a valid +% \texttt{N}-type argument, so we grab it and distinguish the case +% of a character from a control sequence: in the latter case, +% \cs{str_tail:n} \Arg{token} is non-empty, because the +% escape character is printable. +% +% \begin{macro}{\tl_analysis_i:n} +% We read tokens one by one using \tn{futurelet}. +% While performing the loop, we keep track of the number of +% true begin-group characters minus the number of +% true end-group characters in \cs{l_tl_analysis_nesting_int}. +% This reaches $-1$ when we read the closing brace. +% \begin{macrocode} +\cs_new_protected:Npn \tl_analysis_i:n #1 + { + \int_zero:N \l_tl_analysis_normal_int + \int_zero:N \l_tl_analysis_index_int + \int_zero:N \l_tl_analysis_nesting_int + \if_false: { \fi: \tl_analysis_i_loop:w #1 } + \int_decr:N \l_tl_analysis_index_int + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int]{\tl_analysis_i_loop:w} +% Read one character and check its type. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \tl_analysis_i_loop:w + { \tex_futurelet:D \l_tl_analysis_token \tl_analysis_i_type:w } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int]{\tl_analysis_i_type:w} +% At this point, \cs{l_tl_analysis_token} holds the meaning of the following +% token. We store in \cs{l_tl_analysis_type_int} the meaning of the token ahead: +% \begin{itemize} +% \item[0] space token; +% \item[1] begin-group token; +% \item[-1] end-group token; +% \item[2] other. +% \end{itemize} +% The values $0$, $1$, $-1$ correspond to how much a true such character +% changes the nesting level ($2$ is used only here, and is irrelevant later). +% Then call the auxiliary for each case. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \tl_analysis_i_type:w + { + \l_tl_analysis_type_int = + \if_meaning:w \l_tl_analysis_token \c_space_token + \c_zero + \else: + \if_catcode:w \exp_not:N \l_tl_analysis_token \c_group_begin_token + \c_one + \else: + \if_catcode:w \exp_not:N \l_tl_analysis_token \c_group_end_token + \c_minus_one + \else: + \c_two + \fi: + \fi: + \fi: + \if_case:w \l_tl_analysis_type_int + \exp_after:wN \tl_analysis_i_space:w + \or: \exp_after:wN \tl_analysis_i_bgroup:w + \or: \exp_after:wN \tl_analysis_i_safe:N + \else: \exp_after:wN \tl_analysis_i_egroup:w + \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int]{\tl_analysis_i_safe:N} +% \begin{macro}[aux]{\tl_analysis_i_cs:ww} +% The upcoming token is safe: we can simply grab it in a second pass. +% Since other branches of the code must pass their tokens through +% \tn{string}, we do it here as well. It's been optimized in an evil way, +% sorry. +%^^A todo: document +% \begin{macrocode} +\cs_new_protected:Npn \tl_analysis_i_safe:N #1 + { + \if_charcode:w + \scan_stop: + \exp_after:wN \use_none:n \token_to_str:N #1 \prg_do_nothing: + \scan_stop: + \int_incr:N \l_tl_analysis_normal_int + \else: + \tl_analysis_cs_space_count:NN \tl_analysis_i_cs:ww #1 + \fi: + \tl_analysis_i_loop:w + } +\cs_new_protected:Npn \tl_analysis_i_cs:ww #1; #2; + { + \if_num:w #1 > \c_zero + \tex_skip:D \l_tl_analysis_index_int + = \int_eval:w \l_tl_analysis_normal_int + \c_one sp \scan_stop: + \tex_advance:D \l_tl_analysis_index_int #1 \exp_stop_f: + \l_tl_analysis_normal_int #2 \exp_stop_f: + \else: + \tex_advance:D \l_tl_analysis_normal_int #2 \exp_stop_f: + \fi: + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[aux]{\tl_analysis_i_space:w} +% \begin{macro}[aux]{\tl_analysis_i_space_test:w} +% In this branch, the following token's meaning is a blank space. +% Apply \tn{string} to that token: if it is a control sequence +% the result starts with the escape character; otherwise it is +% a true blank space, whose string representation is also a blank space. +% We test for that in \cs{tl_analysis_i_space_test:w}, +% after grabbing the first character of the string representation +% as \cs{l_tl_analysis_token}. +% Also, since \cs{tl_analysis_i_store:} expects the special token to be +% stored in the relevant \tn{toks} register, we do that. The extra +% \cs{exp_not:n} is unnecessary of course, but it makes the treatment +% of all tokens more homogeneous. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \tl_analysis_i_space:w + { + \tex_afterassignment:D \tl_analysis_i_space_test:w + \exp_after:wN \cs_set_eq:NN + \exp_after:wN \l_tl_analysis_token + \token_to_str:N + } +\cs_new_protected_nopar:Npn \tl_analysis_i_space_test:w + { + \if_meaning:w \l_tl_analysis_token \c_space_token + \tex_toks:D \l_tl_analysis_index_int { \exp_not:n { ~ } } + \tl_analysis_i_store: + \else: + \int_incr:N \l_tl_analysis_normal_int + \fi: + \tl_analysis_i_loop:w + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int]{\tl_analysis_i_bgroup:w} +% \begin{macro}[int]{\tl_analysis_i_egroup:w} +% \begin{macro}[aux]{\tl_analysis_i_group:nw} +% \begin{macro}[aux]{\tl_analysis_i_group_test:w} +% The token might be either a true character token with +% catcode $1$ or $2$, or it could be a control sequence. +% The only tricky case is if the character code happens +% to be equal to the escape character: then we change +% the escape character, so that the string representation +% of the true character and of a control sequence set equal +% to it start differently. +% \begin{macrocode} +\group_begin: + \char_set_catcode_group_begin:N \^^@ + \char_set_catcode_group_end:N \^^E + \cs_new_protected_nopar:Npn \tl_analysis_i_bgroup:w + { \tl_analysis_i_group:nw { \exp_after:wN ^^@ \if_false: ^^E \fi: } } + \char_set_catcode_group_begin:N \^^B + \char_set_catcode_group_end:N \^^@ + \cs_new_protected_nopar:Npn \tl_analysis_i_egroup:w + { \tl_analysis_i_group:nw { \if_false: ^^B \fi: ^^@ } } +\group_end: +\cs_new_protected_nopar:Npn \tl_analysis_i_group:nw #1 + { + \l_tl_analysis_char_int = \tl_analysis_extract_charcode: \scan_stop: + \tex_lccode:D \c_zero \l_tl_analysis_char_int + \tl_to_lowercase:n { \tex_toks:D \l_tl_analysis_index_int {#1} } + \if_num:w \l_tl_analysis_char_int = \tex_escapechar:D + \int_set:Nn \tex_escapechar:D { 139 - \tex_escapechar:D } + \fi: + \tex_afterassignment:D \tl_analysis_i_group_test:w + \exp_after:wN \cs_set_eq:NN + \exp_after:wN \l_tl_analysis_token + \token_to_str:N + } +\cs_new_protected_nopar:Npn \tl_analysis_i_group_test:w + { + \if_num:w \tl_analysis_extract_charcode: = \l_tl_analysis_char_int + \tl_analysis_i_store: + \else: + \int_incr:N \l_tl_analysis_normal_int + \fi: + \tl_analysis_i_loop:w + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int]{\tl_analysis_i_store:} +% This function is called each time we meet a special token, +% and the value of \cs{l_tl_analysis_type_int} indicates which case +% we are in: +% \begin{itemize} +% \item[-1] end-group character; +% \item[0] space character; +% \item[1] begin-group character. +% \end{itemize} +% We need to distinguish further the case of a space character +% (code $32$) from other character codes, because those will +% behave differently in the second pass. Namely, we change the +% cases above to +% \begin{itemize} +% \item[-2] space end-group character; +% \item[-1] non-space end-group character; +% \item[0] space blank space character; +% \item[1] non-space begin-group character; +% \item[2] space begin-group character. +% \end{itemize} +% This has the property that non-space characters correspond to odd +% values of \cs{l_tl_analysis_type_int}. +% Also, the \tn{toks} register number \cs{l_tl_analysis_index_int} +% holds a token list which expands to the given special token. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \tl_analysis_i_store: + { + \tex_advance:D \l_tl_analysis_nesting_int \l_tl_analysis_type_int + \if_num:w \l_tl_analysis_char_int = \c_thirty_two + \tex_multiply:D \l_tl_analysis_type_int \c_two + \fi: + \tex_skip:D \l_tl_analysis_index_int + = \l_tl_analysis_normal_int sp plus \l_tl_analysis_type_int sp \scan_stop: + \int_incr:N \l_tl_analysis_index_int + \int_zero:N \l_tl_analysis_normal_int + \if_num:w \l_tl_analysis_nesting_int = \c_minus_one + \cs_set_eq:NN \tl_analysis_i_loop:w \scan_stop: + \fi: + } +% \end{macrocode} +% \end{macro} +% +% \subsection{Second pass} +% +% The second pass is an exercise in expandable loops. +% +% \begin{macro}[int]{\tl_analysis_ii:n} +% \begin{macro}[aux]{\tl_analysis_ii_loop:w} +% Start the loop with the index $0$. No need for an end-marker: +% the loop will stop by itself when the last index is read. +% We will repeatedly oscillate between reading long stretches +% of normal tokens, and reading special tokens. +% \begin{macrocode} +\cs_new_protected:Npn \tl_analysis_ii:n #1 + { + \tl_gset:Nx \g_tl_analysis_result_tl + { + \tl_analysis_ii_loop:w 0; #1 + \prg_break_point:n { } + } + } +\cs_new:Npn \tl_analysis_ii_loop:w #1; + { + \exp_after:wN \tl_analysis_ii_normals:ww + \int_value:w \tex_skip:D #1 ; #1 ; + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[aux]{\tl_analysis_ii_normals:ww} +% \begin{macro}[aux]{\tl_analysis_ii_normal:wwN} +% The first argument is the number of normal tokens which remain +% to be read, and the second argument is the index in the array +% produced in the first step. +% A character's string representation is always one character long, +% while a control sequence is always longer (we have set the escape +% character to a printable value). In both cases, we leave +% \cs{exp_not:n} \Arg{token} \cs{q_mark} in the input stream +% (after \texttt{x}-expansion). +% \begin{macrocode} +\cs_new:Npn \tl_analysis_ii_normals:ww #1; + { + \if_num:w #1 = \c_zero + \tl_analysis_ii_special:w + \fi: + \tl_analysis_ii_normal:wwN #1; + } +\cs_new:Npn \tl_analysis_ii_normal:wwN #1; #2; #3 + { + \exp_not:n { \exp_not:n { #3 } } \exp_not:N \q_mark + \if_charcode:w + \scan_stop: + \exp_after:wN \use_none:n \token_to_str:N #3 \prg_do_nothing: + \scan_stop: + \exp_after:wN \tl_analysis_ii_char:Nww + \else: + \exp_after:wN \tl_analysis_ii_cs:Nww + \fi: + #3 #1; #2; + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[aux]{\tl_analysis_ii_char:Nww} +% If the normal token we grab is a character, leave +% \meta{catcode} \meta{charcode} followed by a comma +% in the input stream, and call \cs{tl_analysis_ii_normals:ww} +% with its first argument decremented. +% \begin{macrocode} +\cs_new:Npn \tl_analysis_ii_char:Nww #1 + { + \if_meaning:w #1 \c_undefined:D D \else: + \if_catcode:w #1 \c_catcode_other_token C \else: + \if_catcode:w #1 \c_catcode_letter_token B \else: + \if_catcode:w #1 \c_math_toggle_token 3 \else: + \if_catcode:w #1 \c_alignment_token 4 \else: + \if_catcode:w #1 \c_math_superscript_token 7 \else: + \if_catcode:w #1 \c_math_subscript_token 8 \else: + 6 + \fi: \fi: \fi: \fi: \fi: \fi: \fi: + \int_value:w `#1 , + \exp_after:wN \tl_analysis_ii_normals:ww + \int_use:N \int_eval:w \c_minus_one + + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\tl_analysis_ii_cs:Nww} +% If the token we grab is a control sequence, leave +% |0-1,| (as category code and character code) in the input stream, +% and call \cs{tl_analysis_ii_normals:ww} with updated arguments. +% \begin{macrocode} +\cs_new:Npn \tl_analysis_ii_cs:Nww #1 + { + 0 -1 , + \tl_analysis_cs_space_count:NN \tl_analysis_ii_cs_test:ww #1 + } +\cs_new:Npn \tl_analysis_ii_cs_test:ww #1 ; #2 ; #3; #4; + { + \exp_after:wN \tl_analysis_ii_normals:ww + \int_use:N \int_eval:w + \if_num:w #1 = \c_zero + #3 + \else: + \tex_skip:D \int_eval:w #4 + #1 \int_eval_end: + \fi: + - #2 + \exp_after:wN ; + \int_use:N \int_eval:w #4 + #1; + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\tl_analysis_ii_special:w} +% \begin{macro}[aux]{\tl_analysis_ii_special_char:wN} +% \begin{macro}[aux]{\tl_analysis_ii_special_space:w} +% Here, |#1| is the current index in the array built in the first pass. +% Check whether we reached the end (we shouldn't insert the trailing +% end-group character that marked the end of the token list in the +% first pass). +% Unpack the \tn{toks} register: when \texttt{x}-expanding again, +% we will get the special token. +% Then leave the category code in the input stream, followed by +% the character code, and call \cs{tl_analysis_ii_loop:w} with the next index. +% \begin{macrocode} +\cs_new:Npn \tl_analysis_ii_special:w \fi: \tl_analysis_ii_normal:wwN 0; #1; + { + \fi: + \if_num:w #1 = \l_tl_analysis_index_int + \exp_after:wN \prg_map_break: + \fi: + \tex_the:D \tex_toks:D #1 \exp_stop_f: \exp_not:N \q_mark + \if_case:w \etex_gluestretch:D \tex_skip:D #1 \exp_stop_f: + A + \or: 1 + \or: 1 + \else: 2 + \fi: + \if_int_odd:w \etex_gluestretch:D \tex_skip:D #1 \exp_stop_f: + \exp_after:wN \tl_analysis_ii_special_char:wN \int_use:N + \else: + \exp_after:wN \tl_analysis_ii_special_space:w \int_use:N + \fi: + \int_eval:w \c_one + #1 \exp_after:wN ; + \token_to_str:N + } +\cs_new:Npn \tl_analysis_ii_special_char:wN #1 ; #2 + { + \int_value:w `#2 , + \tl_analysis_ii_loop:w #1 ; + } +\cs_new:Npn \tl_analysis_ii_special_space:w #1 ; ~ + { + 32 , + \tl_analysis_ii_loop:w #1 ; + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \subsection{Showing the results} +% +% \begin{macro}{\tl_show_analysis:N} +% \begin{macro}{\tl_show_analysis:n} +% Add to \cs{tl_analysis:n} a third pass to display tokens to the terminal. +% \begin{macrocode} +\cs_new_protected:Npn \tl_show_analysis:N #1 + { + \exp_args:No \tl_analysis:n {#1} + \msg_aux_show:Nnx #1 + { tl-analysis } + { + \exp_after:wN \tl_show_analysis_loop:wNw \g_tl_analysis_result_tl + \q_mark { ? \prg_map_break: } , + \prg_break_point:n { } + } + } +\cs_new_protected:Npn \tl_show_analysis:n #1 + { + \tl_set:Nn \l_tl_tmpa_tl {#1} + \tl_show_analysis:N \l_tl_tmpa_tl + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[aux, EXP]{\tl_show_analysis_loop:wNw} +% Here, |#1| \texttt{o}-expands to the token; +% |#2| is the category code (one hexadecimal digit), +% $0$ for control sequences; +% |#3| is the character code, which we ignore. +% In the cases of control sequences and active characters, +% the meaning may overflow one line, and we want to truncate +% it. Those cases are thus separated out. +% \begin{macrocode} +\cs_new:Npn \tl_show_analysis_loop:wNw #1 \q_mark #2 #3, + { + \use_none:n #2 + \iow_newline: > \c_space_tl \c_space_tl + \if_num:w "#2 = \c_zero + \exp_after:wN \tl_show_analysis_cs:n + \else: + \if_num:w "#2 = \c_thirteen + \exp_after:wN \exp_after:wN + \exp_after:wN \tl_show_analysis_active:n + \else: + \exp_after:wN \exp_after:wN + \exp_after:wN \tl_show_analysis_normal:n + \fi: + \fi: + {#1} + \tl_show_analysis_loop:wNw + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\tl_show_analysis_normal:n} +% Non-active characters are a simple matter of printing +% the character, and its meaning. One can check that +% begin-group and end-group characters do not mess up +% \TeX{}'s alignment status. +% \begin{macrocode} +\cs_new:Npn \tl_show_analysis_normal:n #1 + { + \exp_after:wN \token_to_str:N #1 ~ + ( \exp_after:wN \token_to_meaning:N #1 ) + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux]{\tl_show_analysis_cs:n} +% \begin{macro}[aux]{\tl_show_analysis_active:n} +% \begin{macro}[aux]{\tl_show_analysis_long:nn} +% \begin{macro}[aux]{\tl_show_analysis_long_aux:nnn} +% Control sequences and active characters are printed in the same way, +% making sure not to go beyond the \cs{l_iow_line_length_int}. In case +% of an overflow, we replace the last characters by +% \cs{c_tl_show_analysis_etc_str}. +% \begin{macrocode} +\cs_new:Npn \tl_show_analysis_cs:n #1 + { \exp_args:No \tl_show_analysis_long:nn {#1} { control~sequence= } } +\cs_new:Npn \tl_show_analysis_active:n #1 + { \exp_args:No \tl_show_analysis_long:nn {#1} { active~character= } } +\cs_new:Npn \tl_show_analysis_long:nn #1 + { + \exp_args:Noo \tl_show_analysis_long_aux:nnn + { \token_to_str:N #1 } + { \token_to_meaning:N #1 } + } +\cs_new_nopar:Npn \tl_show_analysis_long_aux:nnn #1#2#3 + { + \int_compare:nNnTF + { \str_length:n { >>> #1 ~ ( #3 #2 ) } } + > { \l_iow_line_length_int } + { + \str_substr:nnn { >>> #1 ~ ( #3 #2 ) } \c_three + { \l_iow_line_length_int - \str_length:N \c_tl_show_analysis_etc_str } + \c_tl_show_analysis_etc_str + } + {#1~(#3#2)} + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \begin{variable}{\c_tl_show_analysis_etc_str} +% When a control sequence (or active character) +% and its meaning are too long to fit in one line +% of the terminal, the end is replaced by this token list. +% \begin{macrocode} +\str_const:Nx \c_tl_show_analysis_etc_str % ( + { \iow_char:N \\ETC. ) } +% \end{macrocode} +% \end{variable} +% +% \begin{macrocode} +\msg_kernel_new:nnn { tl-analysis } { show } + { + Token~list~ + \str_if_eq:nnF {#1} { \l_tl_tmpa_tl } { \token_to_str:N #1~} + \tl_if_empty:NTF #1 + { is~empty } + { contains~the~tokens: } + } +% \end{macrocode} +% +% \begin{macrocode} +%</initex|package> +% \end{macrocode} +% +% \end{implementation} +% +% \PrintIndex diff --git a/Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx b/Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx index 57fbb9ecca4..0d3fdafcdf8 100644 --- a/Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx +++ b/Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx @@ -37,7 +37,7 @@ % %<*driver|package> \RequirePackage{l3names} -\GetIdInfo$Id: l3galley.dtx 2938 2011-10-25 18:41:20Z joseph $ +\GetIdInfo$Id: l3galley.dtx 3071 2011-12-20 20:14:00Z bruno $ {L3 Experimental galley code} %</driver|package> %<*driver> @@ -653,17 +653,6 @@ \package_check_loaded_expl: %</package> % \end{macrocode} -% -% \subsection{TO BE MOVED} -% -% \begin{macro}{\scan_marker_new:N, \scan_marker_new:c} -% Temporary: needs to go somewhere else if this concept is retained. -% \begin{macrocode} -\cs_new_protected:Npn \scan_marker_new:N #1 - { \cs_new_eq:NN #1 \scan_stop: } -\cs_generate_variant:Nn \scan_marker_new:N { c } -% \end{macrocode} -% \end{macro} % % \subsection{Support items} % @@ -1049,11 +1038,11 @@ % \end{macro} % % \subsection{The \cs{par} token} -% +% % \begin{variable}{\s_par_omit} % Used to indicate that a paragraph should be omitted. % \begin{macrocode} -\scan_marker_new:N \s_par_omit +\scan_new:N \s_par_omit % \end{macrocode} % \end{variable} % @@ -1085,7 +1074,7 @@ } } % \end{macrocode} -% No marker, so really insert a paragraph. In vertical mode, +% No marker, so really insert a paragraph. In vertical mode, % \begin{macrocode} \cs_new_protected_nopar:Npn \galley_std_par_aux_ii: { diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3flag.sty b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3flag.sty new file mode 100644 index 00000000000..40e2c899682 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3flag.sty @@ -0,0 +1,100 @@ +%% +%% This is file `l3flag.sty', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3flag.dtx (with options: `package') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3flag.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\RequirePackage{expl3} +\GetIdInfo$Id: l3flag.dtx 3039 2011-12-08 09:22:35Z bruno $ + {L3 Experimental Flags} +\ProvidesExplPackage + {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription} +\tl_new:N \g_flag_list_tl +\cs_new_protected:Npn \flag_new:N #1 + { + \tl_if_in:NnTF \g_flag_list_tl {#1} + { + \msg_kernel_error:nnx + { flag } { already-defined } + { \token_to_str:N #1 } + } + { + \tl_gput_right:Nn \g_flag_list_tl {#1} + \cs_new_eq:NN #1 \c_undefined:D + } + } +\cs_generate_variant:Nn \flag_new:N { c } +\prg_new_conditional:Npnn \flag_test:N #1 { p , T , F , TF } + { + \if_cs_exist:N #1 + \prg_return_true: + \else: + \prg_return_false: + \fi: + } +\prg_new_conditional:Npnn \flag_test:c #1 { p , T , F , TF } + { + \if_cs_exist:w #1 \cs_end: + \prg_return_true: + \else: + \prg_return_false: + \fi: + } +\cs_new:Npn \flag_raise:N #1 + { \exp_after:wN \use_none:n \cs:w \cs_to_str:N #1 \cs_end: } +\cs_new:Npn \flag_raise:c #1 + { \exp_after:wN \use_none:n \cs:w #1 \cs_end: } +\cs_new_protected:Npn \flag_lower:N #1 + { \cs_set_eq:NN #1 \c_undefined:D } +\cs_generate_variant:Nn \flag_lower:N { c } +\msg_kernel_new:nnnn { flag } { already-defined } + { The~control~sequence~#1~is~already~declared~as~a~flag. } + { + LaTeX~was~asked~to~define~the~flag~#1,~but~it~has~already~ + been~defined~as~a~flag.~The~flag~module~is~mostly~meant~ + for~kernel~use,~and~booleans~should~be~preferred. + } +%% +%% +%% End of file `l3flag.sty'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex.sty b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex.sty index f3dc0b5a358..facf764c168 100644 --- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex.sty +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex.sty @@ -46,17 +46,19 @@ %% %% ----------------------------------------------------------------------- \RequirePackage{expl3} -\GetIdInfo$Id: l3regex.dtx 3017 2011-11-28 00:10:02Z bruno $ +\GetIdInfo$Id: l3regex.dtx 3073 2011-12-21 01:53:06Z bruno $ {L3 Experimental Regular Expressions} \ProvidesExplPackage {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription} -\RequirePackage{l3str} +\RequirePackage{l3str, l3tl-analysis, l3flag} \cs_new:Npn \regex_tmp:w { } -\tl_new:N \g_regex_tmpa_tl \tl_new:N \l_regex_tmpa_tl \tl_new:N \l_regex_tmpb_tl +\tl_new:N \l_regex_tmpc_tl \int_new:N \l_regex_tmpa_int \int_new:N \l_regex_tmpb_int +\flag_new:N \l_regex_group_begin_flag +\flag_new:N \l_regex_group_end_flag \int_new:N \l_regex_max_state_int \int_new:N \l_regex_left_state_int \int_new:N \l_regex_right_state_int @@ -67,7 +69,10 @@ \int_new:N \l_regex_capturing_group_int \seq_new:N \l_regex_capturing_group_seq \int_new:N \l_regex_capturing_group_max_int +\tl_new:N \l_regex_build_or_class_tl \tl_new:N \l_regex_one_or_group_tl +\bool_new:N \l_regex_in_csname_bool +\tl_new:N \l_regex_tmpa_regex_tl \cs_new_eq:NN \regex_build_tmp_class:n \use_none:n \bool_new:N \l_regex_class_bool \tl_new:N \l_regex_class_tl @@ -133,12 +138,16 @@ \regex_item_geq:n { \c_eleven } % > lf \regex_item_range:nn { \c_zero } { \c_nine } % < lf } -\tl_new:N \l_regex_query_other_str +\int_new:N \l_regex_nesting_int +\int_new:N \l_regex_min_step_int +\int_new:N \l_regex_max_step_int \int_new:N \l_regex_current_step_int \int_new:N \l_regex_start_step_int \int_new:N \l_regex_success_step_int -\int_new:N \l_regex_unique_step_int +\int_new:N \l_regex_unique_id_int \int_new:N \l_regex_current_char_int +\int_new:N \l_regex_current_catcode_int +\tl_new:N \l_regex_current_token_tl \int_new:N \l_regex_last_char_int \int_new:N \l_regex_case_changed_char_int \bool_new:N \l_regex_caseless_bool @@ -147,23 +156,23 @@ \prop_new:N \l_regex_success_submatches_prop \int_new:N \l_regex_max_index_int \tl_new:N \l_regex_every_match_tl -\bool_new:N \l_regex_success_bool +\bool_new:N \g_regex_success_bool +\bool_new:N \l_regex_saved_success_bool +\bool_new:N \l_regex_success_match_bool \cs_new_protected:Npn \regex_last_match_empty_no:F #1 {#1} \cs_new_protected:Npn \regex_last_match_empty_yes:F { \int_compare:nNnF \l_regex_start_step_int = \l_regex_current_step_int } \cs_new_eq:NN \regex_last_match_empty:F \regex_last_match_empty_no:F \bool_new:N \l_regex_success_empty_bool \bool_new:N \l_regex_fresh_thread_bool -\seq_new:N \g_regex_submatches_seq -\int_new:N \g_regex_match_count_int -\seq_new:N \g_regex_extract_all_seq -\seq_new:N \g_regex_split_seq -\tl_new:N \l_regex_replacement_tl -\tl_new:N \g_regex_replaced_str -\cs_new_eq:NN \s_regex_stop \scan_stop: +\int_new:N \l_regex_replacement_int +\int_new:N \l_regex_replacement_csnames_int +\int_new:N \l_regex_submatch_int +\int_new:N \l_regex_submatch_start_int +\int_new:N \l_regex_match_count_int \cs_new_protected:Npn \regex_toks_put_left:Nx #1#2 { - \cs_set_nopar:Npx \regex_tmp:w \s_regex_stop { \s_regex_stop #2 } + \cs_set_nopar:Npx \regex_tmp:w \s_stop { \s_stop #2 } \tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN { \exp_after:wN \regex_tmp:w \tex_the:D \tex_toks:D #1 } } @@ -173,18 +182,24 @@ \tex_toks:D #1 \exp_after:wN { \tex_the:D \tex_toks:D \exp_after:wN #1 \regex_tmp:w } } -\cs_new_nopar:Npn \regex_query_substr:NN #1#2 +\cs_new:Npn \regex_query_substr:nn #1#2 { - \exp_after:wN \regex_query_substr_aux:NN - \exp_after:wN #1 - \exp_after:wN #2 - \l_regex_query_other_str - \q_stop + \if_num:w \int_eval:w #1 < \int_eval:w #2 \int_eval_end: + \str_aux_toks_range:nn {#1} {#2} + \fi: + } +\cs_new:Npn \regex_query_submatch:nn #1#2 + { + \if_num:w #1 < \l_regex_capturing_group_int + \exp_after:wN \regex_query_submatch:w + \int_use:N \int_eval:w #1 + #2 ; + \fi: } -\cs_new:Npn \regex_query_substr_aux:NN #1#2 +\cs_new:Npn \regex_query_submatch:w #1 ; { - \str_skip_do:nn {#1} - { \str_collect_do:nn { #2 - #1 } { \use_i_delimit_by_q_stop:nw } } + \regex_query_substr:nn + { \tex_skip:D #1 } + { \etex_gluestretch:D \tex_skip:D #1 } } \cs_new_protected:Npn \regex_seq_pop_int:NN #1#2 { @@ -208,8 +223,27 @@ \seq_get:NN #1 \l_regex_tmpa_tl \l_regex_tmpa_tl } +\cs_new:Npn \regex_seq_item:Nn #1#2 + { + \exp_after:wN \regex_seq_item_aux:wNn + \int_use:N \int_eval:w #2 \exp_after:wN ; + #1 { ? \prg_map_break: } { } + \prg_break_point:n { } + } +\cs_new:Npn \regex_seq_item_aux:wNn #1; #2#3 + { + \use_none:n #2 + \if_num:w #1 = \c_zero + \exp_after:wN \prg_map_break:n + \else: + \exp_after:wN \use_none:n + \fi: + { \exp_not:n {#3} } + \exp_after:wN \regex_seq_item_aux:wNn + \int_use:N \int_eval:w #1 - \c_one ; + } \cs_new_protected_nopar:Npn \regex_item_dot:T - { \int_compare:nNnF \l_regex_current_char_int = \c_minus_one } + { \int_compare:nNnF \l_regex_current_char_int < \c_minus_one } \cs_new_protected_nopar:Npn \regex_break_true:w #1 \regex_break_point:TF #2 #3 {#2} \cs_new_protected_nopar:Npn \regex_break_false:w @@ -283,6 +317,37 @@ \cs_set:Npn \regex_item_range:nn { \regex_item_caseful_range:nn } \cs_set:Npn \regex_item_geq:n { \regex_item_caseful_geq:n } } +\cs_new_protected_nopar:Npn \regex_item_catcode:n #1 + { + \if_num:w #1 = \l_regex_current_catcode_int + \exp_after:wN \regex_break_true:w + \fi: + } +\cs_new_protected_nopar:Npn \regex_item_catcode_neq:n #1 + { + \if_num:w #1 = \l_regex_current_catcode_int + \else: + \exp_after:wN \regex_break_true:w + \fi: + } +\cs_new_protected:Npn \regex_item_cs:n #1 + { + \int_compare:nNnT \l_regex_current_catcode_int = \c_zero + { + \tl_set:Nn \l_regex_tmpa_regex_tl {#1} + \bool_set_eq:NN \l_regex_saved_success_bool \g_regex_success_bool + \exp_args:NNx \regex_match:NnTF \l_regex_tmpa_regex_tl + { + \exp_after:wN \exp_after:wN + \exp_after:wN \cs_to_str:N \l_regex_current_token_tl + } + { + \bool_gset_eq:NN \g_regex_success_bool \l_regex_saved_success_bool + \regex_break_true:w + } + { \bool_gset_eq:NN \g_regex_success_bool \l_regex_saved_success_bool } + } + } \cs_new_protected:Npn \regex_get_digits:nw #1 { \tex_afterassignment:D \regex_tmp:w @@ -362,14 +427,10 @@ } \cs_new_protected_nopar:Npn \regex_build_new_state: { - \int_compare:nNnTF \l_regex_max_state_int > { 32766 } - { \msg_error:nn { regex } { -997 } } - { - \int_incr:N \l_regex_max_state_int - \tex_toks:D \l_regex_max_state_int { \s_regex_stop } - } + \tex_toks:D \l_regex_max_state_int { \s_stop } \int_set_eq:NN \l_regex_left_state_int \l_regex_right_state_int \int_set_eq:NN \l_regex_right_state_int \l_regex_max_state_int + \int_incr:N \l_regex_max_state_int } \cs_new_protected_nopar:Npn \regex_build_transition_aux:NN #1#2 { @@ -388,16 +449,18 @@ } \cs_new_protected:Npn \regex_build:n #1 { - \regex_build_setup: - \str_aux_escape:NNNn - \regex_build_i_unescaped:N - \regex_build_i_escaped:N - \regex_build_i_raw:N - { #1 } + \bool_set_false:N \l_regex_in_csname_bool + \regex_build:w + \regex_build_new_state: + \regex_toks_put_right:Nx \l_regex_left_state_int + { \regex_action_start_wildcard:nn {0} {1} } \regex_build_open_aux: - \g_str_result_tl \prg_do_nothing: \prg_do_nothing: - \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_zero - \regex_build_close_aux: \regex_build_group_: + \str_escape_use:NNNn + \regex_build_i_unescaped:N + \regex_build_i_escaped:N + \regex_build_i_raw:N + { #1 } + \prg_do_nothing: \prg_do_nothing: \regex_build_end: } \cs_new_nopar:Npn \regex_build_i_unescaped:N #1 @@ -414,38 +477,39 @@ } \cs_new_nopar:Npn \regex_build_i_raw:N #1 { \exp_not:N \regex_build_raw:N #1 } -\cs_new_protected_nopar:Npn \regex_build_default_control:N #1 +\cs_new_protected_nopar:Npn \regex_build_control:N #1 { - \cs_if_exist_use:cF { regex_build_#1: } - { \regex_build_default_raw:N #1 } + \cs_if_exist_use:cF { regex_ \l_regex_build_or_class_tl _#1: } + { \use:c { regex_ \l_regex_build_or_class_tl _default_raw:N } #1 } } -\cs_new_protected_nopar:Npn \regex_build_setup: +\cs_new_protected_nopar:Npn \regex_build:w { - \cs_set_eq:NN \regex_build_control:N \regex_build_default_control:N \cs_set_eq:NN \regex_build_raw:N \regex_build_default_raw:N + \tl_set:Nn \l_regex_build_or_class_tl { build } \int_set_eq:NN \l_regex_capturing_group_int \c_zero \int_zero:N \l_regex_max_state_int \regex_build_new_state: - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int - { \regex_action_start_wildcard:nn {0} {1} } } \cs_new_protected_nopar:Npn \regex_build_end: { + \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_zero + \regex_build_close_aux: \regex_build_group_: \seq_if_empty:NF \l_regex_capturing_group_seq { - \msg_error:nn { regex } {22} + \msg_kernel_error:nnx { regex } { missing-rparen } + { \seq_length:N \l_regex_capturing_group_seq } \prg_replicate:nn { \seq_length:N \l_regex_capturing_group_seq } { \regex_build_close_aux: \regex_build_group_: } } \regex_toks_put_right:Nx \l_regex_right_state_int { \regex_action_success: } + \int_incr:N \l_regex_capturing_group_int } \cs_new_protected_nopar:cpn { regex_build_^: } - { \regex_build_anchor_start:N \c_zero } + { \regex_build_anchor_start:N \l_regex_min_step_int } \cs_new_protected_nopar:Npn \regex_build_A: - { \regex_build_anchor_start:N \c_zero } + { \regex_build_anchor_start:N \l_regex_min_step_int } \cs_new_protected_nopar:Npn \regex_build_G: { \regex_build_anchor_start:N \l_regex_start_step_int } \cs_new_protected_nopar:Npn \regex_build_anchor_start:N #1 @@ -453,7 +517,7 @@ \regex_build_new_state: \regex_toks_put_right:Nx \l_regex_left_state_int { - \exp_not:N \int_compare:nNnT {#1} = \l_regex_current_step_int + \exp_not:N \int_compare:nNnT #1 = \l_regex_current_step_int { \regex_action_free:n { @@ -469,7 +533,7 @@ \regex_toks_put_right:Nx \l_regex_left_state_int { \exp_not:N \int_compare:nNnT - \c_minus_one = \l_regex_current_char_int + \l_regex_current_char_int < \c_minus_one { \regex_action_free:n { @@ -519,61 +583,52 @@ \int_set_eq:NN \l_regex_current_char_int \l_regex_last_char_int \c_regex_w_tl \regex_break_point:TF - { \group_end: \c_regex_W_tl \regex_item_equal:n { -1 } } + { \group_end: \c_regex_W_tl \regex_item_equal:n { -2 } } { \group_end: \c_regex_w_tl } \regex_break_point:TF } -\cs_new_protected_nopar:Npn \regex_build_default_raw:N #1 +\cs_new_protected:Npn \regex_build_one:n #1 + { \regex_build_one:x { \exp_not:n {#1} } } +\cs_new_protected:Npn \regex_build_one:x #1 { - \cs_set:Npx \regex_build_tmp_class:n ##1 + \cs_set_nopar:Npx \regex_build_tmp_class:n ##1 { - \regex_item_equal:n { \int_value:w `#1 ~ } - \regex_break_point:TF { \regex_action_cost:n { ##1 } } { } + \exp_not:N \exp_not:n {#1} + \regex_break_point:TF { \regex_action_cost:n {##1} } { } } \regex_build_one_quantifier: } +\cs_new_protected_nopar:Npn \regex_build_default_raw:N #1 + { \regex_build_one:x { \regex_item_equal:n { \int_value:w `#1 ~ } } } \cs_new_protected_nopar:cpn { regex_build_.: } { \cs_set:Npn \regex_build_tmp_class:n ##1 { \regex_item_dot:T { \regex_action_cost:n {##1} } } \regex_build_one_quantifier: } -\cs_new_protected_nopar:Npn \regex_build_char_type:N #1 - { - \cs_set:Npn \regex_build_tmp_class:n ##1 - { - \exp_not:N #1 - \regex_break_point:TF { \regex_action_cost:n {##1} } { } - } - \regex_build_one_quantifier: - } \tl_map_inline:nn { dDhHsSvVwWN } { \cs_new_protected_nopar:cpx { regex_build_#1: } - { - \exp_not:N \regex_build_char_type:N - \exp_not:c { c_regex_#1_tl } - } + { \regex_build_one:n \exp_not:c { c_regex_#1_tl } } } +\cs_new_protected:Npn \regex_class_one:n + { \tl_put_right:Nn \l_regex_class_tl } +\cs_new_protected:Npn \regex_class_one:x + { \tl_put_right:Nx \l_regex_class_tl } \cs_new_protected_nopar:cpn { regex_build_[: } { \tl_clear:N \l_regex_class_tl - \cs_set_eq:NN \regex_build_control:N \regex_class_control:N - \cs_set_eq:NN \regex_build_raw:N \regex_class_raw:N + \cs_set_eq:NN \regex_build_raw:N \regex_class_default_raw:N + \tl_set:Nn \l_regex_build_or_class_tl { class } \regex_class_first:NN } -\cs_new_protected_nopar:Npn \regex_class_control:N #1 - { - \cs_if_exist_use:cF { regex_class_#1: } - { \regex_class_raw:N #1 } - } \cs_new_protected_nopar:cpn { regex_class_]: } { \tl_if_empty:NTF \l_regex_class_tl %[ - { \regex_class_raw:N ] } + { \regex_class_default_raw:N ] } { - \cs_set_eq:NN \regex_build_control:N \regex_build_default_control:N \cs_set_eq:NN \regex_build_raw:N \regex_build_default_raw:N + \tl_set:Nn \l_regex_build_or_class_tl { build } \cs_set:Npn \regex_build_tmp_class:n ##1 { \exp_not:o \l_regex_class_tl @@ -593,7 +648,7 @@ #1 #2 } } -\cs_new_protected_nopar:Npn \regex_class_raw:N #1#2#3 +\cs_new_protected_nopar:Npn \regex_class_default_raw:N #1#2#3 { \str_if_eq:nnTF {#2#3} { \regex_build_control:N - } { \regex_class_range:Nw #1 } @@ -603,10 +658,7 @@ } } \cs_new_protected_nopar:Npn \regex_class_single:N #1 - { - \tl_put_right:Nx \l_regex_class_tl - { \regex_item_equal:n { \int_value:w `#1 } } - } + { \regex_class_one:x { \regex_item_equal:n { \int_value:w `#1 } } } \cs_new_protected_nopar:Npn \regex_class_range:Nw #1#2#3 { \token_if_eq_meaning:NNTF #2 \regex_build_control:N @@ -624,9 +676,9 @@ \cs_new_protected_nopar:Npn \regex_class_range_put:NN #1#2 { \if_num:w `#1 > `#2 \exp_stop_f: - \msg_error:nn { regex } {8} + \msg_kernel_error:nnxx { regex } { backwards-range } {#1} {#2} \else: - \tl_put_right:Nx \l_regex_class_tl + \regex_class_one:x { \if_num:w `#1 = `#2 \exp_stop_f: \regex_item_equal:n @@ -640,10 +692,93 @@ \tl_map_inline:nn { dDhHsSvVwWN } { \cs_new_protected_nopar:cpx { regex_class_#1: } + { \regex_class_one:n \exp_not:c { c_regex_#1_tl } } + } +\cs_new_protected_nopar:Npn \regex_build_c: + { \regex_aux_c:NNNN \regex_item_catcode:n \use_i:nn } +\cs_new_protected_nopar:Npn \regex_build_C: + { \regex_aux_c:NNNN \regex_item_catcode_neq:n \use_ii:nn } +\cs_new_eq:NN \regex_class_c: \regex_build_c: +\cs_new_eq:NN \regex_class_C: \regex_build_C: +\cs_new_protected:Npn \regex_aux_c:NNNN #1#2#3#4 + { + \token_if_eq_meaning:NNTF #3 \regex_build_raw:N + { + \regex_aux_c_test:NTF #4 + { + \use:c { regex_ \l_regex_build_or_class_tl _one:x } + { #1 { \int_use:N \l_regex_current_catcode_int } } + } + { + \msg_kernel_error:nnx { regex } { build-c } { #4 } + #3 #4 + } + } + { + \if_charcode:w \c_lbrace_str \exp_not:N #4 + \exp_after:wN #2 + \else: + \exp_after:wN \use_ii:nn + \fi: + { + \group_begin: + \bool_set_true:N \l_regex_in_csname_bool + \regex_build:w + } + { + \msg_kernel_error:nnx { regex } { build-c } { #4 } + #3 #4 + } + } + } +\cs_new_protected:Npn \regex_aux_c_test:NTF #1 + { + \int_set:Nn \l_regex_current_catcode_int + { + \if_case:w \int_eval:w `#1 - `A \int_eval_end: + \c_thirteen % Active + \or: \c_one % Begin-group + \or: \c_zero % Control sequence + \or: \c_eight % Down (subscript) + \or: \c_two % End-group + \else: + \if_case:w \int_eval:w `#1 - `L \int_eval_end: + \c_eleven % Letter + \or: \c_three % Math toggle + \or: \c_minus_one + \or: \c_twelve % Other + \or: \c_six % Parameter + \or: \c_minus_one + \or: \c_minus_one + \or: \c_ten % Space + \or: \c_four % Tab (alignment) + \or: \c_seven % Up (superscript) + \else: \c_minus_one + \fi: + \fi: + } + \if_num:w \l_regex_current_catcode_int = \c_minus_one + \exp_after:wN \use_ii:nn + \else: + \exp_after:wN \use_i:nn + \fi: + } +\cs_new_protected:cpn { regex_build_ \c_rbrace_str : } + { + \bool_if:NTF \l_regex_in_csname_bool { - \tl_put_right:Nn \exp_not:N \l_regex_class_tl - { \exp_not:c { c_regex_#1_tl } } + \regex_build_Z: + \regex_toks_put_right:Nx \l_regex_right_state_int + { \regex_action_success: } + \int_incr:N \l_regex_capturing_group_int + \use:x + { + \group_end: + \exp_not:n { \use:c { regex_ \l_regex_build_or_class_tl _one:n } } + { \regex_item_cs:n { \regex_set_aux:N ? } } + } } + { \exp_after:wN \regex_build_raw:N \c_rbrace_str } } \cs_new_protected_nopar:Npn \regex_build_quantifier:w #1#2 { @@ -711,12 +846,8 @@ { \tl_if_empty:nTF {#2} { \regex_build_quantifier_aux:nnNN { {#1} } { n* } } - { - \int_compare:nNnT {#1} > {#2} - { \msg_error:nn { regex } {4} } - \regex_build_quantifier_aux:nnNN { {#1} {#2} } { nn } - } - } + { \regex_build_quantifier_aux:nnNN { {#1} {#2} } { nn } } + } %^^A todo: catch the case m>n { \regex_build_quantifier_end:n { } \use:x @@ -833,7 +964,7 @@ \cs_new_protected_nopar:cpn { regex_build_): } { \seq_if_empty:NTF \l_regex_capturing_group_seq - { \msg_error:nn { regex } { 22 } } + { \msg_kernel_error:nn { regex } { extra-rparen } } { \regex_build_close_aux: \regex_build_group_quantifier: @@ -878,7 +1009,8 @@ { \cs_if_exist_use:cF { regex_build_special_group_\token_to_str:N #2 : } { - \msg_error:nn { regex } { -998 } + \msg_kernel_error:nnx { regex } { unsupported } + { (? \token_to_str:N #2 } %) \regex_build_control:N ( % ) \regex_build_raw:N ? #1 #2 @@ -990,7 +1122,7 @@ { \tex_toks:D \l_regex_tmpa_int { - \s_regex_stop + \s_stop #1 { \int_eval:n { \l_regex_right_state_int - \l_regex_tmpa_int } } } } @@ -1052,16 +1184,19 @@ \regex_toks_put_right:Nx \l_regex_right_state_int { \regex_action_free:n - { \int_eval:n { \l_regex_max_state_int - \l_regex_left_state_int } } + { + \int_eval:n %^^A todo: document why that value. + { \l_regex_max_state_int - \c_one - \l_regex_left_state_int } + } } \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int - \int_set:Nn \l_regex_tmpb_int { \l_regex_max_state_int + \c_one } + \int_set_eq:NN \l_regex_tmpb_int \l_regex_max_state_int \int_set:Nn \l_regex_max_state_int { - \l_regex_left_state_int - \c_one - + #1 * ( \l_regex_max_state_int - \l_regex_left_state_int + \c_one ) + \l_regex_left_state_int + + #1 * ( \l_regex_max_state_int - \l_regex_left_state_int ) } - \int_until_do:nNnn \l_regex_tmpb_int > \l_regex_max_state_int + \int_while_do:nNnn \l_regex_tmpb_int < \l_regex_max_state_int { \tex_toks:D \l_regex_tmpb_int = \tex_toks:D \l_regex_tmpa_int \int_incr:N \l_regex_tmpa_int @@ -1114,14 +1249,13 @@ \cs_new_protected_nopar:Npn \regex_state_use_with_submatches: { \regex_state_use_aux:n { } } \cs_new_protected_nopar:Npn \regex_state_use: - { \regex_state_use_aux:n { \exp_after:wN \regex_state_use_aux_ii:w } } -\cs_new_nopar:Npn \regex_state_use_aux_ii:w #1 \s_regex_stop { } + { \regex_state_use_aux:n { \exp_after:wN \use_none_delimit_by_s_stop:w } } \cs_new_protected_nopar:Npn \regex_state_use_aux:n #1 { \if_num:w \tex_dimen:D \l_regex_current_state_int - < \l_regex_unique_step_int + < \l_regex_unique_id_int \tex_dimen:D \l_regex_current_state_int - = \l_regex_unique_step_int sp \scan_stop: + = \l_regex_unique_id_int sp \scan_stop: #1 \tex_the:D \tex_toks:D \exp_after:wN \l_regex_current_state_int \fi: \scan_stop: @@ -1144,73 +1278,105 @@ \tex_the:D \tex_toks:D #1 } } -\cs_new_protected:Npn \regex_store_submatches_aux:w #1 \s_regex_stop +\cs_new_protected:Npn \regex_store_submatches_aux:w #1 \s_stop { \regex_store_submatches_aux_ii:Nnnw #1 \regex_state_submatches:nn \c_minus_one \q_prop - \s_regex_stop + \s_stop } \cs_new_protected:Npn \regex_store_submatches_aux_ii:Nnnw - \regex_state_submatches:nn #1 #2 #3 \s_regex_stop + \regex_state_submatches:nn #1 #2 #3 \s_stop { \exp_after:wN \c_zero \exp_after:wN \regex_state_submatches:nn \exp_after:wN { \int_value:w \int_eval:w - \l_regex_unique_step_int + \c_one + \l_regex_unique_id_int + \c_one \exp_after:wN } \exp_after:wN { \l_regex_current_submatches_prop } \regex_state_submatches:nn {#1} {#2} - \s_regex_stop + \s_stop } \cs_new_protected:Npn \regex_state_submatches:nn #1#2 { - \if_num:w #1 = \l_regex_unique_step_int + \if_num:w #1 = \l_regex_unique_id_int \tl_set:Nn \l_regex_current_submatches_prop { #2 } \fi: } \cs_new_protected:Npn \regex_match:n #1 { - \tl_set:Nx \l_regex_query_other_str { \tl_to_other_str:n {#1} } + \tl_set_analysis:Nn \l_regex_tmpa_tl {#1} + + \int_zero:N \l_regex_nesting_int + \int_set_eq:NN \l_regex_current_step_int \l_regex_max_state_int + \regex_query_set:nnn { } { -1 } { -2 } + \int_set_eq:NN \l_regex_min_step_int \l_regex_current_step_int + \exp_after:wN \regex_query_set_loop:ww + \l_regex_tmpa_tl + \q_mark { ? \prg_map_break: } , + \prg_break_point:n { } + \int_set_eq:NN \l_regex_max_step_int \l_regex_current_step_int + \regex_query_set:nnn { } { -1 } { -2 } + \regex_match_initial_setup: \regex_match_once: } -\cs_new_protected_nopar:Npn \regex_match_once: +\cs_new:Npn \regex_query_set_loop:ww #1 \q_mark #2#3 , { - \regex_match_setup: - \exp_after:wN \regex_match_once_aux: \l_regex_query_other_str - \q_recursion_tail \q_recursion_stop - \bool_if:NT \l_regex_success_bool { \l_regex_every_match_tl } + \use_none:n #2 + \regex_query_set:nnn {#1} {"#2} {#3} + \if_case:w "#2 \exp_stop_f: + \or: \int_incr:N \l_regex_nesting_int + \or: \int_decr:N \l_regex_nesting_int + \fi: + \regex_query_set_loop:ww } -\cs_new_protected_nopar:Npn \regex_match_once_aux: +\cs_new_protected:Npn \regex_query_set:nnn #1#2#3 { - \int_compare:nNnTF \l_regex_start_step_int = \c_zero - { - \int_set_eq:NN \l_regex_current_char_int \c_minus_one - \regex_match_loop:N - } - { - \str_skip_do:nn - { \l_regex_start_step_int - \c_one } - { \regex_match_once_aux_ii:N } - } + \tex_muskip:D \l_regex_current_step_int + = \etex_gluetomu:D + #3 sp + plus #2 sp + minus \l_regex_nesting_int sp + \scan_stop: + \tex_toks:D \l_regex_current_step_int {#1} + \int_incr:N \l_regex_current_step_int } -\cs_new_protected_nopar:Npn \regex_match_once_aux_ii:N #1 +\cs_new_protected_nopar:Npn \regex_query_get: { - \int_set:Nn \l_regex_current_char_int { `#1 } - \regex_match_loop:N + \tl_set:Nx \l_regex_current_token_tl + { \tex_the:D \tex_toks:D \l_regex_current_step_int } + \l_regex_current_char_int + = \etex_mutoglue:D \tex_muskip:D \l_regex_current_step_int + \l_regex_current_catcode_int = \etex_gluestretch:D + \etex_mutoglue:D \tex_muskip:D \l_regex_current_step_int + } +\cs_new_protected_nopar:Npn \regex_match_once: + { + \regex_match_setup: + \regex_query_get: + \regex_match_loop: + \prg_break_point:n { } + \bool_if:NT \l_regex_success_match_bool + { + \bool_gset_true:N \g_regex_success_bool + \l_regex_every_match_tl + } } \cs_new_protected_nopar:Npn \regex_match_initial_setup: { - \prg_stepwise_inline:nnnn {1} {1} { \l_regex_max_state_int } + \prg_stepwise_inline:nnnn {0} {1} { \l_regex_max_state_int - \c_one } { \tex_dimen:D ##1 \c_minus_one sp \scan_stop: } - \int_set_eq:NN \l_regex_unique_step_int \c_minus_one - \int_set_eq:NN \l_regex_start_step_int \c_minus_one - \int_set_eq:NN \l_regex_current_step_int \c_zero - \int_set_eq:NN \l_regex_success_step_int \c_zero + \int_set_eq:NN \l_regex_unique_id_int \c_minus_one + \int_set:Nn \l_regex_start_step_int + { \l_regex_min_step_int - \c_one } + \int_set_eq:NN \l_regex_current_step_int \l_regex_min_step_int + \int_set_eq:NN \l_regex_success_step_int \l_regex_min_step_int + \int_set_eq:NN \l_regex_submatch_int \l_regex_max_state_int \bool_set_false:N \l_regex_success_empty_bool + \bool_gset_false:N \g_regex_success_bool } \cs_new_protected_nopar:Npn \regex_match_setup: { @@ -1219,33 +1385,44 @@ { \cs_set_eq:NN \regex_last_match_empty:F \regex_last_match_empty_yes:F } { \cs_set_eq:NN \regex_last_match_empty:F \regex_last_match_empty_no:F } \int_set_eq:NN \l_regex_start_step_int \l_regex_success_step_int - \int_set_eq:NN \l_regex_current_step_int \l_regex_start_step_int - \int_decr:N \l_regex_current_step_int - \bool_set_false:N \l_regex_success_bool + \int_set:Nn \l_regex_current_step_int + { \l_regex_start_step_int - \c_one } + \bool_set_false:N \l_regex_success_match_bool \int_zero:N \l_regex_max_index_int - \regex_store_state:n {1} + \regex_store_state:n {0} %^^A _state_int! } -\cs_new_protected_nopar:Npn \regex_match_loop:N #1 +\cs_new_protected_nopar:Npn \regex_match_loop: { - \regex_match_loop_setup:N #1 + \int_incr:N \l_regex_current_step_int + \int_incr:N \l_regex_unique_id_int + \bool_set_false:N \l_regex_fresh_thread_bool + \int_set_eq:NN \l_regex_last_char_int \l_regex_current_char_int + \regex_query_get: + \regex_match_loop_case_hook: \cs_set_nopar:Npx \regex_tmp:w { \int_zero:N \l_regex_max_index_int - \prg_stepwise_function:nnnN - {1} {1} { \l_regex_max_index_int } - \regex_match_one_index:n + \regex_match_one_index:w 1 ; \prg_break_point:n { } + \exp_not:N \prg_break_point:n { } } - \regex_tmp:w \prg_break_point:n { } + \regex_tmp:w + \if_num:w \l_regex_current_char_int < \c_minus_one + \exp_after:wN \prg_map_break: + \fi: \if_num:w \l_regex_max_index_int = \c_zero - \exp_after:wN \use_none_delimit_by_q_recursion_stop:w + \exp_after:wN \prg_map_break: \fi: - \quark_if_recursion_tail_stop:N #1 - \regex_match_loop:N + \regex_match_loop: } -\cs_new_nopar:Npn \regex_match_one_index:n #1 +\cs_new_nopar:Npn \regex_match_one_index:w #1; { + \if_num:w #1 > \l_regex_max_index_int + \exp_after:wN \prg_map_break: + \fi: \regex_match_one_index_aux:n { \int_value:w \tex_skip:D #1 } + \exp_after:wN \regex_match_one_index:w + \int_use:N \int_eval:w #1 + \c_one ; } \cs_new_protected_nopar:Npn \regex_match_one_index_aux:n #1 { @@ -1253,19 +1430,6 @@ \prop_clear:N \l_regex_current_submatches_prop \regex_state_use_with_submatches: } -\cs_new_protected_nopar:Npn \regex_match_loop_setup:N #1 - { - \int_incr:N \l_regex_current_step_int - \int_incr:N \l_regex_unique_step_int - \bool_set_false:N \l_regex_fresh_thread_bool - \int_set_eq:NN \l_regex_last_char_int \l_regex_current_char_int - \if_meaning:w #1 \q_recursion_tail - \int_set_eq:NN \l_regex_current_char_int \c_minus_one - \else: - \int_set:Nn \l_regex_current_char_int {`#1} - \fi: - \regex_match_loop_case_hook: - } \cs_new_protected_nopar:Npn \regex_match_loop_case_hook: { } \cs_new_protected_nopar:Npn \regex_match_loop_caseless_hook: { @@ -1293,18 +1457,17 @@ } \cs_new_protected_nopar:Npn \regex_action_cost:n #1 { - \exp_args:Nf \regex_store_state:n %^^A optimize! + \exp_args:Nf \regex_store_state:n { \int_eval:n { \l_regex_current_state_int + #1 } } } \cs_new_protected_nopar:Npn \regex_action_success: { \regex_last_match_empty:F { - \bool_set_true:N \l_regex_success_bool + \bool_set_true:N \l_regex_success_match_bool \bool_set_eq:NN \l_regex_success_empty_bool \l_regex_fresh_thread_bool - \int_set_eq:NN \l_regex_success_step_int - \l_regex_current_step_int + \int_set_eq:NN \l_regex_success_step_int \l_regex_current_step_int \prop_set_eq:NN \l_regex_success_submatches_prop \l_regex_current_submatches_prop \prg_map_break: @@ -1328,504 +1491,668 @@ \prop_put:Nno \l_regex_current_submatches_prop {#1} { \int_use:N \l_regex_current_step_int } } -\cs_new_protected_nopar:Npn \regex_extract: +\cs_new_protected:Npn \regex_submatch_nesting_aux:n #1 { - \seq_gclear:N \g_regex_submatches_seq - \prg_stepwise_inline:nnnn - {0} {1} { \l_regex_capturing_group_int } - { - \regex_extract_aux:nTF { ##1 } - { - \seq_gput_right:Nx \g_regex_submatches_seq - { \regex_query_substr:NN \l_regex_tmpa_tl \l_regex_tmpb_tl } - } - { \seq_gput_right:Nn \g_regex_submatches_seq { } } - } + + \etex_glueshrink:D \etex_mutoglue:D \etex_muexpr:D + \tex_muskip:D \etex_gluestretch:D \tex_skip:D #1 + - \tex_muskip:D \tex_skip:D #1 + \scan_stop: } -\cs_new_protected_nopar:Npn \regex_extract_aux:nTF #1#2#3 +\cs_new:Npn \regex_nesting:n #1 { } %^^A move. Rename? +\tl_new:N \l_regex_nesting_tl +\cs_new_protected:Npn \regex_replacement:n #1 { - \prop_get:NnNTF \l_regex_success_submatches_prop - { #1 < } \l_regex_tmpa_tl + \int_zero:N \l_regex_replacement_int + \int_zero:N \l_regex_nesting_int + \tl_clear:N \l_regex_nesting_tl + \str_escape_use:NNNn + \regex_replacement_unescaped:N + \regex_replacement_escaped:N + \regex_replacement_raw:N + {#1} + \prg_do_nothing: \prg_do_nothing: + \cs_set_nopar:Npx \regex_nesting:n ##1 + { + + \int_use:N \l_regex_nesting_int + \l_regex_nesting_tl + - \regex_submatch_nesting_aux:n {##1} + } + \use:x { - \prop_get:NnNTF \l_regex_success_submatches_prop - { #1 > } \l_regex_tmpb_tl - {#2} - {#3} + \exp_not:n { \cs_set_nopar:Npn \regex_replacement_tl:n ##1 } + { \str_aux_toks_range:nn \c_zero \l_regex_replacement_int } } - {#3} + % ^^A rename! Careful with \cP\#. } -\cs_new_protected_nopar:Npn \regex_set:Nn - { \regex_set_aux:NNn \tl_set_eq:NN } -\cs_new_protected_nopar:Npn \regex_gset:Nn - { \regex_set_aux:NNn \tl_gset_eq:NN } -\cs_new_protected:Npn \regex_set_aux:NNn #1#2#3 +\cs_new_protected:Npn \regex_replacement_raw:N #1 + { \regex_replacement_put:n {#1} } +\cs_new_protected:Npn \regex_replacement_put:n #1 { - \group_begin: - \regex_build:n {#3} - \tl_gset:Nx \g_regex_tmpa_tl - { - \exp_not:N \regex_nfa:Nw \exp_not:N #2 - \l_regex_max_state_int - = \int_use:N \l_regex_max_state_int - \l_regex_capturing_group_int - = \int_use:N \l_regex_capturing_group_int - \token_if_eq_meaning:NNT - \regex_match_loop_case_hook: - \regex_match_loop_caseless_hook: - { - \cs_set_eq:NN \regex_match_loop_case_hook: - \regex_match_loop_caseless_hook: - } - \prg_stepwise_function:nnnN - {1} {1} {\l_regex_max_state_int} - \regex_set_aux:n - \s_regex_stop - } - \group_end: - #1 #2 \g_regex_tmpa_tl + \tex_toks:D \l_regex_replacement_int {#1} + \int_incr:N \l_regex_replacement_int } -\cs_new_nopar:Npn \regex_set_aux:n #1 - { \tex_toks:D #1 { \tex_the:D \tex_toks:D #1 } } -\cs_new:Npn \regex_nfa:Nw #1 #2 \s_regex_stop - { \msg_expandable_kernel_error:nnn { regex } { nfa-misused } {#1} } -\cs_new_protected_nopar:Npn \regex_const:Nn - { \regex_set_aux:NNn \cs_new_eq:NN } -\prg_new_protected_conditional:Npnn \regex_check_nfa:N #1 { T , TF } - { \exp_after:wN \regex_check_nfa_aux:Nw #1 \q_stop } -\cs_new:Npn \regex_check_nfa_aux:Nw #1 #2 \q_stop +\cs_new_protected:Npn \regex_replacement_unescaped:N #1 { - \if_meaning:w \regex_nfa:Nw #1 - \prg_return_true: + \if_charcode:w \c_rbrace_str #1 + \if_num:w \l_regex_replacement_csnames_int > \c_zero + \regex_replacement_put:n \cs_end: + \else: + \regex_replacement_put:n #1 + \fi: \else: - \msg_error:nnx { regex } { not-nfa } { \token_to_str:N #1 } - \prg_return_false: + \regex_replacement_put:n #1 \fi: } -\cs_new_protected_nopar:Npn \regex_use:N #1 - { \exp_after:wN \use_none:nn #1 } -\cs_new_protected:Npn \regex_user_aux:n #1 +\cs_new_protected:Npn \regex_replacement_escaped:N #1 { - \group_begin: - \tl_clear:N \l_regex_every_match_tl - #1 - \group_end: - } -\cs_new_protected_nopar:Npn \regex_return_after_group: - { - \if_meaning:w \c_true_bool \l_regex_success_bool - \group_insert_after:N \prg_return_true: + \if_charcode:w c #1 + \exp_after:wN \regex_replacement_c:w \else: - \group_insert_after:N \prg_return_false: + \if_charcode:w g #1 + \exp_after:wN \exp_after:wN \exp_after:wN \regex_replacement_g:w + \else: + \if_num:w \c_one < 1#1 \exp_stop_f: + \regex_replacement_put_submatch:n {#1} + \else: + \regex_replacement_put:n #1 + \fi: + \fi: \fi: } -\cs_new_protected_nopar:Npn \regex_extract_after_group:N #1 +\cs_new_protected:Npn \regex_replacement_put_submatch:n #1 { - \if_meaning:w \c_true_bool \l_regex_success_bool - \regex_extract: - \group_insert_after:N \seq_set_eq:NN - \group_insert_after:N #1 - \group_insert_after:N \g_regex_submatches_seq + \regex_replacement_put:n + { \regex_query_submatch:nn {#1} {##1} } + \if_num:w \l_regex_replacement_csnames_int = \c_zero + \tl_put_right:Nn \l_regex_nesting_tl + { + \exp_not:N \if_num:w #1 < \l_regex_capturing_group_int + \regex_submatch_nesting_aux:n { \int_eval:w #1+##1 \int_eval_end: } + \exp_not:N \fi: + } \fi: } -\cs_new_protected_nopar:Npn \regex_count_after_group:N #1 +\cs_new_protected:Npn \regex_replacement_error:NNN #1#2#3 { - \group_insert_after:N \int_set_eq:NN - \group_insert_after:N #1 - \group_insert_after:N \g_regex_match_count_int + \msg_kernel_error:nn { regex } { replacement-#1 } + #2 #3 } -\prg_new_protected_conditional:Npnn \regex_match:nn #1#2 { T , F , TF } +\cs_new_protected:Npn \regex_replacement_g:w #1#2 { - \regex_user_aux:n + \str_if_eq:xxTF + { \exp_not:n { #1#2 } } + { \regex_replacement_unescaped:N \c_lbrace_str } { - \regex_disable_submatches: - \regex_build:n {#1} - \regex_match:n {#2} - \regex_return_after_group: + \int_zero:N \l_regex_tmpa_int + \regex_replacement_g_digits:NN } + { \regex_replacement_error:NNN g #1 #2 } } -\prg_new_protected_conditional:Npnn \regex_match:Nn #1#2 { T , F , TF } +\cs_new_protected:Npn \regex_replacement_g_digits:NN #1#2 { - \regex_check_nfa:NTF #1 + \token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N { - \regex_user_aux:n - { - \regex_disable_submatches: - \regex_use:N #1 - \regex_match:n {#2} - \regex_return_after_group: - } + \if_num:w \c_one < 1#2 \exp_stop_f: + \int_set:Nn \l_regex_tmpa_int { \c_ten * \l_regex_tmpa_int + #2 } + \exp_after:wN \use_i:nnn + \exp_after:wN \regex_replacement_g_digits:NN + \else: + \if_charcode:w \c_rbrace_str #2 + \exp_args:No \regex_replacement_put_submatch:n + { \int_use:N \l_regex_tmpa_int } + \exp_after:wN \exp_after:wN \exp_after:wN \use_none:nn + \else: + \exp_after:wN \exp_after:wN + \exp_after:wN \regex_replacement_error:NNN + \exp_after:wN \exp_after:wN \exp_after:wN g + \fi: + \fi: } - { \prg_return_false: } + { \regex_replacement_error:NNN g } + #1 #2 } -\cs_new_protected:Npn \regex_count:nnN #1#2#3 +\cs_new_protected_nopar:Npn \regex_replacement_c:w #1#2 { - \regex_user_aux:n + \token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N { - \regex_disable_submatches: - \int_gzero:N \g_regex_match_count_int - \tl_set:Nn \l_regex_every_match_tl - { - \int_gincr:N \g_regex_match_count_int - \regex_match_once: - } - \regex_build:n {#1} - \regex_match:n {#2} - \regex_count_after_group:N #3 + \cs_if_exist_use:cF { regex_replacement_c_#2:w } + { \regex_replacement_error:NNN c #1#2 } } + { \regex_replacement_error:NNN c #1#2 } } -\cs_new_protected:Npn \regex_count:NnN #1#2#3 +\cs_new_protected_nopar:cpn { regex_replacement_c_ \c_lbrace_str :w } { - \regex_check_nfa:NT #1 - { - \regex_user_aux:n - { - \regex_disable_submatches: - \int_gzero:N \g_regex_match_count_int - \tl_set:Nn \l_regex_every_match_tl - { - \int_gincr:N \g_regex_match_count_int - \regex_match_once: - } - \regex_use:N #1 - \regex_match:n {#2} - \regex_count_after_group:N #3 - } - } + \int_incr:N \l_regex_replacement_csnames_int + \regex_replacement_put:n + { \exp_not:n { \exp_after:wN \regex_replacement_exp_not:N \cs:w } } } -\cs_new_protected:Npn \regex_extract_once:nnN #1#2#3 - { - \regex_user_aux:n - { - \regex_build:n {#1} - \regex_match:n {#2} - \regex_extract_after_group:N #3 - } +\cs_new_nopar:Npn \regex_replacement_exp_not:N #1 { \exp_not:n {#1} } +\group_begin: + + \char_set_catcode_math_superscript:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_U:w + { \regex_replacement_char:nNN { ^^@ } } + + \char_set_catcode_alignment:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_T:w + { \regex_replacement_char:nNN { ^^@ } } + + \cs_new_protected_nopar:Npn \regex_replacement_c_S:w #1#2 + { + \int_compare:nNnTF { `#2 } = \c_zero + { \regex_replacement_error:NNN c #1#2 } + { + \char_set_lccode:nn {32} { `#2 } + \tl_to_lowercase:n { \regex_replacement_put:n {~} } + } + } + + \char_set_catcode_parameter:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_P:w + { \regex_replacement_char:nNN { ^^@^^@ } } + + \char_set_catcode_other:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_O:w + { \regex_replacement_char:nNN { ^^@ } } + + \char_set_catcode_math_toggle:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_M:w + { \regex_replacement_char:nNN { ^^@ } } + + \char_set_catcode_letter:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_L:w + { \regex_replacement_char:nNN { ^^@ } } + + \char_set_catcode_group_end:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_E:w + { + \int_decr:N \l_regex_nesting_int + \regex_replacement_char:nNN { \if_false: { \fi: ^^@ } + } + + \char_set_catcode_math_subscript:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_D:w + { \regex_replacement_char:nNN { ^^@ } } + + \char_set_catcode_group_begin:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_B:w + { + \int_incr:N \l_regex_nesting_int + \regex_replacement_char:nNN { \exp_after:wN ^^@ \if_false: } \fi: } + } + + \char_set_catcode_active:N \^^@ + \cs_new_protected_nopar:Npn \regex_replacement_c_A:w + { \regex_replacement_char:nNN { \exp_not:N ^^@ } } + +\group_end: + +\cs_new_protected_nopar:Npn \regex_replacement_char:nNN #1#2#3 + { + \char_set_lccode:nn \c_zero { `#3 } + \tl_to_lowercase:n + { \regex_replacement_put:n { \exp_not:n {#1} } } } -\prg_new_protected_conditional:Npnn \regex_extract_once:nnN #1#2#3 - { T , F , TF } +\cs_new_protected_nopar:Npn \regex_set:Nn + { \regex_set_aux:NNn \tl_set:Nn } +\cs_new_protected_nopar:Npn \regex_gset:Nn + { \regex_set_aux:NNn \tl_gset:Nn } +\cs_new_protected_nopar:Npn \regex_const:Nn + { \regex_set_aux:NNn \tl_const:Nn } +\cs_new_protected:Npn \regex_set_aux:NNn #1#2#3 { - \regex_user_aux:n - { - \regex_build:n {#1} - \regex_match:n {#2} - \regex_extract_after_group:N #3 - \regex_return_after_group: - } + \group_begin: + \regex_build:n {#3} + \cs_set_nopar:Npx \regex_tmp:w + { #1 \exp_not:N #2 { \regex_set_aux:N #2 } } + \exp_after:wN + \group_end: + \regex_tmp:w } -\cs_new_protected:Npn \regex_extract_once:NnN #1#2#3 +\cs_new:Npn \regex_set_aux:N #1 { - \regex_check_nfa:NT #1 + \exp_not:n { \regex_nfa:Nw #1 } + \l_regex_max_state_int + = \int_use:N \l_regex_max_state_int + \l_regex_capturing_group_int + = \int_use:N \l_regex_capturing_group_int + \token_if_eq_meaning:NNT + \regex_match_loop_case_hook: + \regex_match_loop_caseless_hook: { - \regex_user_aux:n - { - \regex_use:N #1 - \regex_match:n {#2} - \regex_extract_after_group:N #3 - } + \cs_set_eq:NN \regex_match_loop_case_hook: + \regex_match_loop_caseless_hook: } + \prg_stepwise_function:nnnN + {0} {1} {\l_regex_max_state_int - \c_one } + \regex_set_aux:n + \s_stop } -\prg_new_protected_conditional:Npnn \regex_extract_once:NnN #1#2#3 - { T , F , TF } +\cs_new_nopar:Npn \regex_set_aux:n #1 + { \tex_toks:D #1 { \tex_the:D \tex_toks:D #1 } } +\cs_new:Npn \regex_nfa:Nw #1 { - \regex_check_nfa:NTF #1 - { - \regex_user_aux:n - { - \regex_use:N #1 - \regex_match:n {#2} - \regex_extract_after_group:N #3 - \regex_return_after_group: - } - } - { \prg_return_false: } + \msg_expandable_kernel_error:nnn { regex } { nfa-misused } {#1} + \use_none_delimit_by_s_stop:w } -\cs_new_protected:Npn \regex_extract_all:nnN #1#2#3 +\cs_new_protected_nopar:Npn \regex_use:N + { \exp_last_unbraced:No \use_none:nn } +\cs_new_nopar:Npn \regex_to_str:N #1 { - \regex_user_aux:n - { - \seq_gclear:N \g_regex_extract_all_seq - \tl_set:Nn \l_regex_every_match_tl { \regex_extract_all_aux: } - \regex_build:n {#1} - \regex_match:n {#2} - \regex_extract_all_after_group:N #3 - } + \exp_after:wN \regex_to_str_aux:Nw #1 + \tex_toks:D \q_stop \prg_map_break: \tex_toks:D + \prg_break_point:n { } } -\cs_new_protected:Npn \regex_extract_all:NnN #1#2#3 +\cs_new_nopar:Npn \regex_to_str_aux:Nw #1#2\tex_toks:D { - \regex_check_nfa:NT #1 - { - \regex_user_aux:n - { - \seq_gclear:N \g_regex_extract_all_seq - \tl_set:Nn \l_regex_every_match_tl { \regex_extract_all_aux: } - \regex_use:N #1 - \regex_match:n {#2} - \regex_extract_all_after_group:N #3 - } - } + \use_none_delimit_by_q_stop:w #2 \q_stop + \tl_to_str:n {#1#2} \iow_newline: + \regex_to_str_aux:Nw \tex_toks:D } -\cs_new_protected_nopar:Npn \regex_extract_all_aux: +\cs_new_protected_nopar:Npn \regex_aux_return: { - \regex_extract: - \seq_gconcat:NNN \g_regex_extract_all_seq - \g_regex_extract_all_seq \g_regex_submatches_seq - \regex_match_once: + \if_meaning:w \c_true_bool \g_regex_success_bool + \prg_return_true: + \else: + \prg_return_false: + \fi: } -\cs_new_protected_nopar:Npn \regex_extract_all_after_group:N #1 +\cs_new_protected:Npn \regex_aux_build_match:nn #1#2 { - \seq_if_empty:NF \g_regex_extract_all_seq - { - \regex_extract: - \group_insert_after:N \seq_set_eq:NN - \group_insert_after:N #1 - \group_insert_after:N \g_regex_extract_all_seq - } + \regex_build:n {#1} + \regex_match:n {#2} } -\cs_new_protected:Npn \regex_split:nnN #1#2#3 +\cs_new_protected:Npn \regex_aux_use_match:Nn #1#2 { - \regex_user_aux:n + \exp_args:No \tl_if_head_eq_meaning:nNTF {#1} \regex_nfa:Nw { - \seq_gclear:N \g_regex_split_seq - \tl_set:Nn \l_regex_every_match_tl { \regex_split_aux: } - \regex_build:n {#1} + \regex_use:N #1 \regex_match:n {#2} - \regex_split_after_group:N #3 } - } -\cs_new_protected:Npn \regex_split:NnN #1#2#3 - { - \regex_check_nfa:NT #1 - { - \regex_user_aux:n - { - \seq_gclear:N \g_regex_split_seq - \tl_set:Nn \l_regex_every_match_tl { \regex_split_aux: } - \regex_use:N #1 - \regex_match:n {#2} - \regex_split_after_group:N #3 - } - } - } -\cs_new_protected_nopar:Npn \regex_split_aux: - { - \int_compare:nNnF \l_regex_start_step_int = \l_regex_success_step_int { - \regex_extract: - \seq_pop:NN \g_regex_submatches_seq \l_regex_tmpa_tl - \regex_extract_aux:nTF {0} - { - \seq_gput_left:Nx \g_regex_submatches_seq - { - \regex_query_substr:NN - \l_regex_start_step_int \l_regex_tmpa_tl - } - } - { \msg_error:nn { regex } { internal } } - \seq_gconcat:NNN \g_regex_split_seq - \g_regex_split_seq \g_regex_submatches_seq + \msg_kernel_error:nnx { regex } { not-nfa } { \token_to_str:N #1 } + \bool_gset_false:N \g_regex_success_bool } - \regex_match_once: } -\cs_new_protected_nopar:Npn \regex_split_after_group:N #1 +\cs_set_protected:Npn \regex_tmp:w #1#2#3 { - \int_compare:nNnTF \l_regex_start_step_int = \l_regex_current_step_int + \cs_new_protected_nopar:Npn #1 + { #3 \regex_aux_build_match:nn } + \cs_new_protected_nopar:Npn #2 + { #3 \regex_aux_use_match:Nn } + \prg_new_protected_conditional:Npnn #1 ##1##2##3 { T , F , TF } { - \bool_if:NF \l_regex_success_empty_bool - { \seq_gput_right:Nn \g_regex_split_seq { } } + #3 \regex_aux_build_match:nn {##1} {##2} ##3 + \regex_aux_return: } + \prg_new_protected_conditional:Npnn #2 ##1##2##3 { T , F , TF } { - \seq_gput_right:Nx \g_regex_split_seq - { - \regex_query_substr:NN - \l_regex_start_step_int \l_regex_current_step_int - } + #3 \regex_aux_use_match:Nn {##1} {##2} ##3 + \regex_aux_return: } - \group_insert_after:N \seq_set_eq:NN - \group_insert_after:N #1 - \group_insert_after:N \g_regex_split_seq } -\cs_new_protected:Npn \regex_replacement:n #1 +\tl_map_inline:nn { - \str_aux_escape:NNNn - \prg_do_nothing: - \regex_replacement_escaped:N - \prg_do_nothing: - {#1} - \tl_set_eq:NN \l_regex_replacement_tl \g_str_result_tl - \tl_set:Nx \l_regex_replacement_tl - { \l_regex_replacement_tl \prg_do_nothing: } + { extract_once } { extract_all } + { replace_once } { replace_all } + { split } } -\cs_new_nopar:Npn \regex_replacement_escaped:N #1 { - \regex_token_if_other_digit:NTF #1 - { - \exp_not:n - { \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } } {#1} - } - { - \token_if_eq_charcode:NNTF g #1 - { \exp_not:N \regex_replacement_submatch:w } - { #1 } - } + \exp_args:Nccc \regex_tmp:w + { regex_#1:nnN } { regex_#1:NnN } { regex_#1_aux:NnnN } } -\cs_new_nopar:Npn \regex_replacement_submatch:w #1 +\cs_new_protected_nopar:Npn \regex_extract: { - \exp_after:wN \token_if_eq_meaning:NNTF \c_lbrace_str #1 - { \regex_replacement_submatch_aux:nN { } } - { - \regex_token_if_other_digit:NTF #1 - { \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } } - { \msg_expandable_kernel_error:nn { regex } { g-misused } } - #1 - } + \int_set_eq:NN \l_regex_submatch_start_int \l_regex_submatch_int + \if_meaning:w \c_true_bool \g_regex_success_bool + \prg_replicate:nn \l_regex_capturing_group_int + { + \tex_skip:D \l_regex_submatch_int \c_zero sp \scan_stop: + \int_incr:N \l_regex_submatch_int + } + \prop_map_inline:Nn \l_regex_success_submatches_prop + { + \if_num:w ##1 \c_max_int + \exp_after:wN \regex_extract_aux_b:wn \int_use:N + \else: + \exp_after:wN \regex_extract_aux_e:wn \int_use:N + \fi: + \int_eval:w \l_regex_submatch_start_int + ##1 {##2} + } + \fi: } -\cs_new_nopar:Npn \regex_replacement_submatch_aux:nN #1 #2 +\cs_new_protected_nopar:Npn \regex_extract_aux_b:wn #1 < #2 { - \regex_token_if_other_digit:NTF #2 - { \regex_replacement_submatch_aux:nN {#1#2} } - { - \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } {#1} - \exp_after:wN \token_if_eq_meaning:NNF \c_rbrace_str #2 - { - \msg_expandable_kernel_error:nn { regex } { g-misused } - #2 - } - } + \tex_skip:D #1 = #2 sp + plus \etex_gluestretch:D \tex_skip:D #1 \scan_stop: } -\cs_new_protected_nopar:Npn \regex_replace_after_group:N #1 +\cs_new_protected_nopar:Npn \regex_extract_aux_e:wn #1 > #2 { - \group_insert_after:N \tl_set:Nx - \group_insert_after:N #1 - \group_insert_after:N { - \group_insert_after:N \tl_to_str:N - \group_insert_after:N \g_regex_replaced_str - \group_insert_after:N } + \tex_skip:D #1 + = 1 \tex_skip:D #1 plus #2 sp \scan_stop: } -\cs_new_protected:Npn \regex_replace_once_aux:Nn #1#2 +\cs_new_protected:Npn \regex_group_end_extract_seq:N #1 { - \group_begin: - \regex_replace_after_group:N #1 - \tl_clear:N \l_regex_every_match_tl - #2 - \exp_args:No \regex_match:n {#1} - \regex_extract: - \regex_extract_aux:nTF {0} + \cs_set_eq:NN \seq_item:n \scan_stop: + \flag_lower:N \l_regex_group_begin_flag + \flag_lower:N \l_regex_group_end_flag + \tl_set:Nx \l_regex_tmpa_tl { - \cs_set_nopar:Npn \regex_tmp:w - { \str_skip_do:nn { \l_regex_tmpb_tl } { } } - \tl_gset:Nx \g_regex_replaced_str - { - \regex_query_substr:NN \c_zero \l_regex_tmpa_tl - \l_regex_replacement_tl - \exp_after:wN \regex_tmp:w \l_regex_query_other_str - } + \prg_stepwise_function:nnnN + \l_regex_max_state_int + \c_one + { \l_regex_submatch_int - \c_one } + \regex_extract_seq_aux:n } - { \tl_gset_eq:NN \g_regex_replaced_str \l_regex_query_other_str } - \group_end: + \flag_test:NF \l_regex_group_begin_flag + { \flag_test:NF \l_regex_group_end_flag { \use_none:nnn } } + \msg_kernel_error:nn { regex } { sequence-unbalanced } + \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl } + \exp_args:NNNo \group_end: + \tl_set:Nn #1 \l_regex_tmpa_tl } -\cs_new_protected:Npn \regex_replace_once:nnN #1#2#3 +\cs_new_nopar:Npn \regex_extract_seq_aux:n #1 { - \regex_replace_once_aux:Nn #3 + \seq_item:n { - \regex_build:n {#1} - \regex_replacement:n {#2} + \exp_after:wN \regex_extract_seq_aux:ww + \int_value:w \regex_submatch_nesting_aux:n {#1} ; #1; } } -\cs_new_protected:Npn \regex_replace_once:NnN #1#2#3 +\cs_new_nopar:Npn \regex_extract_seq_aux:ww #1; #2; + { %^^A todo: use flags + \if_num:w #1 < \c_zero + \flag_raise:N \l_regex_group_end_flag + \prg_replicate:nn {-#1} { \exp_not:n { { \if_false: } \fi: } } + \fi: + \regex_query_submatch:w #2; + \if_num:w #1 > \c_zero + \flag_raise:N \l_regex_group_begin_flag + \prg_replicate:nn {#1} { \exp_not:n { \if_false: { \fi: } } } + \fi: + } +\prg_new_protected_conditional:Npnn \regex_match:nn #1#2 { T , F , TF } { - \regex_replace_once_aux:Nn #3 - { - \regex_use:N #1 - \regex_replacement:n {#2} - } + \regex_match_aux:n + { \regex_aux_build_match:nn {#1} {#2} } } -\prg_new_protected_conditional:Npnn \regex_replace_once:nnN #1#2#3 {T,F,TF} +\prg_new_protected_conditional:Npnn \regex_match:Nn #1#2 { T , F , TF } { - \regex_replace_once_aux:Nn #3 - { - \regex_build:n {#1} - \regex_replacement:n {#2} - \regex_return_after_group: - } + \regex_match_aux:n + { \regex_aux_use_match:Nn #1 {#2} } } -\prg_new_protected_conditional:Npnn \regex_replace_once:NnN #1#2#3 {T,F,TF} +\cs_new_protected:Npn \regex_match_aux:n #1 { - \regex_replace_once_aux:Nn #3 - { - \regex_use:N #1 - \regex_replacement:n {#2} - \regex_return_after_group: - } + \group_begin: + \tl_clear:N \l_regex_every_match_tl + \regex_disable_submatches: + #1 + \group_end: + \regex_aux_return: } -\cs_new_protected:Npn \regex_replace_all_aux:Nn #1#2 +\cs_new_protected_nopar:Npn \regex_count:nnN + { \regex_count_aux:NnnN \regex_aux_build_match:nn } +\cs_new_protected_nopar:Npn \regex_count:NnN + { \regex_count_aux:NnnN \regex_aux_use_match:Nn } +\cs_new_protected:Npn \regex_count_aux:NnnN #1#2#3#4 { \group_begin: - \regex_replace_after_group:N #1 - \tl_set:Nn \l_regex_every_match_tl { \regex_replace_all_aux: } - \tl_gclear:N \g_regex_replaced_str - #2 - \exp_args:No \regex_match:n {#1} - \cs_set_nopar:Npn \regex_tmp:w - { \str_skip_do:nn { \l_regex_start_step_int } { } } - \tl_gput_right:Nx \g_regex_replaced_str - { \exp_after:wN \regex_tmp:w \l_regex_query_other_str } + \regex_disable_submatches: + \int_zero:N \l_regex_match_count_int + \tl_set:Nn \l_regex_every_match_tl + { + \int_incr:N \l_regex_match_count_int + \regex_match_once: + } + #1 {#2} {#3} + \exp_args:NNNo \group_end: + \int_set:Nn #4 { \int_use:N \l_regex_match_count_int } } -\cs_new_protected_nopar:Npn \regex_replace_all_aux: +\cs_new_protected:Npn \regex_extract_once_aux:NnnN #1#2#3#4 { - \regex_extract: - \regex_extract_aux:nTF {0} - { - \tl_gput_right:Nx \g_regex_replaced_str + \group_begin: + \tl_set:Nn \l_regex_every_match_tl { \regex_extract: } + #1 {#2} {#3} + \regex_group_end_extract_seq:N #4 + } +\cs_new_protected:Npn \regex_extract_all_aux:NnnN #1#2#3#4 + { + \group_begin: + \tl_set:Nn \l_regex_every_match_tl + { \regex_extract: \regex_match_once: } + #1 {#2} {#3} + \regex_group_end_extract_seq:N #4 + } +\cs_new_protected:Npn \regex_split_aux:NnnN #1#2#3#4 + { + \group_begin: + \tl_set:Nn \l_regex_every_match_tl + { + \if_num:w \l_regex_start_step_int < \l_regex_success_step_int + \regex_extract: + \tex_skip:D \l_regex_submatch_start_int + = \l_regex_start_step_int sp + plus \tex_skip:D \l_regex_submatch_start_int \scan_stop: + \fi: + \regex_match_once: + } + #1 {#2} {#3} + \tex_skip:D \l_regex_submatch_int + = \l_regex_start_step_int sp + plus \l_regex_current_step_int sp \scan_stop: + \int_incr:N \l_regex_submatch_int + \if_num:w \l_regex_start_step_int = \l_regex_current_step_int + \if_meaning:w \c_true_bool \l_regex_success_empty_bool + \int_decr:N \l_regex_submatch_int + \fi: + \fi: + \regex_group_end_extract_seq:N #4 + } +\cs_new_protected:Npn \regex_replace_once_aux:NnnN #1#2#3#4 + { + \group_begin: + \tl_clear:N \l_regex_every_match_tl + \regex_replacement:n {#3} + \exp_args:Nno #1 {#2} #4 + \if_meaning:w \c_true_bool \g_regex_success_bool + \regex_extract: + \int_set:Nn \l_regex_tmpa_int + { \regex_nesting:n { \l_regex_submatch_start_int } } + \if_num:w \l_regex_tmpa_int = \c_zero + \else: + \msg_kernel_error:nnx { regex } { replace-unbalanced } + { \l_regex_tmpa_int } + \fi: + \tl_set:Nx \l_regex_tmpa_tl { - \regex_query_substr:NN - \l_regex_start_step_int \l_regex_tmpa_tl - \l_regex_replacement_tl + \if_num:w \l_regex_tmpa_int < \c_zero + \prg_replicate:nn { - \l_regex_tmpa_int } + { \exp_not:n { { \if_false: } \fi: } } + \fi: + \regex_query_substr:nn + { \l_regex_min_step_int } + { \tex_skip:D \l_regex_submatch_start_int } + \regex_replacement_tl:n { \l_regex_submatch_start_int } + \regex_query_substr:nn + { \etex_gluestretch:D \tex_skip:D \l_regex_submatch_start_int } + { \l_regex_max_step_int } + \if_num:w \l_regex_tmpa_int > \c_zero + \prg_replicate:nn { \l_regex_tmpa_int } + { \exp_not:n { \if_false: { \fi: } } } + \fi: } - } - { \msg_error:nn { regex } { internal } } - \regex_match_once: + \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl } + \exp_args:NNNo \group_end: + \tl_set:Nn #4 \l_regex_tmpa_tl + \else: + \group_end: + \fi: } -\cs_new_protected:Npn \regex_replace_all:nnN #1#2#3 +\cs_new_protected_nopar:Npn \regex_replace_once_aux:w #1; { - \regex_replace_all_aux:Nn #3 + } +\cs_new_protected:Npn \regex_replace_all_aux:NnnN #1#2#3#4 + { + \group_begin: + \tl_set:Nn \l_regex_every_match_tl + { + \regex_extract: + \tex_skip:D \l_regex_submatch_start_int + = \tex_the:D \tex_skip:D \l_regex_submatch_start_int + minus \l_regex_start_step_int sp \scan_stop: + \regex_match_once: + } + \regex_replacement:n {#3} + \exp_args:Nno #1 {#2} #4 + \int_set:Nn \l_regex_tmpa_int + { + 0 + \prg_stepwise_function:nnnN + \l_regex_max_state_int + \l_regex_capturing_group_int + { \l_regex_submatch_int - \c_one } + \regex_nesting:n + } + \if_num:w \l_regex_tmpa_int = \c_zero + \else: + \msg_kernel_error:nnx { regex } { replace-unbalanced } + { \l_regex_tmpa_int } + \fi: + \tl_set:Nx \l_regex_tmpa_tl + { + \if_num:w \l_regex_tmpa_int < \c_zero + \prg_replicate:nn { - \l_regex_tmpa_int } + { \exp_not:n { { \if_false: } \fi: } } + \fi: + \prg_stepwise_function:nnnN + \l_regex_max_state_int + \l_regex_capturing_group_int + { \l_regex_submatch_int - \c_one } + \regex_replace_all_aux:n + \regex_query_substr:nn + \l_regex_start_step_int \l_regex_max_step_int + \if_num:w \l_regex_tmpa_int > \c_zero + \prg_replicate:nn { \l_regex_tmpa_int } + { \exp_not:n { \if_false: { \fi: } } } + \fi: + } + \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl } + \exp_args:NNNo \group_end: + \tl_set:Nn #4 \l_regex_tmpa_tl + } +\cs_new_nopar:Npn \regex_replace_all_aux:n #1 + { + \regex_query_substr:nn + { \etex_glueshrink:D \tex_skip:D #1 } { \tex_skip:D #1 } + \regex_replacement_tl:n {#1} + } +\msg_kernel_new:nnnn { regex } { sequence-unbalanced } + { + Missing~ + \flag_test:NTF \l_regex_group_end_flag { - \regex_build:n {#1} - \regex_replacement:n {#2} + left~ + \flag_test:NTF \l_regex_group_begin_flag + { and~right~braces } { brace } } + { right~brace } + \ inserted~in~extracted~match. } -\cs_new_protected:Npn \regex_replace_all:NnN #1#2#3 { - \regex_replace_all_aux:Nn #3 + LaTeX~was~asked~to~extract~submatches~or~split~a~token~list~ + according~to~a~given~regular~expression,~but~some~of~the~resulting~ + items~were~not~balanced. + } +\msg_kernel_new:nnnn { regex } { replace-unbalanced } + { The~result~of~a~replacement~does~not~have~balanced~braces. } + { + LaTeX~was~asked~to~do~some~regular~expression~replacement,~ + and~the~resulting~token~list~would~not~have~the~same~number~ + of~begin-group~and~end-group~tokens. \\ \\ + \ \ \ \ + \prg_case_int:nnn {#1} { - \regex_use:N #1 - \regex_replacement:n {#2} + { -1 } { A~left~brace~was } + { 1 } { A~right~brace~was } } + { + \int_abs:n {#1} ~ + \int_compare:nNnTF {#1} < \c_zero { left } { right } ~ + braces ~ were + } + \ inserted. + } +\msg_kernel_new:nnnn { regex } { missing-rparen } + { Missing~right~parenthesis~added~in~regular~expression. } + { + LaTeX~was~given~a~regular~expression~with~\int_eval:n{#1}~ + more~left~parenthes\int_compare:nTF{#1=1}{i}{e}s~than~right~ + parenthes\int_compare:nTF{#1=1}{i}{e}s. + } +\msg_kernel_new:nnnn { regex } { extra-rparen } + { Extra~right~parenthesis~ignored~in~regular~expression. } + { + LaTeX~came~across~a~closing~parenthesis~when~no~submatch~group~ + was~open.~The~parenthesis~will~be~ignored. } -\msg_new:nnn { regex } { -998 } { Unsupported~construct } -\msg_new:nnn { regex } { -997 } - { The~regular~expression~is~too~large~(32768~states). } -\msg_new:nnn { regex } { 1 } { \iow_char:N\\~at~end~of~pattern } -\msg_new:nnn { regex } { 4 } - { Numbers~out~of~order~in~\iow_char:N\{\iow_char\}~quantifier. } -\msg_new:nnn { regex } { 6 } - { Missing~terminating~\iow_char:N\]~for~character~class } -\msg_new:nnn { regex } { 7 } - { Invalid~escape~sequence~in~character~class } -\msg_new:nnn { regex } { 8 } - { Range~out~of~order~in~character~class } -\msg_new:nnn { regex } { 22 } { Mismatched~parentheses } -\msg_new:nnn { regex } { 34 } - { Character~value~in~\iow_char:N\\x{...}~sequence~is~too~large } -\msg_new:nnn { regex } { not-nfa } - { - LaTeX~was~expecting~a~regular~expression~variable.\\ - Instead,~LaTeX~found~'#1'. +\msg_kernel_new:nnnn { regex } { backwards-range } + { Range~[#1-#2]~out~of~order~in~character~class. } + { + In~ranges~of~characters~[x-y]~appearing~in~character~classes,~ + the~first~character~code~must~not~be~larger~than~the~second.~ + Here,~#1~has~character~code~\int_eval:n {`#1},~while~#2~has~ + character~code~\int_eval:n {`#2}. + } +\msg_kernel_new:nnnn { regex } { unsupported } + { Unsupported~construction~`#1'. } + { + The~construction~`#1'~is~not~supported~by~the~LaTeX~ + regular~expression~module.~Perhaps~some~character~should~ + have~been~escaped? + } +\msg_kernel_new:nnnn { regex } { not-nfa } + { This~is~not~a~regular~expression~variable. } + { + LaTeX~was~expecting~`#1'~to~be~a~regular~expression~variable.\\ + This~control~sequence~is~not~a~regex~variable.~It's~current~meaning~ + is~\\\\ + \ \ \ \ \token_to_str:N #1 = \token_to_meaning:N #1 . + } +\msg_kernel_new:nnnn { regex } { build-c } + { Misused~\iow_char:N\\c~or\iow_char:N\\C~command~in~regular~expression. } + { + The~\iow_char:N\\C~escape~sequence~can~be~followed~by~ + one~of~the~letters~ABCDELMOPSTU.~The~\iow_char:N\\c~ + escape~sequence~can~also~be~followed~by~a~brace~group. + } +\msg_kernel_new:nnnn { regex } { unknown-option } + { Unknown~option~`#1'~for~regular~expressions. } + { + LaTeX~came~across~something~like~`(?#1)'~in~a~regular~expression,~ + but~the~option~`#1'~is~not~known.~It~will~be~ignored. + } +\msg_kernel_new:nnnn { regex } { invalid-in-option } + { Invalid~character~in~option~of~a~regular~expression. } + { + The~character~or~escape~sequence~`#1'~is~not~defined~ + as~an~option~within~regular~expressions. + } +\msg_kernel_new:nnnn { regex } { replacement-g } + { Missing~brace~for~the~\iow_char:N\\g~construction~in~a~replacement~text. } + { + In~the~replacement~text~for~a~regular~expression~search,~ + submatches~are~represented~either~as~\iow_char:N \\g{dd..d},~ + or~\\d,~where~`d'~are~single~digits.~Here,~a~brace~is~missing. + } +\msg_kernel_new:nnnn { regex } { replacement-c } + { Something~is~wrong~with~the~replacement~text. } + { + The~\iow_char:N\\c~escape~sequence~can~only~be~followed~by~ + ABCDELMN~or~by~a~brace. } \msg_kernel_new:nnn { regex } { nfa-misused } { Automaton~#1 used~incorrectly. } -\msg_kernel_new:nnn { regex } { g-misused } - { \g misused~in~replacement~text. } %% %% %% End of file `l3regex.sty'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-hex.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-hex.def index 9af2e4322ad..b24f29458a5 100644 --- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-hex.def +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-hex.def @@ -47,13 +47,16 @@ %% ----------------------------------------------------------------------- \cs_new_protected_nopar:Npn \str_convert_input_hex: { - \tl_gset:Nx \g_str_result_tl - { - \str_output_byte:w " - \exp_after:wN \str_convert_input_hex_aux:N - \g_str_result_tl 0 { ? 0 - \c_one \prg_map_break: } - \prg_break_point:n { \str_output_end: } - } + \group_begin: + \int_set:Nn \tex_escapechar:D { 92 } + \tl_gset:Nx \g_str_result_tl + { + \str_output_byte:w " + \exp_after:wN \str_convert_input_hex_aux:N + \g_str_result_tl 0 { ? 0 - \c_one \prg_map_break: } + \prg_break_point:n { \str_output_end: } + } + \group_end: } \cs_new_nopar:Npn \str_convert_input_hex_aux:N #1 { diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-name.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-name.def index 8d378c87e3d..66e7dddb6b8 100644 --- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-name.def +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-name.def @@ -49,12 +49,15 @@ { \cs_new_protected_nopar:Npn #2 { - \tl_gset:Nx \g_str_result_tl - { - \exp_after:wN #3 \g_str_result_tl - #1 ? { ? \prg_map_break: } - \prg_break_point:n { } - } + \group_begin: + \int_set:Nn \tex_escapechar:D { 92 } + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN #3 \g_str_result_tl + #1 ? { ? \prg_map_break: } + \prg_break_point:n { } + } + \group_end: } \cs_new_nopar:Npn #3 ##1#1##2##3 { diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-string.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-string.def index 7718a0d1b25..5e8fd099b25 100644 --- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-string.def +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-string.def @@ -53,18 +53,21 @@ { \cs_new_protected_nopar:Npn \str_convert_input_string: { - \tl_gset:Nx \g_str_result_tl - { - \exp_after:wN \str_convert_input_string_aux:wN - \g_str_result_tl \prg_map_break: ^^M ? - \prg_break_point:n { } - } - \tl_gset:Nx \g_str_result_tl - { - \exp_after:wN \str_convert_input_string_aux:wNNN - \g_str_result_tl * ?? { ? \prg_map_break: } - \prg_break_point:n { } - } + \group_begin: + \int_set:Nn \tex_escapechar:D { 92 } + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN \str_convert_input_string_aux:wN + \g_str_result_tl \prg_map_break: ^^M ? + \prg_break_point:n { } + } + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN \str_convert_input_string_aux:wNNN + \g_str_result_tl * ?? { ? \prg_map_break: } + \prg_break_point:n { } + } + \group_end: } \cs_new_nopar:Npn \str_convert_input_string_aux:wNNN #1 *#2#3#4 } diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-url.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-url.def index bf6bf96514d..13cbde871fe 100644 --- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-url.def +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-url.def @@ -49,12 +49,15 @@ { \cs_new_protected_nopar:Npn #2 { - \tl_gset:Nx \g_str_result_tl - { - \exp_after:wN #3 \g_str_result_tl - #1 ? { ? \prg_map_break: } - \prg_break_point:n { } - } + \group_begin: + \int_set:Nn \tex_escapechar:D { 92 } + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN #3 \g_str_result_tl + #1 ? { ? \prg_map_break: } + \prg_break_point:n { } + } + \group_end: } \cs_new_nopar:Npn #3 ##1#1##2##3 { diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf16.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf16.def index e2bea7ca224..001b955ac07 100644 --- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf16.def +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf16.def @@ -127,13 +127,13 @@ \if_meaning:w \scan_stop: #2 \str_aux_from_utf_xvi_end:Nw #1 \fi: - \int_use:N \int_eval:w - \if_case:w - \int_eval:w ( \str_tmp:w #1#2 - "D6 ) / \c_four \int_eval_end: - \or: \exp_after:wN \str_aux_from_utf_xvi:wNNN - \or: \exp_after:wN \str_aux_from_utf_xvi_error:wNNw - \fi: - "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 , + \if_case:w + \int_eval:w ( \str_tmp:w #1#2 - "D6 ) / \c_four \int_eval_end: + \or: #1#2 \exp_after:wN \str_aux_from_utf_xvi:wNNN + \or: \exp_after:wN \str_aux_from_utf_xvi_error:wNNw + \else: #1#2 \exp_not:N \q_mark \int_use:N \int_eval:w + \fi: + "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 , \str_aux_from_utf_xvi:NN } \cs_new:Npn \str_aux_from_utf_xvi:wNNN #1, #2#3#4 @@ -142,8 +142,10 @@ \str_aux_from_utf_xvi_end:wNNww #1 , #3 \fi: \if_num:w \int_eval:w ( \str_tmp:w #3#4 - "DA ) / \c_four = \c_one - ( #1 - "D800 ) * "400 - + \str_tmp:w #3#4 * "100 + \str_tmp:w #4#3 - "DC00 , + #3 #4 + \int_use:N \int_eval:w + ( #1 - "D800 ) * "400 + + \str_tmp:w #3#4 * "100 + \str_tmp:w #4#3 - "DC00 , \exp_after:wN \use_i:nnn \else: \str_aux_from_utf_xvi_error:wNNw #1 , @@ -152,6 +154,7 @@ } \cs_new:Npn \str_aux_from_utf_xvi_error:wNNw #1 \str_tmp:w #2#3 #4, { + \exp_not:N \q_mark \msg_expandable_kernel_error:nnn { str } { utf16-surrogate } { #2#3 } \c_str_replacement_char_int , @@ -161,15 +164,17 @@ \fi: \if_meaning:w \scan_stop: #1 \else: + #1 \exp_not:N \q_mark \msg_expandable_kernel_error:nnn { str } { utf16-odd } {#1} - \c_str_replacement_char_int , + \int_use:N \c_str_replacement_char_int , \fi: \prg_map_break: } \cs_new:Npn \str_aux_from_utf_xvi_end:wNNww #1 `#2#3 #4, #5 \fi: { \fi: + \exp_not:N \q_mark \if_meaning:w \scan_stop: #5 \msg_expandable_kernel_error:nnn { str } { utf16-surrogate } { #2#3 } @@ -177,7 +182,7 @@ \msg_expandable_kernel_error:nnn { str } { utf16-odd } { #2#3#5 } \fi: - \c_str_replacement_char_int , + \int_use:N \c_str_replacement_char_int , \prg_map_break: } \group_end: diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf32.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf32.def index 7c56d2e3929..8251d8c44fe 100644 --- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf32.def +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf32.def @@ -107,31 +107,33 @@ \cs_new:Npn \str_aux_from_utf_xxxii:NNNN #1#2#3#4 { \if_meaning:w \scan_stop: #4 - \str_aux_from_utf_xxxii_end:ww #1#2#3#4 + \exp_after:wN \str_aux_from_utf_xxxii_end:w + \fi: + #1#2#3#4 \exp_not:N \q_mark + \if_num:w \int_eval:w + \str_tmp:w #1#4 * \c_two_hundred_fifty_six + + \str_tmp:w #2#3 + > \c_sixteen + \msg_expandable_kernel_error:nnn + { str } { utf32-overflow } { #1#2#3#4 } + \int_use:N \c_str_replacement_char_int + \else: + \int_use:N \int_eval:w + \str_tmp:w #2#3*"10000 + + \str_tmp:w #3#2 * \c_two_hundred_fifty_six + + \str_tmp:w #4#1 \fi: - \int_use:N \int_eval:w - \if_num:w \int_eval:w - \str_tmp:w #1#4 * \c_two_hundred_fifty_six - + \str_tmp:w #2#3 - > \c_sixteen - \msg_expandable_kernel_error:nnn - { str } { utf32-overflow } { #1#2#3#4 } - \c_str_replacement_char_int - \else: - \str_tmp:w #2#3*"10000 - + \str_tmp:w #3#2 * \c_two_hundred_fifty_six - + \str_tmp:w #4#1 - \fi: , \str_aux_from_utf_xxxii:NNNN } - \cs_new:Npn \str_aux_from_utf_xxxii_end:ww #1 \scan_stop: #2 \fi: + \cs_new:Npn \str_aux_from_utf_xxxii_end:w #1 \scan_stop: { - \fi: \tl_if_empty:nF {#1} { \msg_expandable_kernel_error:nnn { str } { utf32-truncated } { #1 } + #1 \exp_not:N \q_mark + \int_use:N \c_str_replacement_char_int , } \prg_map_break: } diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf8.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf8.def index 9e0f26c2346..d6233f22557 100644 --- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf8.def +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf8.def @@ -74,32 +74,33 @@ \tl_gset:Nx \g_str_result_tl { \exp_after:wN \str_aux_from_utf_viii:N \g_str_result_tl - { ? \prg_map_break: \q_stop \str_aux_from_utf_viii_error: } + { \prg_map_break: \str_aux_from_utf_viii_error: } \prg_break_point:n { } } } \cs_new_nopar:Npn \str_aux_from_utf_viii:N #1 { - \use_none:n #1 - \int_use:N \int_eval:w - \if_num:w `#1 < "C0 \exp_stop_f: - \if_num:w `#1 < "80 \exp_stop_f: - `#1 - \else: - \msg_expandable_kernel_error:nnn - { str } { utf8-extra-conti } {#1} - \c_str_replacement_char_int - \fi: + #1 + \if_num:w `#1 < "C0 \exp_stop_f: + \exp_not:N \q_mark + \if_num:w `#1 < "80 \exp_stop_f: + \int_value:w `#1 \else: - \if_num:w `#1 < "F5 \exp_stop_f: - \exp_after:wN \str_aux_from_utf_viii:wwNN - \int_use:N \int_eval:w `#1 - "C0 - \else: - \msg_expandable_kernel_error:nnn - { str } { utf8-invalid-byte } {#1} - \c_str_replacement_char_int - \fi: + \msg_expandable_kernel_error:nnn + { str } { utf8-extra-conti } {#1} + \int_use:N \c_str_replacement_char_int \fi: + \else: + \if_num:w `#1 < "F5 \exp_stop_f: + \exp_after:wN \str_aux_from_utf_viii:wwNN + \int_use:N \int_eval:w `#1 - "C0 + \else: + \exp_not:N \q_mark + \msg_expandable_kernel_error:nnn + { str } { utf8-invalid-byte } {#1} + \int_use:N \c_str_replacement_char_int + \fi: + \fi: , \use_none_delimit_by_q_stop:w "80 , "800 , "10000 , "10FFFF , @@ -108,7 +109,7 @@ } \cs_new_nopar:Npn \str_aux_from_utf_viii:wwNN #1 , #2 \q_stop #3#4 { - \use_none_delimit_by_q_stop:w #4 \q_stop + \use_none:n #4 \exp_after:wN \str_aux_from_utf_viii:wNNww \int_use:N \int_eval:w #1 * "40 + `#4 - "80 , #4 @@ -121,8 +122,9 @@ \if_num:w #1 < #4 \exp_stop_f: \str_aux_from_utf_viii_error:w #1 , \else: + #2 \if_num:w #1 < #5 \exp_stop_f: - #1 + \exp_not:N \q_mark #1 \else: \exp_after:wN \str_aux_from_utf_viii:wwNN \int_use:N \int_eval:w #1 - #5 @@ -136,12 +138,14 @@ } \cs_new_nopar:Npn \str_aux_from_utf_viii_error:w #1 , { + \exp_not:N \q_mark \msg_expandable_kernel_error:nnn { str } { utf8-overlong } {#1} \c_str_replacement_char_int } \cs_new_nopar:Npn \str_aux_from_utf_viii_error:NwN #1 #2 \q_stop #3 { + \exp_not:N \q_mark \msg_expandable_kernel_error:nnn { str } { utf8-missing-conti } {#1} \c_str_replacement_char_int , @@ -149,6 +153,7 @@ } \cs_new_nopar:Npn \str_aux_from_utf_viii_error: { + \exp_not:N \q_mark \msg_expandable_kernel_error:nn { str } { utf8-premature-end } \c_str_replacement_char_int , diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str.sty b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str.sty index c49601841e0..7d527b6d264 100644 --- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str.sty +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str.sty @@ -46,23 +46,10 @@ %% %% ----------------------------------------------------------------------- \RequirePackage{expl3} -\GetIdInfo$Id: l3str.dtx 3035 2011-12-07 11:48:46Z bruno $ +\GetIdInfo$Id: l3str.dtx 3088 2011-12-23 02:00:14Z bruno $ {L3 Experimental Strings} \ProvidesExplPackage {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription} -\cs_if_exist:NF \use_i:nnnnnnnn - { \cs_new:Npn \use_i:nnnnnnnn #1#2#3#4#5#6#7#8 {#1} } -\int_const:Nn \c_forty_eight { 48 } -\int_const:Nn \c_fifty_eight { 58 } -\int_const:Nn \c_sixty_five { 65 } -\int_const:Nn \c_ninety_one { 91 } -\int_const:Nn \c_ninety_seven { 97 } -\int_const:Nn \c_one_hundred_twenty_three { 123 } -\int_const:Nn \c_one_hundred_twenty_seven { 127 } -\pdftex_if_engine:TF - { \int_const:Nn \c_max_char_int { 255 } } - { \int_const:Nn \c_max_char_int { 1114111 } } -\int_const:Nn \c_str_replacement_char_int { "FFFD } \group_begin: \cs_set_protected_nopar:Npn \str_tmp:w #1 { @@ -80,8 +67,19 @@ \group_end: \cs_new_protected_nopar:Npn \str_tmp:w { } \tl_new:N \l_str_tmpa_tl +\int_new:N \l_str_tmpa_int \tl_new:N \g_str_result_tl \int_new:N \l_str_char_int +\int_const:Nn \c_forty_eight { 48 } +\int_const:Nn \c_fifty_eight { 58 } +\int_const:Nn \c_sixty_five { 65 } +\int_const:Nn \c_ninety_one { 91 } +\int_const:Nn \c_ninety_seven { 97 } +\int_const:Nn \c_one_hundred_twenty_three { 123 } +\int_const:Nn \c_one_hundred_twenty_seven { 127 } +\int_const:Nn \c_max_char_int + { \pdftex_if_engine:TF { 255 } { 1114111 } } +\int_const:Nn \c_str_replacement_char_int { "FFFD } \tl_const:Nx \c_backslash_str { \cs_to_str:N \\ } \tl_const:Nx \c_lbrace_str { \cs_to_str:N \{ } \tl_const:Nx \c_rbrace_str { \cs_to_str:N \} } @@ -100,7 +98,6 @@ \prop_gput:Nnn \g_str_aliases_prop { latin9 } { iso885915 } \prop_gput:Nnn \g_str_aliases_prop { latin10 } { iso885916 } \prop_gput:Nnn \g_str_aliases_prop { hexadecimal } { hex } -\prop_gput:Nnn \g_str_aliases_prop { percent } { url } \group_begin: \char_set_lccode:nn { `\* } { `\ } \char_set_lccode:nn { `\A } { `\A } @@ -124,72 +121,53 @@ \cs_new_nopar:Npn \tl_to_other_str_end:w \fi: #1 \q_mark #2 * A #3 \q_stop { \fi: #2 } } -\cs_new_nopar:Npn \tl_to_other_str:N { \exp_args:No \tl_to_other_str:n } -\cs_new:Npn \str_sanitize_args:Nn #1#2 - { \exp_args:Nf #1 { \tl_to_other_str:n {#2} } } -\cs_new:Npn \str_sanitize_args:Nnn #1#2#3 - { - \exp_args:Nff #1 - { \tl_to_other_str:n {#2} } - { \tl_to_other_str:n {#3} } - } \group_begin: \char_set_lccode:nn { `\* } { `\ } \char_set_lccode:nn { `\A } { `\A } \tl_to_lowercase:n { \group_end: - \cs_new_protected:Npn \str_aux_gset_other:Nn #1#2 + \cs_new_protected:Npn \str_gset_other:Nn #1#2 { \tl_gset:Nx #1 { - \exp_after:wN \str_aux_gset_other_loop:w \tl_to_str:n {#2} ~ % + \exp_after:wN \str_gset_other_loop:w \tl_to_str:n {#2} ~ % A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ \q_mark \q_stop } } - \cs_new_nopar:Npn \str_aux_gset_other_loop:w + \cs_new_nopar:Npn \str_gset_other_loop:w #1 ~ #2 ~ #3 ~ #4 ~ #5 ~ #6 ~ #7 ~ #8 ~ #9 ~ { \if_meaning:w A #9 - \str_aux_gset_other_end:w + \str_gset_other_end:w \fi: #1 * #2 * #3 * #4 * #5 * #6 * #7 * #8 * #9 - \str_aux_gset_other_loop:w * + \str_gset_other_loop:w * } - \cs_new_nopar:Npn \str_aux_gset_other_end:w \fi: #1 * A #2 \q_stop + \cs_new_nopar:Npn \str_gset_other_end:w \fi: #1 * A #2 \q_stop { \fi: #1 } } -\cs_new_nopar:Npn \str_map_tokens:Nn - { \exp_args:No \str_map_tokens:nn } -\cs_new:Npn \str_map_tokens:nn #1 - { \exp_args:No \str_map_tokens_aux:nn { \tl_to_str:n {#1} } } -\cs_new:Npn \str_map_tokens_aux:nn #1#2 +\cs_if_exist:NF \use_i:nnnnnnnn + { \cs_new:Npn \use_i:nnnnnnnn #1#2#3#4#5#6#7#8 {#1} } +\cs_new:Npn \str_aux_toks_range:nn #1#2 { - \str_map_tokens_loop:nw {#2} #1 - { ? \use_none_delimit_by_q_recursion_stop:w } - ?~ ?~ ?~ ?~ ?~ ?~ ?~ ?~ \q_recursion_stop + \exp_after:wN \str_aux_toks_range:ww + \int_use:N \int_eval:w #1 \exp_after:wN ; + \int_use:N \int_eval:w #2 ; + \prg_break_point:n { } } -\group_begin: - \char_set_lccode:nn { `* } { `\ } - \tl_to_lowercase:n - { - \group_end: - \cs_new:Npn \str_map_tokens_loop:nw #1 #2~ #3~ #4~ #5~ #6~ #7~ #8~ #9~ - { - \str_map_tokens_loop:nN {#1} #2* #3* #4* #5* #6* #7* #8* #9* - { ? \use_none_delimit_by_q_stop:w } \q_stop - \str_map_tokens_loop:nw {#1} - } - } -\cs_new:Npn \str_map_tokens_loop:nN #1#2 +\cs_new:Npn \str_aux_toks_range:ww #1 ; #2 ; { - \use_none:n #2 - #1 #2 - \str_map_tokens_loop:nN {#1} + \if_num:w #1 = #2 \exp_stop_f: + \exp_after:wN \prg_map_break: + \fi: + \tex_the:D \tex_toks:D #1 \exp_stop_f: + \exp_after:wN \str_aux_toks_range:ww + \int_use:N \int_eval:w #1 + \c_one ; #2 ; } -\cs_new_eq:NN \str_map_break_do:n \use_i_delimit_by_q_recursion_stop:nw \cs_new_nopar:Npn \str_length:N { \exp_args:No \str_length:n } -\cs_new:Npn \str_length:n { \str_sanitize_args:Nn \str_length_unsafe:n } +\cs_new:Npn \str_length:n #1 + { \exp_args:Nf \str_length_unsafe:n { \tl_to_other_str:n {#1} } } \cs_new_nopar:Npn \str_length_unsafe:n #1 { \str_length_aux:n { \str_length_loop:NNNNNNNNN #1 } } \cs_new:Npn \str_length_ignore_spaces:n #1 @@ -308,7 +286,7 @@ \cs_new:Npn \str_item:nn #1#2 { \exp_args:Nf \tl_to_str:n - { \str_sanitize_args:Nn \str_item_unsafe:nn {#1} {#2} } + { \exp_args:Nf \str_item_unsafe:nn { \tl_to_other_str:n {#1} } {#2} } } \cs_new:Npn \str_item_ignore_spaces:nn #1 { \exp_args:No \str_item_unsafe:nn { \tl_to_str:n {#1} } } @@ -344,26 +322,31 @@ \cs_new:Npn \str_substr:nnn #1#2#3 { \exp_args:Nf \tl_to_str:n - { \str_sanitize_args:Nn \str_substr_unsafe:nnn {#1} {#2} {#3} } + { + \exp_args:Nf \str_substr_unsafe:nnn + { \tl_to_other_str:n {#1} } {#2} {#3} + } } \cs_new:Npn \str_substr_ignore_spaces:nnn #1 { \exp_args:No \str_substr_unsafe:nnn { \tl_to_str:n {#1} } } \cs_new:Npn \str_substr_unsafe:nnn #1#2#3 { - \str_aux_eval_args:Nnnn \str_substr_aux:nnnw - { \str_length_unsafe:n {#1} } - {#2} - {#3} + \exp_after:wN \str_substr_aux:www + \int_value:w \str_length_unsafe:n {#1} \exp_after:wN ; + \int_use:N \int_eval:w #2 + \c_zero \exp_after:wN ; + \int_use:N \int_eval:w \exp_args:Nf \str_substr_unsafe:nN {#3} \c_max_int ; #1 \q_stop } -\cs_new:Npn \str_substr_aux:nnnw #1#2#3 +\cs_new:Npn \str_substr_unsafe:nN #1 #2 + { \tl_if_empty:nTF {#1} {#2} {#1} } +\cs_new:Npn \str_substr_aux:www #1; #2; #3; { - \exp_args:Nf \str_substr_aux_ii:nnw + \exp_args:Nf \str_substr_aux:nnw { \str_aux_normalize_range:nn {#2} {#1} } { \str_aux_normalize_range:nn {#3} {#1} } } -\cs_new:Npn \str_substr_aux_ii:nnw #1#2 +\cs_new:Npn \str_substr_aux:nnw #1#2 { \str_skip_do:nn {#1} { @@ -372,13 +355,6 @@ { \use_i_delimit_by_q_stop:nw } } } -\cs_new:Npn \str_aux_eval_args:Nnnn #1#2#3#4 - { - \exp_after:wN #1 - \exp_after:wN { \int_value:w \int_eval:w #2 \exp_after:wN } - \exp_after:wN { \int_value:w \int_eval:w #3 \exp_after:wN } - \exp_after:wN { \int_value:w \int_eval:w #4 } - } \cs_new:Npn \str_aux_normalize_range:nn #1#2 { \int_eval:n @@ -403,69 +379,32 @@ \if_int_compare:w \pdftex_strcmp:D { \tl_to_str:N #1 } { \tl_to_str:N #2 } = \c_zero \prg_return_true: \else: \prg_return_false: \fi: } -\prg_new_conditional:Npnn \str_if_contains_char:NN #1#2 { p , T , F , TF } +\prg_new_conditional:Npnn \str_if_contains_char:NN #1#2 { T , TF } { - \str_map_tokens:Nn #1 { \str_if_contains_char_aux:NN #2 } + \exp_after:wN \str_if_contains_char_aux:NN \exp_after:wN #2 + #1 { \prg_map_break:n { ? \fi: } } + \prg_break_point:n { } \prg_return_false: } -\prg_new_conditional:Npnn \str_if_contains_char:nN #1#2 { p , T , F , TF } +\prg_new_conditional:Npnn \str_if_contains_char:nN #1#2 { TF } { - \str_map_tokens:nn {#1} { \str_if_contains_char_aux:NN #2 } + \exp_after:wN \str_if_contains_char_aux:NN \exp_after:wN #2 + \tl_to_str:n {#1} { \prg_map_break:n { ? \fi: } } + \prg_break_point:n { } \prg_return_false: } \cs_new_nopar:Npn \str_if_contains_char_aux:NN #1#2 { - \if_num:w `#1 = `#2 \exp_stop_f: - \str_if_contains_char_end:w + \if_charcode:w #1 #2 + \exp_after:wN \str_if_contains_char_end: \fi: + \str_if_contains_char_aux:NN #1 } -\cs_new_nopar:Npn \str_if_contains_char_end:w \fi: #1 \prg_return_false: - { \fi: \prg_return_true: } -\pdftex_if_engine:TF - { - \cs_new_eq:NN \str_if_bytes_p:N \c_true_bool - \cs_new_eq:NN \str_if_bytes:NT \use_ii:nn - \cs_new_eq:NN \str_if_bytes:NF \use_none:nn - \cs_new_eq:NN \str_if_bytes:NTF \use_ii:nnn - } - { - \prg_new_conditional:Npnn \str_if_bytes:N #1 { p , T , F , TF } - { - \exp_args:Nf \tl_map_function:nN - { \tl_to_str:N #1 } - \str_if_bytes_aux:N - \prg_return_true: - } - \cs_new_nopar:Npn \str_if_bytes_aux:N #1 - { - \if_num:w `#1 > \c_two_hundred_fifty_five - \exp_after:wN \str_if_bytes_aux: - \fi: - } - \cs_new_nopar:Npn \str_if_bytes_aux: - { \tl_map_break:n { \prg_return_false: \use_none:n } } - } -\prg_new_protected_conditional:Npnn \str_aux_hexadecimal_test:N #1 { TF } - { - \tl_if_in:onTF { \tl_to_str:n { abcdef } } {#1} - { - \int_set:Nn \l_str_char_int - { \c_sixteen * \l_str_char_int + `#1 - 87 } - \prg_return_true: - } - { - \if_num:w \c_fifteen < "1 \exp_not:N #1 \exp_stop_f: - \int_set:Nn \l_str_char_int - { \c_sixteen * \l_str_char_int + "#1 } - \prg_return_true: - \else: - \prg_return_false: - \fi: - } - } +\cs_new_nopar:Npn \str_if_contains_char_end: + { \prg_map_break:n { \prg_return_true: \use_none:n } } \prg_new_conditional:Npnn \str_aux_octal_use:N #1 { TF } { - \if_num:w \c_one < '1 #1 \exp_stop_f: + \if_num:w \c_one < '1 \token_to_str:N #1 \exp_stop_f: #1 \prg_return_true: \else: \prg_return_false: @@ -473,21 +412,27 @@ } \prg_new_conditional:Npnn \str_aux_hexadecimal_use:N #1 { TF } { - \if_num:w \c_two < "1 #1 \exp_stop_f: + \if_num:w \c_two < "1 \token_to_str:N #1 \exp_stop_f: #1 \prg_return_true: \else: - \if_case:w \int_eval:w `#1 - `a \int_eval_end: - A \prg_return_true: - \or: B \prg_return_true: - \or: C \prg_return_true: - \or: D \prg_return_true: - \or: E \prg_return_true: - \or: F \prg_return_true: - \else: \prg_return_false: + \if_case:w \int_eval:w + \exp_after:wN ` \token_to_str:N #1 - `a + \int_eval_end: + A + \or: B + \or: C + \or: D + \or: E + \or: F + \else: + \prg_return_false: + \exp_after:wN \use_none:n \fi: + \prg_return_true: \fi: } \group_begin: + \tl_clear:N \l_str_tmpa_tl \char_set_catcode_other:n { \c_zero } \tl_gset:Nx \g_str_result_tl { \tl_to_str:n { 0123456789ABCDEF } } \tl_map_inline:Nn \g_str_result_tl @@ -496,15 +441,18 @@ { \tl_map_inline:Nn \g_str_result_tl { - \char_set_lccode:nn { \c_zero } {"##1#1} + \char_set_lccode:nn { \c_zero } {"#1##1} \tl_to_lowercase:n { \tl_const:cx - { c_str_byte_ \int_eval:n {"##1#1} _tl } - { ^^@ ##1 #1 } + { c_str_byte_ \int_eval:n {"#1##1} _tl } + { ^^@ #1 ##1 } + \tl_put_right:Nn \l_str_tmpa_tl { ^^@ } } } } + \tl_const:Nx \c_str_positive_bytes_tl + { \exp_after:wN \use_none:n \l_str_tmpa_tl } \group_end: \tl_const:cn { c_str_byte_-1_tl } { { } \use_none:n { } } \cs_new:Npn \str_output_byte:n #1 @@ -545,17 +493,17 @@ { \tl_gset:Nx \g_str_result_tl { - \exp_after:wN \str_aux_gmap_internal_result_loop:Nw + \exp_after:wN \str_aux_gmap_internal_result_loop:Nww \exp_after:wN #1 - \g_str_result_tl \q_stop \prg_map_break: , + \g_str_result_tl \q_mark \q_stop \prg_map_break: , \prg_break_point:n { } } } -\cs_new_nopar:Npn \str_aux_gmap_internal_result_loop:Nw #1#2, +\cs_new_nopar:Npn \str_aux_gmap_internal_result_loop:Nww #1#2\q_mark #3, { - \use_none_delimit_by_q_stop:w #2 \q_stop - #1 {#2} - \str_aux_gmap_internal_result_loop:Nw #1 + \use_none_delimit_by_q_stop:w #3 \q_stop + #1 {#3} + \str_aux_gmap_internal_result_loop:Nww #1 } \cs_new_protected:Npn \str_input:Nn #1#2 { @@ -567,9 +515,11 @@ \str_aux_escape:NNNn \token_to_str:N \token_to_str:N \token_to_str:N {#2} \tl_gset_eq:NN #1 \g_str_result_tl } -\cs_new_eq:NN \str_aux_escape_unescaped:N \use:n -\cs_new_eq:NN \str_aux_escape_escaped:N \use:n -\cs_new_eq:NN \str_aux_escape_raw:N \use:n +\cs_new_protected:Npn \str_escape_use:NNNn #1#2#3#4 + { + \str_aux_escape:NNNn #1#2#3 {#4} + \g_str_result_tl + } \cs_new_protected:Npn \str_aux_escape:NNNn #1#2#3#4 { \group_begin: @@ -577,12 +527,15 @@ \cs_set_nopar:Npn \str_aux_escape_escaped:N { #2 } \cs_set_nopar:Npn \str_aux_escape_raw:N { #3 } \int_set:Nn \tex_escapechar:D { 92 } - \str_aux_gset_other:Nn \g_str_result_tl {#4} + \str_gset_other:Nn \g_str_result_tl {#4} + \int_zero:N \l_str_tmpa_int \tl_gset:Nx \g_str_result_tl { \exp_after:wN \str_aux_escape_loop:N \g_str_result_tl \q_recursion_tail \q_recursion_stop } + \tl_gput_left:Nx \g_str_result_tl + { \str_aux_toks_range:nn \c_zero \l_str_tmpa_int } \group_end: } \cs_new_nopar:Npn \str_aux_escape_loop:N #1 @@ -598,6 +551,9 @@ { \str_aux_escape_escaped:N #1 } \str_aux_escape_loop:N } +\cs_new_eq:NN \str_aux_escape_unescaped:N \prg_do_nothing: +\cs_new_eq:NN \str_aux_escape_escaped:N \prg_do_nothing: +\cs_new_eq:NN \str_aux_escape_raw:N \prg_do_nothing: \cs_new_eq:cN { str_aux_escape_ \c_backslash_str q_recursion_tail :w } \use_none_delimit_by_q_recursion_stop:w @@ -620,6 +576,8 @@ \cs_new_nopar:cpn { str_aux_escape_/x:w } \str_aux_escape_loop:N { \if_false: { \fi: } + \tex_toks:D \l_str_tmpa_int \exp_after:wN { \g_str_result_tl } + \int_incr:N \l_str_tmpa_int \int_zero:N \l_str_char_int \str_aux_escape_x_test:N } @@ -628,35 +586,36 @@ \exp_not:N \token_if_eq_charcode:NNTF \c_space_token #1 { \exp_not:N \str_aux_escape_x_test:N } { - \exp_not:N \token_if_eq_charcode:NNTF \c_lbrace_str #1 - { \exp_not:N \str_aux_escape_x_braced_loop:N } - { \exp_not:N \str_aux_escape_x_unbraced_i:N #1 } + \l_str_char_int = "0 + \exp_not:N \token_if_eq_charcode:NNTF \c_lbrace_str #1 + { \exp_not:N \str_aux_escape_x_braced_loop:N } + { \exp_not:N \str_aux_escape_x_unbraced_i:N #1 } } } -\cs_new_protected_nopar:Npn \str_aux_escape_x_unbraced_i:N #1 +\cs_new_nopar:Npn \str_aux_escape_x_unbraced_i:N #1 { - \str_aux_hexadecimal_test:NTF #1 + \str_aux_hexadecimal_use:NTF #1 { \str_aux_escape_x_unbraced_ii:N } - { \str_aux_escape_x_end: #1 } + { \exp_stop_f: \str_aux_escape_x_end: #1 } } -\cs_new_protected_nopar:Npn \str_aux_escape_x_unbraced_ii:N #1 +\cs_new_nopar:Npn \str_aux_escape_x_unbraced_ii:N #1 { \token_if_eq_charcode:NNTF \c_space_token #1 { \str_aux_escape_x_unbraced_ii:N } { - \str_aux_hexadecimal_test:NTF #1 - { \str_aux_escape_x_end: } - { \str_aux_escape_x_end: #1 } + \str_aux_hexadecimal_use:NTF #1 + { \exp_stop_f: \str_aux_escape_x_end: } + { \exp_stop_f: \str_aux_escape_x_end: #1 } } } -\cs_new_protected_nopar:Npn \str_aux_escape_x_braced_loop:N #1 +\cs_new_nopar:Npn \str_aux_escape_x_braced_loop:N #1 { \token_if_eq_charcode:NNTF \c_space_token #1 { \str_aux_escape_x_braced_loop:N } { - \str_aux_hexadecimal_test:NTF #1 + \str_aux_hexadecimal_use:NTF #1 { \str_aux_escape_x_braced_loop:N } - { \str_aux_escape_x_braced_end:N #1 } + { \exp_stop_f: \str_aux_escape_x_braced_end:N #1 } } } \cs_new_protected_nopar:Npx \str_aux_escape_x_braced_end:N #1 @@ -675,7 +634,7 @@ \tex_lccode:D \c_zero \l_str_char_int \tl_to_lowercase:n { - \tl_gput_right:Nx \g_str_result_tl + \tl_gset:Nx \g_str_result_tl { \if_false: } \fi: \str_aux_escape_raw:N ^^@ \str_aux_escape_loop:N @@ -683,13 +642,13 @@ } \group_end: \cs_new_protected:Npn \str_set_convert:Nnnn - { \str_convert_aux_i:NNnnn \str_set:Nx } + { \str_convert_aux_i:NNnnn \tl_set_eq:NN } \cs_new_protected:Npn \str_gset_convert:Nnnn - { \str_convert_aux_i:NNnnn \str_gset:Nx } + { \str_convert_aux_i:NNnnn \tl_gset_eq:NN } \cs_new_protected:Npn \str_convert_aux_i:NNnnn #1#2#3#4#5 { \group_begin: - \str_aux_gset_other:Nn \g_str_result_tl {#5} + \str_gset_other:Nn \g_str_result_tl {#5} \exp_after:wN \str_convert_aux_ii:wwwnn \tl_to_str:n {#3} /// \q_stop @@ -831,11 +790,10 @@ \char_set_catcode_comment:N \% \int_set:Nn \tex_endlinechar:D {32} } -\int_new:N \l_str_encoding_int \cs_new_protected:Npn \str_convert_from_eight_bit:n #1 { \group_begin: - \int_zero:N \l_str_encoding_int + \int_zero:N \l_str_tmpa_int \exp_last_unbraced:Nx \str_convert_from_eight_bit_load:nn { \tl_use:c { c_str_encoding_#1_tl } } { \q_stop \prg_map_break: } { } @@ -850,24 +808,25 @@ \cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load:nn #1#2 { \use_none_delimit_by_q_stop:w #1 \q_stop - \tex_dimen:D "#1 = \l_str_encoding_int sp \scan_stop: - \tex_skip:D \l_str_encoding_int = "#1 sp \scan_stop: - \tex_toks:D \l_str_encoding_int \exp_after:wN { \int_value:w "#2 } - \tex_advance:D \l_str_encoding_int \c_one + \tex_dimen:D "#1 = \l_str_tmpa_int sp \scan_stop: + \tex_skip:D \l_str_tmpa_int = "#1 sp \scan_stop: + \tex_toks:D \l_str_tmpa_int \exp_after:wN { \int_value:w "#2 } + \tex_advance:D \l_str_tmpa_int \c_one \str_convert_from_eight_bit_load:nn } \cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load_missing:n #1 { \use_none_delimit_by_q_stop:w #1 \q_stop - \tex_dimen:D "#1 = \l_str_encoding_int sp \scan_stop: - \tex_skip:D \l_str_encoding_int = "#1 sp \scan_stop: - \tex_toks:D \l_str_encoding_int \exp_after:wN { \int_value:w "FFFD } - \tex_advance:D \l_str_encoding_int \c_one + \tex_dimen:D "#1 = \l_str_tmpa_int sp \scan_stop: + \tex_skip:D \l_str_tmpa_int = "#1 sp \scan_stop: + \tex_toks:D \l_str_tmpa_int \exp_after:wN { \int_value:w "FFFD } + \tex_advance:D \l_str_tmpa_int \c_one \str_convert_from_eight_bit_load_missing:n } \cs_new:Npn \str_convert_from_eight_bit_aux:N #1 { - \if_num:w \tex_dimen:D `#1 < \l_str_encoding_int + #1 \exp_not:N \q_mark + \if_num:w \tex_dimen:D `#1 < \l_str_tmpa_int \if_num:w \tex_skip:D \tex_dimen:D `#1 = `#1 \exp_stop_f: \tex_the:D \tex_toks:D \tex_dimen:D \fi: @@ -877,7 +836,7 @@ \cs_new_protected:Npn \str_convert_to_eight_bit:n #1 { \group_begin: - \int_zero:N \l_str_encoding_int + \int_zero:N \l_str_tmpa_int \exp_last_unbraced:Nx \str_convert_to_eight_bit_load:nn { \tl_use:c { c_str_encoding_#1_tl } } { \q_stop \prg_map_break: } { } @@ -888,17 +847,17 @@ \cs_new_protected_nopar:Npn \str_convert_to_eight_bit_load:nn #1#2 { \use_none_delimit_by_q_stop:w #1 \q_stop - \tex_dimen:D "#2 = \l_str_encoding_int sp \scan_stop: - \tex_skip:D \l_str_encoding_int = "#2 sp \scan_stop: - \exp_args:NNf \tex_toks:D \l_str_encoding_int + \tex_dimen:D "#2 = \l_str_tmpa_int sp \scan_stop: + \tex_skip:D \l_str_tmpa_int = "#2 sp \scan_stop: + \exp_args:NNf \tex_toks:D \l_str_tmpa_int { \str_output_byte:n { "#1 } } - \tex_advance:D \l_str_encoding_int \c_one + \tex_advance:D \l_str_tmpa_int \c_one \str_convert_to_eight_bit_load:nn } \cs_new:Npn \str_convert_to_eight_bit_aux:n #1 { \if_num:w #1 < "8000 \exp_stop_f: - \if_num:w \tex_dimen:D #1 < \l_str_encoding_int + \if_num:w \tex_dimen:D #1 < \l_str_tmpa_int \if_num:w \tex_skip:D \tex_dimen:D #1 = #1 \exp_stop_f: \tex_the:D \tex_toks:D \tex_dimen:D #1 \exp_stop_f: \exp_after:wN \exp_after:wN \exp_after:wN \use_none:nn @@ -907,7 +866,7 @@ \str_convert_to_eight_bit_aux_ii:n {#1} \else: \msg_expandable_kernel_error:nnn - { str } { eight_bit-to-byte-overflow } {#1} + { str } { eight-bit-to-byte-overflow } {#1} \fi: } \cs_new:Npn \str_convert_to_eight_bit_aux_ii:n #1 @@ -956,30 +915,73 @@ \cs_new_protected_nopar:Npn \str_convert_from_native: { \str_aux_gmap_result:N \str_convert_from_native_aux:N } \cs_new:Npn \str_convert_from_native_aux:N #1 - { \int_value:w `#1 , } + { #1 \exp_not:N \q_mark \int_value:w `#1 , } \cs_new_protected_nopar:Npn \str_convert_to_native: { \tl_set_eq:NN \l_str_tmpa_tl \g_str_result_tl + \str_aux_convert_to_native:n + { + \exp_after:wN \str_convert_to_native_aux:w + \l_str_tmpa_tl \q_mark { \q_stop \prg_map_break: } , + } + } +\cs_new_protected_nopar:Npn \str_convert_to_native_aux:w #1\q_mark #2, + { + \use_none_delimit_by_q_stop:w #2 \q_stop + \str_aux_convert_to_native_step:n {#2} + \str_convert_to_native_aux:w + } +\cs_new_protected_nopar:Npn \str_aux_convert_to_native:n #1 + { + \tex_lccode:D \c_zero \c_zero + \int_zero:N \l_str_tmpa_int \tl_gclear:N \g_str_result_tl - \tex_afterassignment:D \str_convert_to_native_aux:w - \l_str_char_int = \l_str_tmpa_tl \c_zero \prg_map_break: , - \prg_break_point:n { } + #1 + \prg_break_point:n { \str_aux_convert_to_native_flush: } + } +\cs_new_protected_nopar:Npn \str_aux_convert_to_native_step:n #1 + { + \if_num:w \l_str_tmpa_int = \c_two_hundred_fifty_five + \str_aux_convert_to_native_flush: + \l_str_tmpa_int \c_zero + \fi: + \tex_advance:D \l_str_tmpa_int \c_one + \if_int_compare:w #1 > \c_max_char_int + \msg_kernel_error:nnx { str } { overflow } {#1} + \tex_lccode:D \l_str_tmpa_int \c_str_replacement_char_int + \else: + \tex_lccode:D \l_str_tmpa_int \int_eval:w #1 \int_eval_end: + \fi: + } +\cs_new_protected_nopar:Npn \str_aux_convert_to_native_flush: + { + \use:x + { + \tl_to_lowercase:n + { + \tl_gput_right:Nx \exp_not:N \g_str_result_tl + { + \exp_after:wN \str_aux_convert_to_native_filter:N + \c_str_positive_bytes_tl + { ? = \c_zero \fi: \prg_map_break: } + \prg_break_point:n { } + } + } + } } \group_begin: - \char_set_catcode_other:n { `\^^@ } - \cs_new_protected_nopar:Npn \str_convert_to_native_aux:w #1 , + \char_set_catcode_other:n { 0 } + \cs_new:Npn \str_aux_convert_to_native_filter:N #1 { - #1 - \if_int_compare:w \l_str_char_int > \c_max_char_int - \msg_kernel_error:nnx { str } { overflow } - { \int_use:N \l_str_char_int } + \if_num:w `#1 > \l_str_tmpa_int + \exp_after:wN \prg_map_break: + \fi: + \if_num:w \tex_lccode:D `#1 = \c_zero + ^^@ \else: - \tex_lccode:D \c_zero \l_str_char_int - \tl_to_lowercase:n - { \tex_xdef:D \g_str_result_tl { \g_str_result_tl ^^@ } } + #1 \fi: - \tex_afterassignment:D \str_convert_to_native_aux:w - \l_str_char_int = + \str_aux_convert_to_native_filter:N } \group_end: \msg_kernel_new:nnnn { str } { x-missing-brace } diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3tl-analysis.sty b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3tl-analysis.sty new file mode 100644 index 00000000000..84479cc1543 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3tl-analysis.sty @@ -0,0 +1,402 @@ +%% +%% This is file `l3tl-analysis.sty', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3tl-analysis.dtx (with options: `package') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3tl-analysis.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\RequirePackage{expl3} +\GetIdInfo$Id: l3tl-analysis.dtx 3039 2011-12-08 09:22:35Z bruno $ + {L3 Experimental Analysis of Token lists} +\ProvidesExplPackage + {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription} +\RequirePackage{l3str} +\int_new:N \l_tl_analysis_normal_int +\int_new:N \l_tl_analysis_index_int +\int_new:N \l_tl_analysis_nesting_int +\int_new:N \l_tl_analysis_char_int +\int_new:N \l_tl_analysis_type_int +\tl_new:N \g_tl_analysis_result_tl +\cs_new_protected_nopar:Npn \tl_analysis_extract_charcode: + { + \exp_after:wN \tl_analysis_extract_charcode_aux:w + \token_to_meaning:N \l_tl_analysis_token + } +\cs_new_protected_nopar:Npn \tl_analysis_extract_charcode_aux:w #1 ~ #2 ~ { ` } +\cs_new:Npn \tl_analysis_cs_space_count:NN #1 #2 + { + \exp_after:wN #1 + \int_value:w \int_eval:w \c_zero + \exp_after:wN \tl_analysis_cs_space_count:w + \token_to_str:N #2 + \fi: \tl_analysis_cs_space_count_end:w ; ~ ! + } +\cs_new:Npn \tl_analysis_cs_space_count:w #1 ~ + { + \if_false: #1 #1 \fi: + + \c_one + \tl_analysis_cs_space_count:w + } +\cs_new:Npn \tl_analysis_cs_space_count_end:w ; #1 \fi: #2 ! + { \exp_after:wN ; \int_value:w \str_length_ignore_spaces:n {#1} ; } +\cs_new_protected:Npn \tl_analysis:n #1 + { + \group_begin: + \group_align_safe_begin: + \tl_analysis_setup:n {#1} + \tl_analysis_i:n {#1} + \tl_analysis_ii:n {#1} + \group_align_safe_end: + \group_end: + } +\cs_new_protected:Npn \tl_set_analysis:Nn #1#2 + { + \tl_analysis:n {#2} + \tl_set_eq:NN #1 \g_tl_analysis_result_tl + } +\cs_new_protected:Npn \tl_analysis_setup:n #1 + { + \int_set:Nn \tex_escapechar:D { 92 } + \exp_after:wN \tl_analysis_disable_loop:N + \tl_to_str:n {#1} { ~ } { ? ~ \prg_map_break: } + \prg_break_point:n { } + } +\group_begin: + \char_set_catcode_active:N \^^@ + \cs_new_protected_nopar:Npn \tl_analysis_disable_loop:N #1 + { + \tex_lccode:D \c_zero `#1 ~ + \tl_to_lowercase:n { \tex_let:D ^^@ } \c_undefined:D + \tl_analysis_disable_loop:N + } +\group_end: +\cs_new_protected:Npn \tl_analysis_i:n #1 + { + \int_zero:N \l_tl_analysis_normal_int + \int_zero:N \l_tl_analysis_index_int + \int_zero:N \l_tl_analysis_nesting_int + \if_false: { \fi: \tl_analysis_i_loop:w #1 } + \int_decr:N \l_tl_analysis_index_int + } +\cs_new_protected_nopar:Npn \tl_analysis_i_loop:w + { \tex_futurelet:D \l_tl_analysis_token \tl_analysis_i_type:w } +\cs_new_protected_nopar:Npn \tl_analysis_i_type:w + { + \l_tl_analysis_type_int = + \if_meaning:w \l_tl_analysis_token \c_space_token + \c_zero + \else: + \if_catcode:w \exp_not:N \l_tl_analysis_token \c_group_begin_token + \c_one + \else: + \if_catcode:w \exp_not:N \l_tl_analysis_token \c_group_end_token + \c_minus_one + \else: + \c_two + \fi: + \fi: + \fi: + \if_case:w \l_tl_analysis_type_int + \exp_after:wN \tl_analysis_i_space:w + \or: \exp_after:wN \tl_analysis_i_bgroup:w + \or: \exp_after:wN \tl_analysis_i_safe:N + \else: \exp_after:wN \tl_analysis_i_egroup:w + \fi: + } +\cs_new_protected:Npn \tl_analysis_i_safe:N #1 + { + \if_charcode:w + \scan_stop: + \exp_after:wN \use_none:n \token_to_str:N #1 \prg_do_nothing: + \scan_stop: + \int_incr:N \l_tl_analysis_normal_int + \else: + \tl_analysis_cs_space_count:NN \tl_analysis_i_cs:ww #1 + \fi: + \tl_analysis_i_loop:w + } +\cs_new_protected:Npn \tl_analysis_i_cs:ww #1; #2; + { + \if_num:w #1 > \c_zero + \tex_skip:D \l_tl_analysis_index_int + = \int_eval:w \l_tl_analysis_normal_int + \c_one sp \scan_stop: + \tex_advance:D \l_tl_analysis_index_int #1 \exp_stop_f: + \l_tl_analysis_normal_int #2 \exp_stop_f: + \else: + \tex_advance:D \l_tl_analysis_normal_int #2 \exp_stop_f: + \fi: + } +\cs_new_protected_nopar:Npn \tl_analysis_i_space:w + { + \tex_afterassignment:D \tl_analysis_i_space_test:w + \exp_after:wN \cs_set_eq:NN + \exp_after:wN \l_tl_analysis_token + \token_to_str:N + } +\cs_new_protected_nopar:Npn \tl_analysis_i_space_test:w + { + \if_meaning:w \l_tl_analysis_token \c_space_token + \tex_toks:D \l_tl_analysis_index_int { \exp_not:n { ~ } } + \tl_analysis_i_store: + \else: + \int_incr:N \l_tl_analysis_normal_int + \fi: + \tl_analysis_i_loop:w + } +\group_begin: + \char_set_catcode_group_begin:N \^^@ + \char_set_catcode_group_end:N \^^E + \cs_new_protected_nopar:Npn \tl_analysis_i_bgroup:w + { \tl_analysis_i_group:nw { \exp_after:wN ^^@ \if_false: ^^E \fi: } } + \char_set_catcode_group_begin:N \^^B + \char_set_catcode_group_end:N \^^@ + \cs_new_protected_nopar:Npn \tl_analysis_i_egroup:w + { \tl_analysis_i_group:nw { \if_false: ^^B \fi: ^^@ } } +\group_end: +\cs_new_protected_nopar:Npn \tl_analysis_i_group:nw #1 + { + \l_tl_analysis_char_int = \tl_analysis_extract_charcode: \scan_stop: + \tex_lccode:D \c_zero \l_tl_analysis_char_int + \tl_to_lowercase:n { \tex_toks:D \l_tl_analysis_index_int {#1} } + \if_num:w \l_tl_analysis_char_int = \tex_escapechar:D + \int_set:Nn \tex_escapechar:D { 139 - \tex_escapechar:D } + \fi: + \tex_afterassignment:D \tl_analysis_i_group_test:w + \exp_after:wN \cs_set_eq:NN + \exp_after:wN \l_tl_analysis_token + \token_to_str:N + } +\cs_new_protected_nopar:Npn \tl_analysis_i_group_test:w + { + \if_num:w \tl_analysis_extract_charcode: = \l_tl_analysis_char_int + \tl_analysis_i_store: + \else: + \int_incr:N \l_tl_analysis_normal_int + \fi: + \tl_analysis_i_loop:w + } +\cs_new_protected_nopar:Npn \tl_analysis_i_store: + { + \tex_advance:D \l_tl_analysis_nesting_int \l_tl_analysis_type_int + \if_num:w \l_tl_analysis_char_int = \c_thirty_two + \tex_multiply:D \l_tl_analysis_type_int \c_two + \fi: + \tex_skip:D \l_tl_analysis_index_int + = \l_tl_analysis_normal_int sp plus \l_tl_analysis_type_int sp \scan_stop: + \int_incr:N \l_tl_analysis_index_int + \int_zero:N \l_tl_analysis_normal_int + \if_num:w \l_tl_analysis_nesting_int = \c_minus_one + \cs_set_eq:NN \tl_analysis_i_loop:w \scan_stop: + \fi: + } +\cs_new_protected:Npn \tl_analysis_ii:n #1 + { + \tl_gset:Nx \g_tl_analysis_result_tl + { + \tl_analysis_ii_loop:w 0; #1 + \prg_break_point:n { } + } + } +\cs_new:Npn \tl_analysis_ii_loop:w #1; + { + \exp_after:wN \tl_analysis_ii_normals:ww + \int_value:w \tex_skip:D #1 ; #1 ; + } +\cs_new:Npn \tl_analysis_ii_normals:ww #1; + { + \if_num:w #1 = \c_zero + \tl_analysis_ii_special:w + \fi: + \tl_analysis_ii_normal:wwN #1; + } +\cs_new:Npn \tl_analysis_ii_normal:wwN #1; #2; #3 + { + \exp_not:n { \exp_not:n { #3 } } \exp_not:N \q_mark + \if_charcode:w + \scan_stop: + \exp_after:wN \use_none:n \token_to_str:N #3 \prg_do_nothing: + \scan_stop: + \exp_after:wN \tl_analysis_ii_char:Nww + \else: + \exp_after:wN \tl_analysis_ii_cs:Nww + \fi: + #3 #1; #2; + } +\cs_new:Npn \tl_analysis_ii_char:Nww #1 + { + \if_meaning:w #1 \c_undefined:D D \else: + \if_catcode:w #1 \c_catcode_other_token C \else: + \if_catcode:w #1 \c_catcode_letter_token B \else: + \if_catcode:w #1 \c_math_toggle_token 3 \else: + \if_catcode:w #1 \c_alignment_token 4 \else: + \if_catcode:w #1 \c_math_superscript_token 7 \else: + \if_catcode:w #1 \c_math_subscript_token 8 \else: + 6 + \fi: \fi: \fi: \fi: \fi: \fi: \fi: + \int_value:w `#1 , + \exp_after:wN \tl_analysis_ii_normals:ww + \int_use:N \int_eval:w \c_minus_one + + } +\cs_new:Npn \tl_analysis_ii_cs:Nww #1 + { + 0 -1 , + \tl_analysis_cs_space_count:NN \tl_analysis_ii_cs_test:ww #1 + } +\cs_new:Npn \tl_analysis_ii_cs_test:ww #1 ; #2 ; #3; #4; + { + \exp_after:wN \tl_analysis_ii_normals:ww + \int_use:N \int_eval:w + \if_num:w #1 = \c_zero + #3 + \else: + \tex_skip:D \int_eval:w #4 + #1 \int_eval_end: + \fi: + - #2 + \exp_after:wN ; + \int_use:N \int_eval:w #4 + #1; + } +\cs_new:Npn \tl_analysis_ii_special:w \fi: \tl_analysis_ii_normal:wwN 0; #1; + { + \fi: + \if_num:w #1 = \l_tl_analysis_index_int + \exp_after:wN \prg_map_break: + \fi: + \tex_the:D \tex_toks:D #1 \exp_stop_f: \exp_not:N \q_mark + \if_case:w \etex_gluestretch:D \tex_skip:D #1 \exp_stop_f: + A + \or: 1 + \or: 1 + \else: 2 + \fi: + \if_int_odd:w \etex_gluestretch:D \tex_skip:D #1 \exp_stop_f: + \exp_after:wN \tl_analysis_ii_special_char:wN \int_use:N + \else: + \exp_after:wN \tl_analysis_ii_special_space:w \int_use:N + \fi: + \int_eval:w \c_one + #1 \exp_after:wN ; + \token_to_str:N + } +\cs_new:Npn \tl_analysis_ii_special_char:wN #1 ; #2 + { + \int_value:w `#2 , + \tl_analysis_ii_loop:w #1 ; + } +\cs_new:Npn \tl_analysis_ii_special_space:w #1 ; ~ + { + 32 , + \tl_analysis_ii_loop:w #1 ; + } +\cs_new_protected:Npn \tl_show_analysis:N #1 + { + \exp_args:No \tl_analysis:n {#1} + \msg_aux_show:Nnx #1 + { tl-analysis } + { + \exp_after:wN \tl_show_analysis_loop:wNw \g_tl_analysis_result_tl + \q_mark { ? \prg_map_break: } , + \prg_break_point:n { } + } + } +\cs_new_protected:Npn \tl_show_analysis:n #1 + { + \tl_set:Nn \l_tl_tmpa_tl {#1} + \tl_show_analysis:N \l_tl_tmpa_tl + } +\cs_new:Npn \tl_show_analysis_loop:wNw #1 \q_mark #2 #3, + { + \use_none:n #2 + \iow_newline: > \c_space_tl \c_space_tl + \if_num:w "#2 = \c_zero + \exp_after:wN \tl_show_analysis_cs:n + \else: + \if_num:w "#2 = \c_thirteen + \exp_after:wN \exp_after:wN + \exp_after:wN \tl_show_analysis_active:n + \else: + \exp_after:wN \exp_after:wN + \exp_after:wN \tl_show_analysis_normal:n + \fi: + \fi: + {#1} + \tl_show_analysis_loop:wNw + } +\cs_new:Npn \tl_show_analysis_normal:n #1 + { + \exp_after:wN \token_to_str:N #1 ~ + ( \exp_after:wN \token_to_meaning:N #1 ) + } +\cs_new:Npn \tl_show_analysis_cs:n #1 + { \exp_args:No \tl_show_analysis_long:nn {#1} { control~sequence= } } +\cs_new:Npn \tl_show_analysis_active:n #1 + { \exp_args:No \tl_show_analysis_long:nn {#1} { active~character= } } +\cs_new:Npn \tl_show_analysis_long:nn #1 + { + \exp_args:Noo \tl_show_analysis_long_aux:nnn + { \token_to_str:N #1 } + { \token_to_meaning:N #1 } + } +\cs_new_nopar:Npn \tl_show_analysis_long_aux:nnn #1#2#3 + { + \int_compare:nNnTF + { \str_length:n { >>> #1 ~ ( #3 #2 ) } } + > { \l_iow_line_length_int } + { + \str_substr:nnn { >>> #1 ~ ( #3 #2 ) } \c_three + { \l_iow_line_length_int - \str_length:N \c_tl_show_analysis_etc_str } + \c_tl_show_analysis_etc_str + } + {#1~(#3#2)} + } +\str_const:Nx \c_tl_show_analysis_etc_str % ( + { \iow_char:N \\ETC. ) } +\msg_kernel_new:nnn { tl-analysis } { show } + { + Token~list~ + \str_if_eq:nnF {#1} { \l_tl_tmpa_tl } { \token_to_str:N #1~} + \tl_if_empty:NTF #1 + { is~empty } + { contains~the~tokens: } + } +%% +%% +%% End of file `l3tl-analysis.sty'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/xgalley/l3galley.sty b/Master/texmf-dist/tex/latex/l3experimental/xgalley/l3galley.sty index e8a3345204f..28bc6b50dc5 100644 --- a/Master/texmf-dist/tex/latex/l3experimental/xgalley/l3galley.sty +++ b/Master/texmf-dist/tex/latex/l3experimental/xgalley/l3galley.sty @@ -48,14 +48,11 @@ %% ----------------------------------------------------------------------- %% \RequirePackage{l3names} -\GetIdInfo$Id: l3galley.dtx 2938 2011-10-25 18:41:20Z joseph $ +\GetIdInfo$Id: l3galley.dtx 3071 2011-12-20 20:14:00Z bruno $ {L3 Experimental galley code} \ProvidesExplPackage {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription} \package_check_loaded_expl: -\cs_new_protected:Npn \scan_marker_new:N #1 - { \cs_new_eq:NN #1 \scan_stop: } -\cs_generate_variant:Nn \scan_marker_new:N { c } \cs_new_protected_nopar:Npn \galley_leave_vmode: { \tex_unhbox:D \c_empty_box } \clist_new:N \l_galley_parshape_left_indent_clist @@ -193,7 +190,7 @@ \galley_restore_parameters: \group_end: } -\scan_marker_new:N \s_par_omit +\scan_new:N \s_par_omit \cs_new_protected_nopar:Npn \galley_std_par: { \s_par_omit |