summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKarl Berry <karl@freefriends.org>2011-12-29 23:03:28 +0000
committerKarl Berry <karl@freefriends.org>2011-12-29 23:03:28 +0000
commit0bc9393ae50667d7ff769a76321a9548a70351fc (patch)
tree6da7cc8ebef423a4653c82a19be6d4e1c6652705
parent56ecd0d3d0de54cebdeb29cc8a056001ee93b02e (diff)
l3experimental 3109 (29dec11)
git-svn-id: svn://tug.org/texlive/trunk@24970 c570f23f-e606-0410-a88d-b1316a301751
-rw-r--r--Master/texmf-dist/doc/latex/l3experimental/l3str/l3flag.pdfbin0 -> 408926 bytes
-rw-r--r--Master/texmf-dist/doc/latex/l3experimental/l3str/l3regex.pdfbin644249 -> 602845 bytes
-rw-r--r--Master/texmf-dist/doc/latex/l3experimental/l3str/l3str.pdfbin0 -> 752341 bytes
-rw-r--r--Master/texmf-dist/doc/latex/l3experimental/l3str/l3tl-analysis.pdfbin0 -> 510523 bytes
-rw-r--r--Master/texmf-dist/doc/latex/l3experimental/xcoffins/xcoffins.pdfbin481798 -> 481798 bytes
-rw-r--r--Master/texmf-dist/doc/latex/l3experimental/xgalley/l3galley.pdfbin627162 -> 626291 bytes
-rw-r--r--Master/texmf-dist/doc/latex/l3experimental/xgalley/xgalley.pdfbin566382 -> 566324 bytes
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx236
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx2487
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx1053
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins2
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx901
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx19
-rw-r--r--Master/texmf-dist/tex/latex/l3experimental/l3str/l3flag.sty100
-rw-r--r--Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex.sty1443
-rw-r--r--Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-hex.def17
-rw-r--r--Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-name.def15
-rw-r--r--Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-string.def27
-rw-r--r--Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-url.def15
-rw-r--r--Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf16.def27
-rw-r--r--Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf32.def34
-rw-r--r--Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf8.def47
-rw-r--r--Master/texmf-dist/tex/latex/l3experimental/l3str/l3str.sty392
-rw-r--r--Master/texmf-dist/tex/latex/l3experimental/l3str/l3tl-analysis.sty402
-rw-r--r--Master/texmf-dist/tex/latex/l3experimental/xgalley/l3galley.sty7
25 files changed, 4920 insertions, 2304 deletions
diff --git a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3flag.pdf b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3flag.pdf
new file mode 100644
index 00000000000..681d939c173
--- /dev/null
+++ b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3flag.pdf
Binary files differ
diff --git a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3regex.pdf b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3regex.pdf
index 04338df8609..27ee4afc745 100644
--- a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3regex.pdf
+++ b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3regex.pdf
Binary files differ
diff --git a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3str.pdf b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3str.pdf
new file mode 100644
index 00000000000..d3ae4401372
--- /dev/null
+++ b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3str.pdf
Binary files differ
diff --git a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3tl-analysis.pdf b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3tl-analysis.pdf
new file mode 100644
index 00000000000..620d29b7f91
--- /dev/null
+++ b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3tl-analysis.pdf
Binary files differ
diff --git a/Master/texmf-dist/doc/latex/l3experimental/xcoffins/xcoffins.pdf b/Master/texmf-dist/doc/latex/l3experimental/xcoffins/xcoffins.pdf
index 7035f06a7e1..64244d3391b 100644
--- a/Master/texmf-dist/doc/latex/l3experimental/xcoffins/xcoffins.pdf
+++ b/Master/texmf-dist/doc/latex/l3experimental/xcoffins/xcoffins.pdf
Binary files differ
diff --git a/Master/texmf-dist/doc/latex/l3experimental/xgalley/l3galley.pdf b/Master/texmf-dist/doc/latex/l3experimental/xgalley/l3galley.pdf
index 7864fb8a764..ebfe08a82b2 100644
--- a/Master/texmf-dist/doc/latex/l3experimental/xgalley/l3galley.pdf
+++ b/Master/texmf-dist/doc/latex/l3experimental/xgalley/l3galley.pdf
Binary files differ
diff --git a/Master/texmf-dist/doc/latex/l3experimental/xgalley/xgalley.pdf b/Master/texmf-dist/doc/latex/l3experimental/xgalley/xgalley.pdf
index a2bd04b0349..426fe793fc2 100644
--- a/Master/texmf-dist/doc/latex/l3experimental/xgalley/xgalley.pdf
+++ b/Master/texmf-dist/doc/latex/l3experimental/xgalley/xgalley.pdf
Binary files differ
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx
new file mode 100644
index 00000000000..eed657ad231
--- /dev/null
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx
@@ -0,0 +1,236 @@
+% \iffalse meta-comment
+%
+%% File: l3flag.dtx Copyright (C) 2011 The LaTeX3 Project
+%%
+%% It may be distributed and/or modified under the conditions of the
+%% LaTeX Project Public License (LPPL), either version 1.3c of this
+%% license or (at your option) any later version. The latest version
+%% of this license is in the file
+%%
+%% http://www.latex-project.org/lppl.txt
+%%
+%% This file is part of the "l3experimental bundle" (The Work in LPPL)
+%% and all files in that bundle must be distributed together.
+%%
+%% The released version of this bundle is available from CTAN.
+%%
+%% -----------------------------------------------------------------------
+%%
+%% The development version of the bundle can be found at
+%%
+%% http://www.latex-project.org/svnroot/experimental/trunk/
+%%
+%% for those people who are interested.
+%%
+%%%%%%%%%%%
+%% NOTE: %%
+%%%%%%%%%%%
+%%
+%% Snapshots taken from the repository represent work in progress and may
+%% not work or may contain conflicting material! We therefore ask
+%% people _not_ to put them into distributions, archives, etc. without
+%% prior consultation with the LaTeX3 Project.
+%%
+%% -----------------------------------------------------------------------
+%
+%<*driver|package>
+\RequirePackage{expl3}
+\GetIdInfo$Id: l3flag.dtx 3039 2011-12-08 09:22:35Z bruno $
+ {L3 Experimental Flags}
+%</driver|package>
+%<*driver>
+\documentclass[full]{l3doc}
+\usepackage{amsmath}
+\begin{document}
+ \DocInput{\jobname.dtx}
+\end{document}
+%</driver>
+% \fi
+%
+%
+% \title{^^A
+% The \textsf{l3flag} package: expandable flags^^A
+% \thanks{This file describes v\ExplFileVersion,
+% last revised \ExplFileDate.}^^A
+% }
+%
+% \author{^^A
+% The \LaTeX3 Project\thanks
+% {^^A
+% E-mail:
+% \href{mailto:latex-team@latex-project.org}
+% {latex-team@latex-project.org}^^A
+% }^^A
+% }
+%
+% \date{Released \ExplFileDate}
+%
+% \maketitle
+%
+% \begin{documentation}
+%
+% Flags are the only data structure on which \TeX{} can perform
+% assignments in expansion only contexts. This module is meant
+% mostly for kernel use: in almost all cases, booleans should
+% be preferred to flags, because they are faster.
+%
+% A flag can be in two states: \enquote{lowered} or \enquote{raised}.
+% The status of a flag can be tested expandably (just like booleans).
+% A flag can also be \emph{raised} expandably. However, a flag cannot
+% be lowered expandably.
+%
+% Flag variables are always local.
+%
+% Testing for the existence of flags with \cs{cs_if_exist:NTF} will
+% always result in a negative answer: from the point of view of
+% \TeX{}, flags are either undefined control sequences, or let to
+% \cs{scan_stop:}. This has a second consequence: flags don't need
+% to be declared with \cs{flag_new:N}, although this has error-checking
+% benefits. The ability of using flags without declaring them is used
+% in \pkg{l3rand}.
+%
+% \begin{function}{\flag_new:N, \flag_new:c}
+% \begin{syntax}
+% \cs{flag_new:N} \meta{flag}
+% \end{syntax}
+% Creates a new \meta{flag} or raises an error if the
+% name is already taken. The declaration is global,
+% but \meta{flags} are always local variables.
+% The \meta{flag} will initially be lowered.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\flag_test:N, \flag_test:c}
+% \begin{syntax}
+% \cs{flag_test:N} \meta{flag}
+% \end{syntax}
+% This function returns \texttt{true} if the \meta{flag} is raised,
+% and \texttt{false} if the \meta{flag} is lowered.
+% \end{function}
+%
+% \begin{function}[EXP]{\flag_raise:N}
+% \begin{syntax}
+% \cs{flag_raise:N} \meta{flag}
+% \end{syntax}
+% The \meta{flag} is raised. The assignment is local.
+% This function is expandable, despite being an assignment.
+% \end{function}
+%
+% \begin{function}{\flag_lower:N}
+% \begin{syntax}
+% \cs{flag_lower:N} \meta{flag}
+% \end{syntax}
+% The \meta{flag} is lowered. The assignment is local.
+% \end{function}
+%
+% \end{documentation}
+%
+% \begin{implementation}
+%
+% \section{\pkg{l3str} implementation}
+%
+% \begin{macrocode}
+%<*initex|package>
+% \end{macrocode}
+%
+% \begin{macrocode}
+\ProvidesExplPackage
+ {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
+% \end{macrocode}
+%
+% \begin{variable}{\g_flag_list_tl}
+% We keep track of the list of all defined flags in a token list.
+% This is only used by \cs{flag_new:N} to check that the flag
+% has not yet been defined.
+% \begin{macrocode}
+\tl_new:N \g_flag_list_tl
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{macro}{\flag_new:N, \flag_new:c}
+% If the control sequence is already defined, then we raise an error.
+% Otherwise we check if it is already a flag, raising an appropriate
+% error, finally lower the flag globally (this is the only global
+% assignment to a flag\ldots{} perhaps an error?).
+% The \texttt{c} variant can be created without worrying for this function.
+% \begin{macrocode}
+\cs_new_protected:Npn \flag_new:N #1
+ {
+ \tl_if_in:NnTF \g_flag_list_tl {#1}
+ {
+ \msg_kernel_error:nnx
+ { flag } { already-defined }
+ { \token_to_str:N #1 }
+ }
+ {
+ \tl_gput_right:Nn \g_flag_list_tl {#1}
+ \cs_new_eq:NN #1 \c_undefined:D
+ }
+ }
+\cs_generate_variant:Nn \flag_new:N { c }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[EXP, pTF]{\flag_test:N, \flag_test:c}
+% Test if the control sequence is \texttt{undefined} or not.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \flag_test:N #1 { p , T , F , TF }
+ {
+ \if_cs_exist:N #1
+ \prg_return_true:
+ \else:
+ \prg_return_false:
+ \fi:
+ }
+\prg_new_conditional:Npnn \flag_test:c #1 { p , T , F , TF }
+ {
+ \if_cs_exist:w #1 \cs_end:
+ \prg_return_true:
+ \else:
+ \prg_return_false:
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[EXP]{\flag_raise:N, \flag_raise:c}
+% Raising a flag expandably relies on the fact that \TeX{}
+% automatically lets undefined control sequences to \tn{relax}
+% when building an unknown csname. We build such a csname, then
+% remove it with \cs{use_none:n}, essentially doing
+% \cs{use_none:c}, but optimized slightly. When a flag is given
+% as an \texttt{N}-type argument, a little more work is needed,
+% converting to a csname before raising that.
+% \begin{macrocode}
+\cs_new:Npn \flag_raise:N #1
+ { \exp_after:wN \use_none:n \cs:w \cs_to_str:N #1 \cs_end: }
+\cs_new:Npn \flag_raise:c #1
+ { \exp_after:wN \use_none:n \cs:w #1 \cs_end: }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\flag_lower:N, \flag_lower:c}
+% Simply undefine the control sequence, locally.
+% \begin{macrocode}
+\cs_new_protected:Npn \flag_lower:N #1
+ { \cs_set_eq:NN #1 \c_undefined:D }
+\cs_generate_variant:Nn \flag_lower:N { c }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macrocode}
+\msg_kernel_new:nnnn { flag } { already-defined }
+ { The~control~sequence~#1~is~already~declared~as~a~flag. }
+ {
+ LaTeX~was~asked~to~define~the~flag~#1,~but~it~has~already~
+ been~defined~as~a~flag.~The~flag~module~is~mostly~meant~
+ for~kernel~use,~and~booleans~should~be~preferred.
+ }
+% \end{macrocode}
+%
+% \begin{macrocode}
+%</initex|package>
+% \end{macrocode}
+%
+% \end{implementation}
+%
+% \PrintIndex
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx
index 94bd054f0ce..9c276b6922d 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx
@@ -35,7 +35,7 @@
%
%<*driver|package>
\RequirePackage{expl3}
-\GetIdInfo$Id: l3regex.dtx 3017 2011-11-28 00:10:02Z bruno $
+\GetIdInfo$Id: l3regex.dtx 3073 2011-12-21 01:53:06Z bruno $
{L3 Experimental Regular Expressions}
%</driver|package>
%<*driver>
@@ -74,36 +74,53 @@
% \section{\pkg{l3regex} documentation}
%
% The \pkg{l3regex} package provides regular expression testing,
-% extraction of submatches, splitting, and replacement, all acting on strings
-% of characters. The syntax of regular expressions is mostly a subset
-% of the PCRE syntax (and very close to POSIX). For performance
-% reasons, only a limited set of features are implemented. Notably,
-% back-references are not supported.
+% extraction of submatches, splitting, and replacement, all acting
+% on token lists. The syntax of regular expressions is mostly a subset
+% of the PCRE syntax (and very close to POSIX), with some additions
+% due to the fact that we act on lists of tokens rather than characters.
+% For performance reasons, only a limited set of features are implemented.
+% Notably, back-references are not supported.
%
% Let us give a few examples. After
% \begin{verbatim}
-% \str_set:Nn \l_my_str { That~cat. }
-% \regex_replace_once:nnN { at } { is } \l_my_str
+% \tl_set:Nn \l_my_tl { That~cat. }
+% \regex_replace_once:nnN { at } { is } \l_my_tl
% \end{verbatim}
-% the string variable \cs{l_my_str} holds the text
+% the token list variable \cs{l_my_tl} holds the text
% \enquote{\texttt{This cat.}}, where the first
% occurrence of \enquote{\texttt{at}} was replaced
% by \enquote{\texttt{is}}. A more complicated example is
% a pattern to add a comma at the end of each word:
% \begin{verbatim}
-% \regex_replace_all:nnN { \w+ } { \0 , } \l_my_str
+% \regex_replace_all:nnN { \w+ } { \0 , } \l_my_tl
% \end{verbatim}
% The |\w| sequence represents any \enquote{word} character,
% and |+| indicates that the |\w| sequence should be repeated
% as many times as possible (at least once), hence matching a word in the
-% input string. In the replacement text, |\0| denotes the full match
+% input token list. In the replacement text, |\0| denotes the full match
% (here, a word).
%
+% If a regular expression is to be used several times,
+% it can be compiled once, and stored in a token list
+% variable using \cs{regex_const:Nn}. For example,
+% \begin{verbatim}
+% \regex_const:Nn \l_foo_regex_tl { \c{begin} \cB (\CB*?) \cE }
+% \end{verbatim}
+% stores in \cs{l_foo_regex_tl} a regular expression which matches
+% the starting marker for an environment: \cs{begin}, followed
+% by a begin-group token (|\cB|), then tokens which are not
+% begin-group characters (|\CB|), as few as possible (|*?|),
+% followed by an end-group token (|\cE|). As explained in the
+% next section, the parentheses \enquote{capture} the result of
+% |\CB*?|, giving us access to the name of the environment
+% when doing replacements.
+%
% \subsection{Syntax of regular expressions}
%
-% Most characters match exactly themselves. Some characters are
+% Most characters match exactly themselves,
+% with an arbitrary category code. Some characters are
% special and must be escaped with a backslash (\emph{e.g.}, |\*|
-% matches an explicit star character). Some escape sequences of
+% matches a star character). Some escape sequences of
% the form backslash--letter also have a special meaning
% (for instance |\d| matches any digit). As a rule,
% \begin{itemize}
@@ -113,7 +130,7 @@
% |\A|, |\B|, \ldots{} have special meanings;
% \item non-alphanumeric printable ascii characters can (and should)
% always be escaped: many of them have special meanings (\emph{e.g.},
-% |(|, |)|, |?|, |.|);
+% use |\(|, |\)|, |\?|, |\.|);
% \item spaces should always be escaped (even in character
% classes);
% \item any other character may be escaped or not, without any
@@ -121,12 +138,11 @@
% \end{itemize}
% Note that these rules play nicely with the fact that many
% non-alphanumeric characters are difficult to input into \TeX{}
-% under normal category codes. For instance, |\$\%\^\\abc\#|
-% matches the literal string |$%^\abc#|.
-% \begin{texnote}
-% When converting the regular expression to a string,
-% the value of the escape character is set to be a backslash.
-% \end{texnote}
+% under normal category codes. For instance, |\\abc\%|
+% matches the characters |\abc%| (with arbitrary category codes),
+% but does not match the control sequence |\abc| followed by a
+% percent character. Matching control sequences can be done
+% using the |\c|\Arg{regex} syntax (see below).
%
% Any special character which appears at a place where its special
% behaviour cannot apply matches itself instead (for instance,
@@ -146,7 +162,7 @@
%
% Character types.
% \begin{l3regex-syntax}
-% \item[.] A single period matches any character, including newlines.
+% \item[.] A single period matches any token.
% \item[\\d] Any decimal digit.
% \item[\\h] Any horizontal space character,
% equivalent to |[\ \^^I]|: space and tab.
@@ -157,22 +173,27 @@
% but not a space, for compatibility with Perl.
% \item[\\w] Any word character, \emph{i.e.},
% alpha-numerics and underscore, equivalent to |[A-Za-z0-9\_]|.
-% \item[\\D] Any character not matched by |\d|.
-% \item[\\H] Any character not matched by |\h|.
-% \item[\\N] Any character other than the |\n| character (hex 0A).
-% \item[\\S] Any character not matched by |\s|.
-% \item[\\V] Any character not matched by |\v|.
-% \item[\\W] Any character not matched by |\w|.
+% \item[\\D] Any token not matched by |\d|.
+% \item[\\H] Any token not matched by |\h|.
+% \item[\\N] Any token other than the |\n| character (hex 0A).
+% \item[\\S] Any token not matched by |\s|.
+% \item[\\V] Any token not matched by |\v|.
+% \item[\\W] Any token not matched by |\w|.
% \end{l3regex-syntax}
+% Of those, |.|, |\D|, |\H|, |\N|, |\S|, |\V|, and |\W| will match arbitrary
+% control sequences.
%
% Character classes match exactly one character in the subject string.
% \begin{l3regex-syntax}
% \item[{[\ldots{}]}] Positive character class.
-% Matches any of the specified characters.
+% Matches any of the specified tokens.
% \item[{[\char`\^\ldots{}]}] Negative character class.
-% Matches any character other than the specified characters.
+% Matches any token other than the specified characters.
% \item[{[x-y]}] Range (can be used with escaped characters).
% \end{l3regex-syntax}
+% For instance, |[a-oq-z\cC]| matches any lowercase latin letter
+% except |p|, as well as control sequences (see below for a description
+% of |\c|).
%
% Quantifiers (repetition).
% \begin{l3regex-syntax}
@@ -191,12 +212,15 @@
%
% Anchors and simple assertions.
% \begin{l3regex-syntax}
-% \item[\\b] Word boundary.
-% \item[\\B] Not a word boundary.
+% \item[\\b] Word boundary: either the previous token is matched by
+% |\w| and the next by |\W|, or the opposite. For this purpose,
+% the ends of the token list are considered as |\W|.
+% \item[\\B] Not a word boundary: between two |\w| tokens
+% or two |\W| tokens (including the boundary).
% \item[\char`^ \textrm{or} \\A]
-% Start of the subject string.
+% Start of the subject token list.
% \item[\char`$\textrm{,} \\Z \textrm{or} \\z]
-% End of the subject string.
+% End of the subject token list.
% \item[\\G] Start of the current match. This is only different from |^|
% in the case of multiple matches: for instance
% |\regex_count:nnN { \G a } { aaba } \l_tmpa_int| yields $2$, but
@@ -216,6 +240,30 @@
% group number.
% \end{l3regex-syntax}
%
+% Each catcode is represented by a single uppercase letter:
+% |C| for control sequences;
+% |B| for begin-group tokens;
+% |E| for end-group tokens;
+% |M| for math shift;
+% |T| for alignment tab tokens;
+% |P| for macro parameter tokens;
+% |U| for superscript tokens (up);
+% |D| for subscript tokens (down);
+% |S| for spaces;
+% |L| for letters;
+% |O| for others; and
+% |A| for active characters.
+% \begin{l3regex-syntax}
+% \item[\\cX] Any character token with catcode |X|,
+% where |X| can be any of |CBEMTPUDSLOA|.
+% \item[\\CX] Any token with catcode different from |X|,
+% where |X| can be any of |CBEMTPUDSLOA|.
+% \item[\\c\Arg{regex}] A control sequence whose csname matches
+% the \meta{regex}, anchored at the beginning and end,
+% so that |\c{begin}| matches exactly \cs{begin}, and nothing
+% else.
+% \end{l3regex-syntax}
+%
% Options can be set with |(?|\meta{option}|)| and
% unset with |(?-|\meta{option}|)|. Options are local
% to the group in which they are set, and revert to their
@@ -234,7 +282,7 @@
% In character classes, only |^|, |-|, |]|, |\| and spaces are special,
% and should be escaped. Other non-alphanumeric characters can
% still be escaped without harm. The escape sequences |\d|,
-% |\D|, |\w|, |\W| are also supported in character classes.
+% |\D|, etc. are also supported in character classes.
% If the first character is |^|, then the meaning of the character
% class is inverted. Ranges of characters can be expressed using
% |-|, for instance, |[\D 0-5]| is equivalent to |[^6-9]|.
@@ -256,10 +304,24 @@
%
% For instance,
% \begin{verbatim}
-% \str_set:Nn \l_my_str { Hello,~world! }
-% \regex_replace_all:nnN { ([er]?l|o) . } { \(\0\-\-\1\) } \l_my_str
+% \tl_set:Nn \l_my_tl { Hello,~world! }
+% \regex_replace_all:nnN { ([er]?l|o) . } { \(\0\-\-\1\) } \l_my_tl
% \end{verbatim}
-% results in \cs{l_my_str} holding |H(ell--el)(o,--o) w(or--o)(ld--l)!|
+% results in \cs{l_my_tl} holding |H(ell--el)(o,--o) w(or--o)(ld--l)!|
+%
+% The characters inserted by the replacement have category code $12$
+% (other) by default. The escape sequence |\c| allows to insert characters
+% with arbitrary category codes, as well as control sequences.
+% \begin{l3regex-syntax}
+% \item[\\cXY] Produces the character |Y| (which can be given as
+% an escape sequence such as |\t| for tab) with category code |X|,
+% which must be one of |CBEMTPUDSLOA|.
+% \item[\\c\Arg{text}] Produces the control sequence with csname
+% \meta{text}. The \meta{text} may contain references to the submatches
+% |\0|, |\1| \emph{etc.} As an experimental feature, |\c{...}| can be
+% nested, but the behaviour is not yet fully specified. Use at own risks.
+%^^A todo: decide what the right behaviour should be.
+% \end{l3regex-syntax}
%
% \subsection{Precompiling regular expressions}
%
@@ -285,14 +347,32 @@
% The assignment is local for \cs{regex_set:Nn} and global for
% \cs{regex_gset:Nn}. Use \cs{regex_const:Nn} for precompiled expressions
% which will never change.
-% \begin{texnote}
-% Precompiled regular expressions can be safely written to a file
-% and read when the \LaTeX3 syntax is active (as triggered by
-% \cs{ExplSyntaxOn}).
-% \end{texnote}
% \end{function}
%
-% \subsection{String matching}
+% \begin{function}{\regex_to_str:N}
+% \begin{syntax}
+% \cs{regex_to_str:N} \meta{regex var}
+% \end{syntax}
+% Converts the \meta{regex var}, previously defined using
+% \cs{regex_(g)set:Nn}, to a string, which can be safely
+% read back when the \LaTeX3 syntax is active
+% (as triggered by \cs{ExplSyntaxOn}).
+% Line breaks are inserted at various places, to \emph{try}
+% and keep the line length short.
+% For instance, if \cs{l_my_stream} is an open stream for writing,
+% and \cs{l_regex_tl} is a regex variable, you can save its definition
+% using
+% \begin{verbatim}
+% \iow_now:Nx \l_my_stream
+% {
+% \tl_to_str:n { \tl_set:Nn \l_my_regex_tl } { \iow_newline:
+% \regex_to_str:N \l_my_regex_tl
+% }
+% }
+% \end{verbatim}
+% \end{function}
+%
+% \subsection{Matching}
%
% All regular expression functions are available in both |:n| and |:N|
% variants. The former require a \enquote{standard} regular expression,
@@ -301,10 +381,10 @@
%
% \begin{function}[TF]{\regex_match:nn, \regex_match:Nn}
% \begin{syntax}
-% \cs{regex_match:nnTF} \Arg{regex} \Arg{string} \Arg{true code} \Arg{false code}
+% \cs{regex_match:nnTF} \Arg{regex} \Arg{token list} \Arg{true code} \Arg{false code}
% \end{syntax}
-% Tests whether the \meta{regular expression} matches any substring
-% of \meta{string}. For instance,
+% Tests whether the \meta{regular expression} matches any part
+% of the \meta{token list}. For instance,
% \begin{verbatim}
% \regex_match:nnTF { b [cde]* } { abecdcx } { TRUE } { FALSE }
% \regex_match:nnTF { [b-dq-w] } { example } { TRUE } { FALSE }
@@ -314,15 +394,18 @@
%
% \begin{function}{\regex_count:nnN, \regex_count:NnN}
% \begin{syntax}
-% \cs{regex_count:nnN} \Arg{regex} \Arg{string} \meta{int var}
+% \cs{regex_count:nnN} \Arg{regex} \Arg{token list} \meta{int var}
% \end{syntax}
% Sets \meta{int var} within the current \TeX{} group level
% equal to the number of times
-% \meta{regular expression} appears in \meta{string}.
+% \meta{regular expression} appears in \meta{token list}.
% The search starts by finding the left-most longest match,
% respecting greedy and ungreedy operators. Then the search
% starts again from the character following the last character
-% of the previous match, until reaching the end of the string.
+% of the previous match, until reaching the end of the token list.
+% Infinite loops are prevented in the case where the regular expression
+% can match an empty string: then we count one match between each
+% pair of characters.
% For instance,
% \begin{verbatim}
% \int_new:N \l_foo_int
@@ -335,25 +418,25 @@
%
% \begin{function}[TF]{\regex_extract_once:nnN, \regex_extract_once:NnN}
% \begin{syntax}
-% \cs{regex_extract_once:nnN} \Arg{regex} \Arg{string} \meta{seq~var}
-% \cs{regex_extract_once:nnNTF} \Arg{regex} \Arg{string} \meta{seq~var} \Arg{true code} \Arg{false code}
+% \cs{regex_extract_once:nnN} \Arg{regex} \Arg{token list} \meta{seq~var}
+% \cs{regex_extract_once:nnNTF} \Arg{regex} \Arg{token list} \meta{seq~var} \Arg{true code} \Arg{false code}
% \end{syntax}
% Finds the first match of the \meta{regular expression}
-% in the \meta{string}. If it exists, the match is stored
+% in the \meta{token list}. If it exists, the match is stored
% as the zeroeth item of the \meta{seq~var}, and further
% items are the contents of capturing groups, in the order
% of their opening parenthesis. The \meta{seq~var}
% is assigned locally. If there is no match,
-% the \meta{seq~var} is not altered.
+% the \meta{seq~var} is cleared.
% The testing versions insert the \meta{true code} into the input
% stream if a match was found, and the \meta{false code} otherwise.
% For instance, assume that you type
% \begin{verbatim}
-% \regex_extract_once:nnNTF { ^(La)?TeX(!*)$ } { LaTeX!!! }
-% \l_foo_seq { true } { false }
+% \regex_extract_once:nnNTF { \A(La)?TeX(!*)\Z } { LaTeX!!! } \l_foo_seq
+% { true } { false }
% \end{verbatim}
-% Then the regular expression (anchored at the start with |^| and
-% at the end with |$|) will match the whole string. The first
+% Then the regular expression (anchored at the start with |\A| and
+% at the end with |\Z|) will match the whole token list. The first
% capturing group, |(La)?|, matches |La|, and the second capturing
% group, |(!*)|, matches |!!!|. Thus, |\l_foo_seq| will contain
% the items |{LaTeX!!!}|, |{La}|, and |{!!!}|, and the \texttt{true}
@@ -362,64 +445,79 @@
%
% \begin{function}[TF]{\regex_extract_all:nnN, \regex_extract_all:NnN}
% \begin{syntax}
-% \cs{regex_extract_all:nnNTF} \Arg{regex} \Arg{string} \meta{seq~var} \Arg{true code} \Arg{false code}
+% \cs{regex_extract_all:nnN} \Arg{regex} \Arg{token list} \meta{seq~var}
+% \cs{regex_extract_all:nnNTF} \Arg{regex} \Arg{token list} \meta{seq~var} \Arg{true code} \Arg{false code}
% \end{syntax}
% Finds all matches of the \meta{regular expression}
-% in the \meta{string}, and stores all the submatch information
+% in the \meta{token list}, and stores all the submatch information
% in a single sequence (concatenating the results of
% multiple \cs{regex_extract_once:nnN} calls).
+% The \meta{seq~var} is assigned locally. If there is no match,
+% the \meta{seq~var} is cleared.
+% The testing versions insert the \meta{true code} into the input
+% stream if a match was found, and the \meta{false code} otherwise.
% For instance, assume that you type
% \begin{verbatim}
-% \regex_extract_all:nnN { \w+ } { Hello,~world! } \l_foo_seq
+% \regex_extract_all:nnNTF { \w+ } { Hello,~world! } \l_foo_seq
+% { true } { false }
% \end{verbatim}
% Then the regular expression will match twice, and the resulting
-% sequence contains the two items \texttt{Hello} and \texttt{world}.
+% sequence contains the two items |{Hello}| and |{world}|,
+% and the \texttt{true} branch is left in the input stream.
% \end{function}
%
-% \begin{function}{\regex_split:nnN, \regex_split:NnN}
+% \begin{function}[TF]{\regex_split:nnN, \regex_split:NnN}
% \begin{syntax}
-% \cs{regex_split:nnN} \Arg{regular expression} \meta{string} \meta{seq~var}
+% \cs{regex_split:nnN} \Arg{regular expression} \Arg{token list} \meta{seq~var}
+% \cs{regex_split:nnNTF} \Arg{regular expression} \Arg{token list} \meta{seq~var} \Arg{true code} \Arg{false code}
% \end{syntax}
-% Searches the \meta{string} into a sequence of substrings, delimited by
+% Splits the \meta{token list} into a sequence of parts, delimited by
% matches of the \meta{regular expression}. If the \meta{regular expression}
-% has capturing groups, then the substrings that they match are stored as
+% has capturing groups, then the token lists that they match are stored as
% items of the sequence as well. The assignment to \meta{seq~var} is local.
-% If no match is found the resulting \meta{seq~var} has the \meta{string} as
-% its sole item. If the \meta{regular expression} matches the empty string,
-% then the \meta{string} is split into single character substrings.
+% If no match is found the resulting \meta{seq~var} has the
+% \meta{token list} as its sole item. If the \meta{regular expression}
+% matches the empty token list, then the \meta{token list} is split
+% into single tokens.
+% The testing versions insert the \meta{true code} into the input
+% stream if a match was found, and the \meta{false code} otherwise.
% For example, after
% \begin{verbatim}
% \seq_new:N \l_path_seq
-% \regex_split:nnN { / } { the/path/for/this/file.tex } \l_path_seq
+% \regex_split:nnNTF { / } { the/path/for/this/file.tex } \l_path_seq
+% { true } { false }
% \end{verbatim}
% the sequence |\l_path_seq| contains the items |{the}|, |{path}|,
-% |{for}|, |{this}|, and |{file.tex}|.
+% |{for}|, |{this}|, and |{file.tex}|, and the \texttt{true} branch
+% is left in the input stream.
% \end{function}
%
-% \subsection{String replacement}
+% \subsection{Replacement}
%
% \begin{function}[TF]{\regex_replace_once:nnN,\regex_replace_once:NnN}
% \begin{syntax}
-% \cs{regex_replace_once:nnN} \Arg{regular expression} \Arg{replacement} \meta{str~var}
+% \cs{regex_replace_once:nnN} \Arg{regular expression} \Arg{replacement} \meta{tl~var}
+% \cs{regex_replace_once:nnNTF} \Arg{regular expression} \Arg{replacement} \meta{tl~var} \Arg{true code} \Arg{false code}
% \end{syntax}
-% Searches for the \meta{regular expression} in the \meta{string}
-% and replaces the matching part with the \meta{replacement}. The result
-% is assigned locally to \meta{str~var}. In the \meta{replacement},
+% Searches for the \meta{regular expression} in the \meta{token list}
+% and replaces the first match with the \meta{replacement}. The result
+% is assigned locally to \meta{tl~var}. In the \meta{replacement},
% |\0| represents the full match, |\1| represent the contents
% of the first capturing group, |\2| of the second, \emph{etc.}
% \end{function}
%
-% \begin{function}{\regex_replace_all:nnN, \regex_replace_all:NnN}
+% \begin{function}[TF]{\regex_replace_all:nnN, \regex_replace_all:NnN}
% \begin{syntax}
-% \cs{regex_replace_all:nnN} \Arg{regular expression} \Arg{replacement} \meta{str~var}
+% \cs{regex_replace_all:nnN} \Arg{regular expression} \Arg{replacement} \meta{tl~var}
+% \cs{regex_replace_all:nnNTF} \Arg{regular expression} \Arg{replacement} \meta{tl~var} \Arg{true code} \Arg{false code}
% \end{syntax}
% Replaces all occurrences of the \cs{regular expression}
-% in the \meta{string} by the \meta{replacement}, where
+% in the \meta{token list} by the \meta{replacement}, where
% |\0| represents the full match, |\1|
% represent the contents of the first capturing group,
% |\2| of the second, \emph{etc.} Every match
% is treated independently, and matches cannot overlap.
-% The result is assigned locally to \meta{str~var}.
+% The result is assigned locally to \meta{tl~var}.
% \end{function}
%
% \subsection{Bugs, misfeatures, future work, and other possibilities}
@@ -427,11 +525,27 @@
% The following need to be done now.
% \begin{itemize}
% \item The \texttt{\{\}} quantifiers are only partially implemented.
-% \item Check the catcode of spaces everywhere.
-% \item Newline conventions are not done.
-% In particular, we should have an option for |.| not to match newlines.
-% Also, |\A| should differ from |^|, and |\Z|, |\z| and |$| should
-% differ.
+% \item Clean up the use of messages.
+% \item Add tests for every regex and replacement feature.
+% \item Make sure the documentation is correct, and the code comments
+% are as well.
+% \end{itemize}
+%
+% Code improvements to come.
+% \begin{itemize}
+% \item Test for the maximum register \cs{c_max_register_int}.
+% \item Use \tn{dimen} registers rather than \cs{l_regex_nesting_tl}
+% to build \cs{regex_nesting:n}.
+% \item Reduce the number of epsilon-transitions in alternatives.
+% \item Merge the \texttt{build} and \texttt{class} functions where possible.
+% \item Improve nesting of |\c| in the replacement text.
+% \item Move the \enquote{reconstruction} part of \pkg{l3regex}
+% to \pkg{l3tl-analysis}.
+% \item Optimize regexes for csnames when the regex is a simple string.
+% \item Optimize simple strings: use less states
+% (|abcade| should give two states, for |abc| and |ade|).
+% \item Optimize groups with no alternative.
+% \item Optimize the use of \cs{prg_stepwise_...} functions.
% \end{itemize}
%
% The following features are likely to be implemented at some point
@@ -441,30 +555,33 @@
% \item Regex matching on external files.
% \item Conditional subpatterns with look ahead/behind: \enquote{if
% what follows is [\ldots{}], then [\ldots{}]}.
-% \item |(*..)| and |(?..)| sequences to set some options.
-% \item Reduce the number of epsilon-transitions in alternatives.
-% \item Optimize simple strings: use less states
-% (|abcade| should give two states, for |abc| and |ade|).
-% \item Optimize groups with no alternative.
-% \item Optimize the use of \cs{prg_stepwise_...} functions.
+% \item |(*..)| and |(?..)| sequences to set some options
+% (partially implemented).
% \item |\K| for resetting the beginning of the match.
% \item UTF-8 mode for pdf\TeX{}.
+% \item Newline conventions are not done.
+% In particular, we should have an option for |.| not to match newlines.
+% Also, |\A| should differ from |^|, and |\Z|, |\z| and |$| should
+% differ.
+% \item Unicode properties: |\p{..}| and |\P{..}|;
+% |\X| which should match any \enquote{extended} Unicode sequence.
+% This requires to manipulate a lot of data, hence a lot of optimization
+% ahead.
% \end{itemize}
%
% The following features of PCRE or Perl will probably not be implemented.
% \begin{itemize}
-% \item |\cx|, similar to \TeX{}'s own |\^^x|;
% \item |\ddd|, matching the character with code \texttt{ddd} in octal;
-% \item POSIX character classes |[:alpha:]| \emph{etc.};
-% \item Unicode properties: |\p{..}| and |\P{..}|;
-% \item |\X| which should match any \enquote{extended} Unicode sequence;
-% \item Callout with |(?C...)|;
+% \item POSIX character classes |[:alpha:]| \emph{etc.}, this is redundant;
+% \item Callout with |(?C...)|, we cannot run arbitrary user code during
+% the matching, because the regex code uses registers in an unsafe way;
% \item Conditional subpatterns (other than with a look-ahead
% or look-behind condition): this is non-regular, isn't it?
% \end{itemize}
%
% The following features of PCRE or perl will definitely not be implemented.
% \begin{itemize}
+% \item |\cx|, similar to \TeX{}'s own |\^^x|;
% \item Comments: \TeX{} already has its own system for comments.
% \item Named subpatterns: \TeX{} programmers have lived so far without
% any need for named macro parameters.
@@ -495,7 +612,7 @@
% \begin{macrocode}
\ProvidesExplPackage
{\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
-\RequirePackage{l3str}
+\RequirePackage{l3str, l3tl-analysis, l3flag}
% \end{macrocode}
%
% Most regex engines use backtracking. This allows to provide very
@@ -506,8 +623,8 @@
%
% Given a regular expression of $n$ characters, we build a
% non-deterministic finite automaton (NFA) with roughly $n$ states,
-% which accepts precisely those strings matching that regular expression.
-% We then run the string through the NFA, and check the return value.
+% which accepts precisely those token lists matching that regular expression.
+% We then run the token list through the NFA, and check the return value.
%
% The code is structured as follows. Various helper functions are
% introduced in the next subsection, to limit the clutter in later
@@ -520,14 +637,14 @@
% \subsection{Constants and variables}
%
% \begin{macro}{\regex_tmp:w}
-% \begin{variable}{\g_regex_tmpa_tl, \l_regex_tmpa_tl, \l_regex_tmpb_tl}
+% \begin{variable}{\l_regex_tmpa_tl, \l_regex_tmpb_tl, \l_regex_tmpc_tl}
% \begin{variable}{\l_regex_tmpa_int, \l_regex_tmpb_int}
% Temporary variables.
% \begin{macrocode}
\cs_new:Npn \regex_tmp:w { }
-\tl_new:N \g_regex_tmpa_tl
\tl_new:N \l_regex_tmpa_tl
\tl_new:N \l_regex_tmpb_tl
+\tl_new:N \l_regex_tmpc_tl
\int_new:N \l_regex_tmpa_int
\int_new:N \l_regex_tmpb_int
% \end{macrocode}
@@ -535,14 +652,27 @@
% \end{variable}
% \end{macro}
%
+% \begin{variable}{\l_regex_group_begin_flag}
+% \begin{variable}{\l_regex_group_end_flag}
+% Those flags are raised to indicate extra begin-group
+% or end-group tokens when extracting submatches.
+% \begin{macrocode}
+\flag_new:N \l_regex_group_begin_flag
+\flag_new:N \l_regex_group_end_flag
+% \end{macrocode}
+% \end{variable}
+% \end{variable}
+%
% \subsubsection{Variables used while building}
%
% \begin{variable}{\l_regex_max_state_int}
% \begin{variable}{\l_regex_left_state_int, \l_regex_right_state_int}
-% The last state that was allocated is stored in \cs{l_regex_max_state_int},
-% and \cs{l_regex_left/right_state_int} point to both end-points of the
-% last group (which any quantifier would repeat). For simple strings of
-% characters, the left and right pointers only differ by one.
+% The end-point states of the last group (which any quantifier
+% would repeat) are stored as \cs{l_regex_left/right_state_int}.
+% For simple strings of characters, the left and right pointers
+% only differ by one.
+% The last state that was allocated is $\cs{l_regex_max_state_int}-1$,
+% so that \cs{l_regex_max_state_int} always points to a free state.
% \begin{macrocode}
\int_new:N \l_regex_max_state_int
\int_new:N \l_regex_left_state_int
@@ -593,6 +723,16 @@
% \end{variable}
% \end{variable}
%
+% \begin{variable}{\l_regex_build_or_class_tl}
+% The actions of various escape sequences and special characters
+% are different within a class and outside a class. Rather than
+% using a boolean, we store either \texttt{build} or \texttt{class}
+% in this token list.
+% \begin{macrocode}
+\tl_new:N \l_regex_build_or_class_tl
+% \end{macrocode}
+% \end{variable}
+%
% \begin{variable}{\l_regex_one_or_group_tl}
% When looking for quantifiers, this variable holds either
% \enquote{one} or \enquote{group} depending on whether the
@@ -603,6 +743,22 @@
% \end{macrocode}
% \end{variable}
%
+% \begin{variable}{\l_regex_in_csname_bool}
+% This boolean is true when building the regular expression
+% for a given csname.
+% \begin{macrocode}
+\bool_new:N \l_regex_in_csname_bool
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_regex_tmpa_regex_tl}
+% This holds a temporary pre-compiled regular expression
+% when matching a control sequence name.
+% \begin{macrocode}
+\tl_new:N \l_regex_tmpa_regex_tl
+% \end{macrocode}
+% \end{variable}
+%
% \subsubsection{Character classes}
%
% \begin{macro}{\regex_build_tmp_class:n}
@@ -710,19 +866,37 @@
%
% \subsubsection{Variables used when matching}
%
-% \begin{variable}{\l_regex_query_other_str}
-% The string that is being matched.
-% In that string, spaces have category code \enquote{other}.
+% \begin{variable}{\l_regex_nesting_int}
+% This integer is used to keep track of begin and end-group tokens.
+% \begin{macrocode}
+\int_new:N \l_regex_nesting_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_regex_min_step_int}
+% \begin{variable}{\l_regex_max_step_int}
+% This integer holds the value of the step corresponding to
+% the left end of the token list, \emph{i.e.}, when the token
+% list is stored in \tn{toks} registers, the \tn{toks} register
+% \cs{l_regex_min_step_int} holds the first token in the token
+% list. In fact, this number is always one more than
+% \cs{l_regex_max_state_int}, but it is more practical to give
+% that a name.
% \begin{macrocode}
-\tl_new:N \l_regex_query_other_str
+\int_new:N \l_regex_min_step_int
+\int_new:N \l_regex_max_step_int
% \end{macrocode}
% \end{variable}
+% \end{variable}
%
% \begin{variable}{\l_regex_current_step_int}
% \begin{variable}{\l_regex_start_step_int}
% \begin{variable}{\l_regex_success_step_int}
-% While reading through the query string, \cs{l_regex_current_step_int}
-% is the position in the string. Each match begins at the position
+% While reading through the query token list,
+% \cs{l_regex_current_step_int} is the position in the
+% token list, starting at \cs{l_regex_min_step_int} for
+% the left-most token.
+% Each match begins at the position
% given by \cs{l_regex_start_step_int}. Whenever an execution thread
% succeeds, the corresponding step is stored into
% \cs{l_regex_success_step_int}, which will be the next starting step
@@ -736,24 +910,26 @@
% \end{variable}
% \end{variable}
%
-% \begin{variable}{\l_regex_unique_step_int}
+% \begin{variable}{\l_regex_unique_id_int}
% In the case of repeated matches, \cs{l_regex_current_step_int}
% is reset to the end-position of the previous match. In contrast,
-% \cs{l_regex_unique_step_int} is simply incremented to provide
+% \cs{l_regex_unique_id_int} is simply incremented to provide
% a unique number for each iteration of the matching loop. This
% is handy to attach each set of submatch information to a given
% iteration (and automatically discard it when it corresponds to
% a past iteration).
% \begin{macrocode}
-\int_new:N \l_regex_unique_step_int
+\int_new:N \l_regex_unique_id_int
% \end{macrocode}
% \end{variable}
%
% \begin{variable}{\l_regex_current_char_int}
+% \begin{variable}{\l_regex_current_catcode_int}
+% \begin{variable}{\l_regex_current_token_tl}
% \begin{variable}{\l_regex_last_char_int}
% \begin{variable}{\l_regex_case_changed_char_int}
% The character codes of the character at the current position
-% in the string, and at the previous position, and the current
+% in the token list, and at the previous position, and the current
% character with its case changed (|A-Z|$\leftrightarrow$|a-z|).
% The \cs{l_regex_last_char_int} is used to test for word boundaries
% (|\b| and |\B|). The \cs{l_regex_case_changed_char_int} is
@@ -761,12 +937,16 @@
% is used in the regex.
% \begin{macrocode}
\int_new:N \l_regex_current_char_int
+\int_new:N \l_regex_current_catcode_int
+\tl_new:N \l_regex_current_token_tl
\int_new:N \l_regex_last_char_int
\int_new:N \l_regex_case_changed_char_int
% \end{macrocode}
% \end{variable}
% \end{variable}
% \end{variable}
+% \end{variable}
+% \end{variable}
%
% \begin{variable}{\l_regex_caseless_bool}
% True if caseless matching is used within the regular expression.
@@ -777,7 +957,7 @@
% \end{variable}
%
% \begin{variable}{\l_regex_current_state_int}
-% For every character in the string, each of the active states is
+% For every character in the token list, each of the active states is
% considered in turn.
% The variable \cs{l_regex_current_state_int} holds the state
% of the NFA which is currently considered: transitions are then
@@ -823,26 +1003,38 @@
% \begin{macro}[int]{\l_regex_every_match_tl}
% Every time a match is found, this token list is used.
% For single matching, the token list is set to removing
-% the remainder of the query string. For multiple matching,
+% the remainder of the query token list. For multiple matching,
% the token list is set to repeat the matching.
% \begin{macrocode}
\tl_new:N \l_regex_every_match_tl
% \end{macrocode}
% \end{macro}
%
-% \begin{variable}{\l_regex_success_bool}
-% The boolean \cs{l_regex_success_bool} is true if the current match
-% attempt was successful.
+% \begin{variable}{\g_regex_success_bool}
+% \begin{variable}{\l_regex_saved_success_bool}
+% \begin{variable}{\l_regex_success_match_bool}
+% The boolean \cs{g_regex_success_bool} is true if there was
+% at least one match, and \cs{l_regex_success_match_bool} is
+% true if the current match attempt was successful.
+% The variable \cs{g_regex_success_bool} is the only global
+% variable in this whole module. When nesting \cs{regex}
+% functions internally, the value of \cs{g_regex_success_bool}
+% is saved into \cs{l_regex_saved_success_bool}, which is local,
+% hence not affected by the changes due to inner regex functions.
% \begin{macrocode}
-\bool_new:N \l_regex_success_bool
+\bool_new:N \g_regex_success_bool
+\bool_new:N \l_regex_saved_success_bool
+\bool_new:N \l_regex_success_match_bool
% \end{macrocode}
% \end{variable}
+% \end{variable}
+% \end{variable}
%
% \begin{macro}{\regex_last_match_empty:F}
% \begin{macro}[aux]{\regex_last_match_empty_no:F}
% \begin{macro}[aux]{\regex_last_match_empty_yes:F}
% When doing multiple matches, we need to avoid infinite loops where
-% each iteration matches the same empty string. When we detect such
+% each iteration matches the same empty token list. When we detect such
% a situation, the next match attempt is shifted by one character.
% Namely, an empty match is discarded if it follows an empty match
% at the same position. If the previous match was non-empty,
@@ -876,86 +1068,61 @@
% \end{variable}
% \end{variable}
%
-% \subsubsection{Variables used for user functions}
+% \subsubsection{Variables used when building the replacement}
%
-% \begin{variable}{\g_regex_submatches_seq}
-% This holds temporarily a sequence of submatches,
-% global so that it exits the group.
+% \begin{variable}{\l_regex_replacement_int}
% \begin{macrocode}
-\seq_new:N \g_regex_submatches_seq
+\int_new:N \l_regex_replacement_int
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\g_regex_match_count_int}
-% The number of matches found so far is stored
-% in \cs{g_regex_match_count_int}. This is only used
-% in the \cs{regex_count:nnN} functions.
+% \begin{variable}{\l_regex_replacement_csnames_int}
% \begin{macrocode}
-\int_new:N \g_regex_match_count_int
+\int_new:N \l_regex_replacement_csnames_int
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\g_regex_extract_all_seq}
-% The \cs{regex_extract_all:nnN} function stores its result
-% in that sequence variable before assigning it to the
-% variable provided by the user.
+% \begin{variable}{\l_regex_submatch_int}
+% \begin{variable}{\l_regex_submatch_start_int}
% \begin{macrocode}
-\seq_new:N \g_regex_extract_all_seq
+\int_new:N \l_regex_submatch_int
+\int_new:N \l_regex_submatch_start_int
% \end{macrocode}
% \end{variable}
-%
-% \begin{variable}{\g_regex_split_seq}
-% The \cs{regex_split:nnN} function stores its result
-% in that sequence variable before assigning it to the
-% variable provided by the user.
-% \begin{macrocode}
-\seq_new:N \g_regex_split_seq
-% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\l_regex_replacement_tl}
-% \begin{variable}{\g_regex_replaced_str}
-% The replacement code stores a processed version
-% of the user's argument in \cs{l_regex_replacement_tl}.
-% The result of the replacement is stored in \cs{g_regex_replaced_str},
-% global to exit the group.
+% \subsubsection{Variables used for user functions}
+%
+% \begin{variable}{\l_regex_match_count_int}
+% The number of matches found so far is stored
+% in \cs{l_regex_match_count_int}. This is only used
+% in the \cs{regex_count:nnN} functions.
% \begin{macrocode}
-\tl_new:N \l_regex_replacement_tl
-\tl_new:N \g_regex_replaced_str
+\int_new:N \l_regex_match_count_int
% \end{macrocode}
% \end{variable}
-% \end{variable}
%
% \subsection{Helpers}
%
-% \subsubsection{Scan markers}
-%
-% \begin{macro}[int]{\s_regex_stop}
-% Unexpandable markers, which does nothing if it reaches \TeX{}'s stomach.
-% When performing the matching, the \tn{toks} registers hold submatch
-% information, followed by the instruction for a given state of the NFA.
-% The two parts are separated by \cs{s_regex_stop}.
-% See \cs{regex_nfa:Nw} for another use.
-% \begin{macrocode}
-\cs_new_eq:NN \s_regex_stop \scan_stop:
-% \end{macrocode}
-% \end{macro}
-%
% \subsubsection{Toks}
%
+% When performing the matching, the \tn{toks} registers hold submatch
+% information, followed by the instruction for a given state of the NFA.
+% The two parts are separated by \cs{s_stop}.
+%
% \begin{macro}[int]{\regex_toks_put_left:Nx}
% \begin{macro}[int]{\regex_toks_put_right:Nx}
% During the building phase, every \tn{toks} register starts with
-% \cs{s_regex_stop}, and we wish to add \texttt{x}-expanded material
+% \cs{s_stop}, and we wish to add \texttt{x}-expanded material
% to those registers. The expansion is done \enquote{by hand} for
% optimization (these operations are used quite a lot). When adding
% material to the left, we define \cs{regex_tmp:w} to remove the
-% \cs{s_regex_stop} marker and put it back to the left of the new
+% \cs{s_stop} marker and put it back to the left of the new
% material.
% \begin{macrocode}
\cs_new_protected:Npn \regex_toks_put_left:Nx #1#2
{
- \cs_set_nopar:Npx \regex_tmp:w \s_regex_stop { \s_regex_stop #2 }
+ \cs_set_nopar:Npx \regex_tmp:w \s_stop { \s_stop #2 }
\tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN
{ \exp_after:wN \regex_tmp:w \tex_the:D \tex_toks:D #1 }
}
@@ -969,30 +1136,37 @@
% \end{macro}
% \end{macro}
%
-% \subsubsection{Unsafe substring extraction}
+% \subsubsection{Extracting parts of the query token list}
%
-% \begin{macro}[int]{\regex_query_substr:NN}
-% \begin{macro}[aux]{\regex_query_substr_aux:nnn}
-% Extracting submatches from the query string is done
-% in a way similar to \cs{str_substr:Nnn}, but omitting
-% most checks, hence faster.
+% \begin{macro}[int]{\regex_query_substr:nn}
+% \begin{macro}[int]{\regex_query_submatch:nn}
+% \begin{macro}[int]{\regex_query_submatch:w}
+% The token list is stored in \tn{toks} registers:
+% the first is \cs{l_regex_min_step_int}, the last is (we don't care yet).
% \begin{macrocode}
-\cs_new_nopar:Npn \regex_query_substr:NN #1#2
+\cs_new:Npn \regex_query_substr:nn #1#2
{
- \exp_after:wN \regex_query_substr_aux:NN
- \exp_after:wN #1
- \exp_after:wN #2
- \l_regex_query_other_str
- \q_stop
+ \if_num:w \int_eval:w #1 < \int_eval:w #2 \int_eval_end:
+ \str_aux_toks_range:nn {#1} {#2}
+ \fi:
+ }
+\cs_new:Npn \regex_query_submatch:nn #1#2
+ {
+ \if_num:w #1 < \l_regex_capturing_group_int
+ \exp_after:wN \regex_query_submatch:w
+ \int_use:N \int_eval:w #1 + #2 ;
+ \fi:
}
-\cs_new:Npn \regex_query_substr_aux:NN #1#2
+\cs_new:Npn \regex_query_submatch:w #1 ;
{
- \str_skip_do:nn {#1}
- { \str_collect_do:nn { #2 - #1 } { \use_i_delimit_by_q_stop:nw } }
+ \regex_query_substr:nn
+ { \tex_skip:D #1 }
+ { \etex_gluestretch:D \tex_skip:D #1 }
}
% \end{macrocode}
% \end{macro}
% \end{macro}
+% \end{macro}
%
% \subsubsection{Sequences}
%
@@ -1042,20 +1216,46 @@
% \end{macro}
% \end{macro}
%
+% \begin{macro}[int, EXP]{\regex_seq_item:Nn}
+% \begin{macro}[aux, EXP]{\regex_seq_item_aux:wNn}
+% \begin{macrocode}
+\cs_new:Npn \regex_seq_item:Nn #1#2
+ {
+ \exp_after:wN \regex_seq_item_aux:wNn
+ \int_use:N \int_eval:w #2 \exp_after:wN ;
+ #1 { ? \prg_map_break: } { }
+ \prg_break_point:n { }
+ }
+\cs_new:Npn \regex_seq_item_aux:wNn #1; #2#3
+ {
+ \use_none:n #2
+ \if_num:w #1 = \c_zero
+ \exp_after:wN \prg_map_break:n
+ \else:
+ \exp_after:wN \use_none:n
+ \fi:
+ { \exp_not:n {#3} }
+ \exp_after:wN \regex_seq_item_aux:wNn
+ \int_use:N \int_eval:w #1 - \c_one ;
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
% \subsubsection{Testing characters}
%
% \begin{macro}{\regex_item_dot:T}
% The dot meta-character matches any character,
-% except the end-of-string marker.
+% except the end marker.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \regex_item_dot:T
- { \int_compare:nNnF \l_regex_current_char_int = \c_minus_one }
+ { \int_compare:nNnF \l_regex_current_char_int < \c_minus_one }
% \end{macrocode}
% \end{macro}
%
% \begin{macro}[int]{\regex_break_point:TF}
% \begin{macro}[int]{\regex_break_true:w,\regex_break_false:w}
-% When testing whether a character of the query string matches
+% When testing whether a character of the query token list matches
% a given character class in the regular expression, we often
% have to test it against several ranges of characters, checking
% if any one of those matches. This is done with a structure like
@@ -1187,6 +1387,56 @@
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}[int]{\regex_item_catcode:n}
+% \begin{macro}[int]{\regex_item_catcode_neq:n}
+% Checking the category code is quite straightforward.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \regex_item_catcode:n #1
+ {
+ \if_num:w #1 = \l_regex_current_catcode_int
+ \exp_after:wN \regex_break_true:w
+ \fi:
+ }
+\cs_new_protected_nopar:Npn \regex_item_catcode_neq:n #1
+ {
+ \if_num:w #1 = \l_regex_current_catcode_int
+ \else:
+ \exp_after:wN \regex_break_true:w
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int]{\regex_item_cs:n}
+% Match a control sequence (the argument is a pre-compiled regex).
+% First test the catcode of the current token to be zero.
+% Then perform the matching test, and break if the csname
+% indeed matches. The three \cs{exp_after:wN} expand the contents
+% of \cs{l_regex_current_token_tl} (of the form \cs{exp_not:n}
+% \Arg{control sequence}) to \meta{control sequence}.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_item_cs:n #1
+ {
+ \int_compare:nNnT \l_regex_current_catcode_int = \c_zero
+ {
+ \tl_set:Nn \l_regex_tmpa_regex_tl {#1}
+ \bool_set_eq:NN \l_regex_saved_success_bool \g_regex_success_bool
+ \exp_args:NNx \regex_match:NnTF \l_regex_tmpa_regex_tl
+ {
+ \exp_after:wN \exp_after:wN
+ \exp_after:wN \cs_to_str:N \l_regex_current_token_tl
+ }
+ {
+ \bool_gset_eq:NN \g_regex_success_bool \l_regex_saved_success_bool
+ \regex_break_true:w
+ }
+ { \bool_gset_eq:NN \g_regex_success_bool \l_regex_saved_success_bool }
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+%
% \subsubsection{Grabbing digits}
%
% \begin{macro}[int]{\regex_get_digits:nw}
@@ -1323,19 +1573,15 @@
% \begin{macro}[int]{\regex_build_new_state:}
% Here, we add a new state to the NFA. At the end of the building
% phase, we want every \tn{toks} register to start with
-% \cs{s_regex_stop}, hence initialize the new register appropriately.
+% \cs{s_stop}, hence initialize the new register appropriately.
% Then set \cs{l_regex_left/right_state_int} to their new values.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \regex_build_new_state:
{
- \int_compare:nNnTF \l_regex_max_state_int > { 32766 }
- { \msg_error:nn { regex } { -997 } }
- {
- \int_incr:N \l_regex_max_state_int
- \tex_toks:D \l_regex_max_state_int { \s_regex_stop }
- }
+ \tex_toks:D \l_regex_max_state_int { \s_stop }
\int_set_eq:NN \l_regex_left_state_int \l_regex_right_state_int
\int_set_eq:NN \l_regex_right_state_int \l_regex_max_state_int
+ \int_incr:N \l_regex_max_state_int
}
% \end{macrocode}
% \end{macro}
@@ -1377,7 +1623,9 @@
% First, reset a few variables. Then use the generic framework defined
% in \pkg{l3str} to parse the regular expression once, recognizing
% which characters are raw characters, and which have special meanings.
-% The result is stored in \cs{g_str_result_tl}, and can be run directly.
+% The search is not anchored: to achieve that, we insert state(s)
+% responsible for repeating the match attempt on every token
+% in the token list.
% The trailing \cs{prg_do_nothing:} ensure that the look-ahead done by
% some of the operations is harmless.
% Finally, \cs{regex_build_end:} adds the finishing code
@@ -1385,16 +1633,18 @@
% \begin{macrocode}
\cs_new_protected:Npn \regex_build:n #1
{
- \regex_build_setup:
- \str_aux_escape:NNNn
- \regex_build_i_unescaped:N
- \regex_build_i_escaped:N
- \regex_build_i_raw:N
- { #1 }
+ \bool_set_false:N \l_regex_in_csname_bool
+ \regex_build:w
+ \regex_build_new_state:
+ \regex_toks_put_right:Nx \l_regex_left_state_int
+ { \regex_action_start_wildcard:nn {0} {1} }
\regex_build_open_aux:
- \g_str_result_tl \prg_do_nothing: \prg_do_nothing:
- \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_zero
- \regex_build_close_aux: \regex_build_group_:
+ \str_escape_use:NNNn
+ \regex_build_i_unescaped:N
+ \regex_build_i_escaped:N
+ \regex_build_i_raw:N
+ { #1 }
+ \prg_do_nothing: \prg_do_nothing:
\regex_build_end:
}
% \end{macrocode}
@@ -1403,7 +1653,7 @@
% \begin{macro}[aux]{\regex_build_i_unescaped:N}
% \begin{macro}[aux]{\regex_build_i_escaped:N}
% \begin{macro}[aux]{\regex_build_i_raw:N}
-% The \pkg{l3str} function \cs{str_aux_escape:NNNn} goes through
+% The \pkg{l3str} function \cs{str_escape_use:NNNn} goes through
% the regular expression and finds the |\a|, |\e|, |\f|, |\n|, |\r|,
% |\t|, and |\x| escape sequences, then distinguishes three cases:
% non-escaped characters, escaped characters, and \enquote{raw}
@@ -1432,35 +1682,36 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_default_control:N}
-% If the control character has a particular meaning in regular expressions,
-% the corresponding function is used. Otherwise, it is interpreted as a raw
-% character. The \cs{regex_build_default_raw:N} function is defined later.
+% \begin{macro}[aux]{\regex_build_control:N}
+% If the control character has a particular meaning in regexes,
+% the corresponding function is used. Otherwise, it is interpreted
+% as a raw character. The \cs{regex_build/class_default_raw:N}
+% functions are defined later.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_default_control:N #1
+\cs_new_protected_nopar:Npn \regex_build_control:N #1
{
- \cs_if_exist_use:cF { regex_build_#1: }
- { \regex_build_default_raw:N #1 }
+ \cs_if_exist_use:cF { regex_ \l_regex_build_or_class_tl _#1: }
+ { \use:c { regex_ \l_regex_build_or_class_tl _default_raw:N } #1 }
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[int]{\regex_build_setup:}
+%^^A todo: update
+% This function is similar to \cs{regex_build_control:N}. If the control
+% character has a meaning in character classes, call the corresponding
+% function, otherwise, treat it as a raw character, with the
+% \cs{regex_class_default_raw:N} function, defined later.
+%
+% \begin{macro}[int]{\regex_build:w}
% Hopefully, we didn't forget to initialize anything here.
-% The search is not anchored: to achieve that, we insert state(s)
-% responsible for repeating the match attempt on every character
-% of the string.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_setup:
+\cs_new_protected_nopar:Npn \regex_build:w
{
- \cs_set_eq:NN \regex_build_control:N \regex_build_default_control:N
\cs_set_eq:NN \regex_build_raw:N \regex_build_default_raw:N
+ \tl_set:Nn \l_regex_build_or_class_tl { build }
\int_set_eq:NN \l_regex_capturing_group_int \c_zero
\int_zero:N \l_regex_max_state_int
\regex_build_new_state:
- \regex_build_new_state:
- \regex_toks_put_right:Nx \l_regex_left_state_int
- { \regex_action_start_wildcard:nn {0} {1} }
}
% \end{macrocode}
% \end{macro}
@@ -1472,15 +1723,19 @@
% \begin{macrocode}
\cs_new_protected_nopar:Npn \regex_build_end:
{
+ \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_zero
+ \regex_build_close_aux: \regex_build_group_:
\seq_if_empty:NF \l_regex_capturing_group_seq
{
- \msg_error:nn { regex } {22}
+ \msg_kernel_error:nnx { regex } { missing-rparen }
+ { \seq_length:N \l_regex_capturing_group_seq }
\prg_replicate:nn
{ \seq_length:N \l_regex_capturing_group_seq }
{ \regex_build_close_aux: \regex_build_group_: }
}
\regex_toks_put_right:Nx \l_regex_right_state_int
{ \regex_action_success: }
+ \int_incr:N \l_regex_capturing_group_int
}
% \end{macrocode}
% \end{macro}
@@ -1492,14 +1747,14 @@
% \begin{macro}[int]{\regex_build_G:}
% \begin{macro}[aux]{\regex_build_anchor_start:N}
% Anchoring at the start corresponds to checking that the current
-% character is the first in the string. Anchoring to the beginning
+% character is the first in the token list. Anchoring to the beginning
% of the match attempt uses \cs{l_regex_start_step_int} instead of
% \cs{c_zero}.
% \begin{macrocode}
\cs_new_protected_nopar:cpn { regex_build_^: }
- { \regex_build_anchor_start:N \c_zero }
+ { \regex_build_anchor_start:N \l_regex_min_step_int }
\cs_new_protected_nopar:Npn \regex_build_A:
- { \regex_build_anchor_start:N \c_zero }
+ { \regex_build_anchor_start:N \l_regex_min_step_int }
\cs_new_protected_nopar:Npn \regex_build_G:
{ \regex_build_anchor_start:N \l_regex_start_step_int }
\cs_new_protected_nopar:Npn \regex_build_anchor_start:N #1
@@ -1507,7 +1762,7 @@
\regex_build_new_state:
\regex_toks_put_right:Nx \l_regex_left_state_int
{
- \exp_not:N \int_compare:nNnT {#1} = \l_regex_current_step_int
+ \exp_not:N \int_compare:nNnT #1 = \l_regex_current_step_int
{
\regex_action_free:n
{
@@ -1526,7 +1781,7 @@
% \begin{macro}[aux]{\regex_build_Z:}
% \begin{macro}[aux]{\regex_build_z:}
% \begin{macro}[aux]+\regex_build_$:+
-% This matches the end of the string, marked by a character code of $-1$.
+% This matches the end of the token list, marked by a character code of $-2$.
% \begin{macrocode}
\cs_new_protected_nopar:cpn { regex_build_$: } % $
{
@@ -1534,7 +1789,7 @@
\regex_toks_put_right:Nx \l_regex_left_state_int
{
\exp_not:N \int_compare:nNnT
- \c_minus_one = \l_regex_current_char_int
+ \l_regex_current_char_int < \c_minus_one
{
\regex_action_free:n
{
@@ -1555,7 +1810,7 @@
% \begin{macro}[int]{\regex_build_B:}
% \begin{macro}[aux]{\regex_if_word_boundary:TF}
% Contrarily to |^| and |$|, which could be implemented without
-% really knowing what precedes in the string, this requires
+% really knowing what precedes in the token list, this requires
% more information, namely, the knowledge of the last character
% code. Case sensitivity does not change word boundaries.
% \begin{macrocode}
@@ -1597,7 +1852,7 @@
\int_set_eq:NN \l_regex_current_char_int \l_regex_last_char_int
\c_regex_w_tl
\regex_break_point:TF
- { \group_end: \c_regex_W_tl \regex_item_equal:n { -1 } }
+ { \group_end: \c_regex_W_tl \regex_item_equal:n { -2 } }
{ \group_end: \c_regex_w_tl }
\regex_break_point:TF
}
@@ -1608,6 +1863,23 @@
%
% \subsubsection{Normal character, and simple character classes}
%
+% \begin{macro}[int]{\regex_build_one:n, \regex_build_one:x}
+%
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_build_one:n #1
+ { \regex_build_one:x { \exp_not:n {#1} } }
+\cs_new_protected:Npn \regex_build_one:x #1
+ {
+ \cs_set_nopar:Npx \regex_build_tmp_class:n ##1
+ {
+ \exp_not:N \exp_not:n {#1}
+ \regex_break_point:TF { \regex_action_cost:n {##1} } { }
+ }
+ \regex_build_one_quantifier:
+ }
+% \end{macrocode}
+% \end{macro}
+%
% \begin{macro}[aux]{\regex_build_default_raw:N}
% A normal alphanumeric or an escaped non-alphanumeric
% (actually, any unknown combination) will match itself
@@ -1618,20 +1890,13 @@
% yet what this will be before checking for quantifiers.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \regex_build_default_raw:N #1
- {
- \cs_set:Npx \regex_build_tmp_class:n ##1
- {
- \regex_item_equal:n { \int_value:w `#1 ~ }
- \regex_break_point:TF { \regex_action_cost:n { ##1 } } { }
- }
- \regex_build_one_quantifier:
- }
+ { \regex_build_one:x { \regex_item_equal:n { \int_value:w `#1 ~ } } }
% \end{macrocode}
% \end{macro}
%
% \begin{macro}[aux]{\regex_build_.:}
% Similar to \cs{regex_build_default_raw:N} but accepts any character,
-% and refuses $-1$, which marks the end of the string.
+% and refuses $-1$, which marks the end of the token list.
% \begin{macrocode}
\cs_new_protected_nopar:cpn { regex_build_.: }
{
@@ -1642,34 +1907,21 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_d:,\regex_build_D}
-% \begin{macro}[aux]{\regex_build_h:,\regex_build_H}
-% \begin{macro}[aux]{\regex_build_s:,\regex_build_S}
-% \begin{macro}[aux]{\regex_build_v:,\regex_build_V}
-% \begin{macro}[aux]{\regex_build_w:,\regex_build_W}
+% \begin{macro}[aux]{\regex_build_d:,\regex_build_D:}
+% \begin{macro}[aux]{\regex_build_h:,\regex_build_H:}
+% \begin{macro}[aux]{\regex_build_s:,\regex_build_S:}
+% \begin{macro}[aux]{\regex_build_v:,\regex_build_V:}
+% \begin{macro}[aux]{\regex_build_w:,\regex_build_W:}
% \begin{macro}[aux]{\regex_build_N:}
-% \begin{macro}[aux]{\regex_build_char_type:N}
% The constants \cs{c_regex_d_tl}, \emph{etc.} hold
% a list of tests which match the corresponding character
% class, and jump to the \cs{regex_break_point:TF} marker.
% As for a normal character, we check for quantifiers.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_char_type:N #1
- {
- \cs_set:Npn \regex_build_tmp_class:n ##1
- {
- \exp_not:N #1
- \regex_break_point:TF { \regex_action_cost:n {##1} } { }
- }
- \regex_build_one_quantifier:
- }
\tl_map_inline:nn { dDhHsSvVwWN }
{
\cs_new_protected_nopar:cpx { regex_build_#1: }
- {
- \exp_not:N \regex_build_char_type:N
- \exp_not:c { c_regex_#1_tl }
- }
+ { \regex_build_one:n \exp_not:c { c_regex_#1_tl } }
}
% \end{macrocode}
% \end{macro}
@@ -1678,10 +1930,20 @@
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
%
% \subsubsection{Character classes}
%
+% \begin{macro}[aux]{\regex_class_one:n, \regex_class_one:x}
+% This is the function used to add material
+% to the class.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_class_one:n
+ { \tl_put_right:Nn \l_regex_class_tl }
+\cs_new_protected:Npn \regex_class_one:x
+ { \tl_put_right:Nx \l_regex_class_tl }
+% \end{macrocode}
+% \end{macro}
+%
% \begin{macro}[aux]{\regex_build_[:}
% This starts a class. The code for the class is collected
% in \cs{l_regex_class_tl}. The first character is special.
@@ -1689,41 +1951,26 @@
\cs_new_protected_nopar:cpn { regex_build_[: }
{
\tl_clear:N \l_regex_class_tl
- \cs_set_eq:NN \regex_build_control:N \regex_class_control:N
- \cs_set_eq:NN \regex_build_raw:N \regex_class_raw:N
+ \cs_set_eq:NN \regex_build_raw:N \regex_class_default_raw:N
+ \tl_set:Nn \l_regex_build_or_class_tl { class }
\regex_class_first:NN
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_class_control:N}
-% This function is similar to \cs{regex_build_control:N}. If the control
-% character has a meaning in character classes, call the corresponding
-% function, otherwise, treat it as a raw character, with the
-% \cs{regex_class_raw:N} function, defined later.
-% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_class_control:N #1
- {
- \cs_if_exist_use:cF { regex_class_#1: }
- { \regex_class_raw:N #1 }
- }
-% \end{macrocode}
-% \end{macro}
-%
% \begin{macro}[aux]{\regex_class_]:}
% If \texttt{]} appears as the first item of a class, then
% it doesn't end the class. Otherwise, it's the end,
% act just as for a single character, but with a more
-% complicated test. And restore \cs{regex_build_control:N}
-% and \cs{regex_build_raw:N}.
+% complicated test. And restore \cs{regex_build_raw:N}.
% \begin{macrocode}
\cs_new_protected_nopar:cpn { regex_class_]: }
{
\tl_if_empty:NTF \l_regex_class_tl %[
- { \regex_class_raw:N ] }
+ { \regex_class_default_raw:N ] }
{
- \cs_set_eq:NN \regex_build_control:N \regex_build_default_control:N
\cs_set_eq:NN \regex_build_raw:N \regex_build_default_raw:N
+ \tl_set:Nn \l_regex_build_or_class_tl { build }
\cs_set:Npn \regex_build_tmp_class:n ##1
{
\exp_not:o \l_regex_class_tl
@@ -1753,12 +2000,12 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_class_raw:N}
+% \begin{macro}[aux]{\regex_class_default_raw:N}
% \begin{macro}[aux]{\regex_class_single:N}
% Most characters are treated here. We look ahead for an unescaped dash.
% If there is none, then the character matches itself.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_class_raw:N #1#2#3
+\cs_new_protected_nopar:Npn \regex_class_default_raw:N #1#2#3
{
\str_if_eq:nnTF {#2#3} { \regex_build_control:N - }
{ \regex_class_range:Nw #1 }
@@ -1768,10 +2015,7 @@
}
}
\cs_new_protected_nopar:Npn \regex_class_single:N #1
- {
- \tl_put_right:Nx \l_regex_class_tl
- { \regex_item_equal:n { \int_value:w `#1 } }
- }
+ { \regex_class_one:x { \regex_item_equal:n { \int_value:w `#1 } } }
% \end{macrocode}
% \end{macro}
% \end{macro}
@@ -1804,9 +2048,9 @@
\cs_new_protected_nopar:Npn \regex_class_range_put:NN #1#2
{
\if_num:w `#1 > `#2 \exp_stop_f:
- \msg_error:nn { regex } {8}
+ \msg_kernel_error:nnxx { regex } { backwards-range } {#1} {#2}
\else:
- \tl_put_right:Nx \l_regex_class_tl
+ \regex_class_one:x
{
\if_num:w `#1 = `#2 \exp_stop_f:
\regex_item_equal:n
@@ -1834,9 +2078,59 @@
\tl_map_inline:nn { dDhHsSvVwWN }
{
\cs_new_protected_nopar:cpx { regex_class_#1: }
+ { \regex_class_one:n \exp_not:c { c_regex_#1_tl } }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \subsubsection{Catcodes and csnames}
+%
+% \begin{macro}[aux]{\regex_build_c:}
+% \begin{macro}[aux]{\regex_build_C:}
+% \begin{macro}[aux]{\regex_class_c:}
+% \begin{macro}[aux]{\regex_class_C:}
+% \begin{macro}[aux]{\regex_aux_c:nnNN}
+%
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \regex_build_c:
+ { \regex_aux_c:NNNN \regex_item_catcode:n \use_i:nn }
+\cs_new_protected_nopar:Npn \regex_build_C:
+ { \regex_aux_c:NNNN \regex_item_catcode_neq:n \use_ii:nn }
+\cs_new_eq:NN \regex_class_c: \regex_build_c:
+\cs_new_eq:NN \regex_class_C: \regex_build_C:
+\cs_new_protected:Npn \regex_aux_c:NNNN #1#2#3#4
+ {
+ \token_if_eq_meaning:NNTF #3 \regex_build_raw:N
+ {
+ \regex_aux_c_test:NTF #4
+ {
+ \use:c { regex_ \l_regex_build_or_class_tl _one:x }
+ { #1 { \int_use:N \l_regex_current_catcode_int } }
+ }
+ {
+ \msg_kernel_error:nnx { regex } { build-c } { #4 }
+ #3 #4
+ }
+ }
{
- \tl_put_right:Nn \exp_not:N \l_regex_class_tl
- { \exp_not:c { c_regex_#1_tl } }
+ \if_charcode:w \c_lbrace_str \exp_not:N #4
+ \exp_after:wN #2
+ \else:
+ \exp_after:wN \use_ii:nn
+ \fi:
+ {
+ \group_begin:
+ \bool_set_true:N \l_regex_in_csname_bool
+ \regex_build:w
+ }
+ {
+ \msg_kernel_error:nnx { regex } { build-c } { #4 }
+ #3 #4
+ }
}
}
% \end{macrocode}
@@ -1846,6 +2140,81 @@
% \end{macro}
% \end{macro}
%
+% The |\c| and |\C| escape sequences are used both in the
+% regex and the replacement texts. They can be followed either
+% by a capital letter representing a category code, or by a left
+% brace (in the case of |\c| only).
+%
+% \begin{macro}{\regex_aux_c_test:NTF}
+% If the first argument is a known catcode identifier
+% (capital letters listed below), then
+% \cs{l_regex_current_catcode_int} is set to the relevant
+% catcode value (between $0$ for cs and $13$ for active chars),
+% and the \texttt{true} branch is taken.
+% Otherwise, the \texttt{false} branch is taken,
+% and the current catcode is set to $-1$.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_aux_c_test:NTF #1
+ {
+ \int_set:Nn \l_regex_current_catcode_int
+ {
+ \if_case:w \int_eval:w `#1 - `A \int_eval_end:
+ \c_thirteen % Active
+ \or: \c_one % Begin-group
+ \or: \c_zero % Control sequence
+ \or: \c_eight % Down (subscript)
+ \or: \c_two % End-group
+ \else:
+ \if_case:w \int_eval:w `#1 - `L \int_eval_end:
+ \c_eleven % Letter
+ \or: \c_three % Math toggle
+ \or: \c_minus_one
+ \or: \c_twelve % Other
+ \or: \c_six % Parameter
+ \or: \c_minus_one
+ \or: \c_minus_one
+ \or: \c_ten % Space
+ \or: \c_four % Tab (alignment)
+ \or: \c_seven % Up (superscript)
+ \else: \c_minus_one
+ \fi:
+ \fi:
+ }
+ \if_num:w \l_regex_current_catcode_int = \c_minus_one
+ \exp_after:wN \use_ii:nn
+ \else:
+ \exp_after:wN \use_i:nn
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% ^^A{
+% \begin{macro}+\regex_build}:+
+% Non-escaped right braces are only special if they appear
+% when building the regular expression for a csname. Otherwise,
+% replace the brace with an escaped brace.
+% \begin{macrocode}
+\cs_new_protected:cpn { regex_build_ \c_rbrace_str : }
+ {
+ \bool_if:NTF \l_regex_in_csname_bool
+ {
+ \regex_build_Z:
+ \regex_toks_put_right:Nx \l_regex_right_state_int
+ { \regex_action_success: }
+ \int_incr:N \l_regex_capturing_group_int
+ \use:x
+ {
+ \group_end:
+ \exp_not:n { \use:c { regex_ \l_regex_build_or_class_tl _one:n } }
+ { \regex_item_cs:n { \regex_set_aux:N ? } }
+ }
+ }
+ { \exp_after:wN \regex_build_raw:N \c_rbrace_str }
+ }
+% \end{macrocode}
+% \end{macro}
+%
% \subsubsection{Quantifiers}
%
% \begin{macro}[int]{\regex_build_quantifier:w}
@@ -1952,12 +2321,8 @@
{
\tl_if_empty:nTF {#2}
{ \regex_build_quantifier_aux:nnNN { {#1} } { n* } }
- {
- \int_compare:nNnT {#1} > {#2}
- { \msg_error:nn { regex } {4} }
- \regex_build_quantifier_aux:nnNN { {#1} {#2} } { nn }
- }
- }
+ { \regex_build_quantifier_aux:nnNN { {#1} {#2} } { nn } }
+ } %^^A todo: catch the case m>n
{
\regex_build_quantifier_end:n { }
\use:x
@@ -2190,7 +2555,7 @@
\cs_new_protected_nopar:cpn { regex_build_): }
{
\seq_if_empty:NTF \l_regex_capturing_group_seq
- { \msg_error:nn { regex } { 22 } }
+ { \msg_kernel_error:nn { regex } { extra-rparen } }
{
\regex_build_close_aux:
\regex_build_group_quantifier:
@@ -2249,7 +2614,8 @@
{
\cs_if_exist_use:cF { regex_build_special_group_\token_to_str:N #2 : }
{
- \msg_error:nn { regex } { -998 }
+ \msg_kernel_error:nnx { regex } { unsupported }
+ { (? \token_to_str:N #2 } %)
\regex_build_control:N ( % )
\regex_build_raw:N ?
#1 #2
@@ -2433,7 +2799,7 @@
{
\tex_toks:D \l_regex_tmpa_int
{
- \s_regex_stop
+ \s_stop
#1 { \int_eval:n { \l_regex_right_state_int - \l_regex_tmpa_int } }
}
}
@@ -2544,16 +2910,19 @@
\regex_toks_put_right:Nx \l_regex_right_state_int
{
\regex_action_free:n
- { \int_eval:n { \l_regex_max_state_int - \l_regex_left_state_int } }
+ {
+ \int_eval:n %^^A todo: document why that value.
+ { \l_regex_max_state_int - \c_one - \l_regex_left_state_int }
+ }
}
\int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int
- \int_set:Nn \l_regex_tmpb_int { \l_regex_max_state_int + \c_one }
+ \int_set_eq:NN \l_regex_tmpb_int \l_regex_max_state_int
\int_set:Nn \l_regex_max_state_int
{
- \l_regex_left_state_int - \c_one
- + #1 * ( \l_regex_max_state_int - \l_regex_left_state_int + \c_one )
+ \l_regex_left_state_int
+ + #1 * ( \l_regex_max_state_int - \l_regex_left_state_int )
}
- \int_until_do:nNnn \l_regex_tmpb_int > \l_regex_max_state_int
+ \int_while_do:nNnn \l_regex_tmpb_int < \l_regex_max_state_int
{
\tex_toks:D \l_regex_tmpb_int = \tex_toks:D \l_regex_tmpa_int
\int_incr:N \l_regex_tmpa_int
@@ -2640,12 +3009,12 @@
%
% The first step in matching a regular expression is to build
% the corresponding NFA and store its states in the \tn{toks}
-% registers. Then loop through the query string one character
+% registers. Then loop through the query token list one character
% (one \enquote{step}) at a time, exploring in parallel every
% possible path through the NFA. We keep track of an array of
% the states currently \enquote{active}. More precisely,
% \tn{skip} registers hold the state numbers to be considered
-% when the next character of the string is read.
+% when the next token is read.
%
% At every step, we unpack that array of active states and
% empty it. Then loop over all active states, and perform the
@@ -2670,7 +3039,7 @@
% the \enquote{best} match. Execution paths through the NFA
% are ordered by precedence: for instance, the regular
% expression \texttt{a?} creates two paths, matching either
-% an empty string or a single \texttt{a}; the path matching
+% an empty token list or a single \texttt{a}; the path matching
% an \texttt{a} has higher precedence. When two paths collide,
% the path with the highest precedence is kept, and the other
% one is discarded. The submatch information for a given path
@@ -2704,6 +3073,7 @@
%
% The current approach means that stretch and shrink components
% of \tn{skip} registers,
+%^^A todo: update: submatches' end-points are muskips.
% as well as all \tn{muskip} registers are unused. It could seem that
% \tn{count} registers are also free for use, but we still want to be
% able to safely use integers, which are implemented as \tn{count}
@@ -2737,21 +3107,20 @@
% but not by \cs{regex_state_use_with_submatches:}.
% A state is free if it is not marker as taken, namely
% if the corresponding \tn{dimen} register is not
-% \cs{l_regex_unique_step_int} in \texttt{sp}.
+% \cs{l_regex_unique_id_int} in \texttt{sp}.
% The primitive conditional is ended before unpacking
% the \tn{toks} register.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \regex_state_use_with_submatches:
{ \regex_state_use_aux:n { } }
\cs_new_protected_nopar:Npn \regex_state_use:
- { \regex_state_use_aux:n { \exp_after:wN \regex_state_use_aux_ii:w } }
-\cs_new_nopar:Npn \regex_state_use_aux_ii:w #1 \s_regex_stop { }
+ { \regex_state_use_aux:n { \exp_after:wN \use_none_delimit_by_s_stop:w } }
\cs_new_protected_nopar:Npn \regex_state_use_aux:n #1
{
\if_num:w \tex_dimen:D \l_regex_current_state_int
- < \l_regex_unique_step_int
+ < \l_regex_unique_id_int
\tex_dimen:D \l_regex_current_state_int
- = \l_regex_unique_step_int sp \scan_stop:
+ = \l_regex_unique_id_int sp \scan_stop:
#1 \tex_the:D \tex_toks:D \exp_after:wN \l_regex_current_state_int
\fi:
\scan_stop:
@@ -2799,26 +3168,26 @@
\tex_the:D \tex_toks:D #1
}
}
-\cs_new_protected:Npn \regex_store_submatches_aux:w #1 \s_regex_stop
+\cs_new_protected:Npn \regex_store_submatches_aux:w #1 \s_stop
{
\regex_store_submatches_aux_ii:Nnnw
#1
\regex_state_submatches:nn \c_minus_one \q_prop
- \s_regex_stop
+ \s_stop
}
\cs_new_protected:Npn \regex_store_submatches_aux_ii:Nnnw
- \regex_state_submatches:nn #1 #2 #3 \s_regex_stop
+ \regex_state_submatches:nn #1 #2 #3 \s_stop
{
\exp_after:wN \c_zero
\exp_after:wN \regex_state_submatches:nn \exp_after:wN
{
\int_value:w \int_eval:w
- \l_regex_unique_step_int + \c_one
+ \l_regex_unique_id_int + \c_one
\exp_after:wN
}
\exp_after:wN { \l_regex_current_submatches_prop }
\regex_state_submatches:nn {#1} {#2}
- \s_regex_stop
+ \s_stop
}
% \end{macrocode}
% \end{macro}
@@ -2832,7 +3201,7 @@
% \begin{macrocode}
\cs_new_protected:Npn \regex_state_submatches:nn #1#2
{
- \if_num:w #1 = \l_regex_unique_step_int
+ \if_num:w #1 = \l_regex_unique_id_int
\tl_set:Nn \l_regex_current_submatches_prop { #2 }
\fi:
}
@@ -2842,7 +3211,7 @@
% \subsubsection{Matching: framework}
%
% \begin{macro}[int]{\regex_match:n}
-% Store the query string in \cs{l_regex_query_other_str}.
+%^^A todo: update doc.
% Then reset a few variables which should be set only once,
% before the first match, even in the case of multiple matches.
% Then run the NFA (\cs{regex_match_once:} matches multiple times
@@ -2850,70 +3219,95 @@
% \begin{macrocode}
\cs_new_protected:Npn \regex_match:n #1
{
- \tl_set:Nx \l_regex_query_other_str { \tl_to_other_str:n {#1} }
+ \tl_set_analysis:Nn \l_regex_tmpa_tl {#1}
+
+ \int_zero:N \l_regex_nesting_int
+ \int_set_eq:NN \l_regex_current_step_int \l_regex_max_state_int
+ \regex_query_set:nnn { } { -1 } { -2 }
+ \int_set_eq:NN \l_regex_min_step_int \l_regex_current_step_int
+ \exp_after:wN \regex_query_set_loop:ww
+ \l_regex_tmpa_tl
+ \q_mark { ? \prg_map_break: } ,
+ \prg_break_point:n { }
+ \int_set_eq:NN \l_regex_max_step_int \l_regex_current_step_int
+ \regex_query_set:nnn { } { -1 } { -2 }
+
\regex_match_initial_setup:
\regex_match_once:
}
+\cs_new:Npn \regex_query_set_loop:ww #1 \q_mark #2#3 ,
+ {
+ \use_none:n #2
+ \regex_query_set:nnn {#1} {"#2} {#3}
+ \if_case:w "#2 \exp_stop_f:
+ \or: \int_incr:N \l_regex_nesting_int
+ \or: \int_decr:N \l_regex_nesting_int
+ \fi:
+ \regex_query_set_loop:ww
+ }
+\cs_new_protected:Npn \regex_query_set:nnn #1#2#3
+ {
+ \tex_muskip:D \l_regex_current_step_int
+ = \etex_gluetomu:D
+ #3 sp
+ plus #2 sp
+ minus \l_regex_nesting_int sp
+ \scan_stop:
+ \tex_toks:D \l_regex_current_step_int {#1}
+ \int_incr:N \l_regex_current_step_int
+ }
+\cs_new_protected_nopar:Npn \regex_query_get:
+ {
+ \tl_set:Nx \l_regex_current_token_tl
+ { \tex_the:D \tex_toks:D \l_regex_current_step_int }
+ \l_regex_current_char_int
+ = \etex_mutoglue:D \tex_muskip:D \l_regex_current_step_int
+ \l_regex_current_catcode_int = \etex_gluestretch:D
+ \etex_mutoglue:D \tex_muskip:D \l_regex_current_step_int
+ }
% \end{macrocode}
% \end{macro}
%
% \begin{macro}[int]{\regex_match_once:}
-% \begin{macro}[aux]{\regex_match_once_aux:}
-% \begin{macro}[aux]{\regex_match_once_aux_ii:N}
-% After setting up more variables in \cs{regex_match_setup:},
-% skip the \cs{l_regex_start_step_int} first characters of the
-% query string, and loop over it.
+% Set up more variables in \cs{regex_match_setup:}.
% If there was a match, use the token list \cs{l_regex_every_match_tl},
-% which may call \cs{regex_match_once:} to achieve multiple matches.
+% which may call \cs{regex_match_once:} again to achieve multiple matches.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \regex_match_once:
{
\regex_match_setup:
- \exp_after:wN \regex_match_once_aux: \l_regex_query_other_str
- \q_recursion_tail \q_recursion_stop
- \bool_if:NT \l_regex_success_bool { \l_regex_every_match_tl }
- }
-\cs_new_protected_nopar:Npn \regex_match_once_aux:
- {
- \int_compare:nNnTF \l_regex_start_step_int = \c_zero
- {
- \int_set_eq:NN \l_regex_current_char_int \c_minus_one
- \regex_match_loop:N
- }
+ \regex_query_get:
+ \regex_match_loop:
+ \prg_break_point:n { }
+ \bool_if:NT \l_regex_success_match_bool
{
- \str_skip_do:nn
- { \l_regex_start_step_int - \c_one }
- { \regex_match_once_aux_ii:N }
+ \bool_gset_true:N \g_regex_success_bool
+ \l_regex_every_match_tl
}
}
-\cs_new_protected_nopar:Npn \regex_match_once_aux_ii:N #1
- {
- \int_set:Nn \l_regex_current_char_int { `#1 }
- \regex_match_loop:N
- }
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
%
% \begin{macro}[aux]{\regex_match_initial_setup:}
% This function holds the setup that should be done
-% only once for one given pattern matching on a given
-% string. It is called only once for the whole string.
+% only once for one given pattern matching.
+% It is called only once for the whole token list.
% On the other hand, \cs{regex_match_setup:}
-% is called for every match in the string in case of
-% repeated matches, and \cs{regex_match_loop_setup:N}
-% is called at every step.
+% is called for every match in the token list in case of
+% repeated matches.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \regex_match_initial_setup:
{
- \prg_stepwise_inline:nnnn {1} {1} { \l_regex_max_state_int }
+ \prg_stepwise_inline:nnnn {0} {1} { \l_regex_max_state_int - \c_one }
{ \tex_dimen:D ##1 \c_minus_one sp \scan_stop: }
- \int_set_eq:NN \l_regex_unique_step_int \c_minus_one
- \int_set_eq:NN \l_regex_start_step_int \c_minus_one
- \int_set_eq:NN \l_regex_current_step_int \c_zero
- \int_set_eq:NN \l_regex_success_step_int \c_zero
+ \int_set_eq:NN \l_regex_unique_id_int \c_minus_one
+ \int_set:Nn \l_regex_start_step_int
+ { \l_regex_min_step_int - \c_one }
+ \int_set_eq:NN \l_regex_current_step_int \l_regex_min_step_int
+ \int_set_eq:NN \l_regex_success_step_int \l_regex_min_step_int
+ \int_set_eq:NN \l_regex_submatch_int \l_regex_max_state_int
\bool_set_false:N \l_regex_success_empty_bool
+ \bool_gset_false:N \g_regex_success_bool
}
% \end{macrocode}
% \end{macro}
@@ -2929,47 +3323,61 @@
{ \cs_set_eq:NN \regex_last_match_empty:F \regex_last_match_empty_yes:F }
{ \cs_set_eq:NN \regex_last_match_empty:F \regex_last_match_empty_no:F }
\int_set_eq:NN \l_regex_start_step_int \l_regex_success_step_int
- \int_set_eq:NN \l_regex_current_step_int \l_regex_start_step_int
- \int_decr:N \l_regex_current_step_int
- \bool_set_false:N \l_regex_success_bool
+ \int_set:Nn \l_regex_current_step_int
+ { \l_regex_start_step_int - \c_one }
+ \bool_set_false:N \l_regex_success_match_bool
\int_zero:N \l_regex_max_index_int
- \regex_store_state:n {1}
+ \regex_store_state:n {0} %^^A _state_int!
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_match_loop:N}
+% \begin{macro}[aux]{\regex_match_loop:}
% \begin{macro}[aux]{\regex_match_one_index:n}
% \begin{macro}[aux]{\regex_match_one_index_aux:n}
% Setup what needs to be reset at every character,
% then set \cs{l_regex_current_char_int} to the
-% character code of the character that is read
-% (and $-1$ for the end of the string), and loop
+% character code of the token that is read
+% (and $-1$ for the end of the token list), and loop
% over the elements of the \tn{skip} array. Then repeat.
-% There are a couple of tests to stop reading the string
+% There are a couple of tests to stop reading the token list
% when no active state is left, or when the end is reached.
+% At every step in reading the token list, we store the character
+% code of the current character in \cs{l_regex_current_char_int},
+% unless the end was reached: then we store $-1$.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_match_loop:N #1
+\cs_new_protected_nopar:Npn \regex_match_loop:
{
- \regex_match_loop_setup:N #1
+ \int_incr:N \l_regex_current_step_int
+ \int_incr:N \l_regex_unique_id_int
+ \bool_set_false:N \l_regex_fresh_thread_bool
+ \int_set_eq:NN \l_regex_last_char_int \l_regex_current_char_int
+ \regex_query_get:
+ \regex_match_loop_case_hook:
\cs_set_nopar:Npx \regex_tmp:w
{
\int_zero:N \l_regex_max_index_int
- \prg_stepwise_function:nnnN
- {1} {1} { \l_regex_max_index_int }
- \regex_match_one_index:n
+ \regex_match_one_index:w 1 ; \prg_break_point:n { }
+ \exp_not:N \prg_break_point:n { }
}
- \regex_tmp:w \prg_break_point:n { }
+ \regex_tmp:w
+ \if_num:w \l_regex_current_char_int < \c_minus_one
+ \exp_after:wN \prg_map_break:
+ \fi:
\if_num:w \l_regex_max_index_int = \c_zero
- \exp_after:wN \use_none_delimit_by_q_recursion_stop:w
+ \exp_after:wN \prg_map_break:
\fi:
- \quark_if_recursion_tail_stop:N #1
- \regex_match_loop:N
+ \regex_match_loop:
}
-\cs_new_nopar:Npn \regex_match_one_index:n #1
+\cs_new_nopar:Npn \regex_match_one_index:w #1;
{
+ \if_num:w #1 > \l_regex_max_index_int
+ \exp_after:wN \prg_map_break:
+ \fi:
\regex_match_one_index_aux:n
{ \int_value:w \tex_skip:D #1 }
+ \exp_after:wN \regex_match_one_index:w
+ \int_use:N \int_eval:w #1 + \c_one ;
}
\cs_new_protected_nopar:Npn \regex_match_one_index_aux:n #1
{
@@ -2982,28 +3390,6 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_match_loop_setup:N}
-% This is called for every character in the string.
-% At every step in reading the query string, we store the character
-% code of the current character in \cs{l_regex_current_char_int},
-% unless the end was reached: then we store $-1$.
-% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_match_loop_setup:N #1
- {
- \int_incr:N \l_regex_current_step_int
- \int_incr:N \l_regex_unique_step_int
- \bool_set_false:N \l_regex_fresh_thread_bool
- \int_set_eq:NN \l_regex_last_char_int \l_regex_current_char_int
- \if_meaning:w #1 \q_recursion_tail
- \int_set_eq:NN \l_regex_current_char_int \c_minus_one
- \else:
- \int_set:Nn \l_regex_current_char_int {`#1}
- \fi:
- \regex_match_loop_case_hook:
- }
-% \end{macrocode}
-% \end{macro}
-%
% \begin{macro}{\regex_match_loop_case_hook:}
% \begin{macro}{\regex_match_loop_caseless_hook:}
% In the case where the regular expression contains caseless matching,
@@ -3041,7 +3427,7 @@
% The search is made unanchored at the start by putting
% a free transition to the real start of the NFA, and a
% costly transition to the same state, waiting for the
-% next character in the query string. This combination
+% next token in the query. This combination
% could be reused (with some changes). We sometimes need
% to know that the match for a given thread starts at
% this character. For that, we use the boolean
@@ -3063,7 +3449,7 @@
% \begin{macrocode}
\cs_new_protected_nopar:Npn \regex_action_cost:n #1
{
- \exp_args:Nf \regex_store_state:n %^^A optimize!
+ \exp_args:Nf \regex_store_state:n
{ \int_eval:n { \l_regex_current_state_int + #1 } }
}
% \end{macrocode}
@@ -3082,11 +3468,10 @@
{
\regex_last_match_empty:F
{
- \bool_set_true:N \l_regex_success_bool
+ \bool_set_true:N \l_regex_success_match_bool
\bool_set_eq:NN \l_regex_success_empty_bool
\l_regex_fresh_thread_bool
- \int_set_eq:NN \l_regex_success_step_int
- \l_regex_current_step_int
+ \int_set_eq:NN \l_regex_success_step_int \l_regex_current_step_int
\prop_set_eq:NN \l_regex_success_submatches_prop
\l_regex_current_submatches_prop
\prg_map_break:
@@ -3131,46 +3516,252 @@
% \end{macrocode}
% \end{macro}
%
-% \subsection{Submatches, once the correct match is found}
+% \subsection{Replacement}
%
-% \begin{macro}[int]{\regex_extract:}
-% \begin{macro}[aux]{\regex_extract_aux:nTF}
+% \begin{macro}[rEXP]{\regex_submatch_nesting_aux:n}
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_extract:
+\cs_new_protected:Npn \regex_submatch_nesting_aux:n #1
+ {
+ + \etex_glueshrink:D \etex_mutoglue:D \etex_muexpr:D
+ \tex_muskip:D \etex_gluestretch:D \tex_skip:D #1
+ - \tex_muskip:D \tex_skip:D #1
+ \scan_stop:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[int]{\regex_replacement:n}
+% Our goal here is to analyse the replacement text. First take care
+% of detecting escaped and non-escaped characters using
+% \cs{str_escape_use:NNNn} with three protected arguments.
+% This inserts in the input stream a token list of the form
+% \meta{fn 1} \meta{char 1} \ldots{} \meta{fn $N$} \meta{char $N$},
+% where \meta{fn $i$} is one of the three functions, and \meta{char $i$}
+% a character in the string |#1|.
+% \begin{macrocode}
+\cs_new:Npn \regex_nesting:n #1 { } %^^A move. Rename?
+\tl_new:N \l_regex_nesting_tl
+\cs_new_protected:Npn \regex_replacement:n #1
{
- \seq_gclear:N \g_regex_submatches_seq
- \prg_stepwise_inline:nnnn
- {0} {1} { \l_regex_capturing_group_int }
+ \int_zero:N \l_regex_replacement_int
+ \int_zero:N \l_regex_nesting_int
+ \tl_clear:N \l_regex_nesting_tl
+ \str_escape_use:NNNn
+ \regex_replacement_unescaped:N
+ \regex_replacement_escaped:N
+ \regex_replacement_raw:N
+ {#1}
+ \prg_do_nothing: \prg_do_nothing:
+ \cs_set_nopar:Npx \regex_nesting:n ##1
{
- \regex_extract_aux:nTF { ##1 }
- {
- \seq_gput_right:Nx \g_regex_submatches_seq
- { \regex_query_substr:NN \l_regex_tmpa_tl \l_regex_tmpb_tl }
- }
- { \seq_gput_right:Nn \g_regex_submatches_seq { } }
+ + \int_use:N \l_regex_nesting_int
+ \l_regex_nesting_tl
+ - \regex_submatch_nesting_aux:n {##1}
+ }
+ \use:x
+ {
+ \exp_not:n { \cs_set_nopar:Npn \regex_replacement_tl:n ##1 }
+ { \str_aux_toks_range:nn \c_zero \l_regex_replacement_int }
+ }
+ % ^^A rename! Careful with \cP\#.
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_replacement_raw:N}
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_replacement_raw:N #1
+ { \regex_replacement_put:n {#1} }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_replacement_put:n}
+% Raw characters are stored in a toks register.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_replacement_put:n #1
+ {
+ \tex_toks:D \l_regex_replacement_int {#1}
+ \int_incr:N \l_regex_replacement_int
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_replacement_escaped:N}
+% \begin{macro}[aux]{\regex_replacement_submatch:w}
+% \begin{macro}[aux]{\regex_replacement_submatch_aux:nN}
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_replacement_unescaped:N #1
+ {
+ \if_charcode:w \c_rbrace_str #1
+ \if_num:w \l_regex_replacement_csnames_int > \c_zero
+ \regex_replacement_put:n \cs_end:
+ \else:
+ \regex_replacement_put:n #1
+ \fi:
+ \else:
+ \regex_replacement_put:n #1
+ \fi:
+ }
+\cs_new_protected:Npn \regex_replacement_escaped:N #1
+ {
+ \if_charcode:w c #1
+ \exp_after:wN \regex_replacement_c:w
+ \else:
+ \if_charcode:w g #1
+ \exp_after:wN \exp_after:wN \exp_after:wN \regex_replacement_g:w
+ \else:
+ \if_num:w \c_one < 1#1 \exp_stop_f:
+ \regex_replacement_put_submatch:n {#1}
+ \else:
+ \regex_replacement_put:n #1
+ \fi:
+ \fi:
+ \fi:
+ }
+\cs_new_protected:Npn \regex_replacement_put_submatch:n #1
+ {
+ \regex_replacement_put:n
+ { \regex_query_submatch:nn {#1} {##1} }
+ \if_num:w \l_regex_replacement_csnames_int = \c_zero
+ \tl_put_right:Nn \l_regex_nesting_tl
+ {
+ \exp_not:N \if_num:w #1 < \l_regex_capturing_group_int
+ \regex_submatch_nesting_aux:n { \int_eval:w #1+##1 \int_eval_end: }
+ \exp_not:N \fi:
+ }
+ \fi:
+ }
+\cs_new_protected:Npn \regex_replacement_error:NNN #1#2#3
+ {
+ \msg_kernel_error:nn { regex } { replacement-#1 }
+ #2 #3
+ }
+\cs_new_protected:Npn \regex_replacement_g:w #1#2
+ {
+ \str_if_eq:xxTF
+ { \exp_not:n { #1#2 } }
+ { \regex_replacement_unescaped:N \c_lbrace_str }
+ {
+ \int_zero:N \l_regex_tmpa_int
+ \regex_replacement_g_digits:NN
}
+ { \regex_replacement_error:NNN g #1 #2 }
}
-\cs_new_protected_nopar:Npn \regex_extract_aux:nTF #1#2#3
+\cs_new_protected:Npn \regex_replacement_g_digits:NN #1#2
{
- \prop_get:NnNTF \l_regex_success_submatches_prop
- { #1 < } \l_regex_tmpa_tl
+ \token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N
{
- \prop_get:NnNTF \l_regex_success_submatches_prop
- { #1 > } \l_regex_tmpb_tl
- {#2}
- {#3}
+ \if_num:w \c_one < 1#2 \exp_stop_f:
+ \int_set:Nn \l_regex_tmpa_int { \c_ten * \l_regex_tmpa_int + #2 }
+ \exp_after:wN \use_i:nnn
+ \exp_after:wN \regex_replacement_g_digits:NN
+ \else:
+ \if_charcode:w \c_rbrace_str #2
+ \exp_args:No \regex_replacement_put_submatch:n
+ { \int_use:N \l_regex_tmpa_int }
+ \exp_after:wN \exp_after:wN \exp_after:wN \use_none:nn
+ \else:
+ \exp_after:wN \exp_after:wN
+ \exp_after:wN \regex_replacement_error:NNN
+ \exp_after:wN \exp_after:wN \exp_after:wN g
+ \fi:
+ \fi:
}
- {#3}
+ { \regex_replacement_error:NNN g }
+ #1 #2
+ }
+\cs_new_protected_nopar:Npn \regex_replacement_c:w #1#2
+ {
+ \token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N
+ {
+ \cs_if_exist_use:cF { regex_replacement_c_#2:w }
+ { \regex_replacement_error:NNN c #1#2 }
+ }
+ { \regex_replacement_error:NNN c #1#2 }
+ }
+\cs_new_protected_nopar:cpn { regex_replacement_c_ \c_lbrace_str :w }
+ {
+ \int_incr:N \l_regex_replacement_csnames_int
+ \regex_replacement_put:n
+ { \exp_not:n { \exp_after:wN \regex_replacement_exp_not:N \cs:w } }
+ }
+\cs_new_nopar:Npn \regex_replacement_exp_not:N #1 { \exp_not:n {#1} }
+\group_begin:
+
+ \char_set_catcode_math_superscript:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_U:w
+ { \regex_replacement_char:nNN { ^^@ } }
+
+ \char_set_catcode_alignment:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_T:w
+ { \regex_replacement_char:nNN { ^^@ } }
+
+ \cs_new_protected_nopar:Npn \regex_replacement_c_S:w #1#2
+ {
+ \int_compare:nNnTF { `#2 } = \c_zero
+ { \regex_replacement_error:NNN c #1#2 }
+ {
+ \char_set_lccode:nn {32} { `#2 }
+ \tl_to_lowercase:n { \regex_replacement_put:n {~} }
+ }
+ }
+
+ \char_set_catcode_parameter:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_P:w
+ { \regex_replacement_char:nNN { ^^@^^@ } }
+
+ \char_set_catcode_other:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_O:w
+ { \regex_replacement_char:nNN { ^^@ } }
+
+ \char_set_catcode_math_toggle:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_M:w
+ { \regex_replacement_char:nNN { ^^@ } }
+
+ \char_set_catcode_letter:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_L:w
+ { \regex_replacement_char:nNN { ^^@ } }
+
+ \char_set_catcode_group_end:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_E:w
+ {
+ \int_decr:N \l_regex_nesting_int
+ \regex_replacement_char:nNN { \if_false: { \fi: ^^@ }
+ }
+
+ \char_set_catcode_math_subscript:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_D:w
+ { \regex_replacement_char:nNN { ^^@ } }
+
+ \char_set_catcode_group_begin:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_B:w
+ {
+ \int_incr:N \l_regex_nesting_int
+ \regex_replacement_char:nNN { \exp_after:wN ^^@ \if_false: } \fi: }
+ }
+
+ \char_set_catcode_active:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_A:w
+ { \regex_replacement_char:nNN { \exp_not:N ^^@ } }
+
+\group_end:
+
+\cs_new_protected_nopar:Npn \regex_replacement_char:nNN #1#2#3
+ {
+ \char_set_lccode:nn \c_zero { `#3 }
+ \tl_to_lowercase:n
+ { \regex_replacement_put:n { \exp_not:n {#1} } }
}
% \end{macrocode}
% \end{macro}
% \end{macro}
+% \end{macro}
%
% \subsection{User commands}
%
% \subsubsection{Precompiled pattern}
%
-% A given pattern is often reused to match many different strings.
+% A given pattern is often reused to match many different query token lists.
% We thus give a means of storing the NFA corresponding to a given
% pattern in a token list variable of the form
% \begin{quote}
@@ -3179,7 +3770,7 @@
% \cs{tex_toks:D} 0 \{ \meta{instruction0} \} \\
% \ldots{} \\
% \cs{tex_toks:D} $n$ \{ \meta{instruction$\sb{n}$} \} \\
-% \cs{s_regex_stop}
+% \cs{s_stop}
% \end{quote}
% where $n$ is the number of states in the NFA,
% and the various \meta{instruction$\sb{i}$} control
@@ -3191,672 +3782,678 @@
% contents of \meta{nfa var} may overwrite data which is used
% elsewhere, unless everything is done carefully in a group.
%
-% \begin{macro}{\regex_set:Nn}
% \begin{macro}{\regex_gset:Nn}
+% \begin{macro}{\regex_const:Nn}
+% \begin{macro}{\regex_set:Nn}
% \begin{macro}[aux]{\regex_set_aux:NNn}
-% \begin{macro}[aux]{\regex_set_aux:n}
-% \begin{macro}[aux]{\regex_nfa:Nw}
-% Within a group, build the NFA corresponding to the given regular
-% expression, with submatch tracking. Then save the contents of all
-% relevant \tn{toks} registers into \cs{g_regex_tmpa_tl}, then
-% transferred to the user's tl variable.
-% The auxiliary \cs{regex_nfa:Nw} is not protected: this ensures that
-% the NFA will properly be replaced by an error message in expansion
-% contexts.
+% The three user functions only differ with which function is used
+% to assign the pre-compiled regular expression to the user's variable.
+% Internally, they all first build the NFA corresponding to the regex,
+% then store the contents of all the necessary \tn{toks} registers
+% in the user's variable.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \regex_set:Nn
- { \regex_set_aux:NNn \tl_set_eq:NN }
+ { \regex_set_aux:NNn \tl_set:Nn }
\cs_new_protected_nopar:Npn \regex_gset:Nn
- { \regex_set_aux:NNn \tl_gset_eq:NN }
+ { \regex_set_aux:NNn \tl_gset:Nn }
+\cs_new_protected_nopar:Npn \regex_const:Nn
+ { \regex_set_aux:NNn \tl_const:Nn }
\cs_new_protected:Npn \regex_set_aux:NNn #1#2#3
{
\group_begin:
\regex_build:n {#3}
- \tl_gset:Nx \g_regex_tmpa_tl
- {
- \exp_not:N \regex_nfa:Nw \exp_not:N #2
- \l_regex_max_state_int
- = \int_use:N \l_regex_max_state_int
- \l_regex_capturing_group_int
- = \int_use:N \l_regex_capturing_group_int
- \token_if_eq_meaning:NNT
- \regex_match_loop_case_hook:
- \regex_match_loop_caseless_hook:
- {
- \cs_set_eq:NN \regex_match_loop_case_hook:
- \regex_match_loop_caseless_hook:
- }
- \prg_stepwise_function:nnnN
- {1} {1} {\l_regex_max_state_int}
- \regex_set_aux:n
- \s_regex_stop
- }
+ \cs_set_nopar:Npx \regex_tmp:w
+ { #1 \exp_not:N #2 { \regex_set_aux:N #2 } }
+ \exp_after:wN
\group_end:
- #1 #2 \g_regex_tmpa_tl
+ \regex_tmp:w
}
-\cs_new_nopar:Npn \regex_set_aux:n #1
- { \tex_toks:D #1 { \tex_the:D \tex_toks:D #1 } }
-\cs_new:Npn \regex_nfa:Nw #1 #2 \s_regex_stop
- { \msg_expandable_kernel_error:nnn { regex } { nfa-misused } {#1} }
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}{\regex_const:Nn}
-% The same idea as for setting, but using the new function.
-% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_const:Nn
- { \regex_set_aux:NNn \cs_new_eq:NN }
-% \end{macrocode}
-% \end{macro}
-%
-% \begin{macro}[int,TF]{\regex_check_nfa:N}
-% If a token list variable starts with \cs{regex_nfa:Nw},
-% then it most likely holds the data for a precompiled pattern.
+% \begin{macro}[int, rEXP]{\regex_set_aux:N}
+% \begin{macro}[aux, rEXP]{\regex_set_aux:n}
+% \begin{macro}[aux, EXP]{\regex_nfa:Nw}
+% Within a group, build the NFA corresponding to the given regular
+% expression, with submatch tracking. Then save the contents of all
+% relevant \tn{toks} registers into the variable outside the group.
+% The auxiliary \cs{regex_nfa:Nw} is not protected: this ensures that
+% the NFA will properly be replaced by an error message in expansion
+% contexts.
% \begin{macrocode}
-\prg_new_protected_conditional:Npnn \regex_check_nfa:N #1 { T , TF }
- { \exp_after:wN \regex_check_nfa_aux:Nw #1 \q_stop }
-\cs_new:Npn \regex_check_nfa_aux:Nw #1 #2 \q_stop
+\cs_new:Npn \regex_set_aux:N #1
+ {
+ \exp_not:n { \regex_nfa:Nw #1 }
+ \l_regex_max_state_int
+ = \int_use:N \l_regex_max_state_int
+ \l_regex_capturing_group_int
+ = \int_use:N \l_regex_capturing_group_int
+ \token_if_eq_meaning:NNT
+ \regex_match_loop_case_hook:
+ \regex_match_loop_caseless_hook:
+ {
+ \cs_set_eq:NN \regex_match_loop_case_hook:
+ \regex_match_loop_caseless_hook:
+ }
+ \prg_stepwise_function:nnnN
+ {0} {1} {\l_regex_max_state_int - \c_one }
+ \regex_set_aux:n
+ \s_stop
+ }
+\cs_new_nopar:Npn \regex_set_aux:n #1
+ { \tex_toks:D #1 { \tex_the:D \tex_toks:D #1 } }
+\cs_new:Npn \regex_nfa:Nw #1
{
- \if_meaning:w \regex_nfa:Nw #1
- \prg_return_true:
- \else:
- \msg_error:nnx { regex } { not-nfa } { \token_to_str:N #1 }
- \prg_return_false:
- \fi:
+ \msg_expandable_kernel_error:nnn { regex } { nfa-misused } {#1}
+ \use_none_delimit_by_s_stop:w
}
% \end{macrocode}
% \end{macro}
+% \end{macro}
+% \end{macro}
%
% \begin{macro}[int]{\regex_use:N}
% No error-checking.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_use:N #1
- { \exp_after:wN \use_none:nn #1 }
+\cs_new_protected_nopar:Npn \regex_use:N
+ { \exp_last_unbraced:No \use_none:nn }
% \end{macrocode}
% \end{macro}
%
-% \subsubsection{Generic auxiliary functions}
-%
-% \begin{macro}[aux]{\regex_user_aux:n}
-% This is an auxiliary used by most user functions.
-% The first and second arguments control whether we should track
-% submatches, and whether we track one or multiple submatches.
-% Everything is done within a group, so that |#1| can perform
-% \enquote{unsafe} assignments. Most user functions return a
-% result using \cs{group_insert_after:N}.
+% \begin{macro}[int, rEXP]{\regex_to_str:N}
+% \begin{macro}[int, rEXP]{\regex_to_str_aux:w}
+% Of course, we could simply set \cs{regex_to_str:N} equal
+% to \cs{tl_to_str:N}. After all, regex variables are in particular
+% token list variables. We do some more processing to start each
+% line with \cs{tex_toks:D} instead of a single very long line.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_user_aux:n #1
+\cs_new_nopar:Npn \regex_to_str:N #1
{
- \group_begin:
- \tl_clear:N \l_regex_every_match_tl
- #1
- \group_end:
+ \exp_after:wN \regex_to_str_aux:Nw #1
+ \tex_toks:D \q_stop \prg_map_break: \tex_toks:D
+ \prg_break_point:n { }
+ }
+\cs_new_nopar:Npn \regex_to_str_aux:Nw #1#2\tex_toks:D
+ {
+ \use_none_delimit_by_q_stop:w #2 \q_stop
+ \tl_to_str:n {#1#2} \iow_newline:
+ \regex_to_str_aux:Nw \tex_toks:D
}
% \end{macrocode}
% \end{macro}
+% \end{macro}
+%
+% \subsubsection{Generic auxiliary functions}
+%
+% Most of \pkg{l3regex}'s work is done within a group.
%
-% \begin{macro}[aux]{\regex_return_after_group:}
-% Most of \pkg{l3regex}'s work is done within a group.
+% \begin{macro}[aux]{\regex_aux_return:}
% This function triggers either \cs{prg_return_false:}
% or \cs{prg_return_true:} as appropriate to whether a
% match was found or not.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_return_after_group:
+\cs_new_protected_nopar:Npn \regex_aux_return:
{
- \if_meaning:w \c_true_bool \l_regex_success_bool
- \group_insert_after:N \prg_return_true:
+ \if_meaning:w \c_true_bool \g_regex_success_bool
+ \prg_return_true:
\else:
- \group_insert_after:N \prg_return_false:
- \fi:
- }
-% \end{macrocode}
-% \end{macro}
-%
-% \begin{macro}[aux]{\regex_extract_after_group:N}
-% Extract submatches, and store them in the user-given variable
-% after the group has ended.
-% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_extract_after_group:N #1
- {
- \if_meaning:w \c_true_bool \l_regex_success_bool
- \regex_extract:
- \group_insert_after:N \seq_set_eq:NN
- \group_insert_after:N #1
- \group_insert_after:N \g_regex_submatches_seq
+ \prg_return_false:
\fi:
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_count_after_group:N}
-% Same procedure as \cs{regex_extract_after_group:N}, but simpler
-% since the match counting has already taken place.
+% \begin{macro}{\regex_aux_build_match:nn}
+% This auxiliary is used by user functions whose \meta{regex} argument
+% is given as an explicit regular expression within braces. In that
+% case, we need to build the automaton corresponding to that regular
+% expression, then perform the matching on the given token list |#2|.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_count_after_group:N #1
+\cs_new_protected:Npn \regex_aux_build_match:nn #1#2
{
- \group_insert_after:N \int_set_eq:NN
- \group_insert_after:N #1
- \group_insert_after:N \g_regex_match_count_int
+ \regex_build:n {#1}
+ \regex_match:n {#2}
}
% \end{macrocode}
% \end{macro}
%
-% \subsubsection{Matching}
-%
-% \begin{macro}[TF]{\regex_match:nn}
-% \begin{macro}[TF]{\regex_match:Nn}
-% We don't track submatches. Then either build the NFA corresponding
-% to the regular expression, or use a precompiled pattern. Then match,
-% using the internal \cs{regex_match:n}. Finally return the result
-% after closing the group.
+% \begin{macro}{\regex_aux_use_match:Nn}
+% This auxiliary is used by user functions whose \meta{regex} argument
+% is given as a pre-compiled regex variable. We make sure that
+% the token list variable indeed is an automaton (by testing the
+% first token). If not, the match is deemed unsuccessful, after
+% raising an error. If we have an automaton, \enquote{use} it,
+% then perform the matching on the given token list |#2|.
% \begin{macrocode}
-\prg_new_protected_conditional:Npnn \regex_match:nn #1#2 { T , F , TF }
+\cs_new_protected:Npn \regex_aux_use_match:Nn #1#2
{
- \regex_user_aux:n
+ \exp_args:No \tl_if_head_eq_meaning:nNTF {#1} \regex_nfa:Nw
{
- \regex_disable_submatches:
- \regex_build:n {#1}
+ \regex_use:N #1
\regex_match:n {#2}
- \regex_return_after_group:
}
- }
-\prg_new_protected_conditional:Npnn \regex_match:Nn #1#2 { T , F , TF }
- {
- \regex_check_nfa:NTF #1
{
- \regex_user_aux:n
- {
- \regex_disable_submatches:
- \regex_use:N #1
- \regex_match:n {#2}
- \regex_return_after_group:
- }
+ \msg_kernel_error:nnx { regex } { not-nfa } { \token_to_str:N #1 }
+ \bool_gset_false:N \g_regex_success_bool
}
- { \prg_return_false: }
}
% \end{macrocode}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}{\regex_count:nnN}
-% \begin{macro}{\regex_count:NnN}
-% Instead of aborting once the first \enquote{best match} is found,
-% we repeat the search. The code is such that the search will not
-% start on the same character, hence avoiding infinite loops.
+% \begin{macro}
+% {
+% \regex_extract_once:nnN, \regex_extract_once:NnN,
+% \regex_extract_all:nnN, \regex_extract_all:NnN,
+% \regex_replace_once:nnN, \regex_replace_once:NnN,
+% \regex_replace_all:nnN, \regex_replace_all:NnN,
+% \regex_split:nnN, \regex_split:NnN
+% }
+% \begin{macro}[TF]
+% {
+% \regex_extract_once:nnN, \regex_extract_once:NnN,
+% \regex_extract_all:nnN, \regex_extract_all:NnN,
+% \regex_replace_once:nnN, \regex_replace_once:NnN,
+% \regex_replace_all:nnN, \regex_replace_all:NnN,
+% \regex_split:nnN, \regex_split:NnN
+% }
+% We define here $40$ user functions, following a common pattern
+% in terms of an auxiliary such as \cs{regex_extract_once_aux:NnnN}
+% (those auxiliaries are defined in the coming sections).
+% The arguments handed to the auxiliary are
+% \cs{regex_aux_build_match:nn} or \cs{regex_aux_use_match:Nn},
+% followed by the three arguments of the user function.
+% The conditionals call \cs{regex_aux_return:} to return
+% either \texttt{true} or \texttt{false} once matching
+% has been performed.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_count:nnN #1#2#3
+\cs_set_protected:Npn \regex_tmp:w #1#2#3
{
- \regex_user_aux:n
+ \cs_new_protected_nopar:Npn #1
+ { #3 \regex_aux_build_match:nn }
+ \cs_new_protected_nopar:Npn #2
+ { #3 \regex_aux_use_match:Nn }
+ \prg_new_protected_conditional:Npnn #1 ##1##2##3 { T , F , TF }
{
- \regex_disable_submatches:
- \int_gzero:N \g_regex_match_count_int
- \tl_set:Nn \l_regex_every_match_tl
- {
- \int_gincr:N \g_regex_match_count_int
- \regex_match_once:
- }
- \regex_build:n {#1}
- \regex_match:n {#2}
- \regex_count_after_group:N #3
+ #3 \regex_aux_build_match:nn {##1} {##2} ##3
+ \regex_aux_return:
}
- }
-\cs_new_protected:Npn \regex_count:NnN #1#2#3
- {
- \regex_check_nfa:NT #1
+ \prg_new_protected_conditional:Npnn #2 ##1##2##3 { T , F , TF }
{
- \regex_user_aux:n
- {
- \regex_disable_submatches:
- \int_gzero:N \g_regex_match_count_int
- \tl_set:Nn \l_regex_every_match_tl
- {
- \int_gincr:N \g_regex_match_count_int
- \regex_match_once:
- }
- \regex_use:N #1
- \regex_match:n {#2}
- \regex_count_after_group:N #3
- }
+ #3 \regex_aux_use_match:Nn {##1} {##2} ##3
+ \regex_aux_return:
}
}
+\tl_map_inline:nn
+ {
+ { extract_once } { extract_all }
+ { replace_once } { replace_all }
+ { split }
+ }
+ {
+ \exp_args:Nccc \regex_tmp:w
+ { regex_#1:nnN } { regex_#1:NnN } { regex_#1_aux:NnnN }
+ }
% \end{macrocode}
% \end{macro}
% \end{macro}
%
-% \subsubsection{Submatch extraction}
+% \subsubsection{Submatches, once the correct match is found}
+%
+% \begin{macro}[int]{\regex_extract:}
+% \begin{macro}[aux]{\regex_extract_aux_b:wn}
+% \begin{macro}[aux]{\regex_extract_aux_e:wn}
%
-% \begin{macro}{\regex_extract_once:nnN}
-% \begin{macro}{\regex_extract_once:NnN}
-% \begin{macro}[TF]{\regex_extract_once:nnN}
-% \begin{macro}[TF]{\regex_extract_once:NnN}
% \begin{macrocode}
-\cs_new_protected:Npn \regex_extract_once:nnN #1#2#3
+\cs_new_protected_nopar:Npn \regex_extract:
{
- \regex_user_aux:n
- {
- \regex_build:n {#1}
- \regex_match:n {#2}
- \regex_extract_after_group:N #3
- }
+ \int_set_eq:NN \l_regex_submatch_start_int \l_regex_submatch_int
+ \if_meaning:w \c_true_bool \g_regex_success_bool
+ \prg_replicate:nn \l_regex_capturing_group_int
+ {
+ \tex_skip:D \l_regex_submatch_int \c_zero sp \scan_stop:
+ \int_incr:N \l_regex_submatch_int
+ }
+ \prop_map_inline:Nn \l_regex_success_submatches_prop
+ {
+ \if_num:w ##1 \c_max_int
+ \exp_after:wN \regex_extract_aux_b:wn \int_use:N
+ \else:
+ \exp_after:wN \regex_extract_aux_e:wn \int_use:N
+ \fi:
+ \int_eval:w \l_regex_submatch_start_int + ##1 {##2}
+ }
+ \fi:
}
-\prg_new_protected_conditional:Npnn \regex_extract_once:nnN #1#2#3
- { T , F , TF }
+\cs_new_protected_nopar:Npn \regex_extract_aux_b:wn #1 < #2
{
- \regex_user_aux:n
- {
- \regex_build:n {#1}
- \regex_match:n {#2}
- \regex_extract_after_group:N #3
- \regex_return_after_group:
- }
+ \tex_skip:D #1 = #2 sp
+ plus \etex_gluestretch:D \tex_skip:D #1 \scan_stop:
}
-\cs_new_protected:Npn \regex_extract_once:NnN #1#2#3
+\cs_new_protected_nopar:Npn \regex_extract_aux_e:wn #1 > #2
{
- \regex_check_nfa:NT #1
- {
- \regex_user_aux:n
- {
- \regex_use:N #1
- \regex_match:n {#2}
- \regex_extract_after_group:N #3
- }
- }
- }
-\prg_new_protected_conditional:Npnn \regex_extract_once:NnN #1#2#3
- { T , F , TF }
- {
- \regex_check_nfa:NTF #1
- {
- \regex_user_aux:n
- {
- \regex_use:N #1
- \regex_match:n {#2}
- \regex_extract_after_group:N #3
- \regex_return_after_group:
- }
- }
- { \prg_return_false: }
+ \tex_skip:D #1
+ = 1 \tex_skip:D #1 plus #2 sp \scan_stop:
}
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}{\regex_extract_all:nnN}
-% \begin{macro}{\regex_extract_all:NnN}
-% \begin{macro}[aux]{\regex_extract_all_aux:}
-% \begin{macro}[aux]{\regex_extract_all_after_group:N}
-% Similarly to \cs{regex_count:nnN} functions,
-% recurse through matches of the pattern. Then we do
-% something slightly different, extracting submatches.
-% Submatches are not extracted if the pattern matched
-% an empty string at the start of the match attempt
-% (to avoid adding spurious empty items to the resulting
-% sequence).
+% \begin{macro}{\regex_group_end_extract_seq:N}
+% \begin{macro}[aux]{\regex_extract_seq_aux:n}
+% \begin{macro}[aux]{\regex_extract_seq_aux:ww}
% \begin{macrocode}
-\cs_new_protected:Npn \regex_extract_all:nnN #1#2#3
+\cs_new_protected:Npn \regex_group_end_extract_seq:N #1
{
- \regex_user_aux:n
- {
- \seq_gclear:N \g_regex_extract_all_seq
- \tl_set:Nn \l_regex_every_match_tl { \regex_extract_all_aux: }
- \regex_build:n {#1}
- \regex_match:n {#2}
- \regex_extract_all_after_group:N #3
- }
+ \cs_set_eq:NN \seq_item:n \scan_stop:
+ \flag_lower:N \l_regex_group_begin_flag
+ \flag_lower:N \l_regex_group_end_flag
+ \tl_set:Nx \l_regex_tmpa_tl
+ {
+ \prg_stepwise_function:nnnN
+ \l_regex_max_state_int
+ \c_one
+ { \l_regex_submatch_int - \c_one }
+ \regex_extract_seq_aux:n
+ }
+ \flag_test:NF \l_regex_group_begin_flag
+ { \flag_test:NF \l_regex_group_end_flag { \use_none:nnn } }
+ \msg_kernel_error:nn { regex } { sequence-unbalanced }
+ \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl }
+ \exp_args:NNNo \group_end:
+ \tl_set:Nn #1 \l_regex_tmpa_tl
}
-\cs_new_protected:Npn \regex_extract_all:NnN #1#2#3
+\cs_new_nopar:Npn \regex_extract_seq_aux:n #1
{
- \regex_check_nfa:NT #1
+ \seq_item:n
{
- \regex_user_aux:n
- {
- \seq_gclear:N \g_regex_extract_all_seq
- \tl_set:Nn \l_regex_every_match_tl { \regex_extract_all_aux: }
- \regex_use:N #1
- \regex_match:n {#2}
- \regex_extract_all_after_group:N #3
- }
+ \exp_after:wN \regex_extract_seq_aux:ww
+ \int_value:w \regex_submatch_nesting_aux:n {#1} ; #1;
}
}
-\cs_new_protected_nopar:Npn \regex_extract_all_aux:
- {
- \regex_extract:
- \seq_gconcat:NNN \g_regex_extract_all_seq
- \g_regex_extract_all_seq \g_regex_submatches_seq
- \regex_match_once:
- }
-\cs_new_protected_nopar:Npn \regex_extract_all_after_group:N #1
- {
- \seq_if_empty:NF \g_regex_extract_all_seq
- {
- \regex_extract:
- \group_insert_after:N \seq_set_eq:NN
- \group_insert_after:N #1
- \group_insert_after:N \g_regex_extract_all_seq
- }
+\cs_new_nopar:Npn \regex_extract_seq_aux:ww #1; #2;
+ { %^^A todo: use flags
+ \if_num:w #1 < \c_zero
+ \flag_raise:N \l_regex_group_end_flag
+ \prg_replicate:nn {-#1} { \exp_not:n { { \if_false: } \fi: } }
+ \fi:
+ \regex_query_submatch:w #2;
+ \if_num:w #1 > \c_zero
+ \flag_raise:N \l_regex_group_begin_flag
+ \prg_replicate:nn {#1} { \exp_not:n { \if_false: { \fi: } } }
+ \fi:
}
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
%
-% \subsubsection{Splitting a string by matches of a regex}
+% \subsubsection{Matching}
%
-% \begin{macro}{\regex_split:nnN}
-% \begin{macro}{\regex_split:NnN}
-% \begin{macro}[aux]{\regex_split_aux:}
-% \begin{macro}[aux]{\regex_split_after_group:N}
-% Similarly to \cs{regex_count:nnN} functions,
-% recurse through matches of the pattern. Then we do
-% something slightly different, extracting submatches.
-% Submatches are not extracted if the pattern matched
-% an empty string at the start of the match attempt
-% (to avoid adding spurious empty items to the resulting
-% sequence).
+% \begin{macro}[TF]{\regex_match:nn}
+% \begin{macro}[TF]{\regex_match:Nn}
+% \begin{macro}{\regex_match_aux:n}
+% We don't track submatches. Then either build the NFA corresponding
+% to the regular expression, or use a precompiled pattern. Then match,
+% using the internal \cs{regex_match:n}. Finally return the result
+% after closing the group.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_split:nnN #1#2#3
+\prg_new_protected_conditional:Npnn \regex_match:nn #1#2 { T , F , TF }
{
- \regex_user_aux:n
- {
- \seq_gclear:N \g_regex_split_seq
- \tl_set:Nn \l_regex_every_match_tl { \regex_split_aux: }
- \regex_build:n {#1}
- \regex_match:n {#2}
- \regex_split_after_group:N #3
- }
+ \regex_match_aux:n
+ { \regex_aux_build_match:nn {#1} {#2} }
}
-\cs_new_protected:Npn \regex_split:NnN #1#2#3
+\prg_new_protected_conditional:Npnn \regex_match:Nn #1#2 { T , F , TF }
{
- \regex_check_nfa:NT #1
- {
- \regex_user_aux:n
- {
- \seq_gclear:N \g_regex_split_seq
- \tl_set:Nn \l_regex_every_match_tl { \regex_split_aux: }
- \regex_use:N #1
- \regex_match:n {#2}
- \regex_split_after_group:N #3
- }
- }
+ \regex_match_aux:n
+ { \regex_aux_use_match:Nn #1 {#2} }
}
-\cs_new_protected_nopar:Npn \regex_split_aux:
+\cs_new_protected:Npn \regex_match_aux:n #1
{
- \int_compare:nNnF \l_regex_start_step_int = \l_regex_success_step_int
- {
- \regex_extract:
- \seq_pop:NN \g_regex_submatches_seq \l_regex_tmpa_tl
- \regex_extract_aux:nTF {0}
- {
- \seq_gput_left:Nx \g_regex_submatches_seq
- {
- \regex_query_substr:NN
- \l_regex_start_step_int \l_regex_tmpa_tl
- }
- }
- { \msg_error:nn { regex } { internal } }
- \seq_gconcat:NNN \g_regex_split_seq
- \g_regex_split_seq \g_regex_submatches_seq
- }
- \regex_match_once:
+ \group_begin:
+ \tl_clear:N \l_regex_every_match_tl
+ \regex_disable_submatches:
+ #1
+ \group_end:
+ \regex_aux_return:
}
-\cs_new_protected_nopar:Npn \regex_split_after_group:N #1
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}{\regex_count:nnN}
+% \begin{macro}{\regex_count:NnN}
+% \begin{macro}{\regex_count_aux:n}
+% Instead of aborting once the first \enquote{longest match} is found,
+% we repeat the search. The code is such that the search will not
+% start on the same character, hence avoiding infinite loops.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \regex_count:nnN
+ { \regex_count_aux:NnnN \regex_aux_build_match:nn }
+\cs_new_protected_nopar:Npn \regex_count:NnN
+ { \regex_count_aux:NnnN \regex_aux_use_match:Nn }
+\cs_new_protected:Npn \regex_count_aux:NnnN #1#2#3#4
{
- \int_compare:nNnTF \l_regex_start_step_int = \l_regex_current_step_int
- {
- \bool_if:NF \l_regex_success_empty_bool
- { \seq_gput_right:Nn \g_regex_split_seq { } }
- }
- {
- \seq_gput_right:Nx \g_regex_split_seq
- {
- \regex_query_substr:NN
- \l_regex_start_step_int \l_regex_current_step_int
- }
- }
- \group_insert_after:N \seq_set_eq:NN
- \group_insert_after:N #1
- \group_insert_after:N \g_regex_split_seq
+ \group_begin:
+ \regex_disable_submatches:
+ \int_zero:N \l_regex_match_count_int
+ \tl_set:Nn \l_regex_every_match_tl
+ {
+ \int_incr:N \l_regex_match_count_int
+ \regex_match_once:
+ }
+ #1 {#2} {#3}
+ \exp_args:NNNo
+ \group_end:
+ \int_set:Nn #4 { \int_use:N \l_regex_match_count_int }
}
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
%
-% \subsubsection{String replacement}
+% \subsubsection{Submatch extraction}
%
-% \begin{macro}[int]{\regex_replacement:n}
-% \begin{macro}[aux]{\regex_replacement_escaped:N}
-% \begin{macro}[aux]{\regex_replacement_submatch:w}
-% \begin{macro}[aux]{\regex_replacement_submatch_aux:nN}
+% \begin{macro}[aux]{\regex_extract_once_aux:NnnN}
+% As announced, here comes the auxiliary for extracting one match.
+% Since we only want one match, \cs{l_regex_every_match_tl}
+% is empty, and does not trigger the matching code again.
+% After matching, \cs{regex_extract:} extracts submatches
+% into various \tn{skip} registers, and those are then concatenated
+% into a sequence by \cs{regex_group_end_extract_seq:N}. That function
+% is also responsible for closing the group.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_replacement:n #1
- {
- \str_aux_escape:NNNn
- \prg_do_nothing:
- \regex_replacement_escaped:N
- \prg_do_nothing:
- {#1}
- \tl_set_eq:NN \l_regex_replacement_tl \g_str_result_tl
- \tl_set:Nx \l_regex_replacement_tl
- { \l_regex_replacement_tl \prg_do_nothing: }
- }
-\cs_new_nopar:Npn \regex_replacement_escaped:N #1
+\cs_new_protected:Npn \regex_extract_once_aux:NnnN #1#2#3#4
{
- \regex_token_if_other_digit:NTF #1
- {
- \exp_not:n
- { \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } } {#1}
- }
- {
- \token_if_eq_charcode:NNTF g #1
- { \exp_not:N \regex_replacement_submatch:w }
- { #1 }
- }
+ \group_begin:
+ \tl_set:Nn \l_regex_every_match_tl { \regex_extract: }
+ #1 {#2} {#3}
+ \regex_group_end_extract_seq:N #4
}
-\cs_new_nopar:Npn \regex_replacement_submatch:w #1
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_extract_all_aux:NnnN}
+%^^A todo: update doc
+% The set of submatches will be built progressively
+% in \cs{l_regex_result_seq}. For each match, extract
+% the submatches, and concatenate that to the right of
+% the result sequence, then start matching again.
+% Finally, copy the result in the user's sequence variable.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_extract_all_aux:NnnN #1#2#3#4
{
- \exp_after:wN \token_if_eq_meaning:NNTF \c_lbrace_str #1
- { \regex_replacement_submatch_aux:nN { } }
- {
- \regex_token_if_other_digit:NTF #1
- { \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } }
- { \msg_expandable_kernel_error:nn { regex } { g-misused } }
- #1
- }
- }
-\cs_new_nopar:Npn \regex_replacement_submatch_aux:nN #1 #2
+ \group_begin:
+ \tl_set:Nn \l_regex_every_match_tl
+ { \regex_extract: \regex_match_once: }
+ #1 {#2} {#3}
+ \regex_group_end_extract_seq:N #4
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \subsubsection{Splitting a token list by matches of a regex}
+%
+% \begin{macro}[aux]{\regex_split_aux:NnnN}
+% Recurse through the matches, and for each, do the following.
+% Extract the submatches into various \tn{skip} registers,
+% then replace the match |\0|, which should not be kept in the
+% final result, and replace it by the part of the token list
+% before the match.
+% This process must be inhibited to avoid creating empty items
+% if the regex matched an empty token list at the place where
+% the match attempt started.
+% After the last successful match, we need to add to the result
+% the part of the token list after the last match, unless the
+% last match was empty and at the very end.
+% Finally, \cs{regex_group_end_extract_seq:N} builds a sequence
+% from all the \tn{skip} registers, and assigns it to |#4|
+% after closing the group.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_split_aux:NnnN #1#2#3#4
{
- \regex_token_if_other_digit:NTF #2
- { \regex_replacement_submatch_aux:nN {#1#2} }
- {
- \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } {#1}
- \exp_after:wN \token_if_eq_meaning:NNF \c_rbrace_str #2
- {
- \msg_expandable_kernel_error:nn { regex } { g-misused }
- #2
- }
- }
+ \group_begin:
+ \tl_set:Nn \l_regex_every_match_tl
+ {
+ \if_num:w \l_regex_start_step_int < \l_regex_success_step_int
+ \regex_extract:
+ \tex_skip:D \l_regex_submatch_start_int
+ = \l_regex_start_step_int sp
+ plus \tex_skip:D \l_regex_submatch_start_int \scan_stop:
+ \fi:
+ \regex_match_once:
+ }
+ #1 {#2} {#3}
+ \tex_skip:D \l_regex_submatch_int
+ = \l_regex_start_step_int sp
+ plus \l_regex_current_step_int sp \scan_stop:
+ \int_incr:N \l_regex_submatch_int
+ \if_num:w \l_regex_start_step_int = \l_regex_current_step_int
+ \if_meaning:w \c_true_bool \l_regex_success_empty_bool
+ \int_decr:N \l_regex_submatch_int
+ \fi:
+ \fi:
+ \regex_group_end_extract_seq:N #4
}
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}[aux]{\regex_replace_after_group:N}
-% I wonder why I implemented things with \tn{aftergroup}
-% rather than simply placing that code after the explicit
-% \cs{group_end:} that I have. (General remark.)
+% \subsubsection{Replacement}
+%
+% \begin{macro}[aux]{\regex_replace_once_aux:NnnN}
+% The replacement text is analysed by \cs{regex_replacement:n},
+% which defines \cs{regex_replacement_tl:n} to expand to the
+% replaced token list, assuming that submatches are stored in
+% various \tn{skip} registers, as done by \cs{regex_extract:}.
+% If there is a match, we grab
+% the parts before and after it, and get the result
+% by \texttt{x}-expanding twice.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_replace_after_group:N #1
+\cs_new_protected:Npn \regex_replace_once_aux:NnnN #1#2#3#4
+ {
+ \group_begin:
+ \tl_clear:N \l_regex_every_match_tl
+ \regex_replacement:n {#3}
+ \exp_args:Nno #1 {#2} #4
+ \if_meaning:w \c_true_bool \g_regex_success_bool
+ \regex_extract:
+ \int_set:Nn \l_regex_tmpa_int
+ { \regex_nesting:n { \l_regex_submatch_start_int } }
+ \if_num:w \l_regex_tmpa_int = \c_zero
+ \else:
+ \msg_kernel_error:nnx { regex } { replace-unbalanced }
+ { \l_regex_tmpa_int }
+ \fi:
+ \tl_set:Nx \l_regex_tmpa_tl
+ {
+ \if_num:w \l_regex_tmpa_int < \c_zero
+ \prg_replicate:nn { - \l_regex_tmpa_int }
+ { \exp_not:n { { \if_false: } \fi: } }
+ \fi:
+ \regex_query_substr:nn
+ { \l_regex_min_step_int }
+ { \tex_skip:D \l_regex_submatch_start_int }
+ \regex_replacement_tl:n { \l_regex_submatch_start_int }
+ \regex_query_substr:nn
+ { \etex_gluestretch:D \tex_skip:D \l_regex_submatch_start_int }
+ { \l_regex_max_step_int }
+ \if_num:w \l_regex_tmpa_int > \c_zero
+ \prg_replicate:nn { \l_regex_tmpa_int }
+ { \exp_not:n { \if_false: { \fi: } } }
+ \fi:
+ }
+ \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl }
+ \exp_args:NNNo \group_end:
+ \tl_set:Nn #4 \l_regex_tmpa_tl
+ \else:
+ \group_end:
+ \fi:
+ }
+\cs_new_protected_nopar:Npn \regex_replace_once_aux:w #1;
{
- \group_insert_after:N \tl_set:Nx
- \group_insert_after:N #1
- \group_insert_after:N {
- \group_insert_after:N \tl_to_str:N
- \group_insert_after:N \g_regex_replaced_str
- \group_insert_after:N }
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}{\regex_replace_once:nnN}
-% \begin{macro}{\regex_replace_once:NnN}
-% \begin{macro}[TF]{\regex_replace_once:nnN}
-% \begin{macro}[TF]{\regex_replace_once:NnN}
-% \begin{macro}[aux]{\regex_replace_once_aux:Nn}
+% \begin{macro}[aux]{\regex_replace_all_aux:NnnN}
+% For every match, extract submatches, and add the part before
+% the beginning of the match, as well as the replacement,
+% to the result. After the last match, extract the end
+% of the token list, and add it to the replaced token list.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_replace_once_aux:Nn #1#2
+\cs_new_protected:Npn \regex_replace_all_aux:NnnN #1#2#3#4
{
\group_begin:
- \regex_replace_after_group:N #1
- \tl_clear:N \l_regex_every_match_tl
- #2
- \exp_args:No \regex_match:n {#1}
- \regex_extract:
- \regex_extract_aux:nTF {0}
+ \tl_set:Nn \l_regex_every_match_tl
{
- \cs_set_nopar:Npn \regex_tmp:w
- { \str_skip_do:nn { \l_regex_tmpb_tl } { } }
- \tl_gset:Nx \g_regex_replaced_str
- {
- \regex_query_substr:NN \c_zero \l_regex_tmpa_tl
- \l_regex_replacement_tl
- \exp_after:wN \regex_tmp:w \l_regex_query_other_str
- }
+ \regex_extract:
+ \tex_skip:D \l_regex_submatch_start_int
+ = \tex_the:D \tex_skip:D \l_regex_submatch_start_int
+ minus \l_regex_start_step_int sp \scan_stop:
+ \regex_match_once:
}
- { \tl_gset_eq:NN \g_regex_replaced_str \l_regex_query_other_str }
- \group_end:
+ \regex_replacement:n {#3}
+ \exp_args:Nno #1 {#2} #4
+ \int_set:Nn \l_regex_tmpa_int
+ {
+ 0
+ \prg_stepwise_function:nnnN
+ \l_regex_max_state_int
+ \l_regex_capturing_group_int
+ { \l_regex_submatch_int - \c_one }
+ \regex_nesting:n
+ }
+ \if_num:w \l_regex_tmpa_int = \c_zero
+ \else:
+ \msg_kernel_error:nnx { regex } { replace-unbalanced }
+ { \l_regex_tmpa_int }
+ \fi:
+ \tl_set:Nx \l_regex_tmpa_tl
+ {
+ \if_num:w \l_regex_tmpa_int < \c_zero
+ \prg_replicate:nn { - \l_regex_tmpa_int }
+ { \exp_not:n { { \if_false: } \fi: } }
+ \fi:
+ \prg_stepwise_function:nnnN
+ \l_regex_max_state_int
+ \l_regex_capturing_group_int
+ { \l_regex_submatch_int - \c_one }
+ \regex_replace_all_aux:n
+ \regex_query_substr:nn
+ \l_regex_start_step_int \l_regex_max_step_int
+ \if_num:w \l_regex_tmpa_int > \c_zero
+ \prg_replicate:nn { \l_regex_tmpa_int }
+ { \exp_not:n { \if_false: { \fi: } } }
+ \fi:
+ }
+ \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl }
+ \exp_args:NNNo \group_end:
+ \tl_set:Nn #4 \l_regex_tmpa_tl
}
-\cs_new_protected:Npn \regex_replace_once:nnN #1#2#3
+\cs_new_nopar:Npn \regex_replace_all_aux:n #1
{
- \regex_replace_once_aux:Nn #3
- {
- \regex_build:n {#1}
- \regex_replacement:n {#2}
- }
+ \regex_query_substr:nn
+ { \etex_glueshrink:D \tex_skip:D #1 } { \tex_skip:D #1 }
+ \regex_replacement_tl:n {#1}
}
-\cs_new_protected:Npn \regex_replace_once:NnN #1#2#3
+% \end{macrocode}
+% \end{macro}
+%
+% \subsection{Messages}
+%
+% \begin{macrocode}
+\msg_kernel_new:nnnn { regex } { sequence-unbalanced }
{
- \regex_replace_once_aux:Nn #3
+ Missing~
+ \flag_test:NTF \l_regex_group_end_flag
{
- \regex_use:N #1
- \regex_replacement:n {#2}
+ left~
+ \flag_test:NTF \l_regex_group_begin_flag
+ { and~right~braces } { brace }
}
+ { right~brace }
+ \ inserted~in~extracted~match.
}
-\prg_new_protected_conditional:Npnn \regex_replace_once:nnN #1#2#3 {T,F,TF}
{
- \regex_replace_once_aux:Nn #3
- {
- \regex_build:n {#1}
- \regex_replacement:n {#2}
- \regex_return_after_group:
- }
+ LaTeX~was~asked~to~extract~submatches~or~split~a~token~list~
+ according~to~a~given~regular~expression,~but~some~of~the~resulting~
+ items~were~not~balanced.
}
-\prg_new_protected_conditional:Npnn \regex_replace_once:NnN #1#2#3 {T,F,TF}
+\msg_kernel_new:nnnn { regex } { replace-unbalanced }
+ { The~result~of~a~replacement~does~not~have~balanced~braces. }
{
- \regex_replace_once_aux:Nn #3
+ LaTeX~was~asked~to~do~some~regular~expression~replacement,~
+ and~the~resulting~token~list~would~not~have~the~same~number~
+ of~begin-group~and~end-group~tokens. \\ \\
+ \ \ \ \
+ \prg_case_int:nnn {#1}
{
- \regex_use:N #1
- \regex_replacement:n {#2}
- \regex_return_after_group:
+ { -1 } { A~left~brace~was }
+ { 1 } { A~right~brace~was }
}
+ {
+ \int_abs:n {#1} ~
+ \int_compare:nNnTF {#1} < \c_zero { left } { right } ~
+ braces ~ were
+ }
+ \ inserted.
}
% \end{macrocode}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}{\regex_replace_all:nnN}
-% \begin{macro}{\regex_replace_all:NnN}
-% \begin{macro}[aux]{\regex_replace_all_aux:Nn}
-% \begin{macro}[aux]{\regex_replace_all_aux:}
% \begin{macrocode}
-\cs_new_protected:Npn \regex_replace_all_aux:Nn #1#2
+\msg_kernel_new:nnnn { regex } { missing-rparen }
+ { Missing~right~parenthesis~added~in~regular~expression. }
{
- \group_begin:
- \regex_replace_after_group:N #1
- \tl_set:Nn \l_regex_every_match_tl { \regex_replace_all_aux: }
- \tl_gclear:N \g_regex_replaced_str
- #2
- \exp_args:No \regex_match:n {#1}
- \cs_set_nopar:Npn \regex_tmp:w
- { \str_skip_do:nn { \l_regex_start_step_int } { } }
- \tl_gput_right:Nx \g_regex_replaced_str
- { \exp_after:wN \regex_tmp:w \l_regex_query_other_str }
- \group_end:
+ LaTeX~was~given~a~regular~expression~with~\int_eval:n{#1}~
+ more~left~parenthes\int_compare:nTF{#1=1}{i}{e}s~than~right~
+ parenthes\int_compare:nTF{#1=1}{i}{e}s.
}
-\cs_new_protected_nopar:Npn \regex_replace_all_aux:
+\msg_kernel_new:nnnn { regex } { extra-rparen }
+ { Extra~right~parenthesis~ignored~in~regular~expression. }
{
- \regex_extract:
- \regex_extract_aux:nTF {0}
- {
- \tl_gput_right:Nx \g_regex_replaced_str
- {
- \regex_query_substr:NN
- \l_regex_start_step_int \l_regex_tmpa_tl
- \l_regex_replacement_tl
- }
- }
- { \msg_error:nn { regex } { internal } }
- \regex_match_once:
+ LaTeX~came~across~a~closing~parenthesis~when~no~submatch~group~
+ was~open.~The~parenthesis~will~be~ignored.
}
-\cs_new_protected:Npn \regex_replace_all:nnN #1#2#3
+\msg_kernel_new:nnnn { regex } { backwards-range }
+ { Range~[#1-#2]~out~of~order~in~character~class. }
{
- \regex_replace_all_aux:Nn #3
- {
- \regex_build:n {#1}
- \regex_replacement:n {#2}
- }
+ In~ranges~of~characters~[x-y]~appearing~in~character~classes,~
+ the~first~character~code~must~not~be~larger~than~the~second.~
+ Here,~#1~has~character~code~\int_eval:n {`#1},~while~#2~has~
+ character~code~\int_eval:n {`#2}.
}
-\cs_new_protected:Npn \regex_replace_all:NnN #1#2#3
+\msg_kernel_new:nnnn { regex } { unsupported }
+ { Unsupported~construction~`#1'. }
{
- \regex_replace_all_aux:Nn #3
- {
- \regex_use:N #1
- \regex_replacement:n {#2}
- }
+ The~construction~`#1'~is~not~supported~by~the~LaTeX~
+ regular~expression~module.~Perhaps~some~character~should~
+ have~been~escaped?
+ }
+\msg_kernel_new:nnnn { regex } { not-nfa }
+ { This~is~not~a~regular~expression~variable. }
+ {
+ LaTeX~was~expecting~`#1'~to~be~a~regular~expression~variable.\\
+ This~control~sequence~is~not~a~regex~variable.~It's~current~meaning~
+ is~\\\\
+ \ \ \ \ \token_to_str:N #1 = \token_to_meaning:N #1 .
}
% \end{macrocode}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \subsection{Messages}
-%
-% Negative codes are specific to the \LaTeX3 implementation.
-% Other error codes match with the PCRE codes (not all of
-% the PCRE errors can occur, since many constructions are
-% not supported).
% \begin{macrocode}
-% \msg_new:nnn { regex } { -999 } { File~not~found }
-\msg_new:nnn { regex } { -998 } { Unsupported~construct }
-\msg_new:nnn { regex } { -997 }
- { The~regular~expression~is~too~large~(32768~states). }
-\msg_new:nnn { regex } { 1 } { \iow_char:N\\~at~end~of~pattern }
+% \msg_new:nnn { regex } { 1 } { \iow_char:N\\~at~end~of~pattern }
% \msg_new:nnn { regex } { 2 } { \iow_char:N\\c~at~end~of~pattern }
-\msg_new:nnn { regex } { 4 }
- { Numbers~out~of~order~in~\iow_char:N\{\iow_char\}~quantifier. }
-\msg_new:nnn { regex } { 6 }
- { Missing~terminating~\iow_char:N\]~for~character~class }
-\msg_new:nnn { regex } { 7 }
- { Invalid~escape~sequence~in~character~class }
-\msg_new:nnn { regex } { 8 }
- { Range~out~of~order~in~character~class }
-\msg_new:nnn { regex } { 22 } { Mismatched~parentheses }
-\msg_new:nnn { regex } { 34 }
- { Character~value~in~\iow_char:N\\x{...}~sequence~is~too~large }
+% \msg_new:nnn { regex } { 4 }
+% { Numbers~out~of~order~in~\iow_char:N\{\iow_char\}~quantifier. }
+% \msg_new:nnn { regex } { 6 }
+% { Missing~terminating~\iow_char:N\]~for~character~class }
+% \msg_new:nnn { regex } { 7 }
+% { Invalid~escape~sequence~in~character~class }
+% \msg_new:nnn { regex } { 34 }
+% { Character~value~in~\iow_char:N\\x{...}~sequence~is~too~large }
% \msg_new:nnn { regex } { 44 } { Invalid~UTF-8~string }
% \msg_new:nnn { regex } { 46 }
% { Malformed~\iow_char:N\\P~or\iow_char:N\\p~sequence }
@@ -3867,18 +4464,46 @@
% \end{macrocode}
%
% \begin{macrocode}
-\msg_new:nnn { regex } { not-nfa }
+% \end{macrocode}
+%
+% \begin{macrocode}
+\msg_kernel_new:nnnn { regex } { build-c }
+ { Misused~\iow_char:N\\c~or\iow_char:N\\C~command~in~regular~expression. }
+ {
+ The~\iow_char:N\\C~escape~sequence~can~be~followed~by~
+ one~of~the~letters~ABCDELMOPSTU.~The~\iow_char:N\\c~
+ escape~sequence~can~also~be~followed~by~a~brace~group.
+ }
+\msg_kernel_new:nnnn { regex } { unknown-option }
+ { Unknown~option~`#1'~for~regular~expressions. }
+ {
+ LaTeX~came~across~something~like~`(?#1)'~in~a~regular~expression,~
+ but~the~option~`#1'~is~not~known.~It~will~be~ignored.
+ }
+\msg_kernel_new:nnnn { regex } { invalid-in-option }
+ { Invalid~character~in~option~of~a~regular~expression. }
+ {
+ The~character~or~escape~sequence~`#1'~is~not~defined~
+ as~an~option~within~regular~expressions.
+ }
+\msg_kernel_new:nnnn { regex } { replacement-g }
+ { Missing~brace~for~the~\iow_char:N\\g~construction~in~a~replacement~text. }
+ {
+ In~the~replacement~text~for~a~regular~expression~search,~
+ submatches~are~represented~either~as~\iow_char:N \\g{dd..d},~
+ or~\\d,~where~`d'~are~single~digits.~Here,~a~brace~is~missing.
+ }
+\msg_kernel_new:nnnn { regex } { replacement-c }
+ { Something~is~wrong~with~the~replacement~text. }
{
- LaTeX~was~expecting~a~regular~expression~variable.\\
- Instead,~LaTeX~found~'#1'.
+ The~\iow_char:N\\c~escape~sequence~can~only~be~followed~by~
+ ABCDELMN~or~by~a~brace.
}
% \end{macrocode}
%
% \begin{macrocode}
\msg_kernel_new:nnn { regex } { nfa-misused }
{ Automaton~#1 used~incorrectly. }
-\msg_kernel_new:nnn { regex } { g-misused }
- { \g misused~in~replacement~text. }
% \end{macrocode}
%
% \begin{macrocode}
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx
index db73291a70b..a33d4909a1a 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx
@@ -35,7 +35,7 @@
%
%<*driver|package>
\RequirePackage{expl3}
-\GetIdInfo$Id: l3str.dtx 3035 2011-12-07 11:48:46Z bruno $
+\GetIdInfo$Id: l3str.dtx 3088 2011-12-23 02:00:14Z bruno $
{L3 Experimental Strings}
%</driver|package>
%<*driver>
@@ -76,25 +76,25 @@
% String variables are simply specialised token lists, but by convention
% should be named with the suffix \texttt{\ldots str}. Such variables should
% contain characters with category code $12$ (other), except spaces, which
-% have category code $10$ (blank space). All the \enquote{safe} functions in
+% have category code $10$ (blank space). All the functions in
% this module first convert their argument to a string for internal processing,
% and will not treat a token list or the corresponding string representation
% differently.
%
% Most functions in this module come in three flavours:
% \begin{itemize}
-% \item \cs{str_...:N...}, which expect a token list
+% \item \cs{str_...:N...}, which expect a token list or string
% variable as their argument;
% \item \cs{str_...:n...}, taking any token list (or string) as an argument;
% \item \cs{str_..._ignore_spaces:n...}, which ignores any space encountered
% during the operation: these functions are faster than those
% which take care of escaping spaces appropriately;
% \end{itemize}
-% When performance is important, the internal \cs{str_..._unsafe:n...}
-% functions, which expect a \enquote{safe} string in which spaces
+% When performance is critical, the internal \cs{str_..._unsafe:n...}
+% functions, which expect an \enquote{other string} in which spaces
% have category code $12$ instead of $10$, might be useful.
%
-% \section{Conversion and input of strings}
+% \section{Building strings}
%
% \begin{variable}
% {
@@ -106,6 +106,7 @@
% \c_percent_str
% }
% Constant strings, containing a single character, with category code $12$.
+% Other characters can be accessed as \cs{iow_char:N} |\|\meta{character}.
% \end{variable}
%
% \begin{function}[EXP]{\tl_to_str:N, \tl_to_str:n}
@@ -124,77 +125,49 @@
% Creates a new constant \meta{str~var} or raises an error
% if the name is already taken. The value of the
% \meta{str~var} will be set globally to the
-% \meta{token list}, after being converted to a string.
-% \end{function}
-%
-% \begin{function}{\str_set:Nn, \str_set:Nx, \str_set:cn, \str_set:cx}
-% \begin{syntax}
-% \cs{str_set:Nn} \meta{str var} \Arg{token list}
-% \end{syntax}
-% Converts the \meta{token list} to a \meta{string},
-% and saves the result in \meta{str var}.
-% \end{function}
-%
-% \begin{function}{\str_gset:Nn, \str_gset:Nx, \str_gset:cn, \str_gset:cx}
-% \begin{syntax}
-% \cs{str_gset:Nn} \meta{str~var} \Arg{token list}
-% \end{syntax}
-% Converts the \meta{token list} to a \meta{string},
-% and saves the result in \meta{str~var} globally.
+% \meta{token list}, converted to a string.
% \end{function}
%
% \begin{function}
% {
-% \str_put_left:Nn, \str_put_left:Nx,
-% \str_put_left:cn, \str_put_left:cx
+% \str_set:Nn, \str_set:Nx, \str_set:cn, \str_set:cx,
+% \str_gset:Nn, \str_gset:Nx, \str_gset:cn, \str_gset:cx
% }
% \begin{syntax}
-% \cs{str_put_left:Nn} \meta{str var} \Arg{token list}
+% \cs{str_set:Nn} \meta{str var} \Arg{token list}
% \end{syntax}
% Converts the \meta{token list} to a \meta{string},
-% and prepends the result to \meta{str var}.
-% The current contents of the \meta{str var} are not
-% automatically converted to a string.
+% and stores the result in \meta{str var}.
% \end{function}
%
% \begin{function}
% {
+% \str_put_left:Nn, \str_put_left:Nx,
+% \str_put_left:cn, \str_put_left:cx,
% \str_gput_left:Nn, \str_gput_left:Nx,
% \str_gput_left:cn, \str_gput_left:cx
% }
% \begin{syntax}
-% \cs{str_gput_left:Nn} \meta{str var} \Arg{token list}
-% \end{syntax}
-% Converts the \meta{token list} to a \meta{string},
-% and prepends the result to \meta{str var}, globally.
-% The current contents of the \meta{str var} are not
-% automatically converted to a string.
-% \end{function}
-%
-% \begin{function}
-% {
-% \str_put_right:Nn, \str_put_right:Nx,
-% \str_put_right:cn, \str_put_right:cx
-% }
-% \begin{syntax}
-% \cs{str_put_right:Nn} \meta{str var} \Arg{token list}
+% \cs{str_put_left:Nn} \meta{str var} \Arg{token list}
% \end{syntax}
% Converts the \meta{token list} to a \meta{string},
-% and appends the result to \meta{str var}.
+% and prepends the result to \meta{str var}.
% The current contents of the \meta{str var} are not
% automatically converted to a string.
% \end{function}
%
% \begin{function}
% {
+% \str_put_right:Nn, \str_put_right:Nx,
+% \str_put_right:cn, \str_put_right:cx,
% \str_gput_right:Nn, \str_gput_right:Nx,
% \str_gput_right:cn, \str_gput_right:cx
% }
% \begin{syntax}
-% \cs{str_gput_right:Nn} \meta{str var} \Arg{token list}
+% \cs{str_put_right:Nn} \meta{str var} \Arg{token list}
% \end{syntax}
% Converts the \meta{token list} to a \meta{string},
-% and appends the result to \meta{str var}, globally.
+% and appends the result to \meta{str var}.
% The current contents of the \meta{str var} are not
% automatically converted to a string.
% \end{function}
@@ -204,9 +177,7 @@
% \cs{str_input:Nn} \meta{str var} \Arg{token list}
% \end{syntax}
% Converts the \meta{token list} into a \meta{string}, and stores
-% it in the \meta{str var}, within the current \TeX{} group level
-% for the \texttt{input} variant and globally for the \texttt{ginput}
-% version.
+% it in the \meta{str var}.
% Special characters can be input by
% escaping them with a backslash.
% \begin{itemize}
@@ -218,7 +189,7 @@
% are read to give a number between $0$ and $255$.
% \item |\x{hh...}| produces the character with code \texttt{hh...}
% (an arbitrary number of hexadecimal digits are read):
-% this is mostly useful for \LuaTeX{} and \XeTeX{}.
+% this is mostly useful for Unicode-aware engines.
% \item |\a|, |\e|, |\f|, |\n|, |\r|, |\t| stand for specific characters:
% \begin{center}
% \begin{tabular}{cccc}
@@ -292,7 +263,7 @@
% The functions differ in the case where the first character
% is a space: \cs{str_tail:N} and \cs{str_tail:n} will trim
% only that space, while \cs{str_tail_ignore_spaces:n} trims
-% the first non-space character.
+% the first non-space character and any space before it.
% \end{function}
%
% \begin{function}[EXP]{\str_item:Nn, \str_item:nn, \str_item_ignore_spaces:nn}
@@ -321,8 +292,17 @@
% and leaves in the input stream the characters
% between \meta{start index} (inclusive) and \meta{end index} (exclusive).
% If either of \meta{start index} or \meta{end index} is negative,
-% then it is incremented by the length of the list. Both \meta{start index}
-% and \meta{end index} count from $0$ for the first (left most) character.
+% then it is incremented by the length of the list.
+% If either of \meta{start index} or \meta{end index} is empty,
+% it is replaced by the corresponding end-point of the string.
+% Both \meta{start index} and \meta{end index} count from $0$
+% for the first (left most) character.
+% For instance,
+% \begin{verbatim}
+% \iow_term:x { \str_substr:nnn { abcdef } { 1 } { 4 } }
+% \iow_term:x { \str_substr:nnn { abcdef } { -4 } { } }
+% \end{verbatim}
+% will print \texttt{bcd} and \texttt{cdef}.
% \end{function}
%
% \section{String conditionals}
@@ -348,15 +328,6 @@
% expansion).
% \end{function}
%
-% \begin{function}[EXP, pTF]{\str_if_bytes:N}
-% \begin{syntax}
-% \cs{str_if_bytes:NTF} \meta{str var} \Arg{true code} \Arg{false code}
-% \end{syntax}
-% Tests whether the \meta{str var} only contains characters
-% in the range $0$ to $255$. This is automatically true when
-% the pdf\TeX{} engine is in use.
-% \end{function}
-%
% \section{Encoding functions}
%
% Traditionally, string encodings only specify how strings of characters
@@ -370,12 +341,13 @@
% \enquote{encoding}. This can be \textsc{utf-16},
% \textsc{iso 8859-1}, \emph{etc.}
% See Table~\ref{tab:encodings} for a list
-% of encodings supported.
+% of encodings supported.\footnote{Encodings and escapings
+% will be added as they are requested.}
% \item Bytes are translated to \TeX{} tokens through
% a given \enquote{escaping}. Those are defined for
% the most part by the \texttt{pdf} file format.
% See Table~\ref{tab:escapings} for a list
-% of escaping methods supported.
+% of escaping methods supported.\footnotemark
% \end{itemize}
%
% \begin{table}\centering
@@ -408,7 +380,7 @@
% \texttt{iso885916}, \texttt{latin10} & \textsc{iso 8859-16} \\
% \midrule
% \texttt{native} & Native Unicode string. \\
-% \texttt{internal} & Comma-list of integers. \\
+% \texttt{internal} & For curious programmers. \\
% \bottomrule
% \end{tabular}
% \end{table}
@@ -427,7 +399,7 @@
% & see \tn{pdfescapename} \\
% \texttt{string}
% & see \tn{pdfescapestring} \\
-% \texttt{url}, \texttt{percent}
+% \texttt{url}
% & encoding used in \textsc{url}s \\
% \bottomrule
% \end{tabular}
@@ -471,89 +443,101 @@
% an error.
% Characters which cannot be encoded in the \meta{encoding 2}
% are silently ignored.
-%^^A todo: add doc and examples about each encoding and escaping method.
% \end{function}
%
% \section{Internal string functions}
%
-% \begin{function}[EXP]{\tl_to_other_str:N, \tl_to_other_str:n}
+% \begin{function}{\str_escape_use:NNNn}
% \begin{syntax}
-% \cs{tl_to_other_str:n} \Arg{token list}
+% \cs{str_escape_use:NNNn} \meta{fn1} \meta{fn2} \meta{fn3} \Arg{token list}
% \end{syntax}
-% Converts the \meta{token list} to an \meta{other string}, where
-% spaces have category code \enquote{other}. These functions create
-% \enquote{safe} strings.
-% \begin{texnote}
-% These functions can be \texttt{f}-expanded without fear of losing
-% a leading space, since spaces do not have category code $10$ in
-% their result.
-% \end{texnote}
+% The \meta{token list} is converted to a string, then read from
+% left to right, interpreting backslashes as escaping the next character.
+% Unescaped characters are fed to the function \meta{fn1},
+% and escaped characters are fed to the function \meta{fn2}
+% within an \texttt{x}-expansion context (typically those functions
+% perform some tests on their argument to decide how to output them).
+% The escape sequences |\a|, |\e|, |\f|, |\n|, |\r|, |\t| and |\x|
+% are recognized as described for \cs{str_input:Nn}, and those
+% are replaced by the corresponding character, then fed to
+% \meta{fn3}. The result is then left in the input stream.
% \end{function}
%
-% \begin{function}[EXP]{\str_sanitize_args:Nn, \str_sanitize_args:Nnn}
+% \begin{function}[rEXP]{\str_aux_toks_range:nn}
% \begin{syntax}
-% \cs{str_sanitize_args:Nnn} \meta{function}
-% ~~\Arg{token list1} \Arg{token list2}
+% \cs{str_aux_toks_range:nn} \meta{start} \meta{end}
% \end{syntax}
-% Converts the \meta{token lists} to \enquote{safe} strings
-% (where spaces have category code \enquote{other}),
-% and hands-in the result as arguments to \meta{function}.
+% Expands to the contents of the \tn{toks} registers numbered
+% from \meta{start} (inclusive) to \meta{end} (exclusive).
+% This function is used in \pkg{l3regex}.
% \end{function}
%
-% \begin{function}{\str_aux_gset_other:Nn}
+% \begin{function}{\str_aux_hexadecimal_use:NTF}
% \begin{syntax}
-% \cs{str_aux_gset_other:Nn} \meta{tl~var} \Arg{token list}
+% \cs{str_aux_hexadecimal_use:NTF} \meta{token}
+% ~~\Arg{true code} \Arg{false code}
% \end{syntax}
-% Converts the \meta{token list} to an \meta{other string}, where
-% spaces have category code \enquote{other}, and assigns the result
-% to the \meta{tl~var}, globally.
+% If the \meta{token} is a hexdecimal digit (upper case or lower case),
+% its upper-case version is left in the input strem, \emph{followed}
+% by the \meta{true code}. Otherwise, the \meta{false code} is left
+% in the input stream.
+% \begin{texnote}
+% This function will fail if the escape character is a hexadecimal
+% digit, or has a character code in the range $[0,5]$. We are thus
+% careful to set the escape character to a known value before using it.
+% \end{texnote}
% \end{function}
%
-% \begin{function}[rEXP]{\str_map_tokens:Nn, \str_map_tokens:nn}
+% \begin{function}[EXP]{\tl_to_other_str:n}
% \begin{syntax}
-% \cs{str_map_tokens:Nn} \meta{str var} \meta{tokens}
+% \cs{tl_to_other_str:n} \Arg{token list}
% \end{syntax}
-% Maps the \meta{tokens} over every character in the \meta{str var}.
+% Converts the \meta{token list} to a \meta{other string}, where
+% spaces have category code \enquote{other}.
+% \begin{texnote}
+% These functions can be \texttt{f}-expanded without fear of losing
+% a leading space, since spaces do not have category code $10$ in
+% their result.
+% \end{texnote}
% \end{function}
%
-% \begin{function}[EXP, pTF]
-% {\str_if_contains_char:NN, \str_if_contains_char:nN}
+% \begin{function}{\str_gset_other:Nn}
% \begin{syntax}
-% \cs{str_if_contains_char:nN} \Arg{token list} \meta{char}
+% \cs{str_gset_other:Nn} \meta{tl~var} \Arg{token list}
% \end{syntax}
-% Converts the \meta{token list} to a \meta{string}
-% and tests whether the \meta{char} is present in the \meta{string}.
-% The \meta{char} can be given either directly, or as a one
-% letter control sequence.
+% Converts the \meta{token list} to an \meta{other string}, where
+% spaces have category code \enquote{other}, and assigns the result
+% to the \meta{tl~var}, globally.
% \end{function}
%
-% \begin{function}{\str_aux_escape:NNNn}
+% \begin{function}[EXP]
+% {
+% \str_if_contains_char:NNT,
+% \str_if_contains_char:NNTF,
+% \str_if_contains_char:nNTF
+% }
% \begin{syntax}
-% \cs{str_aux_escape:NNNn} \meta{fn1} \meta{fn2} \meta{fn3} \Arg{token list}
+% \cs{str_if_contains_char:nNTF} \Arg{token list} \meta{char}
% \end{syntax}
-% The \meta{token list} is converted to a string, then read from
-% left to right, interpreting backslashes as escaping the next character.
-% Unescaped characters are fed to the function \meta{fn1},
-% and escaped characters are fed to the function \meta{fn2}
-% within an \texttt{x}-expansion context (typically those functions
-% perform some tests on their argument to decide how to output them).
-% The escape sequences |\a|, |\e|, |\f|, |\n|, |\r|, |\t| and |\x|
-% are recognized as described for \cs{str_input:Nn}, and those
-% are replaced by the corresponding character, then fed to
-% \meta{fn3}. The result is assigned globally to \cs{g_str_result_tl}.
+% Converts the \meta{token list} to a \meta{string}
+% and tests whether the \meta{char} is present in the \meta{string}.
+% Spaces are ignored.
% \end{function}
%
-% \section{Possibilities}
+% \section{Possibilities, and things to do}
%
% \begin{itemize}
-% \item More encodings (see Heiko's \pkg{stringenc}).
-% In particular, what is needed for pdf: \textsc{utf-16}?
-% \item CESU
-% \item \cs{str_if_head_eq:nN} alias of \cs{tl_if_head_eq_charcode:nN}
+% \item Use flags rather than expandable errors.
+% \item Add documentation about each encoding and escaping method,
+% and add examples.
+% \item Perhaps use the pdf\TeX{} primitives for escapings when available.
+% But this changes error detection between engines.
+% \item More encodings (see Heiko's \pkg{stringenc}). CESU.
+% \item More escapings: shell escapes, lua escapes, etc?
+% \item \cs{str_if_head_eq:nN}
% \item \cs{str_if_numeric/decimal/integer:n}, perhaps in \pkg{l3fp}?
% \item Should \cs{str_item:Nn} be \cs{str_char:Nn}?
% \end{itemize}
-% Some functionalities of \pkg{stringstrings} and \pkg{xstring} as well.
%
% \end{documentation}
%
@@ -582,52 +566,9 @@
% \item \cs{str_length_loop:NNNNNNNNN} (unchanged)
% \end{itemize}
%
-% \subsection{General functions}
-%
-% \begin{macro}[EXP,aux]{\use_i:nnnnnnnn}
-% A function which may already be defined elsewhere.
-% \begin{macrocode}
-\cs_if_exist:NF \use_i:nnnnnnnn
- { \cs_new:Npn \use_i:nnnnnnnn #1#2#3#4#5#6#7#8 {#1} }
-% \end{macrocode}
-% \end{macro}
+% \subsection{Helpers}
%
-% \begin{variable}
-% {
-% \c_forty_eight, \c_fifty_eight, \c_sixty_five, \c_ninety_one,
-% \c_ninety_seven, \c_one_hundred_twenty_three,
-% \c_one_hundred_twenty_seven
-% }
-% We declare here some integer values which delimit ranges of
-% ASCII characters of various types. This is mostly used in
-% \pkg{l3regex}.
-% \begin{macrocode}
-\int_const:Nn \c_forty_eight { 48 }
-\int_const:Nn \c_fifty_eight { 58 }
-\int_const:Nn \c_sixty_five { 65 }
-\int_const:Nn \c_ninety_one { 91 }
-\int_const:Nn \c_ninety_seven { 97 }
-\int_const:Nn \c_one_hundred_twenty_three { 123 }
-\int_const:Nn \c_one_hundred_twenty_seven { 127 }
-% \end{macrocode}
-% \end{variable}
-%
-% \begin{variable}{\c_max_char_int}
-% The maximum character code depends on the engine.
-% \begin{macrocode}
-\pdftex_if_engine:TF
- { \int_const:Nn \c_max_char_int { 255 } }
- { \int_const:Nn \c_max_char_int { 1114111 } }
-% \end{macrocode}
-% \end{macro}
-%
-% \begin{variable}{\c_str_replacement_char_int}
-% When converting, invalid bytes are replaced by the Unicode
-% replacement character \textsc{u-fffd}.
-% \begin{macrocode}
-\int_const:Nn \c_str_replacement_char_int { "FFFD }
-% \end{macrocode}
-% \end{variable}
+% \subsubsection{Assigning strings}
%
% \begin{macro}
% {
@@ -666,7 +607,7 @@
% \end{macrocode}
% \end{macro}
%
-% \subsection{Variables and constants}
+% \subsubsection{Variables and constants}
%
% \begin{macro}{\str_tmp:w}
% \begin{variable}{\l_str_tmpa_tl}
@@ -674,6 +615,7 @@
% \begin{macrocode}
\cs_new_protected_nopar:Npn \str_tmp:w { }
\tl_new:N \l_str_tmpa_tl
+\int_new:N \l_str_tmpa_int
% \end{macrocode}
% \end{variable}
% \end{macro}
@@ -699,6 +641,42 @@
%
% \begin{variable}
% {
+% \c_forty_eight, \c_fifty_eight, \c_sixty_five, \c_ninety_one,
+% \c_ninety_seven, \c_one_hundred_twenty_three,
+% \c_one_hundred_twenty_seven
+% }
+% We declare here some integer values which delimit ranges of
+% ASCII characters of various types. This is mostly used in
+% \pkg{l3regex}.
+% \begin{macrocode}
+\int_const:Nn \c_forty_eight { 48 }
+\int_const:Nn \c_fifty_eight { 58 }
+\int_const:Nn \c_sixty_five { 65 }
+\int_const:Nn \c_ninety_one { 91 }
+\int_const:Nn \c_ninety_seven { 97 }
+\int_const:Nn \c_one_hundred_twenty_three { 123 }
+\int_const:Nn \c_one_hundred_twenty_seven { 127 }
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\c_max_char_int}
+% The maximum character code depends on the engine.
+% \begin{macrocode}
+\int_const:Nn \c_max_char_int
+ { \pdftex_if_engine:TF { 255 } { 1114111 } }
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\c_str_replacement_char_int}
+% When converting, invalid bytes are replaced by the Unicode
+% replacement character \textsc{u-fffd}.
+% \begin{macrocode}
+\int_const:Nn \c_str_replacement_char_int { "FFFD }
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}
+% {
% \c_backslash_str,
% \c_lbrace_str,
% \c_rbrace_str,
@@ -734,18 +712,17 @@
\prop_gput:Nnn \g_str_aliases_prop { latin9 } { iso885915 }
\prop_gput:Nnn \g_str_aliases_prop { latin10 } { iso885916 }
\prop_gput:Nnn \g_str_aliases_prop { hexadecimal } { hex }
-\prop_gput:Nnn \g_str_aliases_prop { percent } { url }
% \end{macrocode}
% \end{variable}
%
-% \subsection{Internal functions}
-%
% \subsubsection{Escaping spaces}
%
-% \begin{macro}[EXP]{\tl_to_other_str:N, \tl_to_other_str:n}
+% \begin{macro}[EXP]{\tl_to_other_str:n}
% \begin{macro}[EXP,aux]{\tl_to_other_str_loop:w, \tl_to_other_str_end:w}
% Replaces all spaces by \enquote{other} spaces, after converting
% the token list to a string via \cs{tl_to_str:n}.
+% This function is \texttt{f}-expandable, storing the part of the string
+% with escaped spaces between the \cs{q_mark} and \cs{q_stop} markers.
% \begin{macrocode}
\group_begin:
\char_set_lccode:nn { `\* } { `\ }
@@ -770,29 +747,19 @@
\cs_new_nopar:Npn \tl_to_other_str_end:w \fi: #1 \q_mark #2 * A #3 \q_stop
{ \fi: #2 }
}
-\cs_new_nopar:Npn \tl_to_other_str:N { \exp_args:No \tl_to_other_str:n }
% \end{macrocode}
% \end{macro}
% \end{macro}
%
-% \begin{macro}[EXP]{\str_sanitize_args:Nn, \str_sanitize_args:Nnn}
-% Here, \texttt{f}-expansion does not lose leading spaces,
-% since they have catcode \enquote{other} after \cs{str_sanitize:n}.
-% \begin{macrocode}
-\cs_new:Npn \str_sanitize_args:Nn #1#2
- { \exp_args:Nf #1 { \tl_to_other_str:n {#2} } }
-\cs_new:Npn \str_sanitize_args:Nnn #1#2#3
- {
- \exp_args:Nff #1
- { \tl_to_other_str:n {#2} }
- { \tl_to_other_str:n {#3} }
- }
-% \end{macrocode}
-% \end{macro}
-%
-% \begin{macro}[int]{\str_aux_gset_other:Nn}
-% \begin{macro}[aux,EXP]{\str_aux_gset_other_loop:w}
-% \begin{macro}[aux,EXP]{\str_aux_gset_other_end:w}
+% \begin{macro}[int]{\str_gset_other:Nn}
+% \begin{macro}[aux,EXP]{\str_gset_other_loop:w}
+% \begin{macro}[aux,EXP]{\str_gset_other_end:w}
+% This function could be done by using \cs{tl_to_other_str:n}
+% within an \texttt{x}-expansion, but that would take a time
+% quadratic in the size of the string. Instead, we can
+% \enquote{leave the result behind us} in the input stream,
+% to be captured into the expanding assignment. This gives us
+% a linear time.
% \begin{macrocode}
\group_begin:
\char_set_lccode:nn { `\* } { `\ }
@@ -800,24 +767,24 @@
\tl_to_lowercase:n
{
\group_end:
- \cs_new_protected:Npn \str_aux_gset_other:Nn #1#2
+ \cs_new_protected:Npn \str_gset_other:Nn #1#2
{
\tl_gset:Nx #1
{
- \exp_after:wN \str_aux_gset_other_loop:w \tl_to_str:n {#2} ~ %
+ \exp_after:wN \str_gset_other_loop:w \tl_to_str:n {#2} ~ %
A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ \q_mark \q_stop
}
}
- \cs_new_nopar:Npn \str_aux_gset_other_loop:w
+ \cs_new_nopar:Npn \str_gset_other_loop:w
#1 ~ #2 ~ #3 ~ #4 ~ #5 ~ #6 ~ #7 ~ #8 ~ #9 ~
{
\if_meaning:w A #9
- \str_aux_gset_other_end:w
+ \str_gset_other_end:w
\fi:
#1 * #2 * #3 * #4 * #5 * #6 * #7 * #8 * #9
- \str_aux_gset_other_loop:w *
+ \str_gset_other_loop:w *
}
- \cs_new_nopar:Npn \str_aux_gset_other_end:w \fi: #1 * A #2 \q_stop
+ \cs_new_nopar:Npn \str_gset_other_end:w \fi: #1 * A #2 \q_stop
{ \fi: #1 }
}
% \end{macrocode}
@@ -825,54 +792,46 @@
% \end{macro}
% \end{macro}
%
-% \subsubsection{Mapping}
+% \subsubsection{Functions unrelated to strings}
%
-% \begin{macro}[rEXP,int]{\str_map_tokens:nn}
-% \begin{macro}[rEXP,int]{\str_map_tokens:Nn}
-% \begin{macro}[rEXP,aux]{\str_map_tokens_aux:nn}
-% \begin{macro}[rEXP,aux]{\str_map_tokens_loop:nw}
-% \begin{macro}[rEXP,aux]{\str_map_tokens_loop:nN}
-% \begin{macro}[rEXP,int]{\str_map_break_do:n}
-% We simply need to be careful with spaces.
-% Spaces are fed to the mapped tokens with
-% category code $12$, not $10$.
+% \begin{macro}[EXP,aux]{\use_i:nnnnnnnn}
+% A function which may already be defined elsewhere.
% \begin{macrocode}
-\cs_new_nopar:Npn \str_map_tokens:Nn
- { \exp_args:No \str_map_tokens:nn }
-\cs_new:Npn \str_map_tokens:nn #1
- { \exp_args:No \str_map_tokens_aux:nn { \tl_to_str:n {#1} } }
-\cs_new:Npn \str_map_tokens_aux:nn #1#2
+\cs_if_exist:NF \use_i:nnnnnnnn
+ { \cs_new:Npn \use_i:nnnnnnnn #1#2#3#4#5#6#7#8 {#1} }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[int, rEXP]{\str_aux_toks_range:nn}
+% \begin{macro}[aux, rEXP]{\str_aux_toks_range:ww}
+% Some non-expandable functions (in this module,
+% \cs{str_aux_escape:NNNn}, which is used by \cs{str_(g)input:Nn})
+% store pieces of their result in \tn{toks} registers.
+% Those pieces are concatenated using \cs{str_aux_toks_range:nn},
+% which expects two integer registers, \meta{start} and \meta{end},
+% and expands to the contents of the \tn{toks} registers from
+% \meta{start} (inclusive) to \meta{end} (exclusive).
+% This function is also used in the \pkg{l3regex} module.
+% \begin{macrocode}
+\cs_new:Npn \str_aux_toks_range:nn #1#2
{
- \str_map_tokens_loop:nw {#2} #1
- { ? \use_none_delimit_by_q_recursion_stop:w }
- ?~ ?~ ?~ ?~ ?~ ?~ ?~ ?~ \q_recursion_stop
+ \exp_after:wN \str_aux_toks_range:ww
+ \int_use:N \int_eval:w #1 \exp_after:wN ;
+ \int_use:N \int_eval:w #2 ;
+ \prg_break_point:n { }
}
-\group_begin:
- \char_set_lccode:nn { `* } { `\ }
- \tl_to_lowercase:n
- {
- \group_end:
- \cs_new:Npn \str_map_tokens_loop:nw #1 #2~ #3~ #4~ #5~ #6~ #7~ #8~ #9~
- {
- \str_map_tokens_loop:nN {#1} #2* #3* #4* #5* #6* #7* #8* #9*
- { ? \use_none_delimit_by_q_stop:w } \q_stop
- \str_map_tokens_loop:nw {#1}
- }
- }
-\cs_new:Npn \str_map_tokens_loop:nN #1#2
+\cs_new:Npn \str_aux_toks_range:ww #1 ; #2 ;
{
- \use_none:n #2
- #1 #2
- \str_map_tokens_loop:nN {#1}
+ \if_num:w #1 = #2 \exp_stop_f:
+ \exp_after:wN \prg_map_break:
+ \fi:
+ \tex_the:D \tex_toks:D #1 \exp_stop_f:
+ \exp_after:wN \str_aux_toks_range:ww
+ \int_use:N \int_eval:w #1 + \c_one ; #2 ;
}
-\cs_new_eq:NN \str_map_break_do:n \use_i_delimit_by_q_recursion_stop:nw
% \end{macrocode}
% \end{macro}
% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
%
% \subsection{Characters given by their position}
%
@@ -889,7 +848,8 @@
% by \cs{use_none_delimit_by_q_stop:w}.
% \begin{macrocode}
\cs_new_nopar:Npn \str_length:N { \exp_args:No \str_length:n }
-\cs_new:Npn \str_length:n { \str_sanitize_args:Nn \str_length_unsafe:n }
+\cs_new:Npn \str_length:n #1
+ { \exp_args:Nf \str_length_unsafe:n { \tl_to_other_str:n {#1} } }
\cs_new_nopar:Npn \str_length_unsafe:n #1
{ \str_length_aux:n { \str_length_loop:NNNNNNNNN #1 } }
\cs_new:Npn \str_length_ignore_spaces:n #1
@@ -1110,15 +1070,15 @@
% the parameters given to \cs{str_skip_do:nn} are necessarily
% within the bounds of the length of the string.
% The \cs{str_item_ignore_spaces:nn} function cheats a little bit
-% in that it doesn't hand to \cs{str_item_unsafe:nn} a truly
-% \enquote{safe} string. This is alright, as everything else
+% in that it doesn't hand to \cs{str_item_unsafe:nn} an
+% \enquote{other string}. This is alright, as everything else
% is done with undelimited arguments.
% \begin{macrocode}
\cs_new_nopar:Npn \str_item:Nn { \exp_args:No \str_item:nn }
\cs_new:Npn \str_item:nn #1#2
{
\exp_args:Nf \tl_to_str:n
- { \str_sanitize_args:Nn \str_item_unsafe:nn {#1} {#2} }
+ { \exp_args:Nf \str_item_unsafe:nn { \tl_to_other_str:n {#1} } {#2} }
}
\cs_new:Npn \str_item_ignore_spaces:nn #1
{ \exp_args:No \str_item_unsafe:nn { \tl_to_str:n {#1} } }
@@ -1161,13 +1121,21 @@
% \begin{macro}[EXP]{\str_substr:nnn}
% \begin{macro}[EXP]{\str_substr_ignore_spaces:nnn}
% \begin{macro}[EXP]{\str_substr_unsafe:nnn}
-% \begin{macro}[EXP,aux]{\str_substr_aux:nnnw}
-% \begin{macro}[EXP,aux]{\str_substr_aux_ii:nnw}
-% \begin{macro}[EXP,aux]{\str_aux_eval_args:Nnnn}
+% \begin{macro}[EXP,aux]{\str_substr_aux:www}
+% \begin{macro}[EXP,aux]{\str_substr_aux:nnw}
% \begin{macro}[EXP,aux]{\str_aux_normalize_range:nn}
-% Sanitize the string, then limit the second and third arguments
-% to be at most the length of the string (this avoids needing
-% to check for the end of the string when grabbing characters).
+% Sanitize the string. Then evaluate the arguments, replacing
+% them by \cs{c_zero} or \cs{c_max_int} if they are empty.
+% This is done by using the construction
+% \begin{verbatim}
+% \int_use:N \int_eval:w #2 \c_zero \c_zero ;
+% \end{verbatim}
+% which expands to the value of |#2|, followed by |\c_zero \c_zero ;|
+% if |#2| is an expression, and expands to |0\c_zero ;| otherwise.
+% The same is done to the end-point of the range.
+% Then limit the range to be at most the length of the string
+% (this avoids needing to check for the end of the string when
+% grabbing characters).
% Afterwards, skip characters, then keep some more, and finally
% drop the end of the string.
% \begin{macrocode}
@@ -1175,26 +1143,31 @@
\cs_new:Npn \str_substr:nnn #1#2#3
{
\exp_args:Nf \tl_to_str:n
- { \str_sanitize_args:Nn \str_substr_unsafe:nnn {#1} {#2} {#3} }
+ {
+ \exp_args:Nf \str_substr_unsafe:nnn
+ { \tl_to_other_str:n {#1} } {#2} {#3}
+ }
}
\cs_new:Npn \str_substr_ignore_spaces:nnn #1
{ \exp_args:No \str_substr_unsafe:nnn { \tl_to_str:n {#1} } }
\cs_new:Npn \str_substr_unsafe:nnn #1#2#3
{
- \str_aux_eval_args:Nnnn \str_substr_aux:nnnw
- { \str_length_unsafe:n {#1} }
- {#2}
- {#3}
+ \exp_after:wN \str_substr_aux:www
+ \int_value:w \str_length_unsafe:n {#1} \exp_after:wN ;
+ \int_use:N \int_eval:w #2 + \c_zero \exp_after:wN ;
+ \int_use:N \int_eval:w \exp_args:Nf \str_substr_unsafe:nN {#3} \c_max_int ;
#1
\q_stop
}
-\cs_new:Npn \str_substr_aux:nnnw #1#2#3
+\cs_new:Npn \str_substr_unsafe:nN #1 #2
+ { \tl_if_empty:nTF {#1} {#2} {#1} }
+\cs_new:Npn \str_substr_aux:www #1; #2; #3;
{
- \exp_args:Nf \str_substr_aux_ii:nnw
+ \exp_args:Nf \str_substr_aux:nnw
{ \str_aux_normalize_range:nn {#2} {#1} }
{ \str_aux_normalize_range:nn {#3} {#1} }
}
-\cs_new:Npn \str_substr_aux_ii:nnw #1#2
+\cs_new:Npn \str_substr_aux:nnw #1#2
{
\str_skip_do:nn {#1}
{
@@ -1203,13 +1176,6 @@
{ \use_i_delimit_by_q_stop:nw }
}
}
-\cs_new:Npn \str_aux_eval_args:Nnnn #1#2#3#4
- {
- \exp_after:wN #1
- \exp_after:wN { \int_value:w \int_eval:w #2 \exp_after:wN }
- \exp_after:wN { \int_value:w \int_eval:w #3 \exp_after:wN }
- \exp_after:wN { \int_value:w \int_eval:w #4 }
- }
\cs_new:Npn \str_aux_normalize_range:nn #1#2
{
\int_eval:n
@@ -1237,14 +1203,15 @@
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
%
% \subsection{String conditionals}
%
% \begin{macro}[EXP,pTF]{\str_if_eq:NN}
% \begin{macro}[EXP,pTF]{\str_if_eq:nn,\str_if_eq:xx}
% The \texttt{nn} and \texttt{xx} variants are already
-% defined in \pkg{l3basics}.
+% defined in \pkg{l3basics}. Note that \cs{str_if_eq:NN}
+% is different from \cs{tl_if_eq:NN} because it needs to
+% ignore category codes.
% \begin{macrocode}
\prg_new_conditional:Npnn \str_if_eq:NN #1#2 { p , TF , T , F }
{
@@ -1255,112 +1222,45 @@
% \end{macro}
% \end{macro}
%
-%^^A treat spaces separately, or make that internal and faster.
-% \begin{macro}[EXP,pTF]{\str_if_contains_char:NN}
-% \begin{macro}[EXP,pTF]{\str_if_contains_char:nN}
+% \begin{macro}[EXP]{\str_if_contains_char:NNT, \str_if_contains_char:NNTF}
+% \begin{macro}[EXP]{\str_if_contains_char:nNTF}
% \begin{macro}[EXP,aux]{\str_if_contains_char_aux:NN}
-% \begin{macro}[EXP,aux]{\str_if_contains_char_end:w}
+% \begin{macro}[EXP,aux]{\str_if_contains_char_end:}
% Loop over the characters of the string, comparing character codes.
% We allow |#2| to be a single-character control sequence, hence the
% use of \cs{int_compare:nNnT} rather than \cs{token_if_eq_charcode:NNT}.
% The loop is broken if character codes match. Otherwise we return
% \enquote{false}.
% \begin{macrocode}
-\prg_new_conditional:Npnn \str_if_contains_char:NN #1#2 { p , T , F , TF }
+\prg_new_conditional:Npnn \str_if_contains_char:NN #1#2 { T , TF }
{
- \str_map_tokens:Nn #1 { \str_if_contains_char_aux:NN #2 }
+ \exp_after:wN \str_if_contains_char_aux:NN \exp_after:wN #2
+ #1 { \prg_map_break:n { ? \fi: } }
+ \prg_break_point:n { }
\prg_return_false:
}
-\prg_new_conditional:Npnn \str_if_contains_char:nN #1#2 { p , T , F , TF }
+\prg_new_conditional:Npnn \str_if_contains_char:nN #1#2 { TF }
{
- \str_map_tokens:nn {#1} { \str_if_contains_char_aux:NN #2 }
+ \exp_after:wN \str_if_contains_char_aux:NN \exp_after:wN #2
+ \tl_to_str:n {#1} { \prg_map_break:n { ? \fi: } }
+ \prg_break_point:n { }
\prg_return_false:
}
\cs_new_nopar:Npn \str_if_contains_char_aux:NN #1#2
{
- \if_num:w `#1 = `#2 \exp_stop_f:
- \str_if_contains_char_end:w
+ \if_charcode:w #1 #2
+ \exp_after:wN \str_if_contains_char_end:
\fi:
+ \str_if_contains_char_aux:NN #1
}
-\cs_new_nopar:Npn \str_if_contains_char_end:w \fi: #1 \prg_return_false:
- { \fi: \prg_return_true: }
+\cs_new_nopar:Npn \str_if_contains_char_end:
+ { \prg_map_break:n { \prg_return_true: \use_none:n } }
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
%
-% \begin{macro}[EXP,pTF]{\str_if_bytes:N}
-% \begin{macro}[EXP,aux]{\str_if_bytes_aux:N}
-% \begin{macro}[EXP,aux]{\str_if_bytes_aux:}
-% In the pdf\TeX{} engine, every character is a byte,
-% so there is nothing to check. In other engines, we
-% loop over the string, checking if every character
-% code is less than $256$.
-% \begin{macrocode}
-\pdftex_if_engine:TF
- {
- \cs_new_eq:NN \str_if_bytes_p:N \c_true_bool
- \cs_new_eq:NN \str_if_bytes:NT \use_ii:nn
- \cs_new_eq:NN \str_if_bytes:NF \use_none:nn
- \cs_new_eq:NN \str_if_bytes:NTF \use_ii:nnn
- }
- {
- \prg_new_conditional:Npnn \str_if_bytes:N #1 { p , T , F , TF }
- {
- \exp_args:Nf \tl_map_function:nN
- { \tl_to_str:N #1 }
- \str_if_bytes_aux:N
- \prg_return_true:
- }
- \cs_new_nopar:Npn \str_if_bytes_aux:N #1
- {
- \if_num:w `#1 > \c_two_hundred_fifty_five
- \exp_after:wN \str_if_bytes_aux:
- \fi:
- }
- \cs_new_nopar:Npn \str_if_bytes_aux:
- { \tl_map_break:n { \prg_return_false: \use_none:n } }
- }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-%
-% \subsection{Conversions}
-%
-% \subsubsection{Internal conditionals}
-%
-% \begin{macro}[aux]{\str_aux_hexadecimal_test:NTF}
-% This test is used when reading hexadecimal digits, for the |\x| escape
-% sequence. It returns \meta{true} if the
-% token is a hexadecimal digit, and \meta{false} otherwise. It has the
-% additional side-effect of updating the value of
-% \cs{l_str_char_int} (number formed in base $16$ from the digits
-% read so far).
-%^^A todo: speed tweaks?
-% \begin{macrocode}
-\prg_new_protected_conditional:Npnn \str_aux_hexadecimal_test:N #1 { TF }
- {
- \tl_if_in:onTF { \tl_to_str:n { abcdef } } {#1}
- {
- \int_set:Nn \l_str_char_int
- { \c_sixteen * \l_str_char_int + `#1 - 87 }
- \prg_return_true:
- }
- {
- \if_num:w \c_fifteen < "1 \exp_not:N #1 \exp_stop_f:
- \int_set:Nn \l_str_char_int
- { \c_sixteen * \l_str_char_int + "#1 }
- \prg_return_true:
- \else:
- \prg_return_false:
- \fi:
- }
- }
-% \end{macrocode}
-% \end{macro}
-%
% \begin{macro}[aux, rEXP]{\str_aux_octal_use:NTF}
% \TeX{} dutifully detects octal digits for us: if |#1|
% is an octal digit, then the right-hand side of the
@@ -1370,7 +1270,7 @@
% \begin{macrocode}
\prg_new_conditional:Npnn \str_aux_octal_use:N #1 { TF }
{
- \if_num:w \c_one < '1 #1 \exp_stop_f:
+ \if_num:w \c_one < '1 \token_to_str:N #1 \exp_stop_f:
#1 \prg_return_true:
\else:
\prg_return_false:
@@ -1387,27 +1287,35 @@
% \begin{macrocode}
\prg_new_conditional:Npnn \str_aux_hexadecimal_use:N #1 { TF }
{
- \if_num:w \c_two < "1 #1 \exp_stop_f:
+ \if_num:w \c_two < "1 \token_to_str:N #1 \exp_stop_f:
#1 \prg_return_true:
\else:
- \if_case:w \int_eval:w `#1 - `a \int_eval_end:
- A \prg_return_true:
- \or: B \prg_return_true:
- \or: C \prg_return_true:
- \or: D \prg_return_true:
- \or: E \prg_return_true:
- \or: F \prg_return_true:
- \else: \prg_return_false:
+ \if_case:w \int_eval:w
+ \exp_after:wN ` \token_to_str:N #1 - `a
+ \int_eval_end:
+ A
+ \or: B
+ \or: C
+ \or: D
+ \or: E
+ \or: F
+ \else:
+ \prg_return_false:
+ \exp_after:wN \use_none:n
\fi:
+ \prg_return_true:
\fi:
}
% \end{macrocode}
% \end{macro}
%
+% \subsection{Conversions}
+%
% \subsubsection{Producing one byte or character}
%
% \begin{variable}{\c_str_byte_0_tl}
% \begin{variable}{\c_str_byte_-1_tl}
+% \begin{variable}{\c_str_positive_bytes_tl}
% For each integer $N$ in the range $[0,255]$, we create
% a constant token list which holds three character tokens
% with category code other: the character with character
@@ -1415,8 +1323,11 @@
% two hexadecimal digits.
% The value $-1$ is given a default token list which ensures
% that later functions give an empty result for the input $-1$.
+% Simultaneously, we build a list of all bytes (from which we
+% remove the null byte) in \cs{c_str_positive_bytes_tl}.
% \begin{macrocode}
\group_begin:
+ \tl_clear:N \l_str_tmpa_tl
\char_set_catcode_other:n { \c_zero }
\tl_gset:Nx \g_str_result_tl { \tl_to_str:n { 0123456789ABCDEF } }
\tl_map_inline:Nn \g_str_result_tl
@@ -1425,20 +1336,24 @@
{
\tl_map_inline:Nn \g_str_result_tl
{
- \char_set_lccode:nn { \c_zero } {"##1#1}
+ \char_set_lccode:nn { \c_zero } {"#1##1}
\tl_to_lowercase:n
{
\tl_const:cx
- { c_str_byte_ \int_eval:n {"##1#1} _tl }
- { ^^@ ##1 #1 }
+ { c_str_byte_ \int_eval:n {"#1##1} _tl }
+ { ^^@ #1 ##1 }
+ \tl_put_right:Nn \l_str_tmpa_tl { ^^@ }
}
}
}
+ \tl_const:Nx \c_str_positive_bytes_tl
+ { \exp_after:wN \use_none:n \l_str_tmpa_tl }
\group_end:
\tl_const:cn { c_str_byte_-1_tl } { { } \use_none:n { } }
% \end{macrocode}
% \end{variable}
% \end{variable}
+% \end{variable}
%
% \begin{macro}[int, EXP]{\str_output_byte:n}
% \begin{macro}[aux, EXP]{\str_output_byte:w}
@@ -1476,11 +1391,13 @@
% \end{macro}
% \end{macro}
%
-% \subsubsection{}%^^A here
+% \subsubsection{Mapping functions for encoding conversion}
%
% \begin{macro}{\str_aux_gmap_result:N}
% \begin{macro}[aux, rEXP]{\str_aux_gmap_result_loop:NN}
-%
+% This maps the function |#1| over all characters
+% in \cs{g_str_result_tl}, which should be a byte
+% string in most cases, sometimes a native string.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \str_aux_gmap_result:N #1
{
@@ -1504,23 +1421,25 @@
%
% \begin{macro}{\str_aux_gmap_internal_result:N}
% \begin{macro}[aux, rEXP]{\str_aux_gmap_internal_result_loop:Nw}
-%
+% This maps the function |#1| over all character codes
+% in \cs{g_str_result_tl}, which must be in the internal
+% representation.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \str_aux_gmap_internal_result:N #1
{
\tl_gset:Nx \g_str_result_tl
{
- \exp_after:wN \str_aux_gmap_internal_result_loop:Nw
+ \exp_after:wN \str_aux_gmap_internal_result_loop:Nww
\exp_after:wN #1
- \g_str_result_tl \q_stop \prg_map_break: ,
+ \g_str_result_tl \q_mark \q_stop \prg_map_break: ,
\prg_break_point:n { }
}
}
-\cs_new_nopar:Npn \str_aux_gmap_internal_result_loop:Nw #1#2,
+\cs_new_nopar:Npn \str_aux_gmap_internal_result_loop:Nww #1#2\q_mark #3,
{
- \use_none_delimit_by_q_stop:w #2 \q_stop
- #1 {#2}
- \str_aux_gmap_internal_result_loop:Nw #1
+ \use_none_delimit_by_q_stop:w #3 \q_stop
+ #1 {#3}
+ \str_aux_gmap_internal_result_loop:Nww #1
}
% \end{macrocode}
% \end{macro}
@@ -1566,19 +1485,28 @@
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}[int]{\str_escape_use:NNNn}
+% Use the internal \cs{str_aux_escape:NNNn}, then leave the result
+% in the input stream. This is used in \pkg{l3regex}.
+% \begin{macrocode}
+\cs_new_protected:Npn \str_escape_use:NNNn #1#2#3#4
+ {
+ \str_aux_escape:NNNn #1#2#3 {#4}
+ \g_str_result_tl
+ }
+% \end{macrocode}
+% \end{macro}
+%
% \begin{macro}[int]{\str_aux_escape:NNNn}
-% \begin{macro}[aux]{\str_aux_escape_unescaped:N}
-% \begin{macro}[aux]{\str_aux_escape_escaped:N}
-% \begin{macro}[aux]{\str_aux_escape_raw:N}
% \begin{macro}[aux]{\str_aux_escape_loop:N}
% \begin{macro}[aux]+\str_aux_escape_\:w+
-% Treat the argument as an \meta{escaped string},
-% and store the corresponding \meta{string} globally
-% in \cs{g_str_result_tl}.
+% Most of the work is done within an \texttt{x}-expanding assignment,
+% but the |\x| escape sequence cannot be done in that way. Therefore,
+% we interrupt the assignment at each |\x| escape sequence, store the
+% partial result in a \tn{toks} register, and at the end unpack all
+% of the \tn{toks} registers to the left of the last chunk of
+% \cs{g_str_result_tl}.
% \begin{macrocode}
-\cs_new_eq:NN \str_aux_escape_unescaped:N \use:n
-\cs_new_eq:NN \str_aux_escape_escaped:N \use:n
-\cs_new_eq:NN \str_aux_escape_raw:N \use:n
\cs_new_protected:Npn \str_aux_escape:NNNn #1#2#3#4
{
\group_begin:
@@ -1586,12 +1514,15 @@
\cs_set_nopar:Npn \str_aux_escape_escaped:N { #2 }
\cs_set_nopar:Npn \str_aux_escape_raw:N { #3 }
\int_set:Nn \tex_escapechar:D { 92 }
- \str_aux_gset_other:Nn \g_str_result_tl {#4}
+ \str_gset_other:Nn \g_str_result_tl {#4}
+ \int_zero:N \l_str_tmpa_int
\tl_gset:Nx \g_str_result_tl
{
\exp_after:wN \str_aux_escape_loop:N \g_str_result_tl
\q_recursion_tail \q_recursion_stop
}
+ \tl_gput_left:Nx \g_str_result_tl
+ { \str_aux_toks_range:nn \c_zero \l_str_tmpa_int }
\group_end:
}
\cs_new_nopar:Npn \str_aux_escape_loop:N #1
@@ -1611,6 +1542,17 @@
% \end{macro}
% \end{macro}
% \end{macro}
+%
+% \begin{macro}[aux]{\str_aux_escape_unescaped:N}
+% \begin{macro}[aux]{\str_aux_escape_escaped:N}
+% \begin{macro}[aux]{\str_aux_escape_raw:N}
+% Those functions are never called before being given a new meaning,
+% so their definitions here don't matter.
+% \begin{macrocode}
+\cs_new_eq:NN \str_aux_escape_unescaped:N \prg_do_nothing:
+\cs_new_eq:NN \str_aux_escape_escaped:N \prg_do_nothing:
+\cs_new_eq:NN \str_aux_escape_raw:N \prg_do_nothing:
+% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
@@ -1662,18 +1604,20 @@
% When |\x| is encountered, interrupt the assignment,
% and distinguish the cases of a braced or unbraced syntax.
% In the braced case, collect arbitrarily many hexadecimal digits,
-% building the number in \cs{l_str_char_int} (this is a side effect
-% of \cs{str_aux_hexadecimal_test:NTF}), and check that
-% the run of digits was interrupted by a closing brace.
+% building the number in \cs{l_str_char_int} (using
+% \cs{str_aux_hexadecimal_use:NTF}), and check that
+% the run of digits was stopped by a closing brace.
% In the unbraced case, collect up to two hexadecimal digits,
% possibly less, building the character number in \cs{l_str_char_int}.
% In both cases, once all digits have been collected, use
% the \TeX{} primitive \tn{lowercase} to produce that character,
-% and use a \cs{if_false:} trick to restart the assignment.
+% and use an \cs{if_false:} trick to restart the assignment.
% \begin{macrocode}
\cs_new_nopar:cpn { str_aux_escape_/x:w } \str_aux_escape_loop:N
{
\if_false: { \fi: }
+ \tex_toks:D \l_str_tmpa_int \exp_after:wN { \g_str_result_tl }
+ \int_incr:N \l_str_tmpa_int
\int_zero:N \l_str_char_int
\str_aux_escape_x_test:N
}
@@ -1682,35 +1626,36 @@
\exp_not:N \token_if_eq_charcode:NNTF \c_space_token #1
{ \exp_not:N \str_aux_escape_x_test:N }
{
- \exp_not:N \token_if_eq_charcode:NNTF \c_lbrace_str #1
- { \exp_not:N \str_aux_escape_x_braced_loop:N }
- { \exp_not:N \str_aux_escape_x_unbraced_i:N #1 }
+ \l_str_char_int = "0
+ \exp_not:N \token_if_eq_charcode:NNTF \c_lbrace_str #1
+ { \exp_not:N \str_aux_escape_x_braced_loop:N }
+ { \exp_not:N \str_aux_escape_x_unbraced_i:N #1 }
}
}
-\cs_new_protected_nopar:Npn \str_aux_escape_x_unbraced_i:N #1
+\cs_new_nopar:Npn \str_aux_escape_x_unbraced_i:N #1
{
- \str_aux_hexadecimal_test:NTF #1
+ \str_aux_hexadecimal_use:NTF #1
{ \str_aux_escape_x_unbraced_ii:N }
- { \str_aux_escape_x_end: #1 }
+ { \exp_stop_f: \str_aux_escape_x_end: #1 }
}
-\cs_new_protected_nopar:Npn \str_aux_escape_x_unbraced_ii:N #1
+\cs_new_nopar:Npn \str_aux_escape_x_unbraced_ii:N #1
{
\token_if_eq_charcode:NNTF \c_space_token #1
{ \str_aux_escape_x_unbraced_ii:N }
{
- \str_aux_hexadecimal_test:NTF #1
- { \str_aux_escape_x_end: }
- { \str_aux_escape_x_end: #1 }
+ \str_aux_hexadecimal_use:NTF #1
+ { \exp_stop_f: \str_aux_escape_x_end: }
+ { \exp_stop_f: \str_aux_escape_x_end: #1 }
}
}
-\cs_new_protected_nopar:Npn \str_aux_escape_x_braced_loop:N #1
+\cs_new_nopar:Npn \str_aux_escape_x_braced_loop:N #1
{
\token_if_eq_charcode:NNTF \c_space_token #1
{ \str_aux_escape_x_braced_loop:N }
{
- \str_aux_hexadecimal_test:NTF #1
+ \str_aux_hexadecimal_use:NTF #1
{ \str_aux_escape_x_braced_loop:N }
- { \str_aux_escape_x_braced_end:N #1 }
+ { \exp_stop_f: \str_aux_escape_x_braced_end:N #1 }
}
}
\cs_new_protected_nopar:Npx \str_aux_escape_x_braced_end:N #1
@@ -1729,7 +1674,7 @@
\tex_lccode:D \c_zero \l_str_char_int
\tl_to_lowercase:n
{
- \tl_gput_right:Nx \g_str_result_tl
+ \tl_gset:Nx \g_str_result_tl
{ \if_false: } \fi:
\str_aux_escape_raw:N ^^@
\str_aux_escape_loop:N
@@ -1803,13 +1748,13 @@
% must be empty. Otherwise, the escaping |#2| is performed.
% \begin{macrocode}
\cs_new_protected:Npn \str_set_convert:Nnnn
- { \str_convert_aux_i:NNnnn \str_set:Nx }
+ { \str_convert_aux_i:NNnnn \tl_set_eq:NN }
\cs_new_protected:Npn \str_gset_convert:Nnnn
- { \str_convert_aux_i:NNnnn \str_gset:Nx }
+ { \str_convert_aux_i:NNnnn \tl_gset_eq:NN }
\cs_new_protected:Npn \str_convert_aux_i:NNnnn #1#2#3#4#5
{
\group_begin:
- \str_aux_gset_other:Nn \g_str_result_tl {#5}
+ \str_gset_other:Nn \g_str_result_tl {#5}
\exp_after:wN \str_convert_aux_ii:wwwnn
\tl_to_str:n {#3} /// \q_stop
@@ -2002,11 +1947,6 @@
% \end{macrocode}
% \end{macro}
%
-%^^A todo: document, move.
-% \begin{macrocode}
-\int_new:N \l_str_encoding_int
-% \end{macrocode}
-%
% \begin{macro}[int]{\str_convert_from_eight_bit:n}
% \begin{macro}[aux]{\str_convert_from_eight_bit_load:nn}
% \begin{macro}[aux]{\str_convert_from_eight_bit_load_missing:n}
@@ -2016,7 +1956,7 @@
\cs_new_protected:Npn \str_convert_from_eight_bit:n #1
{
\group_begin:
- \int_zero:N \l_str_encoding_int
+ \int_zero:N \l_str_tmpa_int
\exp_last_unbraced:Nx \str_convert_from_eight_bit_load:nn
{ \tl_use:c { c_str_encoding_#1_tl } }
{ \q_stop \prg_map_break: } { }
@@ -2031,24 +1971,25 @@
\cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load:nn #1#2
{
\use_none_delimit_by_q_stop:w #1 \q_stop
- \tex_dimen:D "#1 = \l_str_encoding_int sp \scan_stop:
- \tex_skip:D \l_str_encoding_int = "#1 sp \scan_stop:
- \tex_toks:D \l_str_encoding_int \exp_after:wN { \int_value:w "#2 }
- \tex_advance:D \l_str_encoding_int \c_one
+ \tex_dimen:D "#1 = \l_str_tmpa_int sp \scan_stop:
+ \tex_skip:D \l_str_tmpa_int = "#1 sp \scan_stop:
+ \tex_toks:D \l_str_tmpa_int \exp_after:wN { \int_value:w "#2 }
+ \tex_advance:D \l_str_tmpa_int \c_one
\str_convert_from_eight_bit_load:nn
}
\cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load_missing:n #1
{
\use_none_delimit_by_q_stop:w #1 \q_stop
- \tex_dimen:D "#1 = \l_str_encoding_int sp \scan_stop:
- \tex_skip:D \l_str_encoding_int = "#1 sp \scan_stop:
- \tex_toks:D \l_str_encoding_int \exp_after:wN { \int_value:w "FFFD }
- \tex_advance:D \l_str_encoding_int \c_one
+ \tex_dimen:D "#1 = \l_str_tmpa_int sp \scan_stop:
+ \tex_skip:D \l_str_tmpa_int = "#1 sp \scan_stop:
+ \tex_toks:D \l_str_tmpa_int \exp_after:wN { \int_value:w "FFFD }
+ \tex_advance:D \l_str_tmpa_int \c_one
\str_convert_from_eight_bit_load_missing:n
}
\cs_new:Npn \str_convert_from_eight_bit_aux:N #1
{
- \if_num:w \tex_dimen:D `#1 < \l_str_encoding_int
+ #1 \exp_not:N \q_mark
+ \if_num:w \tex_dimen:D `#1 < \l_str_tmpa_int
\if_num:w \tex_skip:D \tex_dimen:D `#1 = `#1 \exp_stop_f:
\tex_the:D \tex_toks:D \tex_dimen:D
\fi:
@@ -2070,7 +2011,7 @@
\cs_new_protected:Npn \str_convert_to_eight_bit:n #1
{
\group_begin:
- \int_zero:N \l_str_encoding_int
+ \int_zero:N \l_str_tmpa_int
\exp_last_unbraced:Nx \str_convert_to_eight_bit_load:nn
{ \tl_use:c { c_str_encoding_#1_tl } }
{ \q_stop \prg_map_break: } { }
@@ -2081,17 +2022,17 @@
\cs_new_protected_nopar:Npn \str_convert_to_eight_bit_load:nn #1#2
{
\use_none_delimit_by_q_stop:w #1 \q_stop
- \tex_dimen:D "#2 = \l_str_encoding_int sp \scan_stop:
- \tex_skip:D \l_str_encoding_int = "#2 sp \scan_stop:
- \exp_args:NNf \tex_toks:D \l_str_encoding_int
+ \tex_dimen:D "#2 = \l_str_tmpa_int sp \scan_stop:
+ \tex_skip:D \l_str_tmpa_int = "#2 sp \scan_stop:
+ \exp_args:NNf \tex_toks:D \l_str_tmpa_int
{ \str_output_byte:n { "#1 } }
- \tex_advance:D \l_str_encoding_int \c_one
+ \tex_advance:D \l_str_tmpa_int \c_one
\str_convert_to_eight_bit_load:nn
}
\cs_new:Npn \str_convert_to_eight_bit_aux:n #1
{
\if_num:w #1 < "8000 \exp_stop_f:
- \if_num:w \tex_dimen:D #1 < \l_str_encoding_int
+ \if_num:w \tex_dimen:D #1 < \l_str_tmpa_int
\if_num:w \tex_skip:D \tex_dimen:D #1 = #1 \exp_stop_f:
\tex_the:D \tex_toks:D \tex_dimen:D #1 \exp_stop_f:
\exp_after:wN \exp_after:wN \exp_after:wN \use_none:nn
@@ -2100,7 +2041,7 @@
\str_convert_to_eight_bit_aux_ii:n {#1}
\else:
\msg_expandable_kernel_error:nnn
- { str } { eight_bit-to-byte-overflow } {#1}
+ { str } { eight-bit-to-byte-overflow } {#1}
\fi:
}
\cs_new:Npn \str_convert_to_eight_bit_aux_ii:n #1
@@ -2204,52 +2145,107 @@
\cs_new_protected_nopar:Npn \str_convert_from_native:
{ \str_aux_gmap_result:N \str_convert_from_native_aux:N }
\cs_new:Npn \str_convert_from_native_aux:N #1
- { \int_value:w `#1 , }
+ { #1 \exp_not:N \q_mark \int_value:w `#1 , }
% \end{macrocode}
% \end{macro}
% \end{macro}
%
% \begin{macro}[int]{\str_convert_to_native:}
% \begin{macro}[aux]{\str_convert_to_native_aux:w}
-% Convert the string from the given encoding
-% to the internal format. We grab integers
-% from the internal string using the low-level
-% method to assign integers.
-% Except at the end of the loop, the function
-% \cs{str_convert_to_native_aux:w} is followed by a
-% comma, hence its argument is empty.
-% We are making heavy use of \TeX{} primitives here,
-% because the function can become very slow for large
-% strings.
+% This function is based on the internal functions
+% \cs{str_aux_convert_to_native:n} and
+% \cs{str_aux_convert_to_native_step:n}.
+% Loop through the comma-list, and call
+% \cs{str_aux_convert_to_native_step:n}
+% with the relevant integer argument.
+% We need to store \cs{g_str_result_tl} in a temporary
+% token list, because that global \enquote{result} is
+% cleared silently by \cs{str_aux_convert_to_native:n}.
+%^^A todo: update
% \begin{macrocode}
\cs_new_protected_nopar:Npn \str_convert_to_native:
{
\tl_set_eq:NN \l_str_tmpa_tl \g_str_result_tl
+ \str_aux_convert_to_native:n
+ {
+ \exp_after:wN \str_convert_to_native_aux:w
+ \l_str_tmpa_tl \q_mark { \q_stop \prg_map_break: } ,
+ }
+ }
+\cs_new_protected_nopar:Npn \str_convert_to_native_aux:w #1\q_mark #2,
+ {
+ \use_none_delimit_by_q_stop:w #2 \q_stop
+ \str_aux_convert_to_native_step:n {#2}
+ \str_convert_to_native_aux:w
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int]{\str_aux_convert_to_native:n}
+% \begin{macro}[int]{\str_aux_convert_to_native_step:n}
+% \begin{macro}[aux]{\str_aux_convert_to_native_flush:}
+% \begin{macro}[aux, rEXP]{\str_aux_convert_to_native_filter:N}
+%^^A todo: document.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \str_aux_convert_to_native:n #1
+ {
+ \tex_lccode:D \c_zero \c_zero
+ \int_zero:N \l_str_tmpa_int
\tl_gclear:N \g_str_result_tl
- \tex_afterassignment:D \str_convert_to_native_aux:w
- \l_str_char_int = \l_str_tmpa_tl \c_zero \prg_map_break: ,
- \prg_break_point:n { }
+ #1
+ \prg_break_point:n { \str_aux_convert_to_native_flush: }
+ }
+\cs_new_protected_nopar:Npn \str_aux_convert_to_native_step:n #1
+ {
+ \if_num:w \l_str_tmpa_int = \c_two_hundred_fifty_five
+ \str_aux_convert_to_native_flush:
+ \l_str_tmpa_int \c_zero
+ \fi:
+ \tex_advance:D \l_str_tmpa_int \c_one
+ \if_int_compare:w #1 > \c_max_char_int
+ \msg_kernel_error:nnx { str } { overflow } {#1}
+ \tex_lccode:D \l_str_tmpa_int \c_str_replacement_char_int
+ \else:
+ \tex_lccode:D \l_str_tmpa_int \int_eval:w #1 \int_eval_end:
+ \fi:
+ }
+\cs_new_protected_nopar:Npn \str_aux_convert_to_native_flush:
+ {
+ \use:x
+ {
+ \tl_to_lowercase:n
+ {
+ \tl_gput_right:Nx \exp_not:N \g_str_result_tl
+ {
+ \exp_after:wN \str_aux_convert_to_native_filter:N
+ \c_str_positive_bytes_tl
+ { ? = \c_zero \fi: \prg_map_break: }
+ \prg_break_point:n { }
+ }
+ }
+ }
}
\group_begin:
- \char_set_catcode_other:n { `\^^@ }
- \cs_new_protected_nopar:Npn \str_convert_to_native_aux:w #1 ,
+ \char_set_catcode_other:n { 0 }
+ \cs_new:Npn \str_aux_convert_to_native_filter:N #1
{
- #1
- \if_int_compare:w \l_str_char_int > \c_max_char_int
- \msg_kernel_error:nnx { str } { overflow }
- { \int_use:N \l_str_char_int }
+ \if_num:w `#1 > \l_str_tmpa_int
+ \exp_after:wN \prg_map_break:
+ \fi:
+ \if_num:w \tex_lccode:D `#1 = \c_zero
+ ^^@
\else:
- \tex_lccode:D \c_zero \l_str_char_int
- \tl_to_lowercase:n
- { \tex_xdef:D \g_str_result_tl { \g_str_result_tl ^^@ } }
+ #1
\fi:
- \tex_afterassignment:D \str_convert_to_native_aux:w
- \l_str_char_int =
+ \str_aux_convert_to_native_filter:N
}
\group_end:
% \end{macrocode}
% \end{macro}
% \end{macro}
+% \end{macro}
+% \end{macro}
%
% \subsection{Messages}
%
@@ -2331,10 +2327,8 @@
% as per the pdf\TeX{} primitive \tn{pdfescapehex}
% \item \texttt{name}, as per the pdf\TeX{} primitive \tn{pdfescapename}
% \item \texttt{string}, as per the pdf\TeX{} primitive \tn{pdfescapestring}
-% \item \texttt{url} or \texttt{percent}, as per the percent encoding of urls.
+% \item \texttt{url}, as per the percent encoding of urls.
% \end{itemize}
-%^^A todo: in pdfTeX, use the \tn{pdfescape...}
-%^^A todo: add various shell escapes, lua escapes etc?
%
% \subsubsection{Input methods}
%
@@ -2353,13 +2347,16 @@
%<*hex>
\cs_new_protected_nopar:Npn \str_convert_input_hex:
{
- \tl_gset:Nx \g_str_result_tl
- {
- \str_output_byte:w "
- \exp_after:wN \str_convert_input_hex_aux:N
- \g_str_result_tl 0 { ? 0 - \c_one \prg_map_break: }
- \prg_break_point:n { \str_output_end: }
- }
+ \group_begin:
+ \int_set:Nn \tex_escapechar:D { 92 }
+ \tl_gset:Nx \g_str_result_tl
+ {
+ \str_output_byte:w "
+ \exp_after:wN \str_convert_input_hex_aux:N
+ \g_str_result_tl 0 { ? 0 - \c_one \prg_map_break: }
+ \prg_break_point:n { \str_output_end: }
+ }
+ \group_end:
}
\cs_new_nopar:Npn \str_convert_input_hex_aux:N #1
{
@@ -2388,7 +2385,6 @@
% \begin{macro}[int]{\str_convert_input_name:}
% \begin{macro}[aux, rEXP]{\str_convert_input_name_aux:wNN}
% \begin{macro}[int]{\str_convert_input_url:}
-% \begin{macro}[int]{\str_convert_input_percent:}
% \begin{macro}[aux, rEXP]{\str_convert_input_url_aux:wNN}
% The \cs{str_convert_input_name:} function replaces each
% occurrence of |#| followed by two hexadecimal digits
@@ -2417,12 +2413,15 @@
{
\cs_new_protected_nopar:Npn #2
{
- \tl_gset:Nx \g_str_result_tl
- {
- \exp_after:wN #3 \g_str_result_tl
- #1 ? { ? \prg_map_break: }
- \prg_break_point:n { }
- }
+ \group_begin:
+ \int_set:Nn \tex_escapechar:D { 92 }
+ \tl_gset:Nx \g_str_result_tl
+ {
+ \exp_after:wN #3 \g_str_result_tl
+ #1 ? { ? \prg_map_break: }
+ \prg_break_point:n { }
+ }
+ \group_end:
}
\cs_new_nopar:Npn #3 ##1#1##2##3
{
@@ -2450,7 +2449,6 @@
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
%
% \begin{macro}[int]{\str_convert_input_string:}
% \begin{macro}[aux, rEXP]{\str_convert_input_string_aux:wN}
@@ -2488,18 +2486,21 @@
{
\cs_new_protected_nopar:Npn \str_convert_input_string:
{
- \tl_gset:Nx \g_str_result_tl
- {
- \exp_after:wN \str_convert_input_string_aux:wN
- \g_str_result_tl \prg_map_break: ^^M ?
- \prg_break_point:n { }
- }
- \tl_gset:Nx \g_str_result_tl
- {
- \exp_after:wN \str_convert_input_string_aux:wNNN
- \g_str_result_tl * ?? { ? \prg_map_break: }
- \prg_break_point:n { }
- }
+ \group_begin:
+ \int_set:Nn \tex_escapechar:D { 92 }
+ \tl_gset:Nx \g_str_result_tl
+ {
+ \exp_after:wN \str_convert_input_string_aux:wN
+ \g_str_result_tl \prg_map_break: ^^M ?
+ \prg_break_point:n { }
+ }
+ \tl_gset:Nx \g_str_result_tl
+ {
+ \exp_after:wN \str_convert_input_string_aux:wNNN
+ \g_str_result_tl * ?? { ? \prg_map_break: }
+ \prg_break_point:n { }
+ }
+ \group_end:
}
\cs_new_nopar:Npn \str_convert_input_string_aux:wNNN #1 *#2#3#4
}
@@ -2670,7 +2671,6 @@
% \end{macro}
%
% \begin{macro}[int]{\str_convert_output_url:}
-% \begin{macro}[int]{\str_convert_output_percent:}
% \begin{macro}[aux, rEXP]{\str_convert_output_url_aux:N}
% \begin{macro}[aux, rEXP]{\str_convert_output_url_aux:NTF}
%^^A todo: Figure out what bytes should be percent encoded! Safe could be
@@ -2705,7 +2705,6 @@
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
%
% \subsection{Encoding definition files}
%
@@ -2790,32 +2789,33 @@
\tl_gset:Nx \g_str_result_tl
{
\exp_after:wN \str_aux_from_utf_viii:N \g_str_result_tl
- { ? \prg_map_break: \q_stop \str_aux_from_utf_viii_error: }
+ { \prg_map_break: \str_aux_from_utf_viii_error: }
\prg_break_point:n { }
}
}
\cs_new_nopar:Npn \str_aux_from_utf_viii:N #1
{
- \use_none:n #1
- \int_use:N \int_eval:w
- \if_num:w `#1 < "C0 \exp_stop_f:
- \if_num:w `#1 < "80 \exp_stop_f:
- `#1
- \else:
- \msg_expandable_kernel_error:nnn
- { str } { utf8-extra-conti } {#1}
- \c_str_replacement_char_int
- \fi:
+ #1
+ \if_num:w `#1 < "C0 \exp_stop_f:
+ \exp_not:N \q_mark
+ \if_num:w `#1 < "80 \exp_stop_f:
+ \int_value:w `#1
\else:
- \if_num:w `#1 < "F5 \exp_stop_f:
- \exp_after:wN \str_aux_from_utf_viii:wwNN
- \int_use:N \int_eval:w `#1 - "C0
- \else:
- \msg_expandable_kernel_error:nnn
- { str } { utf8-invalid-byte } {#1}
- \c_str_replacement_char_int
- \fi:
+ \msg_expandable_kernel_error:nnn
+ { str } { utf8-extra-conti } {#1}
+ \int_use:N \c_str_replacement_char_int
\fi:
+ \else:
+ \if_num:w `#1 < "F5 \exp_stop_f:
+ \exp_after:wN \str_aux_from_utf_viii:wwNN
+ \int_use:N \int_eval:w `#1 - "C0
+ \else:
+ \exp_not:N \q_mark
+ \msg_expandable_kernel_error:nnn
+ { str } { utf8-invalid-byte } {#1}
+ \int_use:N \c_str_replacement_char_int
+ \fi:
+ \fi:
,
\use_none_delimit_by_q_stop:w
"80 , "800 , "10000 , "10FFFF ,
@@ -2824,7 +2824,7 @@
}
\cs_new_nopar:Npn \str_aux_from_utf_viii:wwNN #1 , #2 \q_stop #3#4
{
- \use_none_delimit_by_q_stop:w #4 \q_stop
+ \use_none:n #4
\exp_after:wN \str_aux_from_utf_viii:wNNww
\int_use:N \int_eval:w #1 * "40 + `#4 - "80 ,
#4
@@ -2837,8 +2837,9 @@
\if_num:w #1 < #4 \exp_stop_f:
\str_aux_from_utf_viii_error:w #1 ,
\else:
+ #2
\if_num:w #1 < #5 \exp_stop_f:
- #1
+ \exp_not:N \q_mark #1
\else:
\exp_after:wN \str_aux_from_utf_viii:wwNN
\int_use:N \int_eval:w #1 - #5
@@ -2852,12 +2853,14 @@
}
\cs_new_nopar:Npn \str_aux_from_utf_viii_error:w #1 ,
{
+ \exp_not:N \q_mark
\msg_expandable_kernel_error:nnn
{ str } { utf8-overlong } {#1}
\c_str_replacement_char_int
}
\cs_new_nopar:Npn \str_aux_from_utf_viii_error:NwN #1 #2 \q_stop #3
{
+ \exp_not:N \q_mark
\msg_expandable_kernel_error:nnn
{ str } { utf8-missing-conti } {#1}
\c_str_replacement_char_int ,
@@ -2865,6 +2868,7 @@
}
\cs_new_nopar:Npn \str_aux_from_utf_viii_error:
{
+ \exp_not:N \q_mark
\msg_expandable_kernel_error:nn
{ str } { utf8-premature-end }
\c_str_replacement_char_int ,
@@ -3033,13 +3037,13 @@
\if_meaning:w \scan_stop: #2
\str_aux_from_utf_xvi_end:Nw #1
\fi:
- \int_use:N \int_eval:w
- \if_case:w
- \int_eval:w ( \str_tmp:w #1#2 - "D6 ) / \c_four \int_eval_end:
- \or: \exp_after:wN \str_aux_from_utf_xvi:wNNN
- \or: \exp_after:wN \str_aux_from_utf_xvi_error:wNNw
- \fi:
- "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 ,
+ \if_case:w
+ \int_eval:w ( \str_tmp:w #1#2 - "D6 ) / \c_four \int_eval_end:
+ \or: #1#2 \exp_after:wN \str_aux_from_utf_xvi:wNNN
+ \or: \exp_after:wN \str_aux_from_utf_xvi_error:wNNw
+ \else: #1#2 \exp_not:N \q_mark \int_use:N \int_eval:w
+ \fi:
+ "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 ,
\str_aux_from_utf_xvi:NN
}
\cs_new:Npn \str_aux_from_utf_xvi:wNNN #1, #2#3#4
@@ -3048,8 +3052,10 @@
\str_aux_from_utf_xvi_end:wNNww #1 , #3
\fi:
\if_num:w \int_eval:w ( \str_tmp:w #3#4 - "DA ) / \c_four = \c_one
- ( #1 - "D800 ) * "400
- + \str_tmp:w #3#4 * "100 + \str_tmp:w #4#3 - "DC00 ,
+ #3 #4
+ \int_use:N \int_eval:w
+ ( #1 - "D800 ) * "400
+ + \str_tmp:w #3#4 * "100 + \str_tmp:w #4#3 - "DC00 ,
\exp_after:wN \use_i:nnn
\else:
\str_aux_from_utf_xvi_error:wNNw #1 ,
@@ -3058,6 +3064,7 @@
}
\cs_new:Npn \str_aux_from_utf_xvi_error:wNNw #1 \str_tmp:w #2#3 #4,
{
+ \exp_not:N \q_mark
\msg_expandable_kernel_error:nnn
{ str } { utf16-surrogate } { #2#3 }
\c_str_replacement_char_int ,
@@ -3067,15 +3074,17 @@
\fi:
\if_meaning:w \scan_stop: #1
\else:
+ #1 \exp_not:N \q_mark
\msg_expandable_kernel_error:nnn
{ str } { utf16-odd } {#1}
- \c_str_replacement_char_int ,
+ \int_use:N \c_str_replacement_char_int ,
\fi:
\prg_map_break:
}
\cs_new:Npn \str_aux_from_utf_xvi_end:wNNww #1 `#2#3 #4, #5 \fi:
{
\fi:
+ \exp_not:N \q_mark
\if_meaning:w \scan_stop: #5
\msg_expandable_kernel_error:nnn
{ str } { utf16-surrogate } { #2#3 }
@@ -3083,7 +3092,7 @@
\msg_expandable_kernel_error:nnn
{ str } { utf16-odd } { #2#3#5 }
\fi:
- \c_str_replacement_char_int ,
+ \int_use:N \c_str_replacement_char_int ,
\prg_map_break:
}
% \end{macrocode}
@@ -3215,31 +3224,33 @@
\cs_new:Npn \str_aux_from_utf_xxxii:NNNN #1#2#3#4
{
\if_meaning:w \scan_stop: #4
- \str_aux_from_utf_xxxii_end:ww #1#2#3#4
+ \exp_after:wN \str_aux_from_utf_xxxii_end:w
+ \fi:
+ #1#2#3#4 \exp_not:N \q_mark
+ \if_num:w \int_eval:w
+ \str_tmp:w #1#4 * \c_two_hundred_fifty_six
+ + \str_tmp:w #2#3
+ > \c_sixteen
+ \msg_expandable_kernel_error:nnn
+ { str } { utf32-overflow } { #1#2#3#4 }
+ \int_use:N \c_str_replacement_char_int
+ \else:
+ \int_use:N \int_eval:w
+ \str_tmp:w #2#3*"10000
+ + \str_tmp:w #3#2 * \c_two_hundred_fifty_six
+ + \str_tmp:w #4#1
\fi:
- \int_use:N \int_eval:w
- \if_num:w \int_eval:w
- \str_tmp:w #1#4 * \c_two_hundred_fifty_six
- + \str_tmp:w #2#3
- > \c_sixteen
- \msg_expandable_kernel_error:nnn
- { str } { utf32-overflow } { #1#2#3#4 }
- \c_str_replacement_char_int
- \else:
- \str_tmp:w #2#3*"10000
- + \str_tmp:w #3#2 * \c_two_hundred_fifty_six
- + \str_tmp:w #4#1
- \fi:
,
\str_aux_from_utf_xxxii:NNNN
}
- \cs_new:Npn \str_aux_from_utf_xxxii_end:ww #1 \scan_stop: #2 \fi:
+ \cs_new:Npn \str_aux_from_utf_xxxii_end:w #1 \scan_stop:
{
- \fi:
\tl_if_empty:nF {#1}
{
\msg_expandable_kernel_error:nnn
{ str } { utf32-truncated } { #1 }
+ #1 \exp_not:N \q_mark
+ \int_use:N \c_str_replacement_char_int ,
}
\prg_map_break:
}
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins
index cb3e893d52c..55b4066a306 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins
@@ -35,8 +35,10 @@ Do not distribute a modified version of this file.
\keepsilent
+\generate{\file{l3flag.sty} {\from{l3flag.dtx} {package}}}
\generate{\file{l3str.sty} {\from{l3str.dtx} {package}}}
\generate{\file{l3regex.sty} {\from{l3regex.dtx} {package}}}
+\generate{\file{l3tl-analysis.sty} {\from{l3tl-analysis.dtx} {package}}}
%\generate{\file{l3regex-trace.sty} {\from{l3regex.dtx} {package,trace}}}
% Escapings.
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx
new file mode 100644
index 00000000000..c3310545ac0
--- /dev/null
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx
@@ -0,0 +1,901 @@
+% \iffalse meta-comment
+%
+%% File: l3tl-analysis.dtx Copyright (C) 2011 The LaTeX3 Project
+%%
+%% It may be distributed and/or modified under the conditions of the
+%% LaTeX Project Public License (LPPL), either version 1.3c of this
+%% license or (at your option) any later version. The latest version
+%% of this license is in the file
+%%
+%% http://www.latex-project.org/lppl.txt
+%%
+%% This file is part of the "l3experimental bundle" (The Work in LPPL)
+%% and all files in that bundle must be distributed together.
+%%
+%% The released version of this bundle is available from CTAN.
+%%
+%% -----------------------------------------------------------------------
+%%
+%% The development version of the bundle can be found at
+%%
+%% http://www.latex-project.org/svnroot/experimental/trunk/
+%%
+%% for those people who are interested.
+%%
+%%%%%%%%%%%
+%% NOTE: %%
+%%%%%%%%%%%
+%%
+%% Snapshots taken from the repository represent work in progress and may
+%% not work or may contain conflicting material! We therefore ask
+%% people _not_ to put them into distributions, archives, etc. without
+%% prior consultation with the LaTeX3 Project.
+%%
+%% -----------------------------------------------------------------------
+%
+%<*driver|package>
+\RequirePackage{expl3}
+\GetIdInfo$Id: l3tl-analysis.dtx 3039 2011-12-08 09:22:35Z bruno $
+ {L3 Experimental Analysis of Token lists}
+%</driver|package>
+%<*driver>
+\documentclass[full]{l3doc}
+\usepackage{amsmath}
+\begin{document}
+ \DocInput{\jobname.dtx}
+\end{document}
+%</driver>
+% \fi
+%
+%
+% \title{^^A
+% The \textsf{l3tl-analysis} package: analysing token lists^^A
+% \thanks{This file describes v\ExplFileVersion,
+% last revised \ExplFileDate.}^^A
+% }
+%
+% \author{^^A
+% The \LaTeX3 Project\thanks
+% {^^A
+% E-mail:
+% \href{mailto:latex-team@latex-project.org}
+% {latex-team@latex-project.org}^^A
+% }^^A
+% }
+%
+% \date{Released \ExplFileDate}
+%
+% \maketitle
+%
+% \begin{documentation}
+%
+% \section{\pkg{l3tl-analysis} documentation}
+%
+% This module moslty provides internal functions for use
+% in the \pkg{l3regex} module. However, it provides as a
+% side-effect a user debugging function, very similar to
+% the \cs{ShowTokens} macro from the \pkg{ted} package.
+%
+% \begin{function}{\tl_show_analysis:N, \tl_show_analysis:n}
+% \begin{syntax}
+% \cs{tl_show_analysis:n} \Arg{token list}
+% \end{syntax}
+% Displays to the terminal the detailed decomposition of
+% the \meta{token list} into tokens,
+% showing the category code of each character token,
+% and the meaning of control sequences and active characters.
+% \end{function}
+%
+% \subsection{Internal}
+%
+% \begin{function}{\tl_set_analysis:Nn}
+% \begin{syntax}
+% \cs{tl_analysis:Nn} \meta{tl var} \Arg{token list}
+% \end{syntax}
+% This function analysiss each token in the \meta{token list},
+% converting each \meta{token} to
+% \begin{quote}
+% \Arg{tokens} \cs{q_mark} \meta{catcode} \meta{char code} |,|
+% \end{quote}
+% where \meta{tokens} expands in one step to the \meta{token},
+% and \meta{catcode} and \meta{char code} are the category code
+% and character code of the token if it is a character token,
+% and otherwise are $0$ and $-1$ respectively.
+% The result of the conversion is then stored in \meta{tl var}.
+% \end{function}
+%
+% \end{documentation}
+%
+% \begin{implementation}
+%
+% \section{\pkg{l3tl-analysis} implementation}
+%
+% \begin{macrocode}
+%<*initex|package>
+% \end{macrocode}
+%
+% \begin{macrocode}
+\ProvidesExplPackage
+ {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
+\RequirePackage{l3str}
+% \end{macrocode}
+%
+% \subsection{Variables and helper functions}
+%
+% \begin{variable}{\l_tl_analysis_normal_int}
+% The number of normal (\texttt{N}-type argument) tokens
+% since the last special token.
+% \begin{macrocode}
+\int_new:N \l_tl_analysis_normal_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_tl_analysis_index_int}
+% During the first pass, this is the index in the array being built.
+% During the second pass, it is equal to the maximum index in the
+% array from the first pass.
+% \begin{macrocode}
+\int_new:N \l_tl_analysis_index_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_tl_analysis_nesting_int}
+% Nesting depth of explicit begin-group and end-group characters
+% during the first pass. This lets us detect the end of the token
+% list without a reserved end-marker.
+% \begin{macrocode}
+\int_new:N \l_tl_analysis_nesting_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_tl_analysis_char_int}
+% Holds the character code of the token currently considered,
+% in cases where it is a character.
+% \begin{macrocode}
+\int_new:N \l_tl_analysis_char_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_tl_analysis_type_int}
+% When encountering special characters, we record their \enquote{type}
+% in this integer.
+% \begin{macrocode}
+\int_new:N \l_tl_analysis_type_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\g_tl_analysis_result_tl}
+% The result of the conversion is stored in this token list,
+% with a succession of items of the form
+% \begin{quote}
+% \meta{tokens} \cs{q_mark} \meta{catcode} \meta{char code} |,|
+% \end{quote}
+% \begin{macrocode}
+\tl_new:N \g_tl_analysis_result_tl
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{macro}[int]{\tl_analysis_extract_charcode:}
+% \begin{macro}[aux]{\tl_analysis_extract_charcode_aux:w}
+% Extracting the character code from the meaning of \cs{l_tl_analysis_token}.
+% This has no error checking, and should only be assumed to work
+% for begin-group and end-group tokens.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \tl_analysis_extract_charcode:
+ {
+ \exp_after:wN \tl_analysis_extract_charcode_aux:w
+ \token_to_meaning:N \l_tl_analysis_token
+ }
+\cs_new_protected_nopar:Npn \tl_analysis_extract_charcode_aux:w #1 ~ #2 ~ { ` }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int]{\tl_analysis_cs_space_count:NN}
+% \begin{macro}[aux]{\tl_analysis_cs_space_count:w}
+% \begin{macro}[aux]{\tl_analysis_cs_space_count_end:w}
+% Counts the number of spaces in the string representation of its
+% second argument, as well as the number of characters following
+% the last space in that representation, and feeds the two numbers
+% as semicolon-delimited arguments to the first argument.
+% \begin{macrocode}
+\cs_new:Npn \tl_analysis_cs_space_count:NN #1 #2
+ {
+ \exp_after:wN #1
+ \int_value:w \int_eval:w \c_zero
+ \exp_after:wN \tl_analysis_cs_space_count:w
+ \token_to_str:N #2
+ \fi: \tl_analysis_cs_space_count_end:w ; ~ !
+ }
+\cs_new:Npn \tl_analysis_cs_space_count:w #1 ~
+ {
+ \if_false: #1 #1 \fi:
+ + \c_one
+ \tl_analysis_cs_space_count:w
+ }
+\cs_new:Npn \tl_analysis_cs_space_count_end:w ; #1 \fi: #2 !
+ { \exp_after:wN ; \int_value:w \str_length_ignore_spaces:n {#1} ; }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \subsection{Plan of attack}
+%
+% Our goal is to produce a token list of the form roughly
+% \begin{quote}
+% \Arg{token 1} \cs{q_mark} \meta{catcode 1} \meta{char code 1} |,| \\
+% \Arg{token 2} \cs{q_mark} \meta{catcode 2} \meta{char code 2} |,| \\
+% \ldots{}
+% \Arg{token N} \cs{q_mark} \meta{catcode N} \meta{char code N} |,|
+% \end{quote}
+% Most but not all tokens can be grabbed as an undelimited
+% (\texttt{N}-type) argument by \TeX{}.
+% The plan is to have a two pass system. In the first pass,
+% locate special tokens, and store them in various \tn{toks} registers.
+% In the second pass, which is done within an \texttt{x}-expanding
+% assignment, normal tokens are taken in as \texttt{N}-type
+% arguments, and special tokens are retrieved from the \tn{toks}
+% registers, and removed from the input stream by some means.
+%
+% To ease the difficult first pass, we first do some setup
+% with \cs{tl_analysis_i_setup:n}.
+% Active characters set equal to non-active characters
+% cause trouble, so we disable all active characters by
+% setting them equal to \texttt{undefined}.
+% We also set there the escape character to be printable
+% (backslash, but this later changes to a forward slash):
+% this makes it possible to distinguish characters from
+% control sequences.
+%
+% A token has two characteristics: its \tn{meaning},
+% and what it looks like for \TeX{} when it is in
+% scanning mode (\emph{e.g.}, when capturing parameters
+% for a macro).
+% For our purposes, the token's meaning can be
+% \begin{itemize}
+% \item a begin-group token (category code $1$),
+% either space (character code $32$), or non-space;
+% \item an end-group token (category code $2$),
+% either space (character code $32$), or non-space;
+% \item a space token (category code $10$, character code $32$);
+%^^A \item a character with category code $3$, $4$, $6$, $7$, $8$, $11$,
+%^^A or $12$, or with category code $10$ but character code $\neq 32$;
+%^^A \item a non-character (primitive, macro, register, font,
+%^^A undefined, \emph{etc.}).
+% \item anything else (then the token is always an \texttt{N}-type argument).
+% \end{itemize}
+% The token itself can \enquote{look like} one of the following
+% \begin{itemize}
+% \item a non-active character, in which case its meaning
+% is automatically that associated to its character code
+% and category code, we call it \enquote{true} character;
+% \item an active character;
+% \item a control sequence.
+% \end{itemize}
+% The only tokens which are not valid \texttt{N}-type arguments are
+% true begin-group characters, true end-group characters, and true spaces.
+% We will detect those characters by scanning ahead with \tn{futurelet},
+% then distinguishing true characters from tokens set equal to them
+% using the \tn{string} representation.
+%
+% The second pass is a simple exercise in expandable loops.
+%
+% \begin{macro}[int]{\tl_analysis:n}
+% \begin{macro}[int]{\tl_set_analysis:Nn}
+% Everything is done within a group, and all definitions will be local.
+% We use \cs{group_align_safe_begin/end:} to avoid problems in case
+% \cs{tl_analysis:n} is used within an alignment and its argument
+% contains alignment tab tokens.
+% \begin{macrocode}
+\cs_new_protected:Npn \tl_analysis:n #1
+ {
+ \group_begin:
+ \group_align_safe_begin:
+ \tl_analysis_setup:n {#1}
+ \tl_analysis_i:n {#1}
+ \tl_analysis_ii:n {#1}
+ \group_align_safe_end:
+ \group_end:
+ }
+\cs_new_protected:Npn \tl_set_analysis:Nn #1#2
+ {
+ \tl_analysis:n {#2}
+ \tl_set_eq:NN #1 \g_tl_analysis_result_tl
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \subsection{Setup}
+%
+% \begin{macro}[int]{\tl_analysis_setup:n}
+% \begin{macro}[aux]{\tl_analysis_disable_loop:N}
+% Active characters can cause problems later on in the processing,
+% so the first step is to disable them, by setting them to
+% \texttt{undefined}. Since Unicode contains too many characters
+% to loop over all of them, we instead loop over the input token
+% list as a string: any active character in the token list
+% must appear in its string representation.
+% ^^A todo: ask LuaTeX list for an \ifx\undefined <active char>
+% ^^A which does not add the <active char> in memory.
+% \begin{macrocode}
+\cs_new_protected:Npn \tl_analysis_setup:n #1
+ {
+ \int_set:Nn \tex_escapechar:D { 92 }
+ \exp_after:wN \tl_analysis_disable_loop:N
+ \tl_to_str:n {#1} { ~ } { ? ~ \prg_map_break: }
+ \prg_break_point:n { }
+ }
+\group_begin:
+ \char_set_catcode_active:N \^^@
+ \cs_new_protected_nopar:Npn \tl_analysis_disable_loop:N #1
+ {
+ \tex_lccode:D \c_zero `#1 ~
+ \tl_to_lowercase:n { \tex_let:D ^^@ } \c_undefined:D
+ \tl_analysis_disable_loop:N
+ }
+\group_end:
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \subsection{First pass}
+%
+% The goal of this pass is to detect special (non-\texttt{N}-type) tokens,
+% and count how many \texttt{N}-type tokens lie between special tokens.
+% Also, we wish to store some representation of each special token
+% in a \tn{toks} register.
+%
+% After the setup step, we have $11$ types of tokens:
+% \begin{itemize}
+% \item[1] a true non-space begin-group character;
+% \item[2] a true space begin-group character;
+% \item[3] a true non-space end-group character;
+% \item[4] a true space end-group character;
+% \item[5] a true space blank space character;
+% \item[6] an undefined active character;
+% \item[7] any other true character;
+% \item[8] a control sequence equal to a begin-group token (category code $1$);
+% \item[9] a control sequence equal to an end-group token (category code $2$);
+% \item[10] a control sequence equal to a space token
+% (character code $32$, category code $10$);
+% \item[11] any other control sequence.
+% \end{itemize}
+% Our first tool is \tn{futurelet}. This cannot distinguish
+% cases $1$ and $2$ from $8$, nor cases $3$ and $4$ from $9$,
+% nor case $5$ from case $10$. Those cases will be distinguished
+% by applying the \tn{string} primitive to the following token,
+% after possibly changing the escape character to ensure that
+% a control sequence's string representation cannot be mistaken
+% for the true character.
+%
+% In cases $6$, $7$, and $11$, the following token is a valid
+% \texttt{N}-type argument, so we grab it and distinguish the case
+% of a character from a control sequence: in the latter case,
+% \cs{str_tail:n} \Arg{token} is non-empty, because the
+% escape character is printable.
+%
+% \begin{macro}{\tl_analysis_i:n}
+% We read tokens one by one using \tn{futurelet}.
+% While performing the loop, we keep track of the number of
+% true begin-group characters minus the number of
+% true end-group characters in \cs{l_tl_analysis_nesting_int}.
+% This reaches $-1$ when we read the closing brace.
+% \begin{macrocode}
+\cs_new_protected:Npn \tl_analysis_i:n #1
+ {
+ \int_zero:N \l_tl_analysis_normal_int
+ \int_zero:N \l_tl_analysis_index_int
+ \int_zero:N \l_tl_analysis_nesting_int
+ \if_false: { \fi: \tl_analysis_i_loop:w #1 }
+ \int_decr:N \l_tl_analysis_index_int
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[int]{\tl_analysis_i_loop:w}
+% Read one character and check its type.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \tl_analysis_i_loop:w
+ { \tex_futurelet:D \l_tl_analysis_token \tl_analysis_i_type:w }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[int]{\tl_analysis_i_type:w}
+% At this point, \cs{l_tl_analysis_token} holds the meaning of the following
+% token. We store in \cs{l_tl_analysis_type_int} the meaning of the token ahead:
+% \begin{itemize}
+% \item[0] space token;
+% \item[1] begin-group token;
+% \item[-1] end-group token;
+% \item[2] other.
+% \end{itemize}
+% The values $0$, $1$, $-1$ correspond to how much a true such character
+% changes the nesting level ($2$ is used only here, and is irrelevant later).
+% Then call the auxiliary for each case.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \tl_analysis_i_type:w
+ {
+ \l_tl_analysis_type_int =
+ \if_meaning:w \l_tl_analysis_token \c_space_token
+ \c_zero
+ \else:
+ \if_catcode:w \exp_not:N \l_tl_analysis_token \c_group_begin_token
+ \c_one
+ \else:
+ \if_catcode:w \exp_not:N \l_tl_analysis_token \c_group_end_token
+ \c_minus_one
+ \else:
+ \c_two
+ \fi:
+ \fi:
+ \fi:
+ \if_case:w \l_tl_analysis_type_int
+ \exp_after:wN \tl_analysis_i_space:w
+ \or: \exp_after:wN \tl_analysis_i_bgroup:w
+ \or: \exp_after:wN \tl_analysis_i_safe:N
+ \else: \exp_after:wN \tl_analysis_i_egroup:w
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[int]{\tl_analysis_i_safe:N}
+% \begin{macro}[aux]{\tl_analysis_i_cs:ww}
+% The upcoming token is safe: we can simply grab it in a second pass.
+% Since other branches of the code must pass their tokens through
+% \tn{string}, we do it here as well. It's been optimized in an evil way,
+% sorry.
+%^^A todo: document
+% \begin{macrocode}
+\cs_new_protected:Npn \tl_analysis_i_safe:N #1
+ {
+ \if_charcode:w
+ \scan_stop:
+ \exp_after:wN \use_none:n \token_to_str:N #1 \prg_do_nothing:
+ \scan_stop:
+ \int_incr:N \l_tl_analysis_normal_int
+ \else:
+ \tl_analysis_cs_space_count:NN \tl_analysis_i_cs:ww #1
+ \fi:
+ \tl_analysis_i_loop:w
+ }
+\cs_new_protected:Npn \tl_analysis_i_cs:ww #1; #2;
+ {
+ \if_num:w #1 > \c_zero
+ \tex_skip:D \l_tl_analysis_index_int
+ = \int_eval:w \l_tl_analysis_normal_int + \c_one sp \scan_stop:
+ \tex_advance:D \l_tl_analysis_index_int #1 \exp_stop_f:
+ \l_tl_analysis_normal_int #2 \exp_stop_f:
+ \else:
+ \tex_advance:D \l_tl_analysis_normal_int #2 \exp_stop_f:
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[aux]{\tl_analysis_i_space:w}
+% \begin{macro}[aux]{\tl_analysis_i_space_test:w}
+% In this branch, the following token's meaning is a blank space.
+% Apply \tn{string} to that token: if it is a control sequence
+% the result starts with the escape character; otherwise it is
+% a true blank space, whose string representation is also a blank space.
+% We test for that in \cs{tl_analysis_i_space_test:w},
+% after grabbing the first character of the string representation
+% as \cs{l_tl_analysis_token}.
+% Also, since \cs{tl_analysis_i_store:} expects the special token to be
+% stored in the relevant \tn{toks} register, we do that. The extra
+% \cs{exp_not:n} is unnecessary of course, but it makes the treatment
+% of all tokens more homogeneous.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \tl_analysis_i_space:w
+ {
+ \tex_afterassignment:D \tl_analysis_i_space_test:w
+ \exp_after:wN \cs_set_eq:NN
+ \exp_after:wN \l_tl_analysis_token
+ \token_to_str:N
+ }
+\cs_new_protected_nopar:Npn \tl_analysis_i_space_test:w
+ {
+ \if_meaning:w \l_tl_analysis_token \c_space_token
+ \tex_toks:D \l_tl_analysis_index_int { \exp_not:n { ~ } }
+ \tl_analysis_i_store:
+ \else:
+ \int_incr:N \l_tl_analysis_normal_int
+ \fi:
+ \tl_analysis_i_loop:w
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int]{\tl_analysis_i_bgroup:w}
+% \begin{macro}[int]{\tl_analysis_i_egroup:w}
+% \begin{macro}[aux]{\tl_analysis_i_group:nw}
+% \begin{macro}[aux]{\tl_analysis_i_group_test:w}
+% The token might be either a true character token with
+% catcode $1$ or $2$, or it could be a control sequence.
+% The only tricky case is if the character code happens
+% to be equal to the escape character: then we change
+% the escape character, so that the string representation
+% of the true character and of a control sequence set equal
+% to it start differently.
+% \begin{macrocode}
+\group_begin:
+ \char_set_catcode_group_begin:N \^^@
+ \char_set_catcode_group_end:N \^^E
+ \cs_new_protected_nopar:Npn \tl_analysis_i_bgroup:w
+ { \tl_analysis_i_group:nw { \exp_after:wN ^^@ \if_false: ^^E \fi: } }
+ \char_set_catcode_group_begin:N \^^B
+ \char_set_catcode_group_end:N \^^@
+ \cs_new_protected_nopar:Npn \tl_analysis_i_egroup:w
+ { \tl_analysis_i_group:nw { \if_false: ^^B \fi: ^^@ } }
+\group_end:
+\cs_new_protected_nopar:Npn \tl_analysis_i_group:nw #1
+ {
+ \l_tl_analysis_char_int = \tl_analysis_extract_charcode: \scan_stop:
+ \tex_lccode:D \c_zero \l_tl_analysis_char_int
+ \tl_to_lowercase:n { \tex_toks:D \l_tl_analysis_index_int {#1} }
+ \if_num:w \l_tl_analysis_char_int = \tex_escapechar:D
+ \int_set:Nn \tex_escapechar:D { 139 - \tex_escapechar:D }
+ \fi:
+ \tex_afterassignment:D \tl_analysis_i_group_test:w
+ \exp_after:wN \cs_set_eq:NN
+ \exp_after:wN \l_tl_analysis_token
+ \token_to_str:N
+ }
+\cs_new_protected_nopar:Npn \tl_analysis_i_group_test:w
+ {
+ \if_num:w \tl_analysis_extract_charcode: = \l_tl_analysis_char_int
+ \tl_analysis_i_store:
+ \else:
+ \int_incr:N \l_tl_analysis_normal_int
+ \fi:
+ \tl_analysis_i_loop:w
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int]{\tl_analysis_i_store:}
+% This function is called each time we meet a special token,
+% and the value of \cs{l_tl_analysis_type_int} indicates which case
+% we are in:
+% \begin{itemize}
+% \item[-1] end-group character;
+% \item[0] space character;
+% \item[1] begin-group character.
+% \end{itemize}
+% We need to distinguish further the case of a space character
+% (code $32$) from other character codes, because those will
+% behave differently in the second pass. Namely, we change the
+% cases above to
+% \begin{itemize}
+% \item[-2] space end-group character;
+% \item[-1] non-space end-group character;
+% \item[0] space blank space character;
+% \item[1] non-space begin-group character;
+% \item[2] space begin-group character.
+% \end{itemize}
+% This has the property that non-space characters correspond to odd
+% values of \cs{l_tl_analysis_type_int}.
+% Also, the \tn{toks} register number \cs{l_tl_analysis_index_int}
+% holds a token list which expands to the given special token.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \tl_analysis_i_store:
+ {
+ \tex_advance:D \l_tl_analysis_nesting_int \l_tl_analysis_type_int
+ \if_num:w \l_tl_analysis_char_int = \c_thirty_two
+ \tex_multiply:D \l_tl_analysis_type_int \c_two
+ \fi:
+ \tex_skip:D \l_tl_analysis_index_int
+ = \l_tl_analysis_normal_int sp plus \l_tl_analysis_type_int sp \scan_stop:
+ \int_incr:N \l_tl_analysis_index_int
+ \int_zero:N \l_tl_analysis_normal_int
+ \if_num:w \l_tl_analysis_nesting_int = \c_minus_one
+ \cs_set_eq:NN \tl_analysis_i_loop:w \scan_stop:
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \subsection{Second pass}
+%
+% The second pass is an exercise in expandable loops.
+%
+% \begin{macro}[int]{\tl_analysis_ii:n}
+% \begin{macro}[aux]{\tl_analysis_ii_loop:w}
+% Start the loop with the index $0$. No need for an end-marker:
+% the loop will stop by itself when the last index is read.
+% We will repeatedly oscillate between reading long stretches
+% of normal tokens, and reading special tokens.
+% \begin{macrocode}
+\cs_new_protected:Npn \tl_analysis_ii:n #1
+ {
+ \tl_gset:Nx \g_tl_analysis_result_tl
+ {
+ \tl_analysis_ii_loop:w 0; #1
+ \prg_break_point:n { }
+ }
+ }
+\cs_new:Npn \tl_analysis_ii_loop:w #1;
+ {
+ \exp_after:wN \tl_analysis_ii_normals:ww
+ \int_value:w \tex_skip:D #1 ; #1 ;
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[aux]{\tl_analysis_ii_normals:ww}
+% \begin{macro}[aux]{\tl_analysis_ii_normal:wwN}
+% The first argument is the number of normal tokens which remain
+% to be read, and the second argument is the index in the array
+% produced in the first step.
+% A character's string representation is always one character long,
+% while a control sequence is always longer (we have set the escape
+% character to a printable value). In both cases, we leave
+% \cs{exp_not:n} \Arg{token} \cs{q_mark} in the input stream
+% (after \texttt{x}-expansion).
+% \begin{macrocode}
+\cs_new:Npn \tl_analysis_ii_normals:ww #1;
+ {
+ \if_num:w #1 = \c_zero
+ \tl_analysis_ii_special:w
+ \fi:
+ \tl_analysis_ii_normal:wwN #1;
+ }
+\cs_new:Npn \tl_analysis_ii_normal:wwN #1; #2; #3
+ {
+ \exp_not:n { \exp_not:n { #3 } } \exp_not:N \q_mark
+ \if_charcode:w
+ \scan_stop:
+ \exp_after:wN \use_none:n \token_to_str:N #3 \prg_do_nothing:
+ \scan_stop:
+ \exp_after:wN \tl_analysis_ii_char:Nww
+ \else:
+ \exp_after:wN \tl_analysis_ii_cs:Nww
+ \fi:
+ #3 #1; #2;
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[aux]{\tl_analysis_ii_char:Nww}
+% If the normal token we grab is a character, leave
+% \meta{catcode} \meta{charcode} followed by a comma
+% in the input stream, and call \cs{tl_analysis_ii_normals:ww}
+% with its first argument decremented.
+% \begin{macrocode}
+\cs_new:Npn \tl_analysis_ii_char:Nww #1
+ {
+ \if_meaning:w #1 \c_undefined:D D \else:
+ \if_catcode:w #1 \c_catcode_other_token C \else:
+ \if_catcode:w #1 \c_catcode_letter_token B \else:
+ \if_catcode:w #1 \c_math_toggle_token 3 \else:
+ \if_catcode:w #1 \c_alignment_token 4 \else:
+ \if_catcode:w #1 \c_math_superscript_token 7 \else:
+ \if_catcode:w #1 \c_math_subscript_token 8 \else:
+ 6
+ \fi: \fi: \fi: \fi: \fi: \fi: \fi:
+ \int_value:w `#1 ,
+ \exp_after:wN \tl_analysis_ii_normals:ww
+ \int_use:N \int_eval:w \c_minus_one +
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\tl_analysis_ii_cs:Nww}
+% If the token we grab is a control sequence, leave
+% |0-1,| (as category code and character code) in the input stream,
+% and call \cs{tl_analysis_ii_normals:ww} with updated arguments.
+% \begin{macrocode}
+\cs_new:Npn \tl_analysis_ii_cs:Nww #1
+ {
+ 0 -1 ,
+ \tl_analysis_cs_space_count:NN \tl_analysis_ii_cs_test:ww #1
+ }
+\cs_new:Npn \tl_analysis_ii_cs_test:ww #1 ; #2 ; #3; #4;
+ {
+ \exp_after:wN \tl_analysis_ii_normals:ww
+ \int_use:N \int_eval:w
+ \if_num:w #1 = \c_zero
+ #3
+ \else:
+ \tex_skip:D \int_eval:w #4 + #1 \int_eval_end:
+ \fi:
+ - #2
+ \exp_after:wN ;
+ \int_use:N \int_eval:w #4 + #1;
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\tl_analysis_ii_special:w}
+% \begin{macro}[aux]{\tl_analysis_ii_special_char:wN}
+% \begin{macro}[aux]{\tl_analysis_ii_special_space:w}
+% Here, |#1| is the current index in the array built in the first pass.
+% Check whether we reached the end (we shouldn't insert the trailing
+% end-group character that marked the end of the token list in the
+% first pass).
+% Unpack the \tn{toks} register: when \texttt{x}-expanding again,
+% we will get the special token.
+% Then leave the category code in the input stream, followed by
+% the character code, and call \cs{tl_analysis_ii_loop:w} with the next index.
+% \begin{macrocode}
+\cs_new:Npn \tl_analysis_ii_special:w \fi: \tl_analysis_ii_normal:wwN 0; #1;
+ {
+ \fi:
+ \if_num:w #1 = \l_tl_analysis_index_int
+ \exp_after:wN \prg_map_break:
+ \fi:
+ \tex_the:D \tex_toks:D #1 \exp_stop_f: \exp_not:N \q_mark
+ \if_case:w \etex_gluestretch:D \tex_skip:D #1 \exp_stop_f:
+ A
+ \or: 1
+ \or: 1
+ \else: 2
+ \fi:
+ \if_int_odd:w \etex_gluestretch:D \tex_skip:D #1 \exp_stop_f:
+ \exp_after:wN \tl_analysis_ii_special_char:wN \int_use:N
+ \else:
+ \exp_after:wN \tl_analysis_ii_special_space:w \int_use:N
+ \fi:
+ \int_eval:w \c_one + #1 \exp_after:wN ;
+ \token_to_str:N
+ }
+\cs_new:Npn \tl_analysis_ii_special_char:wN #1 ; #2
+ {
+ \int_value:w `#2 ,
+ \tl_analysis_ii_loop:w #1 ;
+ }
+\cs_new:Npn \tl_analysis_ii_special_space:w #1 ; ~
+ {
+ 32 ,
+ \tl_analysis_ii_loop:w #1 ;
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \subsection{Showing the results}
+%
+% \begin{macro}{\tl_show_analysis:N}
+% \begin{macro}{\tl_show_analysis:n}
+% Add to \cs{tl_analysis:n} a third pass to display tokens to the terminal.
+% \begin{macrocode}
+\cs_new_protected:Npn \tl_show_analysis:N #1
+ {
+ \exp_args:No \tl_analysis:n {#1}
+ \msg_aux_show:Nnx #1
+ { tl-analysis }
+ {
+ \exp_after:wN \tl_show_analysis_loop:wNw \g_tl_analysis_result_tl
+ \q_mark { ? \prg_map_break: } ,
+ \prg_break_point:n { }
+ }
+ }
+\cs_new_protected:Npn \tl_show_analysis:n #1
+ {
+ \tl_set:Nn \l_tl_tmpa_tl {#1}
+ \tl_show_analysis:N \l_tl_tmpa_tl
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[aux, EXP]{\tl_show_analysis_loop:wNw}
+% Here, |#1| \texttt{o}-expands to the token;
+% |#2| is the category code (one hexadecimal digit),
+% $0$ for control sequences;
+% |#3| is the character code, which we ignore.
+% In the cases of control sequences and active characters,
+% the meaning may overflow one line, and we want to truncate
+% it. Those cases are thus separated out.
+% \begin{macrocode}
+\cs_new:Npn \tl_show_analysis_loop:wNw #1 \q_mark #2 #3,
+ {
+ \use_none:n #2
+ \iow_newline: > \c_space_tl \c_space_tl
+ \if_num:w "#2 = \c_zero
+ \exp_after:wN \tl_show_analysis_cs:n
+ \else:
+ \if_num:w "#2 = \c_thirteen
+ \exp_after:wN \exp_after:wN
+ \exp_after:wN \tl_show_analysis_active:n
+ \else:
+ \exp_after:wN \exp_after:wN
+ \exp_after:wN \tl_show_analysis_normal:n
+ \fi:
+ \fi:
+ {#1}
+ \tl_show_analysis_loop:wNw
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\tl_show_analysis_normal:n}
+% Non-active characters are a simple matter of printing
+% the character, and its meaning. One can check that
+% begin-group and end-group characters do not mess up
+% \TeX{}'s alignment status.
+% \begin{macrocode}
+\cs_new:Npn \tl_show_analysis_normal:n #1
+ {
+ \exp_after:wN \token_to_str:N #1 ~
+ ( \exp_after:wN \token_to_meaning:N #1 )
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\tl_show_analysis_cs:n}
+% \begin{macro}[aux]{\tl_show_analysis_active:n}
+% \begin{macro}[aux]{\tl_show_analysis_long:nn}
+% \begin{macro}[aux]{\tl_show_analysis_long_aux:nnn}
+% Control sequences and active characters are printed in the same way,
+% making sure not to go beyond the \cs{l_iow_line_length_int}. In case
+% of an overflow, we replace the last characters by
+% \cs{c_tl_show_analysis_etc_str}.
+% \begin{macrocode}
+\cs_new:Npn \tl_show_analysis_cs:n #1
+ { \exp_args:No \tl_show_analysis_long:nn {#1} { control~sequence= } }
+\cs_new:Npn \tl_show_analysis_active:n #1
+ { \exp_args:No \tl_show_analysis_long:nn {#1} { active~character= } }
+\cs_new:Npn \tl_show_analysis_long:nn #1
+ {
+ \exp_args:Noo \tl_show_analysis_long_aux:nnn
+ { \token_to_str:N #1 }
+ { \token_to_meaning:N #1 }
+ }
+\cs_new_nopar:Npn \tl_show_analysis_long_aux:nnn #1#2#3
+ {
+ \int_compare:nNnTF
+ { \str_length:n { >>> #1 ~ ( #3 #2 ) } }
+ > { \l_iow_line_length_int }
+ {
+ \str_substr:nnn { >>> #1 ~ ( #3 #2 ) } \c_three
+ { \l_iow_line_length_int - \str_length:N \c_tl_show_analysis_etc_str }
+ \c_tl_show_analysis_etc_str
+ }
+ {#1~(#3#2)}
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{variable}{\c_tl_show_analysis_etc_str}
+% When a control sequence (or active character)
+% and its meaning are too long to fit in one line
+% of the terminal, the end is replaced by this token list.
+% \begin{macrocode}
+\str_const:Nx \c_tl_show_analysis_etc_str % (
+ { \iow_char:N \\ETC. ) }
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{macrocode}
+\msg_kernel_new:nnn { tl-analysis } { show }
+ {
+ Token~list~
+ \str_if_eq:nnF {#1} { \l_tl_tmpa_tl } { \token_to_str:N #1~}
+ \tl_if_empty:NTF #1
+ { is~empty }
+ { contains~the~tokens: }
+ }
+% \end{macrocode}
+%
+% \begin{macrocode}
+%</initex|package>
+% \end{macrocode}
+%
+% \end{implementation}
+%
+% \PrintIndex
diff --git a/Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx b/Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx
index 57fbb9ecca4..0d3fdafcdf8 100644
--- a/Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx
@@ -37,7 +37,7 @@
%
%<*driver|package>
\RequirePackage{l3names}
-\GetIdInfo$Id: l3galley.dtx 2938 2011-10-25 18:41:20Z joseph $
+\GetIdInfo$Id: l3galley.dtx 3071 2011-12-20 20:14:00Z bruno $
{L3 Experimental galley code}
%</driver|package>
%<*driver>
@@ -653,17 +653,6 @@
\package_check_loaded_expl:
%</package>
% \end{macrocode}
-%
-% \subsection{TO BE MOVED}
-%
-% \begin{macro}{\scan_marker_new:N, \scan_marker_new:c}
-% Temporary: needs to go somewhere else if this concept is retained.
-% \begin{macrocode}
-\cs_new_protected:Npn \scan_marker_new:N #1
- { \cs_new_eq:NN #1 \scan_stop: }
-\cs_generate_variant:Nn \scan_marker_new:N { c }
-% \end{macrocode}
-% \end{macro}
%
% \subsection{Support items}
%
@@ -1049,11 +1038,11 @@
% \end{macro}
%
% \subsection{The \cs{par} token}
-%
+%
% \begin{variable}{\s_par_omit}
% Used to indicate that a paragraph should be omitted.
% \begin{macrocode}
-\scan_marker_new:N \s_par_omit
+\scan_new:N \s_par_omit
% \end{macrocode}
% \end{variable}
%
@@ -1085,7 +1074,7 @@
}
}
% \end{macrocode}
-% No marker, so really insert a paragraph. In vertical mode,
+% No marker, so really insert a paragraph. In vertical mode,
% \begin{macrocode}
\cs_new_protected_nopar:Npn \galley_std_par_aux_ii:
{
diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3flag.sty b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3flag.sty
new file mode 100644
index 00000000000..40e2c899682
--- /dev/null
+++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3flag.sty
@@ -0,0 +1,100 @@
+%%
+%% This is file `l3flag.sty',
+%% generated with the docstrip utility.
+%%
+%% The original source files were:
+%%
+%% l3flag.dtx (with options: `package')
+%%
+%% EXPERIMENTAL CODE
+%%
+%% Do not distribute this file without also distributing the
+%% source files specified above.
+%%
+%% Do not distribute a modified version of this file.
+%%
+%% File: l3flag.dtx Copyright (C) 2011 The LaTeX3 Project
+%%
+%% It may be distributed and/or modified under the conditions of the
+%% LaTeX Project Public License (LPPL), either version 1.3c of this
+%% license or (at your option) any later version. The latest version
+%% of this license is in the file
+%%
+%% http://www.latex-project.org/lppl.txt
+%%
+%% This file is part of the "l3experimental bundle" (The Work in LPPL)
+%% and all files in that bundle must be distributed together.
+%%
+%% The released version of this bundle is available from CTAN.
+%%
+%% -----------------------------------------------------------------------
+%%
+%% The development version of the bundle can be found at
+%%
+%% http://www.latex-project.org/svnroot/experimental/trunk/
+%%
+%% for those people who are interested.
+%%
+%%%%%%%%%%%
+%% NOTE: %%
+%%%%%%%%%%%
+%%
+%% Snapshots taken from the repository represent work in progress and may
+%% not work or may contain conflicting material! We therefore ask
+%% people _not_ to put them into distributions, archives, etc. without
+%% prior consultation with the LaTeX3 Project.
+%%
+%% -----------------------------------------------------------------------
+\RequirePackage{expl3}
+\GetIdInfo$Id: l3flag.dtx 3039 2011-12-08 09:22:35Z bruno $
+ {L3 Experimental Flags}
+\ProvidesExplPackage
+ {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
+\tl_new:N \g_flag_list_tl
+\cs_new_protected:Npn \flag_new:N #1
+ {
+ \tl_if_in:NnTF \g_flag_list_tl {#1}
+ {
+ \msg_kernel_error:nnx
+ { flag } { already-defined }
+ { \token_to_str:N #1 }
+ }
+ {
+ \tl_gput_right:Nn \g_flag_list_tl {#1}
+ \cs_new_eq:NN #1 \c_undefined:D
+ }
+ }
+\cs_generate_variant:Nn \flag_new:N { c }
+\prg_new_conditional:Npnn \flag_test:N #1 { p , T , F , TF }
+ {
+ \if_cs_exist:N #1
+ \prg_return_true:
+ \else:
+ \prg_return_false:
+ \fi:
+ }
+\prg_new_conditional:Npnn \flag_test:c #1 { p , T , F , TF }
+ {
+ \if_cs_exist:w #1 \cs_end:
+ \prg_return_true:
+ \else:
+ \prg_return_false:
+ \fi:
+ }
+\cs_new:Npn \flag_raise:N #1
+ { \exp_after:wN \use_none:n \cs:w \cs_to_str:N #1 \cs_end: }
+\cs_new:Npn \flag_raise:c #1
+ { \exp_after:wN \use_none:n \cs:w #1 \cs_end: }
+\cs_new_protected:Npn \flag_lower:N #1
+ { \cs_set_eq:NN #1 \c_undefined:D }
+\cs_generate_variant:Nn \flag_lower:N { c }
+\msg_kernel_new:nnnn { flag } { already-defined }
+ { The~control~sequence~#1~is~already~declared~as~a~flag. }
+ {
+ LaTeX~was~asked~to~define~the~flag~#1,~but~it~has~already~
+ been~defined~as~a~flag.~The~flag~module~is~mostly~meant~
+ for~kernel~use,~and~booleans~should~be~preferred.
+ }
+%%
+%%
+%% End of file `l3flag.sty'.
diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex.sty b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex.sty
index f3dc0b5a358..facf764c168 100644
--- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex.sty
+++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex.sty
@@ -46,17 +46,19 @@
%%
%% -----------------------------------------------------------------------
\RequirePackage{expl3}
-\GetIdInfo$Id: l3regex.dtx 3017 2011-11-28 00:10:02Z bruno $
+\GetIdInfo$Id: l3regex.dtx 3073 2011-12-21 01:53:06Z bruno $
{L3 Experimental Regular Expressions}
\ProvidesExplPackage
{\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
-\RequirePackage{l3str}
+\RequirePackage{l3str, l3tl-analysis, l3flag}
\cs_new:Npn \regex_tmp:w { }
-\tl_new:N \g_regex_tmpa_tl
\tl_new:N \l_regex_tmpa_tl
\tl_new:N \l_regex_tmpb_tl
+\tl_new:N \l_regex_tmpc_tl
\int_new:N \l_regex_tmpa_int
\int_new:N \l_regex_tmpb_int
+\flag_new:N \l_regex_group_begin_flag
+\flag_new:N \l_regex_group_end_flag
\int_new:N \l_regex_max_state_int
\int_new:N \l_regex_left_state_int
\int_new:N \l_regex_right_state_int
@@ -67,7 +69,10 @@
\int_new:N \l_regex_capturing_group_int
\seq_new:N \l_regex_capturing_group_seq
\int_new:N \l_regex_capturing_group_max_int
+\tl_new:N \l_regex_build_or_class_tl
\tl_new:N \l_regex_one_or_group_tl
+\bool_new:N \l_regex_in_csname_bool
+\tl_new:N \l_regex_tmpa_regex_tl
\cs_new_eq:NN \regex_build_tmp_class:n \use_none:n
\bool_new:N \l_regex_class_bool
\tl_new:N \l_regex_class_tl
@@ -133,12 +138,16 @@
\regex_item_geq:n { \c_eleven } % > lf
\regex_item_range:nn { \c_zero } { \c_nine } % < lf
}
-\tl_new:N \l_regex_query_other_str
+\int_new:N \l_regex_nesting_int
+\int_new:N \l_regex_min_step_int
+\int_new:N \l_regex_max_step_int
\int_new:N \l_regex_current_step_int
\int_new:N \l_regex_start_step_int
\int_new:N \l_regex_success_step_int
-\int_new:N \l_regex_unique_step_int
+\int_new:N \l_regex_unique_id_int
\int_new:N \l_regex_current_char_int
+\int_new:N \l_regex_current_catcode_int
+\tl_new:N \l_regex_current_token_tl
\int_new:N \l_regex_last_char_int
\int_new:N \l_regex_case_changed_char_int
\bool_new:N \l_regex_caseless_bool
@@ -147,23 +156,23 @@
\prop_new:N \l_regex_success_submatches_prop
\int_new:N \l_regex_max_index_int
\tl_new:N \l_regex_every_match_tl
-\bool_new:N \l_regex_success_bool
+\bool_new:N \g_regex_success_bool
+\bool_new:N \l_regex_saved_success_bool
+\bool_new:N \l_regex_success_match_bool
\cs_new_protected:Npn \regex_last_match_empty_no:F #1 {#1}
\cs_new_protected:Npn \regex_last_match_empty_yes:F
{ \int_compare:nNnF \l_regex_start_step_int = \l_regex_current_step_int }
\cs_new_eq:NN \regex_last_match_empty:F \regex_last_match_empty_no:F
\bool_new:N \l_regex_success_empty_bool
\bool_new:N \l_regex_fresh_thread_bool
-\seq_new:N \g_regex_submatches_seq
-\int_new:N \g_regex_match_count_int
-\seq_new:N \g_regex_extract_all_seq
-\seq_new:N \g_regex_split_seq
-\tl_new:N \l_regex_replacement_tl
-\tl_new:N \g_regex_replaced_str
-\cs_new_eq:NN \s_regex_stop \scan_stop:
+\int_new:N \l_regex_replacement_int
+\int_new:N \l_regex_replacement_csnames_int
+\int_new:N \l_regex_submatch_int
+\int_new:N \l_regex_submatch_start_int
+\int_new:N \l_regex_match_count_int
\cs_new_protected:Npn \regex_toks_put_left:Nx #1#2
{
- \cs_set_nopar:Npx \regex_tmp:w \s_regex_stop { \s_regex_stop #2 }
+ \cs_set_nopar:Npx \regex_tmp:w \s_stop { \s_stop #2 }
\tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN
{ \exp_after:wN \regex_tmp:w \tex_the:D \tex_toks:D #1 }
}
@@ -173,18 +182,24 @@
\tex_toks:D #1 \exp_after:wN
{ \tex_the:D \tex_toks:D \exp_after:wN #1 \regex_tmp:w }
}
-\cs_new_nopar:Npn \regex_query_substr:NN #1#2
+\cs_new:Npn \regex_query_substr:nn #1#2
{
- \exp_after:wN \regex_query_substr_aux:NN
- \exp_after:wN #1
- \exp_after:wN #2
- \l_regex_query_other_str
- \q_stop
+ \if_num:w \int_eval:w #1 < \int_eval:w #2 \int_eval_end:
+ \str_aux_toks_range:nn {#1} {#2}
+ \fi:
+ }
+\cs_new:Npn \regex_query_submatch:nn #1#2
+ {
+ \if_num:w #1 < \l_regex_capturing_group_int
+ \exp_after:wN \regex_query_submatch:w
+ \int_use:N \int_eval:w #1 + #2 ;
+ \fi:
}
-\cs_new:Npn \regex_query_substr_aux:NN #1#2
+\cs_new:Npn \regex_query_submatch:w #1 ;
{
- \str_skip_do:nn {#1}
- { \str_collect_do:nn { #2 - #1 } { \use_i_delimit_by_q_stop:nw } }
+ \regex_query_substr:nn
+ { \tex_skip:D #1 }
+ { \etex_gluestretch:D \tex_skip:D #1 }
}
\cs_new_protected:Npn \regex_seq_pop_int:NN #1#2
{
@@ -208,8 +223,27 @@
\seq_get:NN #1 \l_regex_tmpa_tl
\l_regex_tmpa_tl
}
+\cs_new:Npn \regex_seq_item:Nn #1#2
+ {
+ \exp_after:wN \regex_seq_item_aux:wNn
+ \int_use:N \int_eval:w #2 \exp_after:wN ;
+ #1 { ? \prg_map_break: } { }
+ \prg_break_point:n { }
+ }
+\cs_new:Npn \regex_seq_item_aux:wNn #1; #2#3
+ {
+ \use_none:n #2
+ \if_num:w #1 = \c_zero
+ \exp_after:wN \prg_map_break:n
+ \else:
+ \exp_after:wN \use_none:n
+ \fi:
+ { \exp_not:n {#3} }
+ \exp_after:wN \regex_seq_item_aux:wNn
+ \int_use:N \int_eval:w #1 - \c_one ;
+ }
\cs_new_protected_nopar:Npn \regex_item_dot:T
- { \int_compare:nNnF \l_regex_current_char_int = \c_minus_one }
+ { \int_compare:nNnF \l_regex_current_char_int < \c_minus_one }
\cs_new_protected_nopar:Npn \regex_break_true:w
#1 \regex_break_point:TF #2 #3 {#2}
\cs_new_protected_nopar:Npn \regex_break_false:w
@@ -283,6 +317,37 @@
\cs_set:Npn \regex_item_range:nn { \regex_item_caseful_range:nn }
\cs_set:Npn \regex_item_geq:n { \regex_item_caseful_geq:n }
}
+\cs_new_protected_nopar:Npn \regex_item_catcode:n #1
+ {
+ \if_num:w #1 = \l_regex_current_catcode_int
+ \exp_after:wN \regex_break_true:w
+ \fi:
+ }
+\cs_new_protected_nopar:Npn \regex_item_catcode_neq:n #1
+ {
+ \if_num:w #1 = \l_regex_current_catcode_int
+ \else:
+ \exp_after:wN \regex_break_true:w
+ \fi:
+ }
+\cs_new_protected:Npn \regex_item_cs:n #1
+ {
+ \int_compare:nNnT \l_regex_current_catcode_int = \c_zero
+ {
+ \tl_set:Nn \l_regex_tmpa_regex_tl {#1}
+ \bool_set_eq:NN \l_regex_saved_success_bool \g_regex_success_bool
+ \exp_args:NNx \regex_match:NnTF \l_regex_tmpa_regex_tl
+ {
+ \exp_after:wN \exp_after:wN
+ \exp_after:wN \cs_to_str:N \l_regex_current_token_tl
+ }
+ {
+ \bool_gset_eq:NN \g_regex_success_bool \l_regex_saved_success_bool
+ \regex_break_true:w
+ }
+ { \bool_gset_eq:NN \g_regex_success_bool \l_regex_saved_success_bool }
+ }
+ }
\cs_new_protected:Npn \regex_get_digits:nw #1
{
\tex_afterassignment:D \regex_tmp:w
@@ -362,14 +427,10 @@
}
\cs_new_protected_nopar:Npn \regex_build_new_state:
{
- \int_compare:nNnTF \l_regex_max_state_int > { 32766 }
- { \msg_error:nn { regex } { -997 } }
- {
- \int_incr:N \l_regex_max_state_int
- \tex_toks:D \l_regex_max_state_int { \s_regex_stop }
- }
+ \tex_toks:D \l_regex_max_state_int { \s_stop }
\int_set_eq:NN \l_regex_left_state_int \l_regex_right_state_int
\int_set_eq:NN \l_regex_right_state_int \l_regex_max_state_int
+ \int_incr:N \l_regex_max_state_int
}
\cs_new_protected_nopar:Npn \regex_build_transition_aux:NN #1#2
{
@@ -388,16 +449,18 @@
}
\cs_new_protected:Npn \regex_build:n #1
{
- \regex_build_setup:
- \str_aux_escape:NNNn
- \regex_build_i_unescaped:N
- \regex_build_i_escaped:N
- \regex_build_i_raw:N
- { #1 }
+ \bool_set_false:N \l_regex_in_csname_bool
+ \regex_build:w
+ \regex_build_new_state:
+ \regex_toks_put_right:Nx \l_regex_left_state_int
+ { \regex_action_start_wildcard:nn {0} {1} }
\regex_build_open_aux:
- \g_str_result_tl \prg_do_nothing: \prg_do_nothing:
- \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_zero
- \regex_build_close_aux: \regex_build_group_:
+ \str_escape_use:NNNn
+ \regex_build_i_unescaped:N
+ \regex_build_i_escaped:N
+ \regex_build_i_raw:N
+ { #1 }
+ \prg_do_nothing: \prg_do_nothing:
\regex_build_end:
}
\cs_new_nopar:Npn \regex_build_i_unescaped:N #1
@@ -414,38 +477,39 @@
}
\cs_new_nopar:Npn \regex_build_i_raw:N #1
{ \exp_not:N \regex_build_raw:N #1 }
-\cs_new_protected_nopar:Npn \regex_build_default_control:N #1
+\cs_new_protected_nopar:Npn \regex_build_control:N #1
{
- \cs_if_exist_use:cF { regex_build_#1: }
- { \regex_build_default_raw:N #1 }
+ \cs_if_exist_use:cF { regex_ \l_regex_build_or_class_tl _#1: }
+ { \use:c { regex_ \l_regex_build_or_class_tl _default_raw:N } #1 }
}
-\cs_new_protected_nopar:Npn \regex_build_setup:
+\cs_new_protected_nopar:Npn \regex_build:w
{
- \cs_set_eq:NN \regex_build_control:N \regex_build_default_control:N
\cs_set_eq:NN \regex_build_raw:N \regex_build_default_raw:N
+ \tl_set:Nn \l_regex_build_or_class_tl { build }
\int_set_eq:NN \l_regex_capturing_group_int \c_zero
\int_zero:N \l_regex_max_state_int
\regex_build_new_state:
- \regex_build_new_state:
- \regex_toks_put_right:Nx \l_regex_left_state_int
- { \regex_action_start_wildcard:nn {0} {1} }
}
\cs_new_protected_nopar:Npn \regex_build_end:
{
+ \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_zero
+ \regex_build_close_aux: \regex_build_group_:
\seq_if_empty:NF \l_regex_capturing_group_seq
{
- \msg_error:nn { regex } {22}
+ \msg_kernel_error:nnx { regex } { missing-rparen }
+ { \seq_length:N \l_regex_capturing_group_seq }
\prg_replicate:nn
{ \seq_length:N \l_regex_capturing_group_seq }
{ \regex_build_close_aux: \regex_build_group_: }
}
\regex_toks_put_right:Nx \l_regex_right_state_int
{ \regex_action_success: }
+ \int_incr:N \l_regex_capturing_group_int
}
\cs_new_protected_nopar:cpn { regex_build_^: }
- { \regex_build_anchor_start:N \c_zero }
+ { \regex_build_anchor_start:N \l_regex_min_step_int }
\cs_new_protected_nopar:Npn \regex_build_A:
- { \regex_build_anchor_start:N \c_zero }
+ { \regex_build_anchor_start:N \l_regex_min_step_int }
\cs_new_protected_nopar:Npn \regex_build_G:
{ \regex_build_anchor_start:N \l_regex_start_step_int }
\cs_new_protected_nopar:Npn \regex_build_anchor_start:N #1
@@ -453,7 +517,7 @@
\regex_build_new_state:
\regex_toks_put_right:Nx \l_regex_left_state_int
{
- \exp_not:N \int_compare:nNnT {#1} = \l_regex_current_step_int
+ \exp_not:N \int_compare:nNnT #1 = \l_regex_current_step_int
{
\regex_action_free:n
{
@@ -469,7 +533,7 @@
\regex_toks_put_right:Nx \l_regex_left_state_int
{
\exp_not:N \int_compare:nNnT
- \c_minus_one = \l_regex_current_char_int
+ \l_regex_current_char_int < \c_minus_one
{
\regex_action_free:n
{
@@ -519,61 +583,52 @@
\int_set_eq:NN \l_regex_current_char_int \l_regex_last_char_int
\c_regex_w_tl
\regex_break_point:TF
- { \group_end: \c_regex_W_tl \regex_item_equal:n { -1 } }
+ { \group_end: \c_regex_W_tl \regex_item_equal:n { -2 } }
{ \group_end: \c_regex_w_tl }
\regex_break_point:TF
}
-\cs_new_protected_nopar:Npn \regex_build_default_raw:N #1
+\cs_new_protected:Npn \regex_build_one:n #1
+ { \regex_build_one:x { \exp_not:n {#1} } }
+\cs_new_protected:Npn \regex_build_one:x #1
{
- \cs_set:Npx \regex_build_tmp_class:n ##1
+ \cs_set_nopar:Npx \regex_build_tmp_class:n ##1
{
- \regex_item_equal:n { \int_value:w `#1 ~ }
- \regex_break_point:TF { \regex_action_cost:n { ##1 } } { }
+ \exp_not:N \exp_not:n {#1}
+ \regex_break_point:TF { \regex_action_cost:n {##1} } { }
}
\regex_build_one_quantifier:
}
+\cs_new_protected_nopar:Npn \regex_build_default_raw:N #1
+ { \regex_build_one:x { \regex_item_equal:n { \int_value:w `#1 ~ } } }
\cs_new_protected_nopar:cpn { regex_build_.: }
{
\cs_set:Npn \regex_build_tmp_class:n ##1
{ \regex_item_dot:T { \regex_action_cost:n {##1} } }
\regex_build_one_quantifier:
}
-\cs_new_protected_nopar:Npn \regex_build_char_type:N #1
- {
- \cs_set:Npn \regex_build_tmp_class:n ##1
- {
- \exp_not:N #1
- \regex_break_point:TF { \regex_action_cost:n {##1} } { }
- }
- \regex_build_one_quantifier:
- }
\tl_map_inline:nn { dDhHsSvVwWN }
{
\cs_new_protected_nopar:cpx { regex_build_#1: }
- {
- \exp_not:N \regex_build_char_type:N
- \exp_not:c { c_regex_#1_tl }
- }
+ { \regex_build_one:n \exp_not:c { c_regex_#1_tl } }
}
+\cs_new_protected:Npn \regex_class_one:n
+ { \tl_put_right:Nn \l_regex_class_tl }
+\cs_new_protected:Npn \regex_class_one:x
+ { \tl_put_right:Nx \l_regex_class_tl }
\cs_new_protected_nopar:cpn { regex_build_[: }
{
\tl_clear:N \l_regex_class_tl
- \cs_set_eq:NN \regex_build_control:N \regex_class_control:N
- \cs_set_eq:NN \regex_build_raw:N \regex_class_raw:N
+ \cs_set_eq:NN \regex_build_raw:N \regex_class_default_raw:N
+ \tl_set:Nn \l_regex_build_or_class_tl { class }
\regex_class_first:NN
}
-\cs_new_protected_nopar:Npn \regex_class_control:N #1
- {
- \cs_if_exist_use:cF { regex_class_#1: }
- { \regex_class_raw:N #1 }
- }
\cs_new_protected_nopar:cpn { regex_class_]: }
{
\tl_if_empty:NTF \l_regex_class_tl %[
- { \regex_class_raw:N ] }
+ { \regex_class_default_raw:N ] }
{
- \cs_set_eq:NN \regex_build_control:N \regex_build_default_control:N
\cs_set_eq:NN \regex_build_raw:N \regex_build_default_raw:N
+ \tl_set:Nn \l_regex_build_or_class_tl { build }
\cs_set:Npn \regex_build_tmp_class:n ##1
{
\exp_not:o \l_regex_class_tl
@@ -593,7 +648,7 @@
#1 #2
}
}
-\cs_new_protected_nopar:Npn \regex_class_raw:N #1#2#3
+\cs_new_protected_nopar:Npn \regex_class_default_raw:N #1#2#3
{
\str_if_eq:nnTF {#2#3} { \regex_build_control:N - }
{ \regex_class_range:Nw #1 }
@@ -603,10 +658,7 @@
}
}
\cs_new_protected_nopar:Npn \regex_class_single:N #1
- {
- \tl_put_right:Nx \l_regex_class_tl
- { \regex_item_equal:n { \int_value:w `#1 } }
- }
+ { \regex_class_one:x { \regex_item_equal:n { \int_value:w `#1 } } }
\cs_new_protected_nopar:Npn \regex_class_range:Nw #1#2#3
{
\token_if_eq_meaning:NNTF #2 \regex_build_control:N
@@ -624,9 +676,9 @@
\cs_new_protected_nopar:Npn \regex_class_range_put:NN #1#2
{
\if_num:w `#1 > `#2 \exp_stop_f:
- \msg_error:nn { regex } {8}
+ \msg_kernel_error:nnxx { regex } { backwards-range } {#1} {#2}
\else:
- \tl_put_right:Nx \l_regex_class_tl
+ \regex_class_one:x
{
\if_num:w `#1 = `#2 \exp_stop_f:
\regex_item_equal:n
@@ -640,10 +692,93 @@
\tl_map_inline:nn { dDhHsSvVwWN }
{
\cs_new_protected_nopar:cpx { regex_class_#1: }
+ { \regex_class_one:n \exp_not:c { c_regex_#1_tl } }
+ }
+\cs_new_protected_nopar:Npn \regex_build_c:
+ { \regex_aux_c:NNNN \regex_item_catcode:n \use_i:nn }
+\cs_new_protected_nopar:Npn \regex_build_C:
+ { \regex_aux_c:NNNN \regex_item_catcode_neq:n \use_ii:nn }
+\cs_new_eq:NN \regex_class_c: \regex_build_c:
+\cs_new_eq:NN \regex_class_C: \regex_build_C:
+\cs_new_protected:Npn \regex_aux_c:NNNN #1#2#3#4
+ {
+ \token_if_eq_meaning:NNTF #3 \regex_build_raw:N
+ {
+ \regex_aux_c_test:NTF #4
+ {
+ \use:c { regex_ \l_regex_build_or_class_tl _one:x }
+ { #1 { \int_use:N \l_regex_current_catcode_int } }
+ }
+ {
+ \msg_kernel_error:nnx { regex } { build-c } { #4 }
+ #3 #4
+ }
+ }
+ {
+ \if_charcode:w \c_lbrace_str \exp_not:N #4
+ \exp_after:wN #2
+ \else:
+ \exp_after:wN \use_ii:nn
+ \fi:
+ {
+ \group_begin:
+ \bool_set_true:N \l_regex_in_csname_bool
+ \regex_build:w
+ }
+ {
+ \msg_kernel_error:nnx { regex } { build-c } { #4 }
+ #3 #4
+ }
+ }
+ }
+\cs_new_protected:Npn \regex_aux_c_test:NTF #1
+ {
+ \int_set:Nn \l_regex_current_catcode_int
+ {
+ \if_case:w \int_eval:w `#1 - `A \int_eval_end:
+ \c_thirteen % Active
+ \or: \c_one % Begin-group
+ \or: \c_zero % Control sequence
+ \or: \c_eight % Down (subscript)
+ \or: \c_two % End-group
+ \else:
+ \if_case:w \int_eval:w `#1 - `L \int_eval_end:
+ \c_eleven % Letter
+ \or: \c_three % Math toggle
+ \or: \c_minus_one
+ \or: \c_twelve % Other
+ \or: \c_six % Parameter
+ \or: \c_minus_one
+ \or: \c_minus_one
+ \or: \c_ten % Space
+ \or: \c_four % Tab (alignment)
+ \or: \c_seven % Up (superscript)
+ \else: \c_minus_one
+ \fi:
+ \fi:
+ }
+ \if_num:w \l_regex_current_catcode_int = \c_minus_one
+ \exp_after:wN \use_ii:nn
+ \else:
+ \exp_after:wN \use_i:nn
+ \fi:
+ }
+\cs_new_protected:cpn { regex_build_ \c_rbrace_str : }
+ {
+ \bool_if:NTF \l_regex_in_csname_bool
{
- \tl_put_right:Nn \exp_not:N \l_regex_class_tl
- { \exp_not:c { c_regex_#1_tl } }
+ \regex_build_Z:
+ \regex_toks_put_right:Nx \l_regex_right_state_int
+ { \regex_action_success: }
+ \int_incr:N \l_regex_capturing_group_int
+ \use:x
+ {
+ \group_end:
+ \exp_not:n { \use:c { regex_ \l_regex_build_or_class_tl _one:n } }
+ { \regex_item_cs:n { \regex_set_aux:N ? } }
+ }
}
+ { \exp_after:wN \regex_build_raw:N \c_rbrace_str }
}
\cs_new_protected_nopar:Npn \regex_build_quantifier:w #1#2
{
@@ -711,12 +846,8 @@
{
\tl_if_empty:nTF {#2}
{ \regex_build_quantifier_aux:nnNN { {#1} } { n* } }
- {
- \int_compare:nNnT {#1} > {#2}
- { \msg_error:nn { regex } {4} }
- \regex_build_quantifier_aux:nnNN { {#1} {#2} } { nn }
- }
- }
+ { \regex_build_quantifier_aux:nnNN { {#1} {#2} } { nn } }
+ } %^^A todo: catch the case m>n
{
\regex_build_quantifier_end:n { }
\use:x
@@ -833,7 +964,7 @@
\cs_new_protected_nopar:cpn { regex_build_): }
{
\seq_if_empty:NTF \l_regex_capturing_group_seq
- { \msg_error:nn { regex } { 22 } }
+ { \msg_kernel_error:nn { regex } { extra-rparen } }
{
\regex_build_close_aux:
\regex_build_group_quantifier:
@@ -878,7 +1009,8 @@
{
\cs_if_exist_use:cF { regex_build_special_group_\token_to_str:N #2 : }
{
- \msg_error:nn { regex } { -998 }
+ \msg_kernel_error:nnx { regex } { unsupported }
+ { (? \token_to_str:N #2 } %)
\regex_build_control:N ( % )
\regex_build_raw:N ?
#1 #2
@@ -990,7 +1122,7 @@
{
\tex_toks:D \l_regex_tmpa_int
{
- \s_regex_stop
+ \s_stop
#1 { \int_eval:n { \l_regex_right_state_int - \l_regex_tmpa_int } }
}
}
@@ -1052,16 +1184,19 @@
\regex_toks_put_right:Nx \l_regex_right_state_int
{
\regex_action_free:n
- { \int_eval:n { \l_regex_max_state_int - \l_regex_left_state_int } }
+ {
+ \int_eval:n %^^A todo: document why that value.
+ { \l_regex_max_state_int - \c_one - \l_regex_left_state_int }
+ }
}
\int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int
- \int_set:Nn \l_regex_tmpb_int { \l_regex_max_state_int + \c_one }
+ \int_set_eq:NN \l_regex_tmpb_int \l_regex_max_state_int
\int_set:Nn \l_regex_max_state_int
{
- \l_regex_left_state_int - \c_one
- + #1 * ( \l_regex_max_state_int - \l_regex_left_state_int + \c_one )
+ \l_regex_left_state_int
+ + #1 * ( \l_regex_max_state_int - \l_regex_left_state_int )
}
- \int_until_do:nNnn \l_regex_tmpb_int > \l_regex_max_state_int
+ \int_while_do:nNnn \l_regex_tmpb_int < \l_regex_max_state_int
{
\tex_toks:D \l_regex_tmpb_int = \tex_toks:D \l_regex_tmpa_int
\int_incr:N \l_regex_tmpa_int
@@ -1114,14 +1249,13 @@
\cs_new_protected_nopar:Npn \regex_state_use_with_submatches:
{ \regex_state_use_aux:n { } }
\cs_new_protected_nopar:Npn \regex_state_use:
- { \regex_state_use_aux:n { \exp_after:wN \regex_state_use_aux_ii:w } }
-\cs_new_nopar:Npn \regex_state_use_aux_ii:w #1 \s_regex_stop { }
+ { \regex_state_use_aux:n { \exp_after:wN \use_none_delimit_by_s_stop:w } }
\cs_new_protected_nopar:Npn \regex_state_use_aux:n #1
{
\if_num:w \tex_dimen:D \l_regex_current_state_int
- < \l_regex_unique_step_int
+ < \l_regex_unique_id_int
\tex_dimen:D \l_regex_current_state_int
- = \l_regex_unique_step_int sp \scan_stop:
+ = \l_regex_unique_id_int sp \scan_stop:
#1 \tex_the:D \tex_toks:D \exp_after:wN \l_regex_current_state_int
\fi:
\scan_stop:
@@ -1144,73 +1278,105 @@
\tex_the:D \tex_toks:D #1
}
}
-\cs_new_protected:Npn \regex_store_submatches_aux:w #1 \s_regex_stop
+\cs_new_protected:Npn \regex_store_submatches_aux:w #1 \s_stop
{
\regex_store_submatches_aux_ii:Nnnw
#1
\regex_state_submatches:nn \c_minus_one \q_prop
- \s_regex_stop
+ \s_stop
}
\cs_new_protected:Npn \regex_store_submatches_aux_ii:Nnnw
- \regex_state_submatches:nn #1 #2 #3 \s_regex_stop
+ \regex_state_submatches:nn #1 #2 #3 \s_stop
{
\exp_after:wN \c_zero
\exp_after:wN \regex_state_submatches:nn \exp_after:wN
{
\int_value:w \int_eval:w
- \l_regex_unique_step_int + \c_one
+ \l_regex_unique_id_int + \c_one
\exp_after:wN
}
\exp_after:wN { \l_regex_current_submatches_prop }
\regex_state_submatches:nn {#1} {#2}
- \s_regex_stop
+ \s_stop
}
\cs_new_protected:Npn \regex_state_submatches:nn #1#2
{
- \if_num:w #1 = \l_regex_unique_step_int
+ \if_num:w #1 = \l_regex_unique_id_int
\tl_set:Nn \l_regex_current_submatches_prop { #2 }
\fi:
}
\cs_new_protected:Npn \regex_match:n #1
{
- \tl_set:Nx \l_regex_query_other_str { \tl_to_other_str:n {#1} }
+ \tl_set_analysis:Nn \l_regex_tmpa_tl {#1}
+
+ \int_zero:N \l_regex_nesting_int
+ \int_set_eq:NN \l_regex_current_step_int \l_regex_max_state_int
+ \regex_query_set:nnn { } { -1 } { -2 }
+ \int_set_eq:NN \l_regex_min_step_int \l_regex_current_step_int
+ \exp_after:wN \regex_query_set_loop:ww
+ \l_regex_tmpa_tl
+ \q_mark { ? \prg_map_break: } ,
+ \prg_break_point:n { }
+ \int_set_eq:NN \l_regex_max_step_int \l_regex_current_step_int
+ \regex_query_set:nnn { } { -1 } { -2 }
+
\regex_match_initial_setup:
\regex_match_once:
}
-\cs_new_protected_nopar:Npn \regex_match_once:
+\cs_new:Npn \regex_query_set_loop:ww #1 \q_mark #2#3 ,
{
- \regex_match_setup:
- \exp_after:wN \regex_match_once_aux: \l_regex_query_other_str
- \q_recursion_tail \q_recursion_stop
- \bool_if:NT \l_regex_success_bool { \l_regex_every_match_tl }
+ \use_none:n #2
+ \regex_query_set:nnn {#1} {"#2} {#3}
+ \if_case:w "#2 \exp_stop_f:
+ \or: \int_incr:N \l_regex_nesting_int
+ \or: \int_decr:N \l_regex_nesting_int
+ \fi:
+ \regex_query_set_loop:ww
}
-\cs_new_protected_nopar:Npn \regex_match_once_aux:
+\cs_new_protected:Npn \regex_query_set:nnn #1#2#3
{
- \int_compare:nNnTF \l_regex_start_step_int = \c_zero
- {
- \int_set_eq:NN \l_regex_current_char_int \c_minus_one
- \regex_match_loop:N
- }
- {
- \str_skip_do:nn
- { \l_regex_start_step_int - \c_one }
- { \regex_match_once_aux_ii:N }
- }
+ \tex_muskip:D \l_regex_current_step_int
+ = \etex_gluetomu:D
+ #3 sp
+ plus #2 sp
+ minus \l_regex_nesting_int sp
+ \scan_stop:
+ \tex_toks:D \l_regex_current_step_int {#1}
+ \int_incr:N \l_regex_current_step_int
}
-\cs_new_protected_nopar:Npn \regex_match_once_aux_ii:N #1
+\cs_new_protected_nopar:Npn \regex_query_get:
{
- \int_set:Nn \l_regex_current_char_int { `#1 }
- \regex_match_loop:N
+ \tl_set:Nx \l_regex_current_token_tl
+ { \tex_the:D \tex_toks:D \l_regex_current_step_int }
+ \l_regex_current_char_int
+ = \etex_mutoglue:D \tex_muskip:D \l_regex_current_step_int
+ \l_regex_current_catcode_int = \etex_gluestretch:D
+ \etex_mutoglue:D \tex_muskip:D \l_regex_current_step_int
+ }
+\cs_new_protected_nopar:Npn \regex_match_once:
+ {
+ \regex_match_setup:
+ \regex_query_get:
+ \regex_match_loop:
+ \prg_break_point:n { }
+ \bool_if:NT \l_regex_success_match_bool
+ {
+ \bool_gset_true:N \g_regex_success_bool
+ \l_regex_every_match_tl
+ }
}
\cs_new_protected_nopar:Npn \regex_match_initial_setup:
{
- \prg_stepwise_inline:nnnn {1} {1} { \l_regex_max_state_int }
+ \prg_stepwise_inline:nnnn {0} {1} { \l_regex_max_state_int - \c_one }
{ \tex_dimen:D ##1 \c_minus_one sp \scan_stop: }
- \int_set_eq:NN \l_regex_unique_step_int \c_minus_one
- \int_set_eq:NN \l_regex_start_step_int \c_minus_one
- \int_set_eq:NN \l_regex_current_step_int \c_zero
- \int_set_eq:NN \l_regex_success_step_int \c_zero
+ \int_set_eq:NN \l_regex_unique_id_int \c_minus_one
+ \int_set:Nn \l_regex_start_step_int
+ { \l_regex_min_step_int - \c_one }
+ \int_set_eq:NN \l_regex_current_step_int \l_regex_min_step_int
+ \int_set_eq:NN \l_regex_success_step_int \l_regex_min_step_int
+ \int_set_eq:NN \l_regex_submatch_int \l_regex_max_state_int
\bool_set_false:N \l_regex_success_empty_bool
+ \bool_gset_false:N \g_regex_success_bool
}
\cs_new_protected_nopar:Npn \regex_match_setup:
{
@@ -1219,33 +1385,44 @@
{ \cs_set_eq:NN \regex_last_match_empty:F \regex_last_match_empty_yes:F }
{ \cs_set_eq:NN \regex_last_match_empty:F \regex_last_match_empty_no:F }
\int_set_eq:NN \l_regex_start_step_int \l_regex_success_step_int
- \int_set_eq:NN \l_regex_current_step_int \l_regex_start_step_int
- \int_decr:N \l_regex_current_step_int
- \bool_set_false:N \l_regex_success_bool
+ \int_set:Nn \l_regex_current_step_int
+ { \l_regex_start_step_int - \c_one }
+ \bool_set_false:N \l_regex_success_match_bool
\int_zero:N \l_regex_max_index_int
- \regex_store_state:n {1}
+ \regex_store_state:n {0} %^^A _state_int!
}
-\cs_new_protected_nopar:Npn \regex_match_loop:N #1
+\cs_new_protected_nopar:Npn \regex_match_loop:
{
- \regex_match_loop_setup:N #1
+ \int_incr:N \l_regex_current_step_int
+ \int_incr:N \l_regex_unique_id_int
+ \bool_set_false:N \l_regex_fresh_thread_bool
+ \int_set_eq:NN \l_regex_last_char_int \l_regex_current_char_int
+ \regex_query_get:
+ \regex_match_loop_case_hook:
\cs_set_nopar:Npx \regex_tmp:w
{
\int_zero:N \l_regex_max_index_int
- \prg_stepwise_function:nnnN
- {1} {1} { \l_regex_max_index_int }
- \regex_match_one_index:n
+ \regex_match_one_index:w 1 ; \prg_break_point:n { }
+ \exp_not:N \prg_break_point:n { }
}
- \regex_tmp:w \prg_break_point:n { }
+ \regex_tmp:w
+ \if_num:w \l_regex_current_char_int < \c_minus_one
+ \exp_after:wN \prg_map_break:
+ \fi:
\if_num:w \l_regex_max_index_int = \c_zero
- \exp_after:wN \use_none_delimit_by_q_recursion_stop:w
+ \exp_after:wN \prg_map_break:
\fi:
- \quark_if_recursion_tail_stop:N #1
- \regex_match_loop:N
+ \regex_match_loop:
}
-\cs_new_nopar:Npn \regex_match_one_index:n #1
+\cs_new_nopar:Npn \regex_match_one_index:w #1;
{
+ \if_num:w #1 > \l_regex_max_index_int
+ \exp_after:wN \prg_map_break:
+ \fi:
\regex_match_one_index_aux:n
{ \int_value:w \tex_skip:D #1 }
+ \exp_after:wN \regex_match_one_index:w
+ \int_use:N \int_eval:w #1 + \c_one ;
}
\cs_new_protected_nopar:Npn \regex_match_one_index_aux:n #1
{
@@ -1253,19 +1430,6 @@
\prop_clear:N \l_regex_current_submatches_prop
\regex_state_use_with_submatches:
}
-\cs_new_protected_nopar:Npn \regex_match_loop_setup:N #1
- {
- \int_incr:N \l_regex_current_step_int
- \int_incr:N \l_regex_unique_step_int
- \bool_set_false:N \l_regex_fresh_thread_bool
- \int_set_eq:NN \l_regex_last_char_int \l_regex_current_char_int
- \if_meaning:w #1 \q_recursion_tail
- \int_set_eq:NN \l_regex_current_char_int \c_minus_one
- \else:
- \int_set:Nn \l_regex_current_char_int {`#1}
- \fi:
- \regex_match_loop_case_hook:
- }
\cs_new_protected_nopar:Npn \regex_match_loop_case_hook: { }
\cs_new_protected_nopar:Npn \regex_match_loop_caseless_hook:
{
@@ -1293,18 +1457,17 @@
}
\cs_new_protected_nopar:Npn \regex_action_cost:n #1
{
- \exp_args:Nf \regex_store_state:n %^^A optimize!
+ \exp_args:Nf \regex_store_state:n
{ \int_eval:n { \l_regex_current_state_int + #1 } }
}
\cs_new_protected_nopar:Npn \regex_action_success:
{
\regex_last_match_empty:F
{
- \bool_set_true:N \l_regex_success_bool
+ \bool_set_true:N \l_regex_success_match_bool
\bool_set_eq:NN \l_regex_success_empty_bool
\l_regex_fresh_thread_bool
- \int_set_eq:NN \l_regex_success_step_int
- \l_regex_current_step_int
+ \int_set_eq:NN \l_regex_success_step_int \l_regex_current_step_int
\prop_set_eq:NN \l_regex_success_submatches_prop
\l_regex_current_submatches_prop
\prg_map_break:
@@ -1328,504 +1491,668 @@
\prop_put:Nno \l_regex_current_submatches_prop {#1}
{ \int_use:N \l_regex_current_step_int }
}
-\cs_new_protected_nopar:Npn \regex_extract:
+\cs_new_protected:Npn \regex_submatch_nesting_aux:n #1
{
- \seq_gclear:N \g_regex_submatches_seq
- \prg_stepwise_inline:nnnn
- {0} {1} { \l_regex_capturing_group_int }
- {
- \regex_extract_aux:nTF { ##1 }
- {
- \seq_gput_right:Nx \g_regex_submatches_seq
- { \regex_query_substr:NN \l_regex_tmpa_tl \l_regex_tmpb_tl }
- }
- { \seq_gput_right:Nn \g_regex_submatches_seq { } }
- }
+ + \etex_glueshrink:D \etex_mutoglue:D \etex_muexpr:D
+ \tex_muskip:D \etex_gluestretch:D \tex_skip:D #1
+ - \tex_muskip:D \tex_skip:D #1
+ \scan_stop:
}
-\cs_new_protected_nopar:Npn \regex_extract_aux:nTF #1#2#3
+\cs_new:Npn \regex_nesting:n #1 { } %^^A move. Rename?
+\tl_new:N \l_regex_nesting_tl
+\cs_new_protected:Npn \regex_replacement:n #1
{
- \prop_get:NnNTF \l_regex_success_submatches_prop
- { #1 < } \l_regex_tmpa_tl
+ \int_zero:N \l_regex_replacement_int
+ \int_zero:N \l_regex_nesting_int
+ \tl_clear:N \l_regex_nesting_tl
+ \str_escape_use:NNNn
+ \regex_replacement_unescaped:N
+ \regex_replacement_escaped:N
+ \regex_replacement_raw:N
+ {#1}
+ \prg_do_nothing: \prg_do_nothing:
+ \cs_set_nopar:Npx \regex_nesting:n ##1
+ {
+ + \int_use:N \l_regex_nesting_int
+ \l_regex_nesting_tl
+ - \regex_submatch_nesting_aux:n {##1}
+ }
+ \use:x
{
- \prop_get:NnNTF \l_regex_success_submatches_prop
- { #1 > } \l_regex_tmpb_tl
- {#2}
- {#3}
+ \exp_not:n { \cs_set_nopar:Npn \regex_replacement_tl:n ##1 }
+ { \str_aux_toks_range:nn \c_zero \l_regex_replacement_int }
}
- {#3}
+ % ^^A rename! Careful with \cP\#.
}
-\cs_new_protected_nopar:Npn \regex_set:Nn
- { \regex_set_aux:NNn \tl_set_eq:NN }
-\cs_new_protected_nopar:Npn \regex_gset:Nn
- { \regex_set_aux:NNn \tl_gset_eq:NN }
-\cs_new_protected:Npn \regex_set_aux:NNn #1#2#3
+\cs_new_protected:Npn \regex_replacement_raw:N #1
+ { \regex_replacement_put:n {#1} }
+\cs_new_protected:Npn \regex_replacement_put:n #1
{
- \group_begin:
- \regex_build:n {#3}
- \tl_gset:Nx \g_regex_tmpa_tl
- {
- \exp_not:N \regex_nfa:Nw \exp_not:N #2
- \l_regex_max_state_int
- = \int_use:N \l_regex_max_state_int
- \l_regex_capturing_group_int
- = \int_use:N \l_regex_capturing_group_int
- \token_if_eq_meaning:NNT
- \regex_match_loop_case_hook:
- \regex_match_loop_caseless_hook:
- {
- \cs_set_eq:NN \regex_match_loop_case_hook:
- \regex_match_loop_caseless_hook:
- }
- \prg_stepwise_function:nnnN
- {1} {1} {\l_regex_max_state_int}
- \regex_set_aux:n
- \s_regex_stop
- }
- \group_end:
- #1 #2 \g_regex_tmpa_tl
+ \tex_toks:D \l_regex_replacement_int {#1}
+ \int_incr:N \l_regex_replacement_int
}
-\cs_new_nopar:Npn \regex_set_aux:n #1
- { \tex_toks:D #1 { \tex_the:D \tex_toks:D #1 } }
-\cs_new:Npn \regex_nfa:Nw #1 #2 \s_regex_stop
- { \msg_expandable_kernel_error:nnn { regex } { nfa-misused } {#1} }
-\cs_new_protected_nopar:Npn \regex_const:Nn
- { \regex_set_aux:NNn \cs_new_eq:NN }
-\prg_new_protected_conditional:Npnn \regex_check_nfa:N #1 { T , TF }
- { \exp_after:wN \regex_check_nfa_aux:Nw #1 \q_stop }
-\cs_new:Npn \regex_check_nfa_aux:Nw #1 #2 \q_stop
+\cs_new_protected:Npn \regex_replacement_unescaped:N #1
{
- \if_meaning:w \regex_nfa:Nw #1
- \prg_return_true:
+ \if_charcode:w \c_rbrace_str #1
+ \if_num:w \l_regex_replacement_csnames_int > \c_zero
+ \regex_replacement_put:n \cs_end:
+ \else:
+ \regex_replacement_put:n #1
+ \fi:
\else:
- \msg_error:nnx { regex } { not-nfa } { \token_to_str:N #1 }
- \prg_return_false:
+ \regex_replacement_put:n #1
\fi:
}
-\cs_new_protected_nopar:Npn \regex_use:N #1
- { \exp_after:wN \use_none:nn #1 }
-\cs_new_protected:Npn \regex_user_aux:n #1
+\cs_new_protected:Npn \regex_replacement_escaped:N #1
{
- \group_begin:
- \tl_clear:N \l_regex_every_match_tl
- #1
- \group_end:
- }
-\cs_new_protected_nopar:Npn \regex_return_after_group:
- {
- \if_meaning:w \c_true_bool \l_regex_success_bool
- \group_insert_after:N \prg_return_true:
+ \if_charcode:w c #1
+ \exp_after:wN \regex_replacement_c:w
\else:
- \group_insert_after:N \prg_return_false:
+ \if_charcode:w g #1
+ \exp_after:wN \exp_after:wN \exp_after:wN \regex_replacement_g:w
+ \else:
+ \if_num:w \c_one < 1#1 \exp_stop_f:
+ \regex_replacement_put_submatch:n {#1}
+ \else:
+ \regex_replacement_put:n #1
+ \fi:
+ \fi:
\fi:
}
-\cs_new_protected_nopar:Npn \regex_extract_after_group:N #1
+\cs_new_protected:Npn \regex_replacement_put_submatch:n #1
{
- \if_meaning:w \c_true_bool \l_regex_success_bool
- \regex_extract:
- \group_insert_after:N \seq_set_eq:NN
- \group_insert_after:N #1
- \group_insert_after:N \g_regex_submatches_seq
+ \regex_replacement_put:n
+ { \regex_query_submatch:nn {#1} {##1} }
+ \if_num:w \l_regex_replacement_csnames_int = \c_zero
+ \tl_put_right:Nn \l_regex_nesting_tl
+ {
+ \exp_not:N \if_num:w #1 < \l_regex_capturing_group_int
+ \regex_submatch_nesting_aux:n { \int_eval:w #1+##1 \int_eval_end: }
+ \exp_not:N \fi:
+ }
\fi:
}
-\cs_new_protected_nopar:Npn \regex_count_after_group:N #1
+\cs_new_protected:Npn \regex_replacement_error:NNN #1#2#3
{
- \group_insert_after:N \int_set_eq:NN
- \group_insert_after:N #1
- \group_insert_after:N \g_regex_match_count_int
+ \msg_kernel_error:nn { regex } { replacement-#1 }
+ #2 #3
}
-\prg_new_protected_conditional:Npnn \regex_match:nn #1#2 { T , F , TF }
+\cs_new_protected:Npn \regex_replacement_g:w #1#2
{
- \regex_user_aux:n
+ \str_if_eq:xxTF
+ { \exp_not:n { #1#2 } }
+ { \regex_replacement_unescaped:N \c_lbrace_str }
{
- \regex_disable_submatches:
- \regex_build:n {#1}
- \regex_match:n {#2}
- \regex_return_after_group:
+ \int_zero:N \l_regex_tmpa_int
+ \regex_replacement_g_digits:NN
}
+ { \regex_replacement_error:NNN g #1 #2 }
}
-\prg_new_protected_conditional:Npnn \regex_match:Nn #1#2 { T , F , TF }
+\cs_new_protected:Npn \regex_replacement_g_digits:NN #1#2
{
- \regex_check_nfa:NTF #1
+ \token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N
{
- \regex_user_aux:n
- {
- \regex_disable_submatches:
- \regex_use:N #1
- \regex_match:n {#2}
- \regex_return_after_group:
- }
+ \if_num:w \c_one < 1#2 \exp_stop_f:
+ \int_set:Nn \l_regex_tmpa_int { \c_ten * \l_regex_tmpa_int + #2 }
+ \exp_after:wN \use_i:nnn
+ \exp_after:wN \regex_replacement_g_digits:NN
+ \else:
+ \if_charcode:w \c_rbrace_str #2
+ \exp_args:No \regex_replacement_put_submatch:n
+ { \int_use:N \l_regex_tmpa_int }
+ \exp_after:wN \exp_after:wN \exp_after:wN \use_none:nn
+ \else:
+ \exp_after:wN \exp_after:wN
+ \exp_after:wN \regex_replacement_error:NNN
+ \exp_after:wN \exp_after:wN \exp_after:wN g
+ \fi:
+ \fi:
}
- { \prg_return_false: }
+ { \regex_replacement_error:NNN g }
+ #1 #2
}
-\cs_new_protected:Npn \regex_count:nnN #1#2#3
+\cs_new_protected_nopar:Npn \regex_replacement_c:w #1#2
{
- \regex_user_aux:n
+ \token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N
{
- \regex_disable_submatches:
- \int_gzero:N \g_regex_match_count_int
- \tl_set:Nn \l_regex_every_match_tl
- {
- \int_gincr:N \g_regex_match_count_int
- \regex_match_once:
- }
- \regex_build:n {#1}
- \regex_match:n {#2}
- \regex_count_after_group:N #3
+ \cs_if_exist_use:cF { regex_replacement_c_#2:w }
+ { \regex_replacement_error:NNN c #1#2 }
}
+ { \regex_replacement_error:NNN c #1#2 }
}
-\cs_new_protected:Npn \regex_count:NnN #1#2#3
+\cs_new_protected_nopar:cpn { regex_replacement_c_ \c_lbrace_str :w }
{
- \regex_check_nfa:NT #1
- {
- \regex_user_aux:n
- {
- \regex_disable_submatches:
- \int_gzero:N \g_regex_match_count_int
- \tl_set:Nn \l_regex_every_match_tl
- {
- \int_gincr:N \g_regex_match_count_int
- \regex_match_once:
- }
- \regex_use:N #1
- \regex_match:n {#2}
- \regex_count_after_group:N #3
- }
- }
+ \int_incr:N \l_regex_replacement_csnames_int
+ \regex_replacement_put:n
+ { \exp_not:n { \exp_after:wN \regex_replacement_exp_not:N \cs:w } }
}
-\cs_new_protected:Npn \regex_extract_once:nnN #1#2#3
- {
- \regex_user_aux:n
- {
- \regex_build:n {#1}
- \regex_match:n {#2}
- \regex_extract_after_group:N #3
- }
+\cs_new_nopar:Npn \regex_replacement_exp_not:N #1 { \exp_not:n {#1} }
+\group_begin:
+
+ \char_set_catcode_math_superscript:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_U:w
+ { \regex_replacement_char:nNN { ^^@ } }
+
+ \char_set_catcode_alignment:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_T:w
+ { \regex_replacement_char:nNN { ^^@ } }
+
+ \cs_new_protected_nopar:Npn \regex_replacement_c_S:w #1#2
+ {
+ \int_compare:nNnTF { `#2 } = \c_zero
+ { \regex_replacement_error:NNN c #1#2 }
+ {
+ \char_set_lccode:nn {32} { `#2 }
+ \tl_to_lowercase:n { \regex_replacement_put:n {~} }
+ }
+ }
+
+ \char_set_catcode_parameter:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_P:w
+ { \regex_replacement_char:nNN { ^^@^^@ } }
+
+ \char_set_catcode_other:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_O:w
+ { \regex_replacement_char:nNN { ^^@ } }
+
+ \char_set_catcode_math_toggle:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_M:w
+ { \regex_replacement_char:nNN { ^^@ } }
+
+ \char_set_catcode_letter:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_L:w
+ { \regex_replacement_char:nNN { ^^@ } }
+
+ \char_set_catcode_group_end:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_E:w
+ {
+ \int_decr:N \l_regex_nesting_int
+ \regex_replacement_char:nNN { \if_false: { \fi: ^^@ }
+ }
+
+ \char_set_catcode_math_subscript:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_D:w
+ { \regex_replacement_char:nNN { ^^@ } }
+
+ \char_set_catcode_group_begin:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_B:w
+ {
+ \int_incr:N \l_regex_nesting_int
+ \regex_replacement_char:nNN { \exp_after:wN ^^@ \if_false: } \fi: }
+ }
+
+ \char_set_catcode_active:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_A:w
+ { \regex_replacement_char:nNN { \exp_not:N ^^@ } }
+
+\group_end:
+
+\cs_new_protected_nopar:Npn \regex_replacement_char:nNN #1#2#3
+ {
+ \char_set_lccode:nn \c_zero { `#3 }
+ \tl_to_lowercase:n
+ { \regex_replacement_put:n { \exp_not:n {#1} } }
}
-\prg_new_protected_conditional:Npnn \regex_extract_once:nnN #1#2#3
- { T , F , TF }
+\cs_new_protected_nopar:Npn \regex_set:Nn
+ { \regex_set_aux:NNn \tl_set:Nn }
+\cs_new_protected_nopar:Npn \regex_gset:Nn
+ { \regex_set_aux:NNn \tl_gset:Nn }
+\cs_new_protected_nopar:Npn \regex_const:Nn
+ { \regex_set_aux:NNn \tl_const:Nn }
+\cs_new_protected:Npn \regex_set_aux:NNn #1#2#3
{
- \regex_user_aux:n
- {
- \regex_build:n {#1}
- \regex_match:n {#2}
- \regex_extract_after_group:N #3
- \regex_return_after_group:
- }
+ \group_begin:
+ \regex_build:n {#3}
+ \cs_set_nopar:Npx \regex_tmp:w
+ { #1 \exp_not:N #2 { \regex_set_aux:N #2 } }
+ \exp_after:wN
+ \group_end:
+ \regex_tmp:w
}
-\cs_new_protected:Npn \regex_extract_once:NnN #1#2#3
+\cs_new:Npn \regex_set_aux:N #1
{
- \regex_check_nfa:NT #1
+ \exp_not:n { \regex_nfa:Nw #1 }
+ \l_regex_max_state_int
+ = \int_use:N \l_regex_max_state_int
+ \l_regex_capturing_group_int
+ = \int_use:N \l_regex_capturing_group_int
+ \token_if_eq_meaning:NNT
+ \regex_match_loop_case_hook:
+ \regex_match_loop_caseless_hook:
{
- \regex_user_aux:n
- {
- \regex_use:N #1
- \regex_match:n {#2}
- \regex_extract_after_group:N #3
- }
+ \cs_set_eq:NN \regex_match_loop_case_hook:
+ \regex_match_loop_caseless_hook:
}
+ \prg_stepwise_function:nnnN
+ {0} {1} {\l_regex_max_state_int - \c_one }
+ \regex_set_aux:n
+ \s_stop
}
-\prg_new_protected_conditional:Npnn \regex_extract_once:NnN #1#2#3
- { T , F , TF }
+\cs_new_nopar:Npn \regex_set_aux:n #1
+ { \tex_toks:D #1 { \tex_the:D \tex_toks:D #1 } }
+\cs_new:Npn \regex_nfa:Nw #1
{
- \regex_check_nfa:NTF #1
- {
- \regex_user_aux:n
- {
- \regex_use:N #1
- \regex_match:n {#2}
- \regex_extract_after_group:N #3
- \regex_return_after_group:
- }
- }
- { \prg_return_false: }
+ \msg_expandable_kernel_error:nnn { regex } { nfa-misused } {#1}
+ \use_none_delimit_by_s_stop:w
}
-\cs_new_protected:Npn \regex_extract_all:nnN #1#2#3
+\cs_new_protected_nopar:Npn \regex_use:N
+ { \exp_last_unbraced:No \use_none:nn }
+\cs_new_nopar:Npn \regex_to_str:N #1
{
- \regex_user_aux:n
- {
- \seq_gclear:N \g_regex_extract_all_seq
- \tl_set:Nn \l_regex_every_match_tl { \regex_extract_all_aux: }
- \regex_build:n {#1}
- \regex_match:n {#2}
- \regex_extract_all_after_group:N #3
- }
+ \exp_after:wN \regex_to_str_aux:Nw #1
+ \tex_toks:D \q_stop \prg_map_break: \tex_toks:D
+ \prg_break_point:n { }
}
-\cs_new_protected:Npn \regex_extract_all:NnN #1#2#3
+\cs_new_nopar:Npn \regex_to_str_aux:Nw #1#2\tex_toks:D
{
- \regex_check_nfa:NT #1
- {
- \regex_user_aux:n
- {
- \seq_gclear:N \g_regex_extract_all_seq
- \tl_set:Nn \l_regex_every_match_tl { \regex_extract_all_aux: }
- \regex_use:N #1
- \regex_match:n {#2}
- \regex_extract_all_after_group:N #3
- }
- }
+ \use_none_delimit_by_q_stop:w #2 \q_stop
+ \tl_to_str:n {#1#2} \iow_newline:
+ \regex_to_str_aux:Nw \tex_toks:D
}
-\cs_new_protected_nopar:Npn \regex_extract_all_aux:
+\cs_new_protected_nopar:Npn \regex_aux_return:
{
- \regex_extract:
- \seq_gconcat:NNN \g_regex_extract_all_seq
- \g_regex_extract_all_seq \g_regex_submatches_seq
- \regex_match_once:
+ \if_meaning:w \c_true_bool \g_regex_success_bool
+ \prg_return_true:
+ \else:
+ \prg_return_false:
+ \fi:
}
-\cs_new_protected_nopar:Npn \regex_extract_all_after_group:N #1
+\cs_new_protected:Npn \regex_aux_build_match:nn #1#2
{
- \seq_if_empty:NF \g_regex_extract_all_seq
- {
- \regex_extract:
- \group_insert_after:N \seq_set_eq:NN
- \group_insert_after:N #1
- \group_insert_after:N \g_regex_extract_all_seq
- }
+ \regex_build:n {#1}
+ \regex_match:n {#2}
}
-\cs_new_protected:Npn \regex_split:nnN #1#2#3
+\cs_new_protected:Npn \regex_aux_use_match:Nn #1#2
{
- \regex_user_aux:n
+ \exp_args:No \tl_if_head_eq_meaning:nNTF {#1} \regex_nfa:Nw
{
- \seq_gclear:N \g_regex_split_seq
- \tl_set:Nn \l_regex_every_match_tl { \regex_split_aux: }
- \regex_build:n {#1}
+ \regex_use:N #1
\regex_match:n {#2}
- \regex_split_after_group:N #3
}
- }
-\cs_new_protected:Npn \regex_split:NnN #1#2#3
- {
- \regex_check_nfa:NT #1
- {
- \regex_user_aux:n
- {
- \seq_gclear:N \g_regex_split_seq
- \tl_set:Nn \l_regex_every_match_tl { \regex_split_aux: }
- \regex_use:N #1
- \regex_match:n {#2}
- \regex_split_after_group:N #3
- }
- }
- }
-\cs_new_protected_nopar:Npn \regex_split_aux:
- {
- \int_compare:nNnF \l_regex_start_step_int = \l_regex_success_step_int
{
- \regex_extract:
- \seq_pop:NN \g_regex_submatches_seq \l_regex_tmpa_tl
- \regex_extract_aux:nTF {0}
- {
- \seq_gput_left:Nx \g_regex_submatches_seq
- {
- \regex_query_substr:NN
- \l_regex_start_step_int \l_regex_tmpa_tl
- }
- }
- { \msg_error:nn { regex } { internal } }
- \seq_gconcat:NNN \g_regex_split_seq
- \g_regex_split_seq \g_regex_submatches_seq
+ \msg_kernel_error:nnx { regex } { not-nfa } { \token_to_str:N #1 }
+ \bool_gset_false:N \g_regex_success_bool
}
- \regex_match_once:
}
-\cs_new_protected_nopar:Npn \regex_split_after_group:N #1
+\cs_set_protected:Npn \regex_tmp:w #1#2#3
{
- \int_compare:nNnTF \l_regex_start_step_int = \l_regex_current_step_int
+ \cs_new_protected_nopar:Npn #1
+ { #3 \regex_aux_build_match:nn }
+ \cs_new_protected_nopar:Npn #2
+ { #3 \regex_aux_use_match:Nn }
+ \prg_new_protected_conditional:Npnn #1 ##1##2##3 { T , F , TF }
{
- \bool_if:NF \l_regex_success_empty_bool
- { \seq_gput_right:Nn \g_regex_split_seq { } }
+ #3 \regex_aux_build_match:nn {##1} {##2} ##3
+ \regex_aux_return:
}
+ \prg_new_protected_conditional:Npnn #2 ##1##2##3 { T , F , TF }
{
- \seq_gput_right:Nx \g_regex_split_seq
- {
- \regex_query_substr:NN
- \l_regex_start_step_int \l_regex_current_step_int
- }
+ #3 \regex_aux_use_match:Nn {##1} {##2} ##3
+ \regex_aux_return:
}
- \group_insert_after:N \seq_set_eq:NN
- \group_insert_after:N #1
- \group_insert_after:N \g_regex_split_seq
}
-\cs_new_protected:Npn \regex_replacement:n #1
+\tl_map_inline:nn
{
- \str_aux_escape:NNNn
- \prg_do_nothing:
- \regex_replacement_escaped:N
- \prg_do_nothing:
- {#1}
- \tl_set_eq:NN \l_regex_replacement_tl \g_str_result_tl
- \tl_set:Nx \l_regex_replacement_tl
- { \l_regex_replacement_tl \prg_do_nothing: }
+ { extract_once } { extract_all }
+ { replace_once } { replace_all }
+ { split }
}
-\cs_new_nopar:Npn \regex_replacement_escaped:N #1
{
- \regex_token_if_other_digit:NTF #1
- {
- \exp_not:n
- { \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } } {#1}
- }
- {
- \token_if_eq_charcode:NNTF g #1
- { \exp_not:N \regex_replacement_submatch:w }
- { #1 }
- }
+ \exp_args:Nccc \regex_tmp:w
+ { regex_#1:nnN } { regex_#1:NnN } { regex_#1_aux:NnnN }
}
-\cs_new_nopar:Npn \regex_replacement_submatch:w #1
+\cs_new_protected_nopar:Npn \regex_extract:
{
- \exp_after:wN \token_if_eq_meaning:NNTF \c_lbrace_str #1
- { \regex_replacement_submatch_aux:nN { } }
- {
- \regex_token_if_other_digit:NTF #1
- { \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } }
- { \msg_expandable_kernel_error:nn { regex } { g-misused } }
- #1
- }
+ \int_set_eq:NN \l_regex_submatch_start_int \l_regex_submatch_int
+ \if_meaning:w \c_true_bool \g_regex_success_bool
+ \prg_replicate:nn \l_regex_capturing_group_int
+ {
+ \tex_skip:D \l_regex_submatch_int \c_zero sp \scan_stop:
+ \int_incr:N \l_regex_submatch_int
+ }
+ \prop_map_inline:Nn \l_regex_success_submatches_prop
+ {
+ \if_num:w ##1 \c_max_int
+ \exp_after:wN \regex_extract_aux_b:wn \int_use:N
+ \else:
+ \exp_after:wN \regex_extract_aux_e:wn \int_use:N
+ \fi:
+ \int_eval:w \l_regex_submatch_start_int + ##1 {##2}
+ }
+ \fi:
}
-\cs_new_nopar:Npn \regex_replacement_submatch_aux:nN #1 #2
+\cs_new_protected_nopar:Npn \regex_extract_aux_b:wn #1 < #2
{
- \regex_token_if_other_digit:NTF #2
- { \regex_replacement_submatch_aux:nN {#1#2} }
- {
- \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } {#1}
- \exp_after:wN \token_if_eq_meaning:NNF \c_rbrace_str #2
- {
- \msg_expandable_kernel_error:nn { regex } { g-misused }
- #2
- }
- }
+ \tex_skip:D #1 = #2 sp
+ plus \etex_gluestretch:D \tex_skip:D #1 \scan_stop:
}
-\cs_new_protected_nopar:Npn \regex_replace_after_group:N #1
+\cs_new_protected_nopar:Npn \regex_extract_aux_e:wn #1 > #2
{
- \group_insert_after:N \tl_set:Nx
- \group_insert_after:N #1
- \group_insert_after:N {
- \group_insert_after:N \tl_to_str:N
- \group_insert_after:N \g_regex_replaced_str
- \group_insert_after:N }
+ \tex_skip:D #1
+ = 1 \tex_skip:D #1 plus #2 sp \scan_stop:
}
-\cs_new_protected:Npn \regex_replace_once_aux:Nn #1#2
+\cs_new_protected:Npn \regex_group_end_extract_seq:N #1
{
- \group_begin:
- \regex_replace_after_group:N #1
- \tl_clear:N \l_regex_every_match_tl
- #2
- \exp_args:No \regex_match:n {#1}
- \regex_extract:
- \regex_extract_aux:nTF {0}
+ \cs_set_eq:NN \seq_item:n \scan_stop:
+ \flag_lower:N \l_regex_group_begin_flag
+ \flag_lower:N \l_regex_group_end_flag
+ \tl_set:Nx \l_regex_tmpa_tl
{
- \cs_set_nopar:Npn \regex_tmp:w
- { \str_skip_do:nn { \l_regex_tmpb_tl } { } }
- \tl_gset:Nx \g_regex_replaced_str
- {
- \regex_query_substr:NN \c_zero \l_regex_tmpa_tl
- \l_regex_replacement_tl
- \exp_after:wN \regex_tmp:w \l_regex_query_other_str
- }
+ \prg_stepwise_function:nnnN
+ \l_regex_max_state_int
+ \c_one
+ { \l_regex_submatch_int - \c_one }
+ \regex_extract_seq_aux:n
}
- { \tl_gset_eq:NN \g_regex_replaced_str \l_regex_query_other_str }
- \group_end:
+ \flag_test:NF \l_regex_group_begin_flag
+ { \flag_test:NF \l_regex_group_end_flag { \use_none:nnn } }
+ \msg_kernel_error:nn { regex } { sequence-unbalanced }
+ \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl }
+ \exp_args:NNNo \group_end:
+ \tl_set:Nn #1 \l_regex_tmpa_tl
}
-\cs_new_protected:Npn \regex_replace_once:nnN #1#2#3
+\cs_new_nopar:Npn \regex_extract_seq_aux:n #1
{
- \regex_replace_once_aux:Nn #3
+ \seq_item:n
{
- \regex_build:n {#1}
- \regex_replacement:n {#2}
+ \exp_after:wN \regex_extract_seq_aux:ww
+ \int_value:w \regex_submatch_nesting_aux:n {#1} ; #1;
}
}
-\cs_new_protected:Npn \regex_replace_once:NnN #1#2#3
+\cs_new_nopar:Npn \regex_extract_seq_aux:ww #1; #2;
+ { %^^A todo: use flags
+ \if_num:w #1 < \c_zero
+ \flag_raise:N \l_regex_group_end_flag
+ \prg_replicate:nn {-#1} { \exp_not:n { { \if_false: } \fi: } }
+ \fi:
+ \regex_query_submatch:w #2;
+ \if_num:w #1 > \c_zero
+ \flag_raise:N \l_regex_group_begin_flag
+ \prg_replicate:nn {#1} { \exp_not:n { \if_false: { \fi: } } }
+ \fi:
+ }
+\prg_new_protected_conditional:Npnn \regex_match:nn #1#2 { T , F , TF }
{
- \regex_replace_once_aux:Nn #3
- {
- \regex_use:N #1
- \regex_replacement:n {#2}
- }
+ \regex_match_aux:n
+ { \regex_aux_build_match:nn {#1} {#2} }
}
-\prg_new_protected_conditional:Npnn \regex_replace_once:nnN #1#2#3 {T,F,TF}
+\prg_new_protected_conditional:Npnn \regex_match:Nn #1#2 { T , F , TF }
{
- \regex_replace_once_aux:Nn #3
- {
- \regex_build:n {#1}
- \regex_replacement:n {#2}
- \regex_return_after_group:
- }
+ \regex_match_aux:n
+ { \regex_aux_use_match:Nn #1 {#2} }
}
-\prg_new_protected_conditional:Npnn \regex_replace_once:NnN #1#2#3 {T,F,TF}
+\cs_new_protected:Npn \regex_match_aux:n #1
{
- \regex_replace_once_aux:Nn #3
- {
- \regex_use:N #1
- \regex_replacement:n {#2}
- \regex_return_after_group:
- }
+ \group_begin:
+ \tl_clear:N \l_regex_every_match_tl
+ \regex_disable_submatches:
+ #1
+ \group_end:
+ \regex_aux_return:
}
-\cs_new_protected:Npn \regex_replace_all_aux:Nn #1#2
+\cs_new_protected_nopar:Npn \regex_count:nnN
+ { \regex_count_aux:NnnN \regex_aux_build_match:nn }
+\cs_new_protected_nopar:Npn \regex_count:NnN
+ { \regex_count_aux:NnnN \regex_aux_use_match:Nn }
+\cs_new_protected:Npn \regex_count_aux:NnnN #1#2#3#4
{
\group_begin:
- \regex_replace_after_group:N #1
- \tl_set:Nn \l_regex_every_match_tl { \regex_replace_all_aux: }
- \tl_gclear:N \g_regex_replaced_str
- #2
- \exp_args:No \regex_match:n {#1}
- \cs_set_nopar:Npn \regex_tmp:w
- { \str_skip_do:nn { \l_regex_start_step_int } { } }
- \tl_gput_right:Nx \g_regex_replaced_str
- { \exp_after:wN \regex_tmp:w \l_regex_query_other_str }
+ \regex_disable_submatches:
+ \int_zero:N \l_regex_match_count_int
+ \tl_set:Nn \l_regex_every_match_tl
+ {
+ \int_incr:N \l_regex_match_count_int
+ \regex_match_once:
+ }
+ #1 {#2} {#3}
+ \exp_args:NNNo
\group_end:
+ \int_set:Nn #4 { \int_use:N \l_regex_match_count_int }
}
-\cs_new_protected_nopar:Npn \regex_replace_all_aux:
+\cs_new_protected:Npn \regex_extract_once_aux:NnnN #1#2#3#4
{
- \regex_extract:
- \regex_extract_aux:nTF {0}
- {
- \tl_gput_right:Nx \g_regex_replaced_str
+ \group_begin:
+ \tl_set:Nn \l_regex_every_match_tl { \regex_extract: }
+ #1 {#2} {#3}
+ \regex_group_end_extract_seq:N #4
+ }
+\cs_new_protected:Npn \regex_extract_all_aux:NnnN #1#2#3#4
+ {
+ \group_begin:
+ \tl_set:Nn \l_regex_every_match_tl
+ { \regex_extract: \regex_match_once: }
+ #1 {#2} {#3}
+ \regex_group_end_extract_seq:N #4
+ }
+\cs_new_protected:Npn \regex_split_aux:NnnN #1#2#3#4
+ {
+ \group_begin:
+ \tl_set:Nn \l_regex_every_match_tl
+ {
+ \if_num:w \l_regex_start_step_int < \l_regex_success_step_int
+ \regex_extract:
+ \tex_skip:D \l_regex_submatch_start_int
+ = \l_regex_start_step_int sp
+ plus \tex_skip:D \l_regex_submatch_start_int \scan_stop:
+ \fi:
+ \regex_match_once:
+ }
+ #1 {#2} {#3}
+ \tex_skip:D \l_regex_submatch_int
+ = \l_regex_start_step_int sp
+ plus \l_regex_current_step_int sp \scan_stop:
+ \int_incr:N \l_regex_submatch_int
+ \if_num:w \l_regex_start_step_int = \l_regex_current_step_int
+ \if_meaning:w \c_true_bool \l_regex_success_empty_bool
+ \int_decr:N \l_regex_submatch_int
+ \fi:
+ \fi:
+ \regex_group_end_extract_seq:N #4
+ }
+\cs_new_protected:Npn \regex_replace_once_aux:NnnN #1#2#3#4
+ {
+ \group_begin:
+ \tl_clear:N \l_regex_every_match_tl
+ \regex_replacement:n {#3}
+ \exp_args:Nno #1 {#2} #4
+ \if_meaning:w \c_true_bool \g_regex_success_bool
+ \regex_extract:
+ \int_set:Nn \l_regex_tmpa_int
+ { \regex_nesting:n { \l_regex_submatch_start_int } }
+ \if_num:w \l_regex_tmpa_int = \c_zero
+ \else:
+ \msg_kernel_error:nnx { regex } { replace-unbalanced }
+ { \l_regex_tmpa_int }
+ \fi:
+ \tl_set:Nx \l_regex_tmpa_tl
{
- \regex_query_substr:NN
- \l_regex_start_step_int \l_regex_tmpa_tl
- \l_regex_replacement_tl
+ \if_num:w \l_regex_tmpa_int < \c_zero
+ \prg_replicate:nn { - \l_regex_tmpa_int }
+ { \exp_not:n { { \if_false: } \fi: } }
+ \fi:
+ \regex_query_substr:nn
+ { \l_regex_min_step_int }
+ { \tex_skip:D \l_regex_submatch_start_int }
+ \regex_replacement_tl:n { \l_regex_submatch_start_int }
+ \regex_query_substr:nn
+ { \etex_gluestretch:D \tex_skip:D \l_regex_submatch_start_int }
+ { \l_regex_max_step_int }
+ \if_num:w \l_regex_tmpa_int > \c_zero
+ \prg_replicate:nn { \l_regex_tmpa_int }
+ { \exp_not:n { \if_false: { \fi: } } }
+ \fi:
}
- }
- { \msg_error:nn { regex } { internal } }
- \regex_match_once:
+ \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl }
+ \exp_args:NNNo \group_end:
+ \tl_set:Nn #4 \l_regex_tmpa_tl
+ \else:
+ \group_end:
+ \fi:
}
-\cs_new_protected:Npn \regex_replace_all:nnN #1#2#3
+\cs_new_protected_nopar:Npn \regex_replace_once_aux:w #1;
{
- \regex_replace_all_aux:Nn #3
+ }
+\cs_new_protected:Npn \regex_replace_all_aux:NnnN #1#2#3#4
+ {
+ \group_begin:
+ \tl_set:Nn \l_regex_every_match_tl
+ {
+ \regex_extract:
+ \tex_skip:D \l_regex_submatch_start_int
+ = \tex_the:D \tex_skip:D \l_regex_submatch_start_int
+ minus \l_regex_start_step_int sp \scan_stop:
+ \regex_match_once:
+ }
+ \regex_replacement:n {#3}
+ \exp_args:Nno #1 {#2} #4
+ \int_set:Nn \l_regex_tmpa_int
+ {
+ 0
+ \prg_stepwise_function:nnnN
+ \l_regex_max_state_int
+ \l_regex_capturing_group_int
+ { \l_regex_submatch_int - \c_one }
+ \regex_nesting:n
+ }
+ \if_num:w \l_regex_tmpa_int = \c_zero
+ \else:
+ \msg_kernel_error:nnx { regex } { replace-unbalanced }
+ { \l_regex_tmpa_int }
+ \fi:
+ \tl_set:Nx \l_regex_tmpa_tl
+ {
+ \if_num:w \l_regex_tmpa_int < \c_zero
+ \prg_replicate:nn { - \l_regex_tmpa_int }
+ { \exp_not:n { { \if_false: } \fi: } }
+ \fi:
+ \prg_stepwise_function:nnnN
+ \l_regex_max_state_int
+ \l_regex_capturing_group_int
+ { \l_regex_submatch_int - \c_one }
+ \regex_replace_all_aux:n
+ \regex_query_substr:nn
+ \l_regex_start_step_int \l_regex_max_step_int
+ \if_num:w \l_regex_tmpa_int > \c_zero
+ \prg_replicate:nn { \l_regex_tmpa_int }
+ { \exp_not:n { \if_false: { \fi: } } }
+ \fi:
+ }
+ \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl }
+ \exp_args:NNNo \group_end:
+ \tl_set:Nn #4 \l_regex_tmpa_tl
+ }
+\cs_new_nopar:Npn \regex_replace_all_aux:n #1
+ {
+ \regex_query_substr:nn
+ { \etex_glueshrink:D \tex_skip:D #1 } { \tex_skip:D #1 }
+ \regex_replacement_tl:n {#1}
+ }
+\msg_kernel_new:nnnn { regex } { sequence-unbalanced }
+ {
+ Missing~
+ \flag_test:NTF \l_regex_group_end_flag
{
- \regex_build:n {#1}
- \regex_replacement:n {#2}
+ left~
+ \flag_test:NTF \l_regex_group_begin_flag
+ { and~right~braces } { brace }
}
+ { right~brace }
+ \ inserted~in~extracted~match.
}
-\cs_new_protected:Npn \regex_replace_all:NnN #1#2#3
{
- \regex_replace_all_aux:Nn #3
+ LaTeX~was~asked~to~extract~submatches~or~split~a~token~list~
+ according~to~a~given~regular~expression,~but~some~of~the~resulting~
+ items~were~not~balanced.
+ }
+\msg_kernel_new:nnnn { regex } { replace-unbalanced }
+ { The~result~of~a~replacement~does~not~have~balanced~braces. }
+ {
+ LaTeX~was~asked~to~do~some~regular~expression~replacement,~
+ and~the~resulting~token~list~would~not~have~the~same~number~
+ of~begin-group~and~end-group~tokens. \\ \\
+ \ \ \ \
+ \prg_case_int:nnn {#1}
{
- \regex_use:N #1
- \regex_replacement:n {#2}
+ { -1 } { A~left~brace~was }
+ { 1 } { A~right~brace~was }
}
+ {
+ \int_abs:n {#1} ~
+ \int_compare:nNnTF {#1} < \c_zero { left } { right } ~
+ braces ~ were
+ }
+ \ inserted.
+ }
+\msg_kernel_new:nnnn { regex } { missing-rparen }
+ { Missing~right~parenthesis~added~in~regular~expression. }
+ {
+ LaTeX~was~given~a~regular~expression~with~\int_eval:n{#1}~
+ more~left~parenthes\int_compare:nTF{#1=1}{i}{e}s~than~right~
+ parenthes\int_compare:nTF{#1=1}{i}{e}s.
+ }
+\msg_kernel_new:nnnn { regex } { extra-rparen }
+ { Extra~right~parenthesis~ignored~in~regular~expression. }
+ {
+ LaTeX~came~across~a~closing~parenthesis~when~no~submatch~group~
+ was~open.~The~parenthesis~will~be~ignored.
}
-\msg_new:nnn { regex } { -998 } { Unsupported~construct }
-\msg_new:nnn { regex } { -997 }
- { The~regular~expression~is~too~large~(32768~states). }
-\msg_new:nnn { regex } { 1 } { \iow_char:N\\~at~end~of~pattern }
-\msg_new:nnn { regex } { 4 }
- { Numbers~out~of~order~in~\iow_char:N\{\iow_char\}~quantifier. }
-\msg_new:nnn { regex } { 6 }
- { Missing~terminating~\iow_char:N\]~for~character~class }
-\msg_new:nnn { regex } { 7 }
- { Invalid~escape~sequence~in~character~class }
-\msg_new:nnn { regex } { 8 }
- { Range~out~of~order~in~character~class }
-\msg_new:nnn { regex } { 22 } { Mismatched~parentheses }
-\msg_new:nnn { regex } { 34 }
- { Character~value~in~\iow_char:N\\x{...}~sequence~is~too~large }
-\msg_new:nnn { regex } { not-nfa }
- {
- LaTeX~was~expecting~a~regular~expression~variable.\\
- Instead,~LaTeX~found~'#1'.
+\msg_kernel_new:nnnn { regex } { backwards-range }
+ { Range~[#1-#2]~out~of~order~in~character~class. }
+ {
+ In~ranges~of~characters~[x-y]~appearing~in~character~classes,~
+ the~first~character~code~must~not~be~larger~than~the~second.~
+ Here,~#1~has~character~code~\int_eval:n {`#1},~while~#2~has~
+ character~code~\int_eval:n {`#2}.
+ }
+\msg_kernel_new:nnnn { regex } { unsupported }
+ { Unsupported~construction~`#1'. }
+ {
+ The~construction~`#1'~is~not~supported~by~the~LaTeX~
+ regular~expression~module.~Perhaps~some~character~should~
+ have~been~escaped?
+ }
+\msg_kernel_new:nnnn { regex } { not-nfa }
+ { This~is~not~a~regular~expression~variable. }
+ {
+ LaTeX~was~expecting~`#1'~to~be~a~regular~expression~variable.\\
+ This~control~sequence~is~not~a~regex~variable.~It's~current~meaning~
+ is~\\\\
+ \ \ \ \ \token_to_str:N #1 = \token_to_meaning:N #1 .
+ }
+\msg_kernel_new:nnnn { regex } { build-c }
+ { Misused~\iow_char:N\\c~or\iow_char:N\\C~command~in~regular~expression. }
+ {
+ The~\iow_char:N\\C~escape~sequence~can~be~followed~by~
+ one~of~the~letters~ABCDELMOPSTU.~The~\iow_char:N\\c~
+ escape~sequence~can~also~be~followed~by~a~brace~group.
+ }
+\msg_kernel_new:nnnn { regex } { unknown-option }
+ { Unknown~option~`#1'~for~regular~expressions. }
+ {
+ LaTeX~came~across~something~like~`(?#1)'~in~a~regular~expression,~
+ but~the~option~`#1'~is~not~known.~It~will~be~ignored.
+ }
+\msg_kernel_new:nnnn { regex } { invalid-in-option }
+ { Invalid~character~in~option~of~a~regular~expression. }
+ {
+ The~character~or~escape~sequence~`#1'~is~not~defined~
+ as~an~option~within~regular~expressions.
+ }
+\msg_kernel_new:nnnn { regex } { replacement-g }
+ { Missing~brace~for~the~\iow_char:N\\g~construction~in~a~replacement~text. }
+ {
+ In~the~replacement~text~for~a~regular~expression~search,~
+ submatches~are~represented~either~as~\iow_char:N \\g{dd..d},~
+ or~\\d,~where~`d'~are~single~digits.~Here,~a~brace~is~missing.
+ }
+\msg_kernel_new:nnnn { regex } { replacement-c }
+ { Something~is~wrong~with~the~replacement~text. }
+ {
+ The~\iow_char:N\\c~escape~sequence~can~only~be~followed~by~
+ ABCDELMN~or~by~a~brace.
}
\msg_kernel_new:nnn { regex } { nfa-misused }
{ Automaton~#1 used~incorrectly. }
-\msg_kernel_new:nnn { regex } { g-misused }
- { \g misused~in~replacement~text. }
%%
%%
%% End of file `l3regex.sty'.
diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-hex.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-hex.def
index 9af2e4322ad..b24f29458a5 100644
--- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-hex.def
+++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-hex.def
@@ -47,13 +47,16 @@
%% -----------------------------------------------------------------------
\cs_new_protected_nopar:Npn \str_convert_input_hex:
{
- \tl_gset:Nx \g_str_result_tl
- {
- \str_output_byte:w "
- \exp_after:wN \str_convert_input_hex_aux:N
- \g_str_result_tl 0 { ? 0 - \c_one \prg_map_break: }
- \prg_break_point:n { \str_output_end: }
- }
+ \group_begin:
+ \int_set:Nn \tex_escapechar:D { 92 }
+ \tl_gset:Nx \g_str_result_tl
+ {
+ \str_output_byte:w "
+ \exp_after:wN \str_convert_input_hex_aux:N
+ \g_str_result_tl 0 { ? 0 - \c_one \prg_map_break: }
+ \prg_break_point:n { \str_output_end: }
+ }
+ \group_end:
}
\cs_new_nopar:Npn \str_convert_input_hex_aux:N #1
{
diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-name.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-name.def
index 8d378c87e3d..66e7dddb6b8 100644
--- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-name.def
+++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-name.def
@@ -49,12 +49,15 @@
{
\cs_new_protected_nopar:Npn #2
{
- \tl_gset:Nx \g_str_result_tl
- {
- \exp_after:wN #3 \g_str_result_tl
- #1 ? { ? \prg_map_break: }
- \prg_break_point:n { }
- }
+ \group_begin:
+ \int_set:Nn \tex_escapechar:D { 92 }
+ \tl_gset:Nx \g_str_result_tl
+ {
+ \exp_after:wN #3 \g_str_result_tl
+ #1 ? { ? \prg_map_break: }
+ \prg_break_point:n { }
+ }
+ \group_end:
}
\cs_new_nopar:Npn #3 ##1#1##2##3
{
diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-string.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-string.def
index 7718a0d1b25..5e8fd099b25 100644
--- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-string.def
+++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-string.def
@@ -53,18 +53,21 @@
{
\cs_new_protected_nopar:Npn \str_convert_input_string:
{
- \tl_gset:Nx \g_str_result_tl
- {
- \exp_after:wN \str_convert_input_string_aux:wN
- \g_str_result_tl \prg_map_break: ^^M ?
- \prg_break_point:n { }
- }
- \tl_gset:Nx \g_str_result_tl
- {
- \exp_after:wN \str_convert_input_string_aux:wNNN
- \g_str_result_tl * ?? { ? \prg_map_break: }
- \prg_break_point:n { }
- }
+ \group_begin:
+ \int_set:Nn \tex_escapechar:D { 92 }
+ \tl_gset:Nx \g_str_result_tl
+ {
+ \exp_after:wN \str_convert_input_string_aux:wN
+ \g_str_result_tl \prg_map_break: ^^M ?
+ \prg_break_point:n { }
+ }
+ \tl_gset:Nx \g_str_result_tl
+ {
+ \exp_after:wN \str_convert_input_string_aux:wNNN
+ \g_str_result_tl * ?? { ? \prg_map_break: }
+ \prg_break_point:n { }
+ }
+ \group_end:
}
\cs_new_nopar:Npn \str_convert_input_string_aux:wNNN #1 *#2#3#4
}
diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-url.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-url.def
index bf6bf96514d..13cbde871fe 100644
--- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-url.def
+++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-url.def
@@ -49,12 +49,15 @@
{
\cs_new_protected_nopar:Npn #2
{
- \tl_gset:Nx \g_str_result_tl
- {
- \exp_after:wN #3 \g_str_result_tl
- #1 ? { ? \prg_map_break: }
- \prg_break_point:n { }
- }
+ \group_begin:
+ \int_set:Nn \tex_escapechar:D { 92 }
+ \tl_gset:Nx \g_str_result_tl
+ {
+ \exp_after:wN #3 \g_str_result_tl
+ #1 ? { ? \prg_map_break: }
+ \prg_break_point:n { }
+ }
+ \group_end:
}
\cs_new_nopar:Npn #3 ##1#1##2##3
{
diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf16.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf16.def
index e2bea7ca224..001b955ac07 100644
--- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf16.def
+++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf16.def
@@ -127,13 +127,13 @@
\if_meaning:w \scan_stop: #2
\str_aux_from_utf_xvi_end:Nw #1
\fi:
- \int_use:N \int_eval:w
- \if_case:w
- \int_eval:w ( \str_tmp:w #1#2 - "D6 ) / \c_four \int_eval_end:
- \or: \exp_after:wN \str_aux_from_utf_xvi:wNNN
- \or: \exp_after:wN \str_aux_from_utf_xvi_error:wNNw
- \fi:
- "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 ,
+ \if_case:w
+ \int_eval:w ( \str_tmp:w #1#2 - "D6 ) / \c_four \int_eval_end:
+ \or: #1#2 \exp_after:wN \str_aux_from_utf_xvi:wNNN
+ \or: \exp_after:wN \str_aux_from_utf_xvi_error:wNNw
+ \else: #1#2 \exp_not:N \q_mark \int_use:N \int_eval:w
+ \fi:
+ "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 ,
\str_aux_from_utf_xvi:NN
}
\cs_new:Npn \str_aux_from_utf_xvi:wNNN #1, #2#3#4
@@ -142,8 +142,10 @@
\str_aux_from_utf_xvi_end:wNNww #1 , #3
\fi:
\if_num:w \int_eval:w ( \str_tmp:w #3#4 - "DA ) / \c_four = \c_one
- ( #1 - "D800 ) * "400
- + \str_tmp:w #3#4 * "100 + \str_tmp:w #4#3 - "DC00 ,
+ #3 #4
+ \int_use:N \int_eval:w
+ ( #1 - "D800 ) * "400
+ + \str_tmp:w #3#4 * "100 + \str_tmp:w #4#3 - "DC00 ,
\exp_after:wN \use_i:nnn
\else:
\str_aux_from_utf_xvi_error:wNNw #1 ,
@@ -152,6 +154,7 @@
}
\cs_new:Npn \str_aux_from_utf_xvi_error:wNNw #1 \str_tmp:w #2#3 #4,
{
+ \exp_not:N \q_mark
\msg_expandable_kernel_error:nnn
{ str } { utf16-surrogate } { #2#3 }
\c_str_replacement_char_int ,
@@ -161,15 +164,17 @@
\fi:
\if_meaning:w \scan_stop: #1
\else:
+ #1 \exp_not:N \q_mark
\msg_expandable_kernel_error:nnn
{ str } { utf16-odd } {#1}
- \c_str_replacement_char_int ,
+ \int_use:N \c_str_replacement_char_int ,
\fi:
\prg_map_break:
}
\cs_new:Npn \str_aux_from_utf_xvi_end:wNNww #1 `#2#3 #4, #5 \fi:
{
\fi:
+ \exp_not:N \q_mark
\if_meaning:w \scan_stop: #5
\msg_expandable_kernel_error:nnn
{ str } { utf16-surrogate } { #2#3 }
@@ -177,7 +182,7 @@
\msg_expandable_kernel_error:nnn
{ str } { utf16-odd } { #2#3#5 }
\fi:
- \c_str_replacement_char_int ,
+ \int_use:N \c_str_replacement_char_int ,
\prg_map_break:
}
\group_end:
diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf32.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf32.def
index 7c56d2e3929..8251d8c44fe 100644
--- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf32.def
+++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf32.def
@@ -107,31 +107,33 @@
\cs_new:Npn \str_aux_from_utf_xxxii:NNNN #1#2#3#4
{
\if_meaning:w \scan_stop: #4
- \str_aux_from_utf_xxxii_end:ww #1#2#3#4
+ \exp_after:wN \str_aux_from_utf_xxxii_end:w
+ \fi:
+ #1#2#3#4 \exp_not:N \q_mark
+ \if_num:w \int_eval:w
+ \str_tmp:w #1#4 * \c_two_hundred_fifty_six
+ + \str_tmp:w #2#3
+ > \c_sixteen
+ \msg_expandable_kernel_error:nnn
+ { str } { utf32-overflow } { #1#2#3#4 }
+ \int_use:N \c_str_replacement_char_int
+ \else:
+ \int_use:N \int_eval:w
+ \str_tmp:w #2#3*"10000
+ + \str_tmp:w #3#2 * \c_two_hundred_fifty_six
+ + \str_tmp:w #4#1
\fi:
- \int_use:N \int_eval:w
- \if_num:w \int_eval:w
- \str_tmp:w #1#4 * \c_two_hundred_fifty_six
- + \str_tmp:w #2#3
- > \c_sixteen
- \msg_expandable_kernel_error:nnn
- { str } { utf32-overflow } { #1#2#3#4 }
- \c_str_replacement_char_int
- \else:
- \str_tmp:w #2#3*"10000
- + \str_tmp:w #3#2 * \c_two_hundred_fifty_six
- + \str_tmp:w #4#1
- \fi:
,
\str_aux_from_utf_xxxii:NNNN
}
- \cs_new:Npn \str_aux_from_utf_xxxii_end:ww #1 \scan_stop: #2 \fi:
+ \cs_new:Npn \str_aux_from_utf_xxxii_end:w #1 \scan_stop:
{
- \fi:
\tl_if_empty:nF {#1}
{
\msg_expandable_kernel_error:nnn
{ str } { utf32-truncated } { #1 }
+ #1 \exp_not:N \q_mark
+ \int_use:N \c_str_replacement_char_int ,
}
\prg_map_break:
}
diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf8.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf8.def
index 9e0f26c2346..d6233f22557 100644
--- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf8.def
+++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf8.def
@@ -74,32 +74,33 @@
\tl_gset:Nx \g_str_result_tl
{
\exp_after:wN \str_aux_from_utf_viii:N \g_str_result_tl
- { ? \prg_map_break: \q_stop \str_aux_from_utf_viii_error: }
+ { \prg_map_break: \str_aux_from_utf_viii_error: }
\prg_break_point:n { }
}
}
\cs_new_nopar:Npn \str_aux_from_utf_viii:N #1
{
- \use_none:n #1
- \int_use:N \int_eval:w
- \if_num:w `#1 < "C0 \exp_stop_f:
- \if_num:w `#1 < "80 \exp_stop_f:
- `#1
- \else:
- \msg_expandable_kernel_error:nnn
- { str } { utf8-extra-conti } {#1}
- \c_str_replacement_char_int
- \fi:
+ #1
+ \if_num:w `#1 < "C0 \exp_stop_f:
+ \exp_not:N \q_mark
+ \if_num:w `#1 < "80 \exp_stop_f:
+ \int_value:w `#1
\else:
- \if_num:w `#1 < "F5 \exp_stop_f:
- \exp_after:wN \str_aux_from_utf_viii:wwNN
- \int_use:N \int_eval:w `#1 - "C0
- \else:
- \msg_expandable_kernel_error:nnn
- { str } { utf8-invalid-byte } {#1}
- \c_str_replacement_char_int
- \fi:
+ \msg_expandable_kernel_error:nnn
+ { str } { utf8-extra-conti } {#1}
+ \int_use:N \c_str_replacement_char_int
\fi:
+ \else:
+ \if_num:w `#1 < "F5 \exp_stop_f:
+ \exp_after:wN \str_aux_from_utf_viii:wwNN
+ \int_use:N \int_eval:w `#1 - "C0
+ \else:
+ \exp_not:N \q_mark
+ \msg_expandable_kernel_error:nnn
+ { str } { utf8-invalid-byte } {#1}
+ \int_use:N \c_str_replacement_char_int
+ \fi:
+ \fi:
,
\use_none_delimit_by_q_stop:w
"80 , "800 , "10000 , "10FFFF ,
@@ -108,7 +109,7 @@
}
\cs_new_nopar:Npn \str_aux_from_utf_viii:wwNN #1 , #2 \q_stop #3#4
{
- \use_none_delimit_by_q_stop:w #4 \q_stop
+ \use_none:n #4
\exp_after:wN \str_aux_from_utf_viii:wNNww
\int_use:N \int_eval:w #1 * "40 + `#4 - "80 ,
#4
@@ -121,8 +122,9 @@
\if_num:w #1 < #4 \exp_stop_f:
\str_aux_from_utf_viii_error:w #1 ,
\else:
+ #2
\if_num:w #1 < #5 \exp_stop_f:
- #1
+ \exp_not:N \q_mark #1
\else:
\exp_after:wN \str_aux_from_utf_viii:wwNN
\int_use:N \int_eval:w #1 - #5
@@ -136,12 +138,14 @@
}
\cs_new_nopar:Npn \str_aux_from_utf_viii_error:w #1 ,
{
+ \exp_not:N \q_mark
\msg_expandable_kernel_error:nnn
{ str } { utf8-overlong } {#1}
\c_str_replacement_char_int
}
\cs_new_nopar:Npn \str_aux_from_utf_viii_error:NwN #1 #2 \q_stop #3
{
+ \exp_not:N \q_mark
\msg_expandable_kernel_error:nnn
{ str } { utf8-missing-conti } {#1}
\c_str_replacement_char_int ,
@@ -149,6 +153,7 @@
}
\cs_new_nopar:Npn \str_aux_from_utf_viii_error:
{
+ \exp_not:N \q_mark
\msg_expandable_kernel_error:nn
{ str } { utf8-premature-end }
\c_str_replacement_char_int ,
diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str.sty b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str.sty
index c49601841e0..7d527b6d264 100644
--- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str.sty
+++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str.sty
@@ -46,23 +46,10 @@
%%
%% -----------------------------------------------------------------------
\RequirePackage{expl3}
-\GetIdInfo$Id: l3str.dtx 3035 2011-12-07 11:48:46Z bruno $
+\GetIdInfo$Id: l3str.dtx 3088 2011-12-23 02:00:14Z bruno $
{L3 Experimental Strings}
\ProvidesExplPackage
{\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
-\cs_if_exist:NF \use_i:nnnnnnnn
- { \cs_new:Npn \use_i:nnnnnnnn #1#2#3#4#5#6#7#8 {#1} }
-\int_const:Nn \c_forty_eight { 48 }
-\int_const:Nn \c_fifty_eight { 58 }
-\int_const:Nn \c_sixty_five { 65 }
-\int_const:Nn \c_ninety_one { 91 }
-\int_const:Nn \c_ninety_seven { 97 }
-\int_const:Nn \c_one_hundred_twenty_three { 123 }
-\int_const:Nn \c_one_hundred_twenty_seven { 127 }
-\pdftex_if_engine:TF
- { \int_const:Nn \c_max_char_int { 255 } }
- { \int_const:Nn \c_max_char_int { 1114111 } }
-\int_const:Nn \c_str_replacement_char_int { "FFFD }
\group_begin:
\cs_set_protected_nopar:Npn \str_tmp:w #1
{
@@ -80,8 +67,19 @@
\group_end:
\cs_new_protected_nopar:Npn \str_tmp:w { }
\tl_new:N \l_str_tmpa_tl
+\int_new:N \l_str_tmpa_int
\tl_new:N \g_str_result_tl
\int_new:N \l_str_char_int
+\int_const:Nn \c_forty_eight { 48 }
+\int_const:Nn \c_fifty_eight { 58 }
+\int_const:Nn \c_sixty_five { 65 }
+\int_const:Nn \c_ninety_one { 91 }
+\int_const:Nn \c_ninety_seven { 97 }
+\int_const:Nn \c_one_hundred_twenty_three { 123 }
+\int_const:Nn \c_one_hundred_twenty_seven { 127 }
+\int_const:Nn \c_max_char_int
+ { \pdftex_if_engine:TF { 255 } { 1114111 } }
+\int_const:Nn \c_str_replacement_char_int { "FFFD }
\tl_const:Nx \c_backslash_str { \cs_to_str:N \\ }
\tl_const:Nx \c_lbrace_str { \cs_to_str:N \{ }
\tl_const:Nx \c_rbrace_str { \cs_to_str:N \} }
@@ -100,7 +98,6 @@
\prop_gput:Nnn \g_str_aliases_prop { latin9 } { iso885915 }
\prop_gput:Nnn \g_str_aliases_prop { latin10 } { iso885916 }
\prop_gput:Nnn \g_str_aliases_prop { hexadecimal } { hex }
-\prop_gput:Nnn \g_str_aliases_prop { percent } { url }
\group_begin:
\char_set_lccode:nn { `\* } { `\ }
\char_set_lccode:nn { `\A } { `\A }
@@ -124,72 +121,53 @@
\cs_new_nopar:Npn \tl_to_other_str_end:w \fi: #1 \q_mark #2 * A #3 \q_stop
{ \fi: #2 }
}
-\cs_new_nopar:Npn \tl_to_other_str:N { \exp_args:No \tl_to_other_str:n }
-\cs_new:Npn \str_sanitize_args:Nn #1#2
- { \exp_args:Nf #1 { \tl_to_other_str:n {#2} } }
-\cs_new:Npn \str_sanitize_args:Nnn #1#2#3
- {
- \exp_args:Nff #1
- { \tl_to_other_str:n {#2} }
- { \tl_to_other_str:n {#3} }
- }
\group_begin:
\char_set_lccode:nn { `\* } { `\ }
\char_set_lccode:nn { `\A } { `\A }
\tl_to_lowercase:n
{
\group_end:
- \cs_new_protected:Npn \str_aux_gset_other:Nn #1#2
+ \cs_new_protected:Npn \str_gset_other:Nn #1#2
{
\tl_gset:Nx #1
{
- \exp_after:wN \str_aux_gset_other_loop:w \tl_to_str:n {#2} ~ %
+ \exp_after:wN \str_gset_other_loop:w \tl_to_str:n {#2} ~ %
A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ \q_mark \q_stop
}
}
- \cs_new_nopar:Npn \str_aux_gset_other_loop:w
+ \cs_new_nopar:Npn \str_gset_other_loop:w
#1 ~ #2 ~ #3 ~ #4 ~ #5 ~ #6 ~ #7 ~ #8 ~ #9 ~
{
\if_meaning:w A #9
- \str_aux_gset_other_end:w
+ \str_gset_other_end:w
\fi:
#1 * #2 * #3 * #4 * #5 * #6 * #7 * #8 * #9
- \str_aux_gset_other_loop:w *
+ \str_gset_other_loop:w *
}
- \cs_new_nopar:Npn \str_aux_gset_other_end:w \fi: #1 * A #2 \q_stop
+ \cs_new_nopar:Npn \str_gset_other_end:w \fi: #1 * A #2 \q_stop
{ \fi: #1 }
}
-\cs_new_nopar:Npn \str_map_tokens:Nn
- { \exp_args:No \str_map_tokens:nn }
-\cs_new:Npn \str_map_tokens:nn #1
- { \exp_args:No \str_map_tokens_aux:nn { \tl_to_str:n {#1} } }
-\cs_new:Npn \str_map_tokens_aux:nn #1#2
+\cs_if_exist:NF \use_i:nnnnnnnn
+ { \cs_new:Npn \use_i:nnnnnnnn #1#2#3#4#5#6#7#8 {#1} }
+\cs_new:Npn \str_aux_toks_range:nn #1#2
{
- \str_map_tokens_loop:nw {#2} #1
- { ? \use_none_delimit_by_q_recursion_stop:w }
- ?~ ?~ ?~ ?~ ?~ ?~ ?~ ?~ \q_recursion_stop
+ \exp_after:wN \str_aux_toks_range:ww
+ \int_use:N \int_eval:w #1 \exp_after:wN ;
+ \int_use:N \int_eval:w #2 ;
+ \prg_break_point:n { }
}
-\group_begin:
- \char_set_lccode:nn { `* } { `\ }
- \tl_to_lowercase:n
- {
- \group_end:
- \cs_new:Npn \str_map_tokens_loop:nw #1 #2~ #3~ #4~ #5~ #6~ #7~ #8~ #9~
- {
- \str_map_tokens_loop:nN {#1} #2* #3* #4* #5* #6* #7* #8* #9*
- { ? \use_none_delimit_by_q_stop:w } \q_stop
- \str_map_tokens_loop:nw {#1}
- }
- }
-\cs_new:Npn \str_map_tokens_loop:nN #1#2
+\cs_new:Npn \str_aux_toks_range:ww #1 ; #2 ;
{
- \use_none:n #2
- #1 #2
- \str_map_tokens_loop:nN {#1}
+ \if_num:w #1 = #2 \exp_stop_f:
+ \exp_after:wN \prg_map_break:
+ \fi:
+ \tex_the:D \tex_toks:D #1 \exp_stop_f:
+ \exp_after:wN \str_aux_toks_range:ww
+ \int_use:N \int_eval:w #1 + \c_one ; #2 ;
}
-\cs_new_eq:NN \str_map_break_do:n \use_i_delimit_by_q_recursion_stop:nw
\cs_new_nopar:Npn \str_length:N { \exp_args:No \str_length:n }
-\cs_new:Npn \str_length:n { \str_sanitize_args:Nn \str_length_unsafe:n }
+\cs_new:Npn \str_length:n #1
+ { \exp_args:Nf \str_length_unsafe:n { \tl_to_other_str:n {#1} } }
\cs_new_nopar:Npn \str_length_unsafe:n #1
{ \str_length_aux:n { \str_length_loop:NNNNNNNNN #1 } }
\cs_new:Npn \str_length_ignore_spaces:n #1
@@ -308,7 +286,7 @@
\cs_new:Npn \str_item:nn #1#2
{
\exp_args:Nf \tl_to_str:n
- { \str_sanitize_args:Nn \str_item_unsafe:nn {#1} {#2} }
+ { \exp_args:Nf \str_item_unsafe:nn { \tl_to_other_str:n {#1} } {#2} }
}
\cs_new:Npn \str_item_ignore_spaces:nn #1
{ \exp_args:No \str_item_unsafe:nn { \tl_to_str:n {#1} } }
@@ -344,26 +322,31 @@
\cs_new:Npn \str_substr:nnn #1#2#3
{
\exp_args:Nf \tl_to_str:n
- { \str_sanitize_args:Nn \str_substr_unsafe:nnn {#1} {#2} {#3} }
+ {
+ \exp_args:Nf \str_substr_unsafe:nnn
+ { \tl_to_other_str:n {#1} } {#2} {#3}
+ }
}
\cs_new:Npn \str_substr_ignore_spaces:nnn #1
{ \exp_args:No \str_substr_unsafe:nnn { \tl_to_str:n {#1} } }
\cs_new:Npn \str_substr_unsafe:nnn #1#2#3
{
- \str_aux_eval_args:Nnnn \str_substr_aux:nnnw
- { \str_length_unsafe:n {#1} }
- {#2}
- {#3}
+ \exp_after:wN \str_substr_aux:www
+ \int_value:w \str_length_unsafe:n {#1} \exp_after:wN ;
+ \int_use:N \int_eval:w #2 + \c_zero \exp_after:wN ;
+ \int_use:N \int_eval:w \exp_args:Nf \str_substr_unsafe:nN {#3} \c_max_int ;
#1
\q_stop
}
-\cs_new:Npn \str_substr_aux:nnnw #1#2#3
+\cs_new:Npn \str_substr_unsafe:nN #1 #2
+ { \tl_if_empty:nTF {#1} {#2} {#1} }
+\cs_new:Npn \str_substr_aux:www #1; #2; #3;
{
- \exp_args:Nf \str_substr_aux_ii:nnw
+ \exp_args:Nf \str_substr_aux:nnw
{ \str_aux_normalize_range:nn {#2} {#1} }
{ \str_aux_normalize_range:nn {#3} {#1} }
}
-\cs_new:Npn \str_substr_aux_ii:nnw #1#2
+\cs_new:Npn \str_substr_aux:nnw #1#2
{
\str_skip_do:nn {#1}
{
@@ -372,13 +355,6 @@
{ \use_i_delimit_by_q_stop:nw }
}
}
-\cs_new:Npn \str_aux_eval_args:Nnnn #1#2#3#4
- {
- \exp_after:wN #1
- \exp_after:wN { \int_value:w \int_eval:w #2 \exp_after:wN }
- \exp_after:wN { \int_value:w \int_eval:w #3 \exp_after:wN }
- \exp_after:wN { \int_value:w \int_eval:w #4 }
- }
\cs_new:Npn \str_aux_normalize_range:nn #1#2
{
\int_eval:n
@@ -403,69 +379,32 @@
\if_int_compare:w \pdftex_strcmp:D { \tl_to_str:N #1 } { \tl_to_str:N #2 }
= \c_zero \prg_return_true: \else: \prg_return_false: \fi:
}
-\prg_new_conditional:Npnn \str_if_contains_char:NN #1#2 { p , T , F , TF }
+\prg_new_conditional:Npnn \str_if_contains_char:NN #1#2 { T , TF }
{
- \str_map_tokens:Nn #1 { \str_if_contains_char_aux:NN #2 }
+ \exp_after:wN \str_if_contains_char_aux:NN \exp_after:wN #2
+ #1 { \prg_map_break:n { ? \fi: } }
+ \prg_break_point:n { }
\prg_return_false:
}
-\prg_new_conditional:Npnn \str_if_contains_char:nN #1#2 { p , T , F , TF }
+\prg_new_conditional:Npnn \str_if_contains_char:nN #1#2 { TF }
{
- \str_map_tokens:nn {#1} { \str_if_contains_char_aux:NN #2 }
+ \exp_after:wN \str_if_contains_char_aux:NN \exp_after:wN #2
+ \tl_to_str:n {#1} { \prg_map_break:n { ? \fi: } }
+ \prg_break_point:n { }
\prg_return_false:
}
\cs_new_nopar:Npn \str_if_contains_char_aux:NN #1#2
{
- \if_num:w `#1 = `#2 \exp_stop_f:
- \str_if_contains_char_end:w
+ \if_charcode:w #1 #2
+ \exp_after:wN \str_if_contains_char_end:
\fi:
+ \str_if_contains_char_aux:NN #1
}
-\cs_new_nopar:Npn \str_if_contains_char_end:w \fi: #1 \prg_return_false:
- { \fi: \prg_return_true: }
-\pdftex_if_engine:TF
- {
- \cs_new_eq:NN \str_if_bytes_p:N \c_true_bool
- \cs_new_eq:NN \str_if_bytes:NT \use_ii:nn
- \cs_new_eq:NN \str_if_bytes:NF \use_none:nn
- \cs_new_eq:NN \str_if_bytes:NTF \use_ii:nnn
- }
- {
- \prg_new_conditional:Npnn \str_if_bytes:N #1 { p , T , F , TF }
- {
- \exp_args:Nf \tl_map_function:nN
- { \tl_to_str:N #1 }
- \str_if_bytes_aux:N
- \prg_return_true:
- }
- \cs_new_nopar:Npn \str_if_bytes_aux:N #1
- {
- \if_num:w `#1 > \c_two_hundred_fifty_five
- \exp_after:wN \str_if_bytes_aux:
- \fi:
- }
- \cs_new_nopar:Npn \str_if_bytes_aux:
- { \tl_map_break:n { \prg_return_false: \use_none:n } }
- }
-\prg_new_protected_conditional:Npnn \str_aux_hexadecimal_test:N #1 { TF }
- {
- \tl_if_in:onTF { \tl_to_str:n { abcdef } } {#1}
- {
- \int_set:Nn \l_str_char_int
- { \c_sixteen * \l_str_char_int + `#1 - 87 }
- \prg_return_true:
- }
- {
- \if_num:w \c_fifteen < "1 \exp_not:N #1 \exp_stop_f:
- \int_set:Nn \l_str_char_int
- { \c_sixteen * \l_str_char_int + "#1 }
- \prg_return_true:
- \else:
- \prg_return_false:
- \fi:
- }
- }
+\cs_new_nopar:Npn \str_if_contains_char_end:
+ { \prg_map_break:n { \prg_return_true: \use_none:n } }
\prg_new_conditional:Npnn \str_aux_octal_use:N #1 { TF }
{
- \if_num:w \c_one < '1 #1 \exp_stop_f:
+ \if_num:w \c_one < '1 \token_to_str:N #1 \exp_stop_f:
#1 \prg_return_true:
\else:
\prg_return_false:
@@ -473,21 +412,27 @@
}
\prg_new_conditional:Npnn \str_aux_hexadecimal_use:N #1 { TF }
{
- \if_num:w \c_two < "1 #1 \exp_stop_f:
+ \if_num:w \c_two < "1 \token_to_str:N #1 \exp_stop_f:
#1 \prg_return_true:
\else:
- \if_case:w \int_eval:w `#1 - `a \int_eval_end:
- A \prg_return_true:
- \or: B \prg_return_true:
- \or: C \prg_return_true:
- \or: D \prg_return_true:
- \or: E \prg_return_true:
- \or: F \prg_return_true:
- \else: \prg_return_false:
+ \if_case:w \int_eval:w
+ \exp_after:wN ` \token_to_str:N #1 - `a
+ \int_eval_end:
+ A
+ \or: B
+ \or: C
+ \or: D
+ \or: E
+ \or: F
+ \else:
+ \prg_return_false:
+ \exp_after:wN \use_none:n
\fi:
+ \prg_return_true:
\fi:
}
\group_begin:
+ \tl_clear:N \l_str_tmpa_tl
\char_set_catcode_other:n { \c_zero }
\tl_gset:Nx \g_str_result_tl { \tl_to_str:n { 0123456789ABCDEF } }
\tl_map_inline:Nn \g_str_result_tl
@@ -496,15 +441,18 @@
{
\tl_map_inline:Nn \g_str_result_tl
{
- \char_set_lccode:nn { \c_zero } {"##1#1}
+ \char_set_lccode:nn { \c_zero } {"#1##1}
\tl_to_lowercase:n
{
\tl_const:cx
- { c_str_byte_ \int_eval:n {"##1#1} _tl }
- { ^^@ ##1 #1 }
+ { c_str_byte_ \int_eval:n {"#1##1} _tl }
+ { ^^@ #1 ##1 }
+ \tl_put_right:Nn \l_str_tmpa_tl { ^^@ }
}
}
}
+ \tl_const:Nx \c_str_positive_bytes_tl
+ { \exp_after:wN \use_none:n \l_str_tmpa_tl }
\group_end:
\tl_const:cn { c_str_byte_-1_tl } { { } \use_none:n { } }
\cs_new:Npn \str_output_byte:n #1
@@ -545,17 +493,17 @@
{
\tl_gset:Nx \g_str_result_tl
{
- \exp_after:wN \str_aux_gmap_internal_result_loop:Nw
+ \exp_after:wN \str_aux_gmap_internal_result_loop:Nww
\exp_after:wN #1
- \g_str_result_tl \q_stop \prg_map_break: ,
+ \g_str_result_tl \q_mark \q_stop \prg_map_break: ,
\prg_break_point:n { }
}
}
-\cs_new_nopar:Npn \str_aux_gmap_internal_result_loop:Nw #1#2,
+\cs_new_nopar:Npn \str_aux_gmap_internal_result_loop:Nww #1#2\q_mark #3,
{
- \use_none_delimit_by_q_stop:w #2 \q_stop
- #1 {#2}
- \str_aux_gmap_internal_result_loop:Nw #1
+ \use_none_delimit_by_q_stop:w #3 \q_stop
+ #1 {#3}
+ \str_aux_gmap_internal_result_loop:Nww #1
}
\cs_new_protected:Npn \str_input:Nn #1#2
{
@@ -567,9 +515,11 @@
\str_aux_escape:NNNn \token_to_str:N \token_to_str:N \token_to_str:N {#2}
\tl_gset_eq:NN #1 \g_str_result_tl
}
-\cs_new_eq:NN \str_aux_escape_unescaped:N \use:n
-\cs_new_eq:NN \str_aux_escape_escaped:N \use:n
-\cs_new_eq:NN \str_aux_escape_raw:N \use:n
+\cs_new_protected:Npn \str_escape_use:NNNn #1#2#3#4
+ {
+ \str_aux_escape:NNNn #1#2#3 {#4}
+ \g_str_result_tl
+ }
\cs_new_protected:Npn \str_aux_escape:NNNn #1#2#3#4
{
\group_begin:
@@ -577,12 +527,15 @@
\cs_set_nopar:Npn \str_aux_escape_escaped:N { #2 }
\cs_set_nopar:Npn \str_aux_escape_raw:N { #3 }
\int_set:Nn \tex_escapechar:D { 92 }
- \str_aux_gset_other:Nn \g_str_result_tl {#4}
+ \str_gset_other:Nn \g_str_result_tl {#4}
+ \int_zero:N \l_str_tmpa_int
\tl_gset:Nx \g_str_result_tl
{
\exp_after:wN \str_aux_escape_loop:N \g_str_result_tl
\q_recursion_tail \q_recursion_stop
}
+ \tl_gput_left:Nx \g_str_result_tl
+ { \str_aux_toks_range:nn \c_zero \l_str_tmpa_int }
\group_end:
}
\cs_new_nopar:Npn \str_aux_escape_loop:N #1
@@ -598,6 +551,9 @@
{ \str_aux_escape_escaped:N #1 }
\str_aux_escape_loop:N
}
+\cs_new_eq:NN \str_aux_escape_unescaped:N \prg_do_nothing:
+\cs_new_eq:NN \str_aux_escape_escaped:N \prg_do_nothing:
+\cs_new_eq:NN \str_aux_escape_raw:N \prg_do_nothing:
\cs_new_eq:cN
{ str_aux_escape_ \c_backslash_str q_recursion_tail :w }
\use_none_delimit_by_q_recursion_stop:w
@@ -620,6 +576,8 @@
\cs_new_nopar:cpn { str_aux_escape_/x:w } \str_aux_escape_loop:N
{
\if_false: { \fi: }
+ \tex_toks:D \l_str_tmpa_int \exp_after:wN { \g_str_result_tl }
+ \int_incr:N \l_str_tmpa_int
\int_zero:N \l_str_char_int
\str_aux_escape_x_test:N
}
@@ -628,35 +586,36 @@
\exp_not:N \token_if_eq_charcode:NNTF \c_space_token #1
{ \exp_not:N \str_aux_escape_x_test:N }
{
- \exp_not:N \token_if_eq_charcode:NNTF \c_lbrace_str #1
- { \exp_not:N \str_aux_escape_x_braced_loop:N }
- { \exp_not:N \str_aux_escape_x_unbraced_i:N #1 }
+ \l_str_char_int = "0
+ \exp_not:N \token_if_eq_charcode:NNTF \c_lbrace_str #1
+ { \exp_not:N \str_aux_escape_x_braced_loop:N }
+ { \exp_not:N \str_aux_escape_x_unbraced_i:N #1 }
}
}
-\cs_new_protected_nopar:Npn \str_aux_escape_x_unbraced_i:N #1
+\cs_new_nopar:Npn \str_aux_escape_x_unbraced_i:N #1
{
- \str_aux_hexadecimal_test:NTF #1
+ \str_aux_hexadecimal_use:NTF #1
{ \str_aux_escape_x_unbraced_ii:N }
- { \str_aux_escape_x_end: #1 }
+ { \exp_stop_f: \str_aux_escape_x_end: #1 }
}
-\cs_new_protected_nopar:Npn \str_aux_escape_x_unbraced_ii:N #1
+\cs_new_nopar:Npn \str_aux_escape_x_unbraced_ii:N #1
{
\token_if_eq_charcode:NNTF \c_space_token #1
{ \str_aux_escape_x_unbraced_ii:N }
{
- \str_aux_hexadecimal_test:NTF #1
- { \str_aux_escape_x_end: }
- { \str_aux_escape_x_end: #1 }
+ \str_aux_hexadecimal_use:NTF #1
+ { \exp_stop_f: \str_aux_escape_x_end: }
+ { \exp_stop_f: \str_aux_escape_x_end: #1 }
}
}
-\cs_new_protected_nopar:Npn \str_aux_escape_x_braced_loop:N #1
+\cs_new_nopar:Npn \str_aux_escape_x_braced_loop:N #1
{
\token_if_eq_charcode:NNTF \c_space_token #1
{ \str_aux_escape_x_braced_loop:N }
{
- \str_aux_hexadecimal_test:NTF #1
+ \str_aux_hexadecimal_use:NTF #1
{ \str_aux_escape_x_braced_loop:N }
- { \str_aux_escape_x_braced_end:N #1 }
+ { \exp_stop_f: \str_aux_escape_x_braced_end:N #1 }
}
}
\cs_new_protected_nopar:Npx \str_aux_escape_x_braced_end:N #1
@@ -675,7 +634,7 @@
\tex_lccode:D \c_zero \l_str_char_int
\tl_to_lowercase:n
{
- \tl_gput_right:Nx \g_str_result_tl
+ \tl_gset:Nx \g_str_result_tl
{ \if_false: } \fi:
\str_aux_escape_raw:N ^^@
\str_aux_escape_loop:N
@@ -683,13 +642,13 @@
}
\group_end:
\cs_new_protected:Npn \str_set_convert:Nnnn
- { \str_convert_aux_i:NNnnn \str_set:Nx }
+ { \str_convert_aux_i:NNnnn \tl_set_eq:NN }
\cs_new_protected:Npn \str_gset_convert:Nnnn
- { \str_convert_aux_i:NNnnn \str_gset:Nx }
+ { \str_convert_aux_i:NNnnn \tl_gset_eq:NN }
\cs_new_protected:Npn \str_convert_aux_i:NNnnn #1#2#3#4#5
{
\group_begin:
- \str_aux_gset_other:Nn \g_str_result_tl {#5}
+ \str_gset_other:Nn \g_str_result_tl {#5}
\exp_after:wN \str_convert_aux_ii:wwwnn
\tl_to_str:n {#3} /// \q_stop
@@ -831,11 +790,10 @@
\char_set_catcode_comment:N \%
\int_set:Nn \tex_endlinechar:D {32}
}
-\int_new:N \l_str_encoding_int
\cs_new_protected:Npn \str_convert_from_eight_bit:n #1
{
\group_begin:
- \int_zero:N \l_str_encoding_int
+ \int_zero:N \l_str_tmpa_int
\exp_last_unbraced:Nx \str_convert_from_eight_bit_load:nn
{ \tl_use:c { c_str_encoding_#1_tl } }
{ \q_stop \prg_map_break: } { }
@@ -850,24 +808,25 @@
\cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load:nn #1#2
{
\use_none_delimit_by_q_stop:w #1 \q_stop
- \tex_dimen:D "#1 = \l_str_encoding_int sp \scan_stop:
- \tex_skip:D \l_str_encoding_int = "#1 sp \scan_stop:
- \tex_toks:D \l_str_encoding_int \exp_after:wN { \int_value:w "#2 }
- \tex_advance:D \l_str_encoding_int \c_one
+ \tex_dimen:D "#1 = \l_str_tmpa_int sp \scan_stop:
+ \tex_skip:D \l_str_tmpa_int = "#1 sp \scan_stop:
+ \tex_toks:D \l_str_tmpa_int \exp_after:wN { \int_value:w "#2 }
+ \tex_advance:D \l_str_tmpa_int \c_one
\str_convert_from_eight_bit_load:nn
}
\cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load_missing:n #1
{
\use_none_delimit_by_q_stop:w #1 \q_stop
- \tex_dimen:D "#1 = \l_str_encoding_int sp \scan_stop:
- \tex_skip:D \l_str_encoding_int = "#1 sp \scan_stop:
- \tex_toks:D \l_str_encoding_int \exp_after:wN { \int_value:w "FFFD }
- \tex_advance:D \l_str_encoding_int \c_one
+ \tex_dimen:D "#1 = \l_str_tmpa_int sp \scan_stop:
+ \tex_skip:D \l_str_tmpa_int = "#1 sp \scan_stop:
+ \tex_toks:D \l_str_tmpa_int \exp_after:wN { \int_value:w "FFFD }
+ \tex_advance:D \l_str_tmpa_int \c_one
\str_convert_from_eight_bit_load_missing:n
}
\cs_new:Npn \str_convert_from_eight_bit_aux:N #1
{
- \if_num:w \tex_dimen:D `#1 < \l_str_encoding_int
+ #1 \exp_not:N \q_mark
+ \if_num:w \tex_dimen:D `#1 < \l_str_tmpa_int
\if_num:w \tex_skip:D \tex_dimen:D `#1 = `#1 \exp_stop_f:
\tex_the:D \tex_toks:D \tex_dimen:D
\fi:
@@ -877,7 +836,7 @@
\cs_new_protected:Npn \str_convert_to_eight_bit:n #1
{
\group_begin:
- \int_zero:N \l_str_encoding_int
+ \int_zero:N \l_str_tmpa_int
\exp_last_unbraced:Nx \str_convert_to_eight_bit_load:nn
{ \tl_use:c { c_str_encoding_#1_tl } }
{ \q_stop \prg_map_break: } { }
@@ -888,17 +847,17 @@
\cs_new_protected_nopar:Npn \str_convert_to_eight_bit_load:nn #1#2
{
\use_none_delimit_by_q_stop:w #1 \q_stop
- \tex_dimen:D "#2 = \l_str_encoding_int sp \scan_stop:
- \tex_skip:D \l_str_encoding_int = "#2 sp \scan_stop:
- \exp_args:NNf \tex_toks:D \l_str_encoding_int
+ \tex_dimen:D "#2 = \l_str_tmpa_int sp \scan_stop:
+ \tex_skip:D \l_str_tmpa_int = "#2 sp \scan_stop:
+ \exp_args:NNf \tex_toks:D \l_str_tmpa_int
{ \str_output_byte:n { "#1 } }
- \tex_advance:D \l_str_encoding_int \c_one
+ \tex_advance:D \l_str_tmpa_int \c_one
\str_convert_to_eight_bit_load:nn
}
\cs_new:Npn \str_convert_to_eight_bit_aux:n #1
{
\if_num:w #1 < "8000 \exp_stop_f:
- \if_num:w \tex_dimen:D #1 < \l_str_encoding_int
+ \if_num:w \tex_dimen:D #1 < \l_str_tmpa_int
\if_num:w \tex_skip:D \tex_dimen:D #1 = #1 \exp_stop_f:
\tex_the:D \tex_toks:D \tex_dimen:D #1 \exp_stop_f:
\exp_after:wN \exp_after:wN \exp_after:wN \use_none:nn
@@ -907,7 +866,7 @@
\str_convert_to_eight_bit_aux_ii:n {#1}
\else:
\msg_expandable_kernel_error:nnn
- { str } { eight_bit-to-byte-overflow } {#1}
+ { str } { eight-bit-to-byte-overflow } {#1}
\fi:
}
\cs_new:Npn \str_convert_to_eight_bit_aux_ii:n #1
@@ -956,30 +915,73 @@
\cs_new_protected_nopar:Npn \str_convert_from_native:
{ \str_aux_gmap_result:N \str_convert_from_native_aux:N }
\cs_new:Npn \str_convert_from_native_aux:N #1
- { \int_value:w `#1 , }
+ { #1 \exp_not:N \q_mark \int_value:w `#1 , }
\cs_new_protected_nopar:Npn \str_convert_to_native:
{
\tl_set_eq:NN \l_str_tmpa_tl \g_str_result_tl
+ \str_aux_convert_to_native:n
+ {
+ \exp_after:wN \str_convert_to_native_aux:w
+ \l_str_tmpa_tl \q_mark { \q_stop \prg_map_break: } ,
+ }
+ }
+\cs_new_protected_nopar:Npn \str_convert_to_native_aux:w #1\q_mark #2,
+ {
+ \use_none_delimit_by_q_stop:w #2 \q_stop
+ \str_aux_convert_to_native_step:n {#2}
+ \str_convert_to_native_aux:w
+ }
+\cs_new_protected_nopar:Npn \str_aux_convert_to_native:n #1
+ {
+ \tex_lccode:D \c_zero \c_zero
+ \int_zero:N \l_str_tmpa_int
\tl_gclear:N \g_str_result_tl
- \tex_afterassignment:D \str_convert_to_native_aux:w
- \l_str_char_int = \l_str_tmpa_tl \c_zero \prg_map_break: ,
- \prg_break_point:n { }
+ #1
+ \prg_break_point:n { \str_aux_convert_to_native_flush: }
+ }
+\cs_new_protected_nopar:Npn \str_aux_convert_to_native_step:n #1
+ {
+ \if_num:w \l_str_tmpa_int = \c_two_hundred_fifty_five
+ \str_aux_convert_to_native_flush:
+ \l_str_tmpa_int \c_zero
+ \fi:
+ \tex_advance:D \l_str_tmpa_int \c_one
+ \if_int_compare:w #1 > \c_max_char_int
+ \msg_kernel_error:nnx { str } { overflow } {#1}
+ \tex_lccode:D \l_str_tmpa_int \c_str_replacement_char_int
+ \else:
+ \tex_lccode:D \l_str_tmpa_int \int_eval:w #1 \int_eval_end:
+ \fi:
+ }
+\cs_new_protected_nopar:Npn \str_aux_convert_to_native_flush:
+ {
+ \use:x
+ {
+ \tl_to_lowercase:n
+ {
+ \tl_gput_right:Nx \exp_not:N \g_str_result_tl
+ {
+ \exp_after:wN \str_aux_convert_to_native_filter:N
+ \c_str_positive_bytes_tl
+ { ? = \c_zero \fi: \prg_map_break: }
+ \prg_break_point:n { }
+ }
+ }
+ }
}
\group_begin:
- \char_set_catcode_other:n { `\^^@ }
- \cs_new_protected_nopar:Npn \str_convert_to_native_aux:w #1 ,
+ \char_set_catcode_other:n { 0 }
+ \cs_new:Npn \str_aux_convert_to_native_filter:N #1
{
- #1
- \if_int_compare:w \l_str_char_int > \c_max_char_int
- \msg_kernel_error:nnx { str } { overflow }
- { \int_use:N \l_str_char_int }
+ \if_num:w `#1 > \l_str_tmpa_int
+ \exp_after:wN \prg_map_break:
+ \fi:
+ \if_num:w \tex_lccode:D `#1 = \c_zero
+ ^^@
\else:
- \tex_lccode:D \c_zero \l_str_char_int
- \tl_to_lowercase:n
- { \tex_xdef:D \g_str_result_tl { \g_str_result_tl ^^@ } }
+ #1
\fi:
- \tex_afterassignment:D \str_convert_to_native_aux:w
- \l_str_char_int =
+ \str_aux_convert_to_native_filter:N
}
\group_end:
\msg_kernel_new:nnnn { str } { x-missing-brace }
diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3tl-analysis.sty b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3tl-analysis.sty
new file mode 100644
index 00000000000..84479cc1543
--- /dev/null
+++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3tl-analysis.sty
@@ -0,0 +1,402 @@
+%%
+%% This is file `l3tl-analysis.sty',
+%% generated with the docstrip utility.
+%%
+%% The original source files were:
+%%
+%% l3tl-analysis.dtx (with options: `package')
+%%
+%% EXPERIMENTAL CODE
+%%
+%% Do not distribute this file without also distributing the
+%% source files specified above.
+%%
+%% Do not distribute a modified version of this file.
+%%
+%% File: l3tl-analysis.dtx Copyright (C) 2011 The LaTeX3 Project
+%%
+%% It may be distributed and/or modified under the conditions of the
+%% LaTeX Project Public License (LPPL), either version 1.3c of this
+%% license or (at your option) any later version. The latest version
+%% of this license is in the file
+%%
+%% http://www.latex-project.org/lppl.txt
+%%
+%% This file is part of the "l3experimental bundle" (The Work in LPPL)
+%% and all files in that bundle must be distributed together.
+%%
+%% The released version of this bundle is available from CTAN.
+%%
+%% -----------------------------------------------------------------------
+%%
+%% The development version of the bundle can be found at
+%%
+%% http://www.latex-project.org/svnroot/experimental/trunk/
+%%
+%% for those people who are interested.
+%%
+%%%%%%%%%%%
+%% NOTE: %%
+%%%%%%%%%%%
+%%
+%% Snapshots taken from the repository represent work in progress and may
+%% not work or may contain conflicting material! We therefore ask
+%% people _not_ to put them into distributions, archives, etc. without
+%% prior consultation with the LaTeX3 Project.
+%%
+%% -----------------------------------------------------------------------
+\RequirePackage{expl3}
+\GetIdInfo$Id: l3tl-analysis.dtx 3039 2011-12-08 09:22:35Z bruno $
+ {L3 Experimental Analysis of Token lists}
+\ProvidesExplPackage
+ {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
+\RequirePackage{l3str}
+\int_new:N \l_tl_analysis_normal_int
+\int_new:N \l_tl_analysis_index_int
+\int_new:N \l_tl_analysis_nesting_int
+\int_new:N \l_tl_analysis_char_int
+\int_new:N \l_tl_analysis_type_int
+\tl_new:N \g_tl_analysis_result_tl
+\cs_new_protected_nopar:Npn \tl_analysis_extract_charcode:
+ {
+ \exp_after:wN \tl_analysis_extract_charcode_aux:w
+ \token_to_meaning:N \l_tl_analysis_token
+ }
+\cs_new_protected_nopar:Npn \tl_analysis_extract_charcode_aux:w #1 ~ #2 ~ { ` }
+\cs_new:Npn \tl_analysis_cs_space_count:NN #1 #2
+ {
+ \exp_after:wN #1
+ \int_value:w \int_eval:w \c_zero
+ \exp_after:wN \tl_analysis_cs_space_count:w
+ \token_to_str:N #2
+ \fi: \tl_analysis_cs_space_count_end:w ; ~ !
+ }
+\cs_new:Npn \tl_analysis_cs_space_count:w #1 ~
+ {
+ \if_false: #1 #1 \fi:
+ + \c_one
+ \tl_analysis_cs_space_count:w
+ }
+\cs_new:Npn \tl_analysis_cs_space_count_end:w ; #1 \fi: #2 !
+ { \exp_after:wN ; \int_value:w \str_length_ignore_spaces:n {#1} ; }
+\cs_new_protected:Npn \tl_analysis:n #1
+ {
+ \group_begin:
+ \group_align_safe_begin:
+ \tl_analysis_setup:n {#1}
+ \tl_analysis_i:n {#1}
+ \tl_analysis_ii:n {#1}
+ \group_align_safe_end:
+ \group_end:
+ }
+\cs_new_protected:Npn \tl_set_analysis:Nn #1#2
+ {
+ \tl_analysis:n {#2}
+ \tl_set_eq:NN #1 \g_tl_analysis_result_tl
+ }
+\cs_new_protected:Npn \tl_analysis_setup:n #1
+ {
+ \int_set:Nn \tex_escapechar:D { 92 }
+ \exp_after:wN \tl_analysis_disable_loop:N
+ \tl_to_str:n {#1} { ~ } { ? ~ \prg_map_break: }
+ \prg_break_point:n { }
+ }
+\group_begin:
+ \char_set_catcode_active:N \^^@
+ \cs_new_protected_nopar:Npn \tl_analysis_disable_loop:N #1
+ {
+ \tex_lccode:D \c_zero `#1 ~
+ \tl_to_lowercase:n { \tex_let:D ^^@ } \c_undefined:D
+ \tl_analysis_disable_loop:N
+ }
+\group_end:
+\cs_new_protected:Npn \tl_analysis_i:n #1
+ {
+ \int_zero:N \l_tl_analysis_normal_int
+ \int_zero:N \l_tl_analysis_index_int
+ \int_zero:N \l_tl_analysis_nesting_int
+ \if_false: { \fi: \tl_analysis_i_loop:w #1 }
+ \int_decr:N \l_tl_analysis_index_int
+ }
+\cs_new_protected_nopar:Npn \tl_analysis_i_loop:w
+ { \tex_futurelet:D \l_tl_analysis_token \tl_analysis_i_type:w }
+\cs_new_protected_nopar:Npn \tl_analysis_i_type:w
+ {
+ \l_tl_analysis_type_int =
+ \if_meaning:w \l_tl_analysis_token \c_space_token
+ \c_zero
+ \else:
+ \if_catcode:w \exp_not:N \l_tl_analysis_token \c_group_begin_token
+ \c_one
+ \else:
+ \if_catcode:w \exp_not:N \l_tl_analysis_token \c_group_end_token
+ \c_minus_one
+ \else:
+ \c_two
+ \fi:
+ \fi:
+ \fi:
+ \if_case:w \l_tl_analysis_type_int
+ \exp_after:wN \tl_analysis_i_space:w
+ \or: \exp_after:wN \tl_analysis_i_bgroup:w
+ \or: \exp_after:wN \tl_analysis_i_safe:N
+ \else: \exp_after:wN \tl_analysis_i_egroup:w
+ \fi:
+ }
+\cs_new_protected:Npn \tl_analysis_i_safe:N #1
+ {
+ \if_charcode:w
+ \scan_stop:
+ \exp_after:wN \use_none:n \token_to_str:N #1 \prg_do_nothing:
+ \scan_stop:
+ \int_incr:N \l_tl_analysis_normal_int
+ \else:
+ \tl_analysis_cs_space_count:NN \tl_analysis_i_cs:ww #1
+ \fi:
+ \tl_analysis_i_loop:w
+ }
+\cs_new_protected:Npn \tl_analysis_i_cs:ww #1; #2;
+ {
+ \if_num:w #1 > \c_zero
+ \tex_skip:D \l_tl_analysis_index_int
+ = \int_eval:w \l_tl_analysis_normal_int + \c_one sp \scan_stop:
+ \tex_advance:D \l_tl_analysis_index_int #1 \exp_stop_f:
+ \l_tl_analysis_normal_int #2 \exp_stop_f:
+ \else:
+ \tex_advance:D \l_tl_analysis_normal_int #2 \exp_stop_f:
+ \fi:
+ }
+\cs_new_protected_nopar:Npn \tl_analysis_i_space:w
+ {
+ \tex_afterassignment:D \tl_analysis_i_space_test:w
+ \exp_after:wN \cs_set_eq:NN
+ \exp_after:wN \l_tl_analysis_token
+ \token_to_str:N
+ }
+\cs_new_protected_nopar:Npn \tl_analysis_i_space_test:w
+ {
+ \if_meaning:w \l_tl_analysis_token \c_space_token
+ \tex_toks:D \l_tl_analysis_index_int { \exp_not:n { ~ } }
+ \tl_analysis_i_store:
+ \else:
+ \int_incr:N \l_tl_analysis_normal_int
+ \fi:
+ \tl_analysis_i_loop:w
+ }
+\group_begin:
+ \char_set_catcode_group_begin:N \^^@
+ \char_set_catcode_group_end:N \^^E
+ \cs_new_protected_nopar:Npn \tl_analysis_i_bgroup:w
+ { \tl_analysis_i_group:nw { \exp_after:wN ^^@ \if_false: ^^E \fi: } }
+ \char_set_catcode_group_begin:N \^^B
+ \char_set_catcode_group_end:N \^^@
+ \cs_new_protected_nopar:Npn \tl_analysis_i_egroup:w
+ { \tl_analysis_i_group:nw { \if_false: ^^B \fi: ^^@ } }
+\group_end:
+\cs_new_protected_nopar:Npn \tl_analysis_i_group:nw #1
+ {
+ \l_tl_analysis_char_int = \tl_analysis_extract_charcode: \scan_stop:
+ \tex_lccode:D \c_zero \l_tl_analysis_char_int
+ \tl_to_lowercase:n { \tex_toks:D \l_tl_analysis_index_int {#1} }
+ \if_num:w \l_tl_analysis_char_int = \tex_escapechar:D
+ \int_set:Nn \tex_escapechar:D { 139 - \tex_escapechar:D }
+ \fi:
+ \tex_afterassignment:D \tl_analysis_i_group_test:w
+ \exp_after:wN \cs_set_eq:NN
+ \exp_after:wN \l_tl_analysis_token
+ \token_to_str:N
+ }
+\cs_new_protected_nopar:Npn \tl_analysis_i_group_test:w
+ {
+ \if_num:w \tl_analysis_extract_charcode: = \l_tl_analysis_char_int
+ \tl_analysis_i_store:
+ \else:
+ \int_incr:N \l_tl_analysis_normal_int
+ \fi:
+ \tl_analysis_i_loop:w
+ }
+\cs_new_protected_nopar:Npn \tl_analysis_i_store:
+ {
+ \tex_advance:D \l_tl_analysis_nesting_int \l_tl_analysis_type_int
+ \if_num:w \l_tl_analysis_char_int = \c_thirty_two
+ \tex_multiply:D \l_tl_analysis_type_int \c_two
+ \fi:
+ \tex_skip:D \l_tl_analysis_index_int
+ = \l_tl_analysis_normal_int sp plus \l_tl_analysis_type_int sp \scan_stop:
+ \int_incr:N \l_tl_analysis_index_int
+ \int_zero:N \l_tl_analysis_normal_int
+ \if_num:w \l_tl_analysis_nesting_int = \c_minus_one
+ \cs_set_eq:NN \tl_analysis_i_loop:w \scan_stop:
+ \fi:
+ }
+\cs_new_protected:Npn \tl_analysis_ii:n #1
+ {
+ \tl_gset:Nx \g_tl_analysis_result_tl
+ {
+ \tl_analysis_ii_loop:w 0; #1
+ \prg_break_point:n { }
+ }
+ }
+\cs_new:Npn \tl_analysis_ii_loop:w #1;
+ {
+ \exp_after:wN \tl_analysis_ii_normals:ww
+ \int_value:w \tex_skip:D #1 ; #1 ;
+ }
+\cs_new:Npn \tl_analysis_ii_normals:ww #1;
+ {
+ \if_num:w #1 = \c_zero
+ \tl_analysis_ii_special:w
+ \fi:
+ \tl_analysis_ii_normal:wwN #1;
+ }
+\cs_new:Npn \tl_analysis_ii_normal:wwN #1; #2; #3
+ {
+ \exp_not:n { \exp_not:n { #3 } } \exp_not:N \q_mark
+ \if_charcode:w
+ \scan_stop:
+ \exp_after:wN \use_none:n \token_to_str:N #3 \prg_do_nothing:
+ \scan_stop:
+ \exp_after:wN \tl_analysis_ii_char:Nww
+ \else:
+ \exp_after:wN \tl_analysis_ii_cs:Nww
+ \fi:
+ #3 #1; #2;
+ }
+\cs_new:Npn \tl_analysis_ii_char:Nww #1
+ {
+ \if_meaning:w #1 \c_undefined:D D \else:
+ \if_catcode:w #1 \c_catcode_other_token C \else:
+ \if_catcode:w #1 \c_catcode_letter_token B \else:
+ \if_catcode:w #1 \c_math_toggle_token 3 \else:
+ \if_catcode:w #1 \c_alignment_token 4 \else:
+ \if_catcode:w #1 \c_math_superscript_token 7 \else:
+ \if_catcode:w #1 \c_math_subscript_token 8 \else:
+ 6
+ \fi: \fi: \fi: \fi: \fi: \fi: \fi:
+ \int_value:w `#1 ,
+ \exp_after:wN \tl_analysis_ii_normals:ww
+ \int_use:N \int_eval:w \c_minus_one +
+ }
+\cs_new:Npn \tl_analysis_ii_cs:Nww #1
+ {
+ 0 -1 ,
+ \tl_analysis_cs_space_count:NN \tl_analysis_ii_cs_test:ww #1
+ }
+\cs_new:Npn \tl_analysis_ii_cs_test:ww #1 ; #2 ; #3; #4;
+ {
+ \exp_after:wN \tl_analysis_ii_normals:ww
+ \int_use:N \int_eval:w
+ \if_num:w #1 = \c_zero
+ #3
+ \else:
+ \tex_skip:D \int_eval:w #4 + #1 \int_eval_end:
+ \fi:
+ - #2
+ \exp_after:wN ;
+ \int_use:N \int_eval:w #4 + #1;
+ }
+\cs_new:Npn \tl_analysis_ii_special:w \fi: \tl_analysis_ii_normal:wwN 0; #1;
+ {
+ \fi:
+ \if_num:w #1 = \l_tl_analysis_index_int
+ \exp_after:wN \prg_map_break:
+ \fi:
+ \tex_the:D \tex_toks:D #1 \exp_stop_f: \exp_not:N \q_mark
+ \if_case:w \etex_gluestretch:D \tex_skip:D #1 \exp_stop_f:
+ A
+ \or: 1
+ \or: 1
+ \else: 2
+ \fi:
+ \if_int_odd:w \etex_gluestretch:D \tex_skip:D #1 \exp_stop_f:
+ \exp_after:wN \tl_analysis_ii_special_char:wN \int_use:N
+ \else:
+ \exp_after:wN \tl_analysis_ii_special_space:w \int_use:N
+ \fi:
+ \int_eval:w \c_one + #1 \exp_after:wN ;
+ \token_to_str:N
+ }
+\cs_new:Npn \tl_analysis_ii_special_char:wN #1 ; #2
+ {
+ \int_value:w `#2 ,
+ \tl_analysis_ii_loop:w #1 ;
+ }
+\cs_new:Npn \tl_analysis_ii_special_space:w #1 ; ~
+ {
+ 32 ,
+ \tl_analysis_ii_loop:w #1 ;
+ }
+\cs_new_protected:Npn \tl_show_analysis:N #1
+ {
+ \exp_args:No \tl_analysis:n {#1}
+ \msg_aux_show:Nnx #1
+ { tl-analysis }
+ {
+ \exp_after:wN \tl_show_analysis_loop:wNw \g_tl_analysis_result_tl
+ \q_mark { ? \prg_map_break: } ,
+ \prg_break_point:n { }
+ }
+ }
+\cs_new_protected:Npn \tl_show_analysis:n #1
+ {
+ \tl_set:Nn \l_tl_tmpa_tl {#1}
+ \tl_show_analysis:N \l_tl_tmpa_tl
+ }
+\cs_new:Npn \tl_show_analysis_loop:wNw #1 \q_mark #2 #3,
+ {
+ \use_none:n #2
+ \iow_newline: > \c_space_tl \c_space_tl
+ \if_num:w "#2 = \c_zero
+ \exp_after:wN \tl_show_analysis_cs:n
+ \else:
+ \if_num:w "#2 = \c_thirteen
+ \exp_after:wN \exp_after:wN
+ \exp_after:wN \tl_show_analysis_active:n
+ \else:
+ \exp_after:wN \exp_after:wN
+ \exp_after:wN \tl_show_analysis_normal:n
+ \fi:
+ \fi:
+ {#1}
+ \tl_show_analysis_loop:wNw
+ }
+\cs_new:Npn \tl_show_analysis_normal:n #1
+ {
+ \exp_after:wN \token_to_str:N #1 ~
+ ( \exp_after:wN \token_to_meaning:N #1 )
+ }
+\cs_new:Npn \tl_show_analysis_cs:n #1
+ { \exp_args:No \tl_show_analysis_long:nn {#1} { control~sequence= } }
+\cs_new:Npn \tl_show_analysis_active:n #1
+ { \exp_args:No \tl_show_analysis_long:nn {#1} { active~character= } }
+\cs_new:Npn \tl_show_analysis_long:nn #1
+ {
+ \exp_args:Noo \tl_show_analysis_long_aux:nnn
+ { \token_to_str:N #1 }
+ { \token_to_meaning:N #1 }
+ }
+\cs_new_nopar:Npn \tl_show_analysis_long_aux:nnn #1#2#3
+ {
+ \int_compare:nNnTF
+ { \str_length:n { >>> #1 ~ ( #3 #2 ) } }
+ > { \l_iow_line_length_int }
+ {
+ \str_substr:nnn { >>> #1 ~ ( #3 #2 ) } \c_three
+ { \l_iow_line_length_int - \str_length:N \c_tl_show_analysis_etc_str }
+ \c_tl_show_analysis_etc_str
+ }
+ {#1~(#3#2)}
+ }
+\str_const:Nx \c_tl_show_analysis_etc_str % (
+ { \iow_char:N \\ETC. ) }
+\msg_kernel_new:nnn { tl-analysis } { show }
+ {
+ Token~list~
+ \str_if_eq:nnF {#1} { \l_tl_tmpa_tl } { \token_to_str:N #1~}
+ \tl_if_empty:NTF #1
+ { is~empty }
+ { contains~the~tokens: }
+ }
+%%
+%%
+%% End of file `l3tl-analysis.sty'.
diff --git a/Master/texmf-dist/tex/latex/l3experimental/xgalley/l3galley.sty b/Master/texmf-dist/tex/latex/l3experimental/xgalley/l3galley.sty
index e8a3345204f..28bc6b50dc5 100644
--- a/Master/texmf-dist/tex/latex/l3experimental/xgalley/l3galley.sty
+++ b/Master/texmf-dist/tex/latex/l3experimental/xgalley/l3galley.sty
@@ -48,14 +48,11 @@
%% -----------------------------------------------------------------------
%%
\RequirePackage{l3names}
-\GetIdInfo$Id: l3galley.dtx 2938 2011-10-25 18:41:20Z joseph $
+\GetIdInfo$Id: l3galley.dtx 3071 2011-12-20 20:14:00Z bruno $
{L3 Experimental galley code}
\ProvidesExplPackage
{\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
\package_check_loaded_expl:
-\cs_new_protected:Npn \scan_marker_new:N #1
- { \cs_new_eq:NN #1 \scan_stop: }
-\cs_generate_variant:Nn \scan_marker_new:N { c }
\cs_new_protected_nopar:Npn \galley_leave_vmode:
{ \tex_unhbox:D \c_empty_box }
\clist_new:N \l_galley_parshape_left_indent_clist
@@ -193,7 +190,7 @@
\galley_restore_parameters:
\group_end:
}
-\scan_marker_new:N \s_par_omit
+\scan_new:N \s_par_omit
\cs_new_protected_nopar:Npn \galley_std_par:
{
\s_par_omit