summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/source
diff options
context:
space:
mode:
authorKarl Berry <karl@freefriends.org>2012-02-28 00:40:17 +0000
committerKarl Berry <karl@freefriends.org>2012-02-28 00:40:17 +0000
commited49b3dfe19b91f9fd96ab70fe95dcda8fba24c8 (patch)
tree0eb59527efd68f813d629e7d0977fe31d8d6419f /Master/texmf-dist/source
parentb1436d6ee293a02df32a18832d21c54348335565 (diff)
l3experimental 3471 (26feb12)
git-svn-id: svn://tug.org/texlive/trunk@25523 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/source')
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3dt/l3dt.dtx10
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3sort/l3sort.dtx4
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx7450
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx137
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins3
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx54
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-build.dtx301
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins.dtx211
8 files changed, 4576 insertions, 3594 deletions
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3dt/l3dt.dtx b/Master/texmf-dist/source/latex/l3experimental/l3dt/l3dt.dtx
index e5063618228..0fd090d9ef7 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3dt/l3dt.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/l3dt/l3dt.dtx
@@ -36,7 +36,7 @@
%
%<*driver|package>
\RequirePackage{expl3}
-\GetIdInfo$Id: l3dt.dtx 3251 2012-01-28 09:42:44Z joseph $
+\GetIdInfo$Id: l3dt.dtx 3459 2012-02-26 08:58:53Z joseph $
{L3 Experimental data tables}
%</driver|package>
%<*driver>
@@ -500,15 +500,15 @@
% \cs{q_dt_header} \\
% \cs{q_dt_row} \\
% \meta{row$_0$} \\
-% \cs{q_dt} \meta{key$_0$} \cs{q_dt} \Arg{data$_{\0,\0}$} \\
-% \cs{q_dt} \meta{key$_1$} \cs{q_dt} \Arg{data$_{\0,\1}$} \\
+% \cs{q_dt} \meta{key$_0$} \cs{q_dt} \Arg{data$_{0,0}$} \\
+% \cs{q_dt} \meta{key$_1$} \cs{q_dt} \Arg{data$_{0,1}$} \\
% \ldots \\
% \cs{q_dt} \\
% \cs{q_nil} \\
% \cs{q_dt_row} \\
% \meta{row$_2$} \\
-% \cs{q_dt} \meta{key$_0$} \cs{q_dt} \Arg{data$_{\1,\0}$} \\
-% \cs{q_dt} \meta{key$_1$} \cs{q_dt} \Arg{data$_{\1,\1}$} \\
+% \cs{q_dt} \meta{key$_0$} \cs{q_dt} \Arg{data$_{1,0}$} \\
+% \cs{q_dt} \meta{key$_1$} \cs{q_dt} \Arg{data$_{1,1}$} \\
% \ldots \\
% \cs{q_dt} \\
% \cs{q_nil} \\
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3sort/l3sort.dtx b/Master/texmf-dist/source/latex/l3experimental/l3sort/l3sort.dtx
index 65a95dbee29..0d80aebcc51 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3sort/l3sort.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/l3sort/l3sort.dtx
@@ -1,5 +1,5 @@
% \iffalse
-%
+%
%% File l3sort.dtx (C) Copyright 2012 The LaTeX3 Project
%%
%% It may be distributed and/or modified under the conditions of the
@@ -36,7 +36,7 @@
%
%<*driver|package>
\RequirePackage{expl3}
-\GetIdInfo$Id: l3sort.dtx 3295 2012-02-02 09:30:14Z joseph $
+\GetIdInfo$Id: l3sort.dtx 3467 2012-02-26 15:39:35Z bruno $
{L3 Experimental sorting functions}
%</driver|package>
%<*driver>
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx
index d2968338f46..887884d7c3a 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx
@@ -35,7 +35,7 @@
%
%<*driver|package>
\RequirePackage{expl3}
-\GetIdInfo$Id: l3regex.dtx 3326 2012-02-07 00:39:44Z bruno $
+\GetIdInfo$Id: l3regex.dtx 3463 2012-02-26 15:05:14Z bruno $
{L3 Experimental Regular Expressions}
%</driver|package>
%<*driver>
@@ -76,7 +76,8 @@
% The \pkg{l3regex} package provides regular expression testing,
% extraction of submatches, splitting, and replacement, all acting
% on token lists. The syntax of regular expressions is mostly a subset
-% of the PCRE syntax (and very close to POSIX), with some additions
+% of the \textsc{pcre} syntax (and very close to \textsc{posix}),
+% with some additions
% due to the fact that \TeX{} manipulates tokens rather than characters.
% For performance reasons, only a limited set of features are implemented.
% Notably, back-references are not supported.
@@ -180,18 +181,25 @@
% \item[\\S] Any token not matched by |\s|.
% \item[\\V] Any token not matched by |\v|.
% \item[\\W] Any token not matched by |\w|.
-% \item[\\N] Any token not matched by |\n|.
% \end{l3regex-syntax}
% Of those, |.|, |\D|, |\H|, |\N|, |\S|, |\V|, and |\W| will match arbitrary
% control sequences.
%
-% Character classes match exactly one character in the subject string.
+% Character classes match exactly one token in the subject.
% \begin{l3regex-syntax}
% \item[{[\ldots{}]}] Positive character class.
% Matches any of the specified tokens.
% \item[{[\char`\^\ldots{}]}] Negative character class.
% Matches any token other than the specified characters.
-% \item[{[x-y]}] Range (can be used with escaped characters).
+% \item[{x-y}] Within a character class, this denotes a range (can be
+% used with escaped characters).
+% \item[{[:\meta{name}:]}] Within a character class (one more set of
+% brackets), this denotes the \textsc{posix} character class
+% \meta{name}, which can be \texttt{alnum}, \texttt{alpha},
+% \texttt{ascii}, \texttt{blank}, \texttt{cntrl}, \texttt{digit},
+% \texttt{graph}, \texttt{lower}, \texttt{print}, \texttt{punct},
+% \texttt{space}, \texttt{upper}, \texttt{word}, or \texttt{xdigit}.
+% \item[{[:\char`\^\meta{name}:]}] Negative \textsc{posix} character class.
% \end{l3regex-syntax}
% For instance, |[a-oq-z\cC.]| matches any lowercase latin letter
% except |p|, as well as control sequences (see below for a description
@@ -264,25 +272,29 @@
% \item[\\c\Arg{regex}] A control sequence whose csname matches the
% \meta{regex}, anchored at the beginning and end, so that |\c{begin}|
% matches exactly \cs{begin}, and nothing else.
-% \item[\\cX\meta{character or class}] Matches a token with category
-% code |X| (any of |CBEMTPUDSLOA|) if it is also matched by the
-% \meta{character or class}. For instance, |\cL[A-Z]| matches
-% uppercase letters of category code letter, |\cC.| matches any
-% control sequence, and |\cO\d| matches digits of category other.
-% \item[{\\c[XYZ]\meta{character or class}}] Matches a token with
-% category |X|, |Y|, or |Z| (each being any of |CBEMTPUDSLOA|),
-% if it is also matched by the \meta{character or class}. For instance,
-% |\c[LSO].| matches tokens of category letter, space, or other.
-% \item[{\\c[\char`\^XYZ]\meta{character or class}}] Matches a token with
-% category different from |X|, |Y|, or |Z| (each being any of
-% |CBEMTPUDSLOA|), if it is also matched by the \meta{character or
-% class}. For instance, |\c[LSO].| matches tokens of category
-% letter, space, or other.
+% \item[\\cX] Applies to the next object, which can be a character,
+% character property, class, or group, and forces this object to
+% only match tokens with category |X| (any of |CBEMTPUDSLOA|. For
+% instance, |\cL[A-Z\d]| matches uppercase letters and digits of
+% category code letter, |\cC.| matches any control sequence, and
+% |\cO(abc)| matches |abc| where each character has category other.
+% \item[{\\c[XYZ]}] Applies to the next object, and forces it to only
+% match tokens with category |X|, |Y|, or |Z| (each being any of
+% |CBEMTPUDSLOA|). For instance, |\c[LSO](..)| matches two tokens of
+% category letter, space, or other.
+% \item[{\\c[\char`\^XYZ]}] Applies to the next object and prevents it
+% from matching any token with category |X|, |Y|, or |Z| (each being
+% any of |CBEMTPUDSLOA|). For instance, |\c[^O]\d| matches digits
+% which have any category different from other.
% \end{l3regex-syntax}
% The category code tests can be used inside classes; for instance,
% |[\cO\d \c[LO][A-F]]| matches what \TeX{} considers as hexadecimal
% digits, namely digits with category other, or uppercase letters from
-% |A| to |F| with category either letter or other.
+% |A| to |F| with category either letter or other. Within a group
+% affected by a category code test, the outer test can be overridden by
+% a nested test: for instance, |\cL(ab\cO\*cd)| matches |ab*cd| where
+% all characters are of category letter, except |*| which has category
+% other.
%
% The |\u| escape sequence allows to insert the contents of a token list
% directly into a regular expression or a replacement, avoiding the need
@@ -292,37 +304,50 @@
% argument once, in effect performing \cs{tl_to_str:v}. Quantifiers are
% not supported directly: use a group.
%
-% Options can be set with |(?|\meta{option}|)| and
-% unset with |(?-|\meta{option}|)|. Options are local
-% to the group in which they are set, and revert to their
-% previous setting upon reaching the closing parenthesis.
-% For instance, in \verb"(?i)a(b(?-i)c|d)e", the |i| option
-% applies to the letters |a|, |b| and |e|.
-% \begin{l3regex-syntax}
-% \item[(?i) \textrm{and} (?-i)] Toggle to a case
-% insensitive/sensitive mode. This only applies to ascii letters
-% (mapping \texttt{A}--\texttt{Z} to \texttt{a}--\texttt{z}).
-% For instance, |(?i)[Y-\\]| matches the characters |Y|, |Z|, |[|,
-% |\|, and the lower case letters |y| and |z|, while |(?i)[^aeiou]|
-% matches any character which is not a vowel.
-% \end{l3regex-syntax}
-%
-% In character classes, only |^|, |-|, |]|, |\| and spaces are special,
-% and should be escaped. Other non-alphanumeric characters can
-% still be escaped without harm. The escape sequences |\d|,
-% |\D|, etc. are also supported in character classes.
-% If the first character is |^|, then the meaning of the character
-% class is inverted. Ranges of characters can be expressed using
-% |-|, for instance, |[\D 0-5]| is equivalent to |[^6-9]|.
+% The option |(?i)| makes the match case insensitive (identifying
+% \texttt{A}--\texttt{Z} with \texttt{a}--\texttt{z}; no Unicode support
+% yet). This applies until the end of the group in which it appears, and
+% can be reverted using |(?-i)|. For instance, in
+% \verb"(?i)(a(?-i)b|c)d", the letters |a| and |d| are affected by the
+% |i| option. Characters within ranges and classes are affected
+% individually: |(?i)[Y-\\]| is equivalent to |[YZ\[\\yz]|, and
+% |(?i)[^aeiou]| matches any character which is not a vowel. Neither
+% character properties, nor |\c{...}| nor |\u{...}| are affected by the
+% |i| option.
+%
+% In character classes, only |[|, |^|, |-|, |]|, |\| and spaces are
+% special, and should be escaped. Other non-alphanumeric characters can
+% still be escaped without harm. Any escape sequence which matches a
+% single character (|\d|, |\D|, \emph{etc.}) is supported in character
+% classes. If the first character is |^|, then
+% the meaning of the character class is inverted. Ranges of characters
+% can be expressed using |-|, for instance, |[\D 0-5]| and |[^6-9]| are
+% equivalent.
%
% Capturing groups are a means of extracting information about the
% match. Parenthesized groups are labelled in the order of their
% opening parenthesis, starting at $1$. The contents of those groups
% corresponding to the \enquote{best} match (leftmost longest)
-% can be extracted and stored in a sequence of strings using for
+% can be extracted and stored in a sequence of token lists using for
% instance \cs{regex_extract_once:nnNTF}.
%
-% \subsection{Syntax of in the replacement text}
+% The |\K| escape sequence resets the beginning of the match to the
+% current position in the token list. This only affects what is reported
+% as the full match. For instance,
+% \begin{verbatim}
+% \regex_extract_all:nnN { a \K . } { a123aaxyz } \l_foo_seq
+% \end{verbatim}
+% results in \cs{l_foo_seq} containing the items |{1}| and |{a}|: the
+% true matches are |{a1}| and |{aa}|, but they are trimmed by the use of
+% |\K|. The |\K| command does not affect capturing groups: for instance,
+% \begin{verbatim}
+% \regex_extract_once:nnN { (. \K c)+ \d } { acbc3 } \l_foo_seq
+% \end{verbatim}
+% results in \cs{l_foo_seq} containing the items |{c3}| and |{bc}|: the
+% true match is |{acbc3}|, with first submatch |{bc}|, but |\K| resets
+% the beginning of the match to the last position where it appears.
+%
+% \subsection{Syntax of the replacement text}
%
% Most of the features described in regular expressions do not make sense
% within the replacement text. Escaped characters are supported as inside
@@ -349,15 +374,15 @@
% |\0|, |\1| \emph{etc.}
% \end{l3regex-syntax}
%
-% \subsection{Precompiling regular expressions}
+% \subsection{Pre-compiling regular expressions}
%
% If a regular expression is to be used several times,
% it is better to compile it once rather than doing it
-% each time the regular expression is used. The precompiled
+% each time the regular expression is used. The compiled
% regular expression is stored in a variable. All
% of the \pkg{l3regex} module's functions can be given their
% regular expression argument either as an explicit string
-% or as a precompiled regular expression.
+% or as a compiled regular expression.
%
% \begin{function}{\regex_new:N}
% \begin{syntax}
@@ -372,7 +397,7 @@
% \begin{syntax}
% \cs{regex_set:Nn} \meta{regex~var} \Arg{regex}
% \end{syntax}
-% Stores a precompiled version of the \meta{regular expression}
+% Stores a compiled version of the \meta{regular expression}
% in the \meta{regex~var}. For instance, this function can be used
% as
% \begin{verbatim}
@@ -380,15 +405,32 @@
% \regex_set:Nn \l_my_regex { my\ (simple\ )? reg(ex|ular\ expression) }
% \end{verbatim}
% The assignment is local for \cs{regex_set:Nn} and global for
-% \cs{regex_gset:Nn}. Use \cs{regex_const:Nn} for precompiled expressions
+% \cs{regex_gset:Nn}. Use \cs{regex_const:Nn} for compiled expressions
% which will never change.
% \end{function}
%
+% \begin{function}{\regex_show:n, \regex_show:N}
+% \begin{syntax}
+% \cs{regex_show:n} \Arg{regex}
+% \end{syntax}
+% Shows how \pkg{l3regex} interprets the \meta{regex}. For instance,
+% \cs{regex_show:n} \verb+{\A X|Y}+ shows
+% \begin{verbatim}
+% +-branch
+% anchor at start (\A)
+% char code 88
+% +-branch
+% char code 89
+% \end{verbatim}
+% indicating that the anchor |\A| only applies to the first branch:
+% the second branch is not anchored to the beginning of the match.
+% \end{function}
+%
% \subsection{Matching}
%
% All regular expression functions are available in both |:n| and |:N|
% variants. The former require a \enquote{standard} regular expression,
-% while the later require a precompiled expression as generated by
+% while the later require a compiled expression as generated by
% \cs{regex_(g)set:Nn}.
%
% \begin{function}[TF]{\regex_match:nn, \regex_match:Nn}
@@ -416,7 +458,7 @@
% starts again from the character following the last character
% of the previous match, until reaching the end of the token list.
% Infinite loops are prevented in the case where the regular expression
-% can match an empty string: then we count one match between each
+% can match an empty token list: then we count one match between each
% pair of characters.
% For instance,
% \begin{verbatim}
@@ -514,8 +556,8 @@
% Searches for the \meta{regular expression} in the \meta{token list}
% and replaces the first match with the \meta{replacement}. The result
% is assigned locally to \meta{tl~var}. In the \meta{replacement},
-% |\0| represents the full match, |\1| represent the contents
-% of the first capturing group, |\2| of the second, \emph{etc.}
+% |\0| represents the full match, |\1| represent the contents of the
+% first capturing group, |\2| of the second, \emph{etc.}
% \end{function}
%
% \begin{function}[TF]{\regex_replace_all:nnN, \regex_replace_all:NnN}
@@ -523,97 +565,133 @@
% \cs{regex_replace_all:nnN} \Arg{regular expression} \Arg{replacement} \meta{tl~var}
% \cs{regex_replace_all:nnNTF} \Arg{regular expression} \Arg{replacement} \meta{tl~var} \Arg{true code} \Arg{false code}
% \end{syntax}
-% Replaces all occurrences of the \cs{regular expression}
-% in the \meta{token list} by the \meta{replacement}, where
-% |\0| represents the full match, |\1|
-% represent the contents of the first capturing group,
-% |\2| of the second, \emph{etc.} Every match
-% is treated independently, and matches cannot overlap.
-% The result is assigned locally to \meta{tl~var}.
+% Replaces all occurrences of the \cs{regular expression} in the
+% \meta{token list} by the \meta{replacement}, where |\0| represents
+% the full match, |\1| represent the contents of the first capturing
+% group, |\2| of the second, \emph{etc.} Every match is treated
+% independently, and matches cannot overlap. The result is assigned
+% locally to \meta{tl~var}.
% \end{function}
%
% \subsection{Bugs, misfeatures, future work, and other possibilities}
%
% The following need to be done now.
% \begin{itemize}
+% \item Change user function names!
% \item Clean up the use of messages.
-% \item Code comments.
-% \item Rewrite the documentation in a clearer way.
+% \item Rewrite the documentation in a more ordered way, perhaps add a
+% \textsc{bnf}?
+% \end{itemize}
+%
+% Additional error-checking to come.
+% \begin{itemize}
+% \item Detect that a trailing |\c|\meta{category} is an invalid regex.
+% \item Currently, |a{\x34}| is recognized as |a{4}|.
+% \item Cleaner error reporting in the replacement phase.
+% \item Add tracing information.
+% \item Detect attempts to use back-references.
+% \item Test for the maximum register \cs{c_max_register_int}.
+% \item Find out whether the fact that |\W| and friends match the
+% end-marker leads to bugs. Possibly update \cs{regex_item_reverse:n}.
+% \item Enforce that |\cC| can only be followed by a match-all dot.
% \end{itemize}
%
% Code improvements to come.
% \begin{itemize}
-% \item \cs{regex_show:N} to show how a given regular expression is
-% interpreted.
-% \item Test for the maximum register \cs{c_max_register_int}.
-% \item Use \tn{dimen} registers rather than \cs{l_regex_nesting_tl}
-% to build \cs{regex_nesting:n}.
-% \item Reduce the number of epsilon-transitions in alternatives.
-% \item Move the \enquote{reconstruction} part of \pkg{l3regex}
-% to \pkg{l3tl-analysis}.
-% \item Optimize regexes for csnames when the regex is a simple string.
-% \item Optimize simple strings: use less states (|abcade| should give
-% two states, for |abc| and |ade|). [Does that really make sense?]
-% \item Optimize groups with no alternative.
-% \item Optimize the use of \cs{prg_stepwise_...} functions.
-% \item Decide what |\c{\c{...}}| should do in the replacement text.
-% \item Fix the |\c{\cBx}| bug in replacement text.
+% \item Change \tn{skip} to \tn{dimen} for the array of active
+% threads, and shift the array of submatch informations so that it
+% starts at \tn{skip}$0$.
+% \item Optimize |\c{abc}| for matching a specific control sequence.
+% \item Only build \c{...} once.
+% \item Use \tn{skip} for the left and right state stacks when
+% compiling a regex.
+% \item Should \cs{regex_action_free_group:n} only be used for greedy
+% |{n,}| quantifier? (I think not.)
+% \item Quantifiers for |\u| and assertions.
+% \item Improve digit grabbing for the |\g| escape in replacement.
+% Allow arbitrary integer expressions for all those numbers?
+% \item When matching, keep track of an explicit stack of
+% \texttt{current_state} and \texttt{current_submatches}.
+% \item If possible, when a state is reused by the same thread, kill
+% other subthreads.
+% \item Use \tn{dimen} registers rather than \cs{l_regex_balance_tl}
+% to build \cs{regex_replacement_balance_one_match:n}.
+% \item Reduce the number of epsilon-transitions in alternatives.
+% \item Optimize simple strings: use less states (|abcade| should give
+% two states, for |abc| and |ade|). [Does that really make sense?]
+% \item Optimize groups with no alternative.
+% \item Optimize states with a single \cs{regex_action_free:n}.
+% \item Optimize the use of \cs{regex_action_success:} by inserting it
+% in state $2$ directly instead of having an extra transition.
+% \item Optimize the use of \cs{prg_stepwise_...} functions.
+% \item Groups don't capture within regexes for csnames; optimize and
+% document.
+% \item Decide and document what |\c{\c{...}}| should do in the
+% replacement text, similar questions for |\u|.
+% \item Better \enquote{show} for anchors, properties, and catcode tests.
+% \item Does |\K| really need a new state for itself?
+% \item When compiling, use a boolean \texttt{in_cs} and less magic
+% numbers.
+% \item Instead of checking whether the character is special or
+% alphanumeric using its character code, check if it is special in
+% regexes with \cs{cs_if_exist} tests.
% \end{itemize}
%
% The following features are likely to be implemented at some point
% in the future.
% \begin{itemize}
-% \item General look-ahead/behind assertions.
-% \item Regex matching on external files.
-% \item Conditional subpatterns with look ahead/behind: \enquote{if
-% what follows is [\ldots{}], then [\ldots{}]}.
-% \item |(*..)| and |(?..)| sequences to set some options
-% (partially implemented).
-% \item |\K| for resetting the beginning of the match.
-% \item UTF-8 mode for pdf\TeX{}.
-% \item Newline conventions are not done.
-% In particular, we should have an option for |.| not to match newlines.
-% Also, |\A| should differ from |^|, and |\Z|, |\z| and |$| should
-% differ.
-% \item Unicode properties: |\p{..}| and |\P{..}|;
-% |\X| which should match any \enquote{extended} Unicode sequence.
-% This requires to manipulate a lot of data, hence a lot of optimization
-% ahead.
+% \item Allow |\cL(abc)| in replacement text.
+% \item General look-ahead/behind assertions.
+% \item Regex matching on external files.
+% \item Conditional subpatterns with look ahead/behind: \enquote{if
+% what follows is [\ldots{}], then [\ldots{}]}.
+% \item |(*..)| and |(?..)| sequences to set some options.
+% \item UTF-8 mode for pdf\TeX{}.
+% \item Newline conventions are not done.
+% In particular, we should have an option for |.| not to match newlines.
+% Also, |\A| should differ from |^|, and |\Z|, |\z| and |$| should
+% differ.
+% \item Unicode properties: |\p{..}| and |\P{..}|;
+% |\X| which should match any \enquote{extended} Unicode sequence.
+% This requires to manipulate a lot of data, probably using tree-boxes.
% \end{itemize}
%
-% The following features of PCRE or Perl will probably not be implemented.
+% The following features of \textsc{pcre} or Perl will probably not be
+% implemented.
% \begin{itemize}
-% \item |\ddd|, matching the character with code \texttt{ddd} in octal;
-% \item POSIX character classes |[:alpha:]| \emph{etc.}, this is redundant;
-% \item Callout with |(?C...)|, we cannot run arbitrary user code during
-% the matching, because the regex code uses registers in an unsafe way;
-% \item Conditional subpatterns (other than with a look-ahead
-% or look-behind condition): this is non-regular, isn't it?
-% \item Named subpatterns: \TeX{} programmers have lived so far without
-% any need for named macro parameters.
+% \item |\ddd|, matching the character with octal code \texttt{ddd};
+% \item Callout with |(?C...)|, we cannot run arbitrary user code
+% during the matching, because the regex code uses registers in an
+% unsafe way;
+% \item Conditional subpatterns (other than with a look-ahead or
+% look-behind condition): this is non-regular, isn't it?
+% \item Named subpatterns: \TeX{} programmers have lived so far
+% without any need for named macro parameters.
% \end{itemize}
%
-% The following features of PCRE or perl will definitely not be implemented.
+% The following features of \textsc{pcre} or Perl will definitely not be
+% implemented.
% \begin{itemize}
-% \item |\cx|, similar to \TeX{}'s own |\^^x|;
-% \item Comments: \TeX{} already has its own system for comments.
-% \item |\Q...\E| escaping: this would require to read the argument
-% verbatim, which is not in the scope of this module.
-% \item Atomic grouping, possessive quantifiers: those tools, mostly
-% meant to fix catastrophic backtracking, are unnecessary in a
-% non-backtracking algorithm, and difficult to implement.
-% \item Subroutine calls: this syntactic sugar is difficult to include
-% in a non-backtracking algorithm, in particular because the
-% corresponding group should be treated as atomic. Also, we cannot
-% afford to run user code within the regular expression matching,
-% because of our \enquote{misuse} of registers.
-% \item Recursion: this is a non-regular feature.
-% \item Back-references: non-regular feature, this requires backtracking,
-% which is prohibitively slow.
-% \item Backtracking control verbs: intrinsically tied to backtracking.
-% \item |\C| single byte in UTF-8 mode: Xe\TeX{} and Lua\TeX{} serve
-% us characters directly, and splitting those into bytes is tricky,
-% encoding dependent, and most likely not useful anyways.
+% \item |\cx|, similar to \TeX{}'s own |\^^x|;
+% \item Comments: \TeX{} already has its own system for comments.
+% \item |\Q...\E| escaping: this would require to read the argument
+% verbatim, which is not in the scope of this module.
+% \item Atomic grouping, possessive quantifiers: those tools, mostly
+% meant to fix catastrophic backtracking, are unnecessary in a
+% non-backtracking algorithm, and difficult to implement.
+% \item Subroutine calls: this syntactic sugar is difficult to include
+% in a non-backtracking algorithm, in particular because the
+% corresponding group should be treated as atomic. Also, we cannot
+% afford to run user code within the regular expression matching,
+% because of our \enquote{misuse} of registers.
+% \item Recursion: this is a non-regular feature.
+% \item Back-references: non-regular feature, this requires
+% backtracking, which is prohibitively slow.
+% \item Backtracking control verbs: intrinsically tied to
+% backtracking.
+% \item |\C| single byte in UTF-8 mode: Xe\TeX{} and Lua\TeX{} serve
+% us characters directly, and splitting those into bytes is tricky,
+% encoding dependent, and most likely not useful anyways.
% \end{itemize}
%
% \end{documentation}
@@ -626,7 +704,7 @@
% \begin{macrocode}
\ProvidesExplPackage
{\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
-\RequirePackage{l3str, l3tl-analysis, l3flag}
+\RequirePackage{l3tl-build, l3tl-analysis, l3flag, l3str}
% \end{macrocode}
%
% \subsection{Plan of attack}
@@ -638,36 +716,42 @@
% code tends to run slowly, and we must use faster, albeit slightly more
% restrictive, techniques, coming from automata theory.
%
-% Given a regular expression of $n$ characters, we build a
-% non-deterministic finite automaton (NFA) with roughly $n$ states,
-% which accepts precisely those token lists matching that regular
-% expression. Then loop through the query token list one token (one
-% \enquote{index}) at a time, exploring in parallel every possible path
-% through the NFA. When performing this matching, we keep track of an
-% array of the states currently \enquote{active}, which are to be
-% considered in order when the next token is read.
+% Given a regular expression of $n$ characters, we do the following:
+% \begin{itemize}
+% \item (Compiling.) Analyse the regex, finding invalid input, and
+% convert it to an internal representation.
+% \item (Building.) Convert the compiled regex to a non-deterministic
+% finite automaton (\textsc{nfa}) with roughly $n$ states which
+% accepts precisely token lists matching that regex.
+% \item (Matching.) Loop through the query token list one token (one
+% \enquote{position}) at a time, exploring in parallel every
+% possible path (\enquote{active thread}) through the \textsc{nfa},
+% considering active threads in an order determined by the
+% quantifiers' greediness.
+% \end{itemize}
%
% We use the following vocabulary in the code comments (and in variable
% names).
% \begin{itemize}
% \item \emph{Group}: index of the capturing group, $-1$ for
% non-capturing groups.
-% \item \emph{Index}: each token in the query is labelled by an integer
-% \meta{index}, with $\cs{l_regex_min_index_int} - 1 \leq \meta{index}
-% \leq \cs{l_regex_max_index_int}$. The lowest and highest indexes
-% correspond to imaginary begin and end markers (with inacessible
-% category code and character code).
+% \item \emph{Position}: each token in the query is labelled by an
+% integer \meta{position}, with $\texttt{min_pos} - 1 \leq
+% \meta{position} \leq \texttt{max_pos}$. The lowest and highest
+% positions correspond to imaginary begin and end markers (with
+% inacessible category code and character code).
% \item \emph{Query}: the token list to which we apply the regular
% expression.
-% \item \emph{State}: each state of the NFA is labelled by an integer
-% \meta{state} with $0 \leq \meta{state} <
-% \cs{l_regex_max_state_int}$.
-% \item \emph{Active state}: state of the NFA that is reached when
-% reading the query string for the matching. Those states are
-% ordered according to the greediness of quantifiers.
+% \item \emph{State}: each state of the \textsc{nfa} is labelled by an
+% integer \meta{state} with $\texttt{min_state} \leq \meta{state} <
+% \texttt{max_state}$.
+% \item \emph{Active thread}: state of the \textsc{nfa} that is reached
+% when reading the query token list for the matching. Those threads
+% are ordered according to the greediness of quantifiers.
% \item \emph{Step}: used when matching, starts at $0$, incremented
% every time a character is read, and is not reset when searching
-% for repeated matches.
+% for repeated matches. The integer \cs{l_regex_step_int} is a
+% unique id for all the steps of the matching algorithm.
% \end{itemize}
%
% To acheive a good performance, we abuse \TeX{}'s registers in two
@@ -676,764 +760,120 @@
% functions. And we store integers in \tn{dimen} registers in scaled
% points (\texttt{sp}), using \TeX{}'s implicit conversion from
% dimensions to integers in some contexts. Specifically, the registers
-% are used as follows. When building,
+% are used as follows. When compiling, \tn{toks} registers are used
+% under the hood by functions from the \pkg{l3tl-build} module. When
+% building,
% \begin{itemize}
-% \item \tn{toks}\meta{state} has two parts separated by \cs{s_stop}:
-% a property list which holds the submatch information, followed by
-% the tests and actions to perform in the \meta{state} of the NFA.
-% \item \tn{skip}$i$ has the form \meta{group} \texttt{plus}
+% \item \tn{toks}\meta{state} holds the tests and actions to perform
+% in the \meta{state} of the \textsc{nfa}.
+% \item (Not implemented yet.)
+% \tn{skip}$i$ has the form \meta{group id} \texttt{plus}
% \meta{left state} \texttt{minus} \meta{right state}.
-% ^^A todo: not done yet.
% \end{itemize}
% When matching,
% \begin{itemize}
% \item \tn{dimen}\meta{state} is equal to the last \meta{step} in
% which the \meta{state} was active.
-% \item \tn{skip}$i$ holds the \meta{state} number corresponding to
-% the $i$-th active \meta{state} (smaller $i$ is higher precendence)
-% and has no shrink or stretch components.
-% \item \tn{toks}\meta{index} holds \meta{tokens} which \texttt{o}-
-% and \texttt{x}-expand to the \meta{index}-th token in the query.
+% \item (Currently, we use \tn{skip} instead of \tn{dimen}.)
+% \tn{dimen}\meta{thread}, with $\texttt{min_active} \leq
+% \meta{thread} < \texttt{max_active}$, is equal to the
+% \meta{state} in which the \meta{thread} currently is. The
+% \meta{threads} or ordered starting from the best to the least
+% preferred.
+% \item \tn{toks}\meta{thread} holds the submatch information for the
+% \meta{thread}, as the contents of a property list.
+% \item \tn{muskip}\meta{position} holds as its main and stretch
+% components the character and category code of the token at this
+% \meta{position} in the query.
+% \item \tn{toks}\meta{position} holds \meta{tokens} which \texttt{o}-
+% and \texttt{x}-expand to the \meta{position}-th token in the query.
+% \item \tn{skip} registers hold the value of end-points of all
+% submatches as would be extracted by the \cs{regex_extract}
+% functions. Since smaller \tn{skip} registers are used, the minimum
+% index is twice \texttt{max_state}, and the used registers go up to
+% \cs{l_regex_submatch_int}. They are organized in blocks of
+% \texttt{capturing_group}, each block corresponding to one match
+% with all its submatches stored in consecutive \tn{skip}s.
% \end{itemize}
% \tn{count} registers are not abused, which means that we can safely
-% use named integers in this module.
+% use named integers in this module. Note that \tn{box} registers are
+% not abused either; maybe we could leverage those for some purpose.
+%
+% The code is structured as follows. Variables are introduced in the
+% relevant section. First we present some generic helper functions. Then
+% comes the code for compiling a regular expression, and for showing the
+% result of the compilation. The building phase converts a compiled
+% regex to \textsc{nfa} states, and the automaton is run by the code in
+% the following section. The only remaining brick is parsing the
+% replacement text and performing the replacement. We are then ready for
+% all the user functions. Finally, messages, and a little bit of tracing
+% code.
%
-% The code is structured as follows. Variables, constants and various
-% helper functions are introduced first, to limit the clutter in later
-% parts. Then functions pertaining to parsing the regular expression are
-% defined: that part is rather long because of the many bells and
-% whistles of the regex notation. The next subsection takes care of
-% running the NFA. Thereafter, we give tools for the replacements.
-% Finally, user functions.
-%
-% \subsection{Constants and variables}
+% \subsection{Helpers}
%
-% \begin{macro}{\regex_tmp:w}
-% \begin{variable}
-% {\l_regex_internal_a_int, \l_regex_internal_b_int, \l_regex_internal_c_int}
-% \begin{variable}{\l_regex_internal_a_tl, \l_regex_internal_b_tl}
-% \begin{variable}{\g_regex_internal_tl}
-% Temporary variables used for various purposes.
+% \begin{macro}[aux]{\tl_to_str:V}
+% A variant we need for the |\u| escape in the replacement text.
% \begin{macrocode}
-\cs_new:Npn \regex_tmp:w { }
-\tl_new:N \l_regex_internal_a_tl
-\tl_new:N \l_regex_internal_b_tl
-\int_new:N \l_regex_internal_a_int
-\int_new:N \l_regex_internal_b_int
-\int_new:N \l_regex_internal_c_int
-\tl_new:N \g_regex_internal_tl
+\cs_generate_variant:Nn \tl_to_str:n { V }
% \end{macrocode}
-% \end{variable}
-% \end{variable}
-% \end{variable}
% \end{macro}
%
-% \subsubsection{Character properties}
+% \subsubsection{Constants and variables}
%
-% \begin{variable}
-% {
-% \c_regex_d_tl, \c_regex_D_tl, \c_regex_h_tl, \c_regex_H_tl,
-% \c_regex_s_tl, \c_regex_S_tl, \c_regex_v_tl, \c_regex_V_tl,
-% \c_regex_w_tl, \c_regex_W_tl, \c_regex_N_tl
-% }
-%
-% These constant token lists encode which characters are recognized by
-% |\d|, |\D|, |\w|, \emph{etc.} in regular expressions. Namely,
-% |\d=[0-9]|, |\w=[0-9A-Z_a-z]|, \verb*+\s=[\ \^^I\^^J\^^L\^^M]+,
-% \verb*+\h=[\ \^^I]+, |\v=[\^^J-\^^M]|, and the upper case
-% counterparts match anything that the lower case does not match. The
-% order in which the various tests appear is optimized for usual
-% mostly lower case letter text.
-% \begin{macrocode}
-\tl_const:Nn \c_regex_d_tl
- { \regex_item_range:nn \c_forty_eight { 57 } } % 0--9
-\tl_const:Nn \c_regex_h_tl
- {
- \regex_item_equal:n \c_thirty_two % space
- \regex_item_equal:n \c_nine % tab
- }
-\tl_const:Nn \c_regex_s_tl
- {
- \regex_item_equal:n \c_thirty_two % space
- \regex_item_equal:n \c_nine % tab
- \regex_item_equal:n \c_ten % lf
- \regex_item_equal:n \c_twelve % ff
- \regex_item_equal:n \c_thirteen % cr
- }
-\tl_const:Nn \c_regex_v_tl
- { \regex_item_range:nn \c_ten \c_thirteen } % lf, vtab, ff, cr
-\tl_const:Nn \c_regex_w_tl
- {
- \regex_item_range:nn \c_ninety_seven { 122 } % a--z
- \regex_item_range:nn \c_sixty_five { 90 } % A--Z
- \regex_item_range:nn \c_forty_eight { 57 } % 0--9
- \regex_item_equal:n { 95 } % _
- }
-\tl_const:Nn \c_regex_D_tl
- { \c_regex_d_tl \regex_break_point:TF { } \regex_break_true:w }
-\tl_const:Nn \c_regex_H_tl
- { \c_regex_h_tl \regex_break_point:TF { } \regex_break_true:w }
-\tl_const:Nn \c_regex_S_tl
- { \c_regex_s_tl \regex_break_point:TF { } \regex_break_true:w }
-\tl_const:Nn \c_regex_V_tl
- { \c_regex_v_tl \regex_break_point:TF { } \regex_break_true:w }
-\tl_const:Nn \c_regex_W_tl
- { \c_regex_w_tl \regex_break_point:TF { } \regex_break_true:w }
-\tl_const:Nn \c_regex_N_tl
- {
- \regex_item_equal:n \c_ten
- \regex_break_point:TF { } { \regex_break_true:w }
- }
-% \end{macrocode}
-% \end{variable}
-%
-% \begin{variable}{\c_regex_._tl}
-% The dot meta-character matches any character, except the end marker,
-% whose character code is $-2$.
-% \begin{macrocode}
-\tl_const:cn { c_regex_._tl }
- {
- \if_num:w \l_regex_current_char_int > - \c_two
- \exp_after:wN \regex_break_true:w
- \fi:
- }
-% \end{macrocode}
-% \end{variable}
-%
-% \subsubsection{Variables used while building}
-%
-% \begin{variable}{\l_regex_build_mode_int}
-% While building, ten modes are recognized, labelled $-63$, $-23$,
-% $-6$, $-2$, $0$, $2$, $3$, $6$, $23$, $63$. See
-% section~\ref{sec:regex-modes}.
+% \begin{macro}[aux]{\regex_tmp:w}
+% Temporary function used for various short-term purposes.
% \begin{macrocode}
-\int_new:N \l_regex_build_mode_int
-% \end{macrocode}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_max_state_int}
-% \begin{variable}{\l_regex_left_state_int, \l_regex_right_state_int}
-% The last state that was allocated is $\cs{l_regex_max_state_int}-1$,
-% so that \cs{l_regex_max_state_int} always points to a free state.
-% The starting state and end state of the last group (which any
-% quantifier would act on) are stored as
-% \cs{l_regex_left/right_state_int}. In almost all cases, the left
-% and right pointers only differ by~$1$.
-% \begin{macrocode}
-\int_new:N \l_regex_max_state_int
-\int_new:N \l_regex_left_state_int
-\int_new:N \l_regex_right_state_int
-% \end{macrocode}
-% \end{variable}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_left_state_seq, \l_regex_right_state_seq}
-% Alternatives are implemented by branching from a state into the
-% various choices, then merging those into another state. We store
-% information about those states in two sequences.
-% \begin{macrocode}
-\seq_new:N \l_regex_left_state_seq
-\seq_new:N \l_regex_right_state_seq
-% \end{macrocode}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_end_group_seq}
-% \begin{variable}{\l_regex_end_alternation_seq}
-% These sequences hold actions to be performed at the end of a group,
-% and at the end of each branch of the alternation, respectively.
-% Currently, \cs{l_regex_end_group_seq} is used to keep track of
-% letter case, and \cs{l_regex_end_alternation_seq} is used
-% for \verb"(?|...)" groups.
-% \begin{macrocode}
-\seq_new:N \l_regex_end_group_seq
-\seq_new:N \l_regex_end_alternation_seq
-% \end{macrocode}
-% \end{variable}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_capturing_group_int}
-% \begin{variable}{\l_regex_capturing_group_seq}
-% \begin{variable}{\l_regex_capturing_group_max_int}
-% \cs{l_regex_capturing_group_int} is the ID number of the current
-% capturing group, starting at $0$ for a group enclosing the full
-% regular expression, and counting in the order of their left parenthesis.
-% This number is used when a branch of the alternation ends.
-% Capturing groups can be arbitrarily nested, and we keep track of
-% the stack of ID numbers in \cs{l_regex_capturing_group_seq}.
-% The \texttt{max_int} variable is used in the case of the special
-% groups \verb+(?|...|...)+, which reset the capturing group number in
-% each alternative: at the end of the group, we must use a number
-% larger than every capturing group appearing in any alternative.
-% \begin{macrocode}
-\int_new:N \l_regex_capturing_group_int
-\seq_new:N \l_regex_capturing_group_seq
-\int_new:N \l_regex_capturing_group_max_int
-% \end{macrocode}
-% \end{variable}
-% \end{variable}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_one_or_group_tl}
-% When looking for quantifiers, this variable holds either
-% \enquote{one} or \enquote{group} depending on whether the
-% object to which the quantifier applies matches one character
-% (\emph{i.e.}, is a character or character class), or is a group.
-% \begin{macrocode}
-\tl_new:N \l_regex_one_or_group_tl
-% \end{macrocode}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_catcodes_int, \l_regex_catcodes_default_int}
-% \begin{variable}
-% {
-% \c_regex_catcode_C_int, \c_regex_catcode_B_int, \c_regex_catcode_E_int,
-% \c_regex_catcode_M_int, \c_regex_catcode_T_int, \c_regex_catcode_P_int,
-% \c_regex_catcode_U_int, \c_regex_catcode_D_int, \c_regex_catcode_S_int,
-% \c_regex_catcode_L_int, \c_regex_catcode_O_int, \c_regex_catcode_A_int
-% }
-% \begin{variable}{\c_regex_catcodes_all_int}
-% We wish to allow constructions such as |\c[^BE](..\cL[a-z]..)|,
-% matching two tokens which are neither a begin-group nor an end-group
-% token, followed by a token of category letter and character code in
-% |[a-z]|, followed by two more tokens which are neither begin-group
-% nor end-group tokens. For this to work, we need to keep track of
-% lists of allowed category codes: \cs{l_regex_catcodes_int} and
-% \cs{l_regex_catcodes_default_int} are bitmaps, sums of $4^c$, for
-% all allowed catcodes $c$. The latter is local to each capturing
-% group, and we reset \cs{l_regex_catcodes_int} to that value after
-% each character or class, changing it only when encountering a |\c|
-% escape. Errata: the \cs{l_regex_catcodes_default_int} is not used
-% yet, because the idea of having \verb+\cL(...|...)+ act on the whole
-% contents of the group is not yet implemented.
-% \begin{macrocode}
-\int_new:N \l_regex_catcodes_int
-\int_new:N \l_regex_catcodes_default_int
-\int_const:Nn \c_regex_catcode_C_int { "1 }
-\int_const:Nn \c_regex_catcode_B_int { "4 }
-\int_const:Nn \c_regex_catcode_E_int { "10 }
-\int_const:Nn \c_regex_catcode_M_int { "40 }
-\int_const:Nn \c_regex_catcode_T_int { "100 }
-\int_const:Nn \c_regex_catcode_P_int { "1000 }
-\int_const:Nn \c_regex_catcode_U_int { "4000 }
-\int_const:Nn \c_regex_catcode_D_int { "10000 }
-\int_const:Nn \c_regex_catcode_S_int { "100000 }
-\int_const:Nn \c_regex_catcode_L_int { "400000 }
-\int_const:Nn \c_regex_catcode_O_int { "1000000 }
-\int_const:Nn \c_regex_catcode_A_int { "4000000 }
-\int_const:Nn \c_regex_catcodes_all_int { "5515155 }
-% \end{macrocode}
-% \end{variable}
-% \end{variable}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_catcodes_bool}
-% Controls whether the bitmap of category codes built should be
-% inverted or not.
-% \begin{macrocode}
-\bool_new:N \l_regex_catcodes_bool
-% \end{macrocode}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_tests_bool,\l_regex_tests_tl}
-% The tests which should be performed on an item of the token list are
-% stored in \cs{l_regex_tests_tl}. In the case of character classes,
-% \cs{l_regex_tests_bool} is \texttt{false} for negative character
-% classes. In the other case, \cs{l_regex_tests_tl} is directly
-% converted to a state of the NFA. The two variables are cleared
-% (boolean set to \texttt{true}) after looking for a quantifier for
-% the class or single character or property. It would seem cleaner to
-% clear them instead before grabbing the next character or class, but
-% that doesn't interact correctly with constructions like |\cL[...]|
-% where |\cL| inserts some tests in \cs{l_regex_tests_tl}, which
-% should not be eliminated before finding the class.
-% \begin{macrocode}
-\bool_new:N \l_regex_tests_bool
-\tl_new:N \l_regex_tests_tl
+\cs_new:Npn \regex_tmp:w { }
% \end{macrocode}
-% \end{variable}
+% \end{macro}
%
% \begin{variable}
% {
-% \l_regex_tests_saved_bool,
-% \l_regex_tests_saved_tl,
-% \l_regex_catcodes_saved_int
+% \l_regex_internal_a_tl, \l_regex_internal_b_tl,
+% \l_regex_internal_a_int, \l_regex_internal_b_int,
+% \l_regex_internal_c_int, \l_regex_internal_bool,
+% \l_regex_internal_seq, \g_regex_internal_tl,
% }
-% In nested class, which can only occur as |[\cA[...]]|, the variables
-% \cs{l_regex_tests_bool}, \cs{l_regex_tests_tl}, and
-% \cs{l_regex_catcodes_int} from the outer class must be saved while
-% processing the inner class.
-% \begin{macrocode}
-\bool_new:N \l_regex_tests_saved_bool
-\tl_new:N \l_regex_tests_saved_tl
-\int_new:N \l_regex_catcodes_saved_int
-% \end{macrocode}
-% \end{variable}
-%
-% \subsubsection{Variables used when matching}
-%
-% \begin{variable}{\l_regex_min_index_int}
-% \begin{variable}{\l_regex_max_index_int}
-% The tokens in the query are indexed from \cs{l_regex_min_index_int}
-% for the first to $\cs{l_regex_max_index_int}-1$ for the last, and
-% their information is stored in \tn{muskip} and \tn{toks} registers
-% with those numbers. We don't start from $0$ because the \tn{toks}
-% registers with low numbers are used to hold the states of the NFA.
-% \begin{macrocode}
-\int_new:N \l_regex_min_index_int
-\int_new:N \l_regex_max_index_int
-% \end{macrocode}
-% \end{variable}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_nesting_int}
-% The first phase when matching is to go once through the query token
-% list and store the information for each token as \tn{muskip} and
-% \tn{toks} registers. During this phase, \cs{l_regex_nesting_int}
-% counts the balance of begin-group and end-group character tokens
-% which appear before a given point in the string, and is stored as
-% the shrink component of the \tn{muskip} registers. This integer is
-% also used to keep track of the balance in the replacement text.
-% \begin{macrocode}
-\int_new:N \l_regex_nesting_int
-% \end{macrocode}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_current_index_int}
-% \begin{variable}{\l_regex_start_index_int}
-% \begin{variable}{\l_regex_success_index_int}
-% While reading through the query token list,
-% \cs{l_regex_current_index_int} is the position in the token list,
-% starting at \cs{l_regex_min_index_int} for the first token.
-% Each match begins at the position given by
-% \cs{l_regex_start_index_int}. Whenever an execution thread succeeds,
-% the corresponding index is stored into \cs{l_regex_success_index_int},
-% which will be the next starting index (except in the case of empty
-% matches).
-% \begin{macrocode}
-\int_new:N \l_regex_current_index_int
-\int_new:N \l_regex_start_index_int
-\int_new:N \l_regex_success_index_int
-% \end{macrocode}
-% \end{variable}
-% \end{variable}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_current_char_int}
-% \begin{variable}{\l_regex_current_catcode_int}
-% \begin{variable}{\l_regex_current_token_tl}
-% \begin{variable}{\l_regex_last_char_int}
-% \begin{variable}{\l_regex_case_changed_char_int}
-% The character and category codes of the token at the current
-% position; \meta{tokens} which \texttt{o}- and \texttt{x}-expand to
-% the token (as provided by \cs{tl_analysis:n}); the character code of
-% the token at the previous position; and the character code of the
-% result of changing the case of the current token
-% (|A-Z|$\leftrightarrow$|a-z|). This last integer is only computed if
-% the \enquote{case insensitive} option |(?i)| is used in the regex.
-% The \cs{l_regex_current_char_int} variable is also used in various
-% other phases to hold a character code.
-% \begin{macrocode}
-\int_new:N \l_regex_current_char_int
-\int_new:N \l_regex_current_catcode_int
-\tl_new:N \l_regex_current_token_tl
-\int_new:N \l_regex_last_char_int
-\int_new:N \l_regex_case_changed_char_int
-% \end{macrocode}
-% \end{variable}
-% \end{variable}
-% \end{variable}
-% \end{variable}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_caseless_bool}
-% True if caseless matching is used within the regular expression.
-% This controls whether \cs{l_regex_case_changed_char_int} is computed.
-% \begin{macrocode}
-\bool_new:N \l_regex_caseless_bool
-% \end{macrocode}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_current_state_int}
-% For every character in the token list, each of the active states is
-% considered in turn.
-% The variable \cs{l_regex_current_state_int} holds the state
-% of the NFA which is currently considered: transitions are then
-% given as shifts relative to the current state.
-% In some cases of groups with quantifiers,
-% \cs{l_regex_current_state_int} is shifted to a fake value for
-% transitions to point to the correct states.
-% \begin{macrocode}
-\int_new:N \l_regex_current_state_int
-% \end{macrocode}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_current_submatches_prop}
-% \begin{variable}{\l_regex_success_submatches_prop}
-% The submatches for the thread which lies at the
-% \cs{l_regex_current_state_int} are stored in a property
-% list variable. This property list is stored by
-% \cs{regex_action_cost:n} into the \tn{toks} register
-% for the target state of the transition. When a thread
-% succeeds, this property list is copied to
-% \cs{l_regex_success_submatches_prop} and only the last
-% sucessful thread will remain there.
-% \begin{macrocode}
-\prop_new:N \l_regex_current_submatches_prop
-\prop_new:N \l_regex_success_submatches_prop
-% \end{macrocode}
-% \end{variable}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_step_int}
-% In the case of repeated matches, \cs{l_regex_current_index_int}
-% is reset to the end-position of the previous match. In contrast,
-% \cs{l_regex_step_int} is simply incremented to provide
-% a unique number for each iteration of the matching loop. This
-% is handy to attach each set of submatch information to a given
-% iteration (and automatically discard it when it corresponds to
-% a past iteration).
-% \begin{macrocode}
-\int_new:N \l_regex_step_int
-% \end{macrocode}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_max_active_int}
-% All the currently active states are kept in order of precedence
-% in the \tn{skip} registers, which for our purpose serve as an array:
-% the $i$th item of the array is \tn{skip}$i$. The largest index used
-% after treating the previous character is \cs{l_regex_max_active_int}.
-% At the start of every step, the whole array is unpacked, so that the
-% space can immediately be reused, and \cs{l_regex_max_active_int} is
-% reset to zero, effectively clearing the array.
-% \begin{macrocode}
-\int_new:N \l_regex_max_active_int
-% \end{macrocode}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_every_match_tl}
-% Every time a match is found, this token list is used. For single
-% matching, the token list is empty. For multiple matching, the token
-% list is set to repeat the matching.
-% \begin{macrocode}
-\tl_new:N \l_regex_every_match_tl
-% \end{macrocode}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_fresh_thread_bool}
-% \begin{variable}{\l_regex_success_empty_bool}
-% \begin{macro}{\regex_if_two_empty_matches:F}
-% When doing multiple matches, we need to avoid infinite loops where
-% each iteration matches the same empty token list. When an empty
-% token list is matched, the next successful match of the same empty
-% token list is suppressed. We detect empty matches by setting
-% \cs{l_regex_fresh_thread_bool} to \texttt{true} for threads which
-% directly come from the start of the regular expression, and testing
-% that boolean whenever a thread succeeds. The function
-% \cs{regex_if_two_empty_matches:F} is redefined at every match
-% attempt, depending on whether the previous match was empty or not:
-% if it was, then the function must cancel a purported success if it
-% is empty and at the same spot as the previous match; otherwise, we
-% definitely don't have two identical empty matches, so the function
-% is \cs{use:n}.
-% \begin{macrocode}
-\bool_new:N \l_regex_fresh_thread_bool
-\bool_new:N \l_regex_success_empty_bool
-\cs_new_eq:NN \regex_if_two_empty_matches:F \use:n
-% \end{macrocode}
-% \end{macro}
-% \end{variable}
-% \end{variable}
-%
-% \begin{variable}{\g_regex_success_bool}
-% \begin{variable}{\l_regex_saved_success_bool}
-% \begin{variable}{\l_regex_success_match_bool}
-% The boolean \cs{g_regex_success_bool} is true if there was at least
-% one successful match, and \cs{l_regex_success_match_bool} is true if
-% the current match attempt was successful. The variable
-% \cs{g_regex_success_bool} is the only global variable in this whole
-% module. When nesting \cs{regex} functions internally, the value of
-% \cs{g_regex_success_bool} is saved into
-% \cs{l_regex_saved_success_bool}, which is local, hence not affected
-% by the changes due to inner regex functions.
+% Temporary variables used for various purposes.
% \begin{macrocode}
-\bool_new:N \g_regex_success_bool
-\bool_new:N \l_regex_saved_success_bool
-\bool_new:N \l_regex_success_match_bool
+\tl_new:N \l_regex_internal_a_tl
+\tl_new:N \l_regex_internal_b_tl
+\int_new:N \l_regex_internal_a_int
+\int_new:N \l_regex_internal_b_int
+\int_new:N \l_regex_internal_c_int
+\bool_new:N \l_regex_internal_bool
+\seq_new:N \l_regex_internal_seq
+\tl_new:N \g_regex_internal_tl
% \end{macrocode}
% \end{variable}
-% \end{variable}
-% \end{variable}
-%
-% \subsubsection{Regular expression variables}
%
% \begin{variable}{\c_regex_no_match_regex}
% This regular expression matches nothing, but is still a valid
-% regular expression. It is used as the initial value for regular
+% regular expression. We could use a failing assertion, but I went for
+% an empty class. It is used as the initial value for regular
% expressions declared using \cs{regex_new:N}.
% \begin{macrocode}
\tl_const:Nn \c_regex_no_match_regex
{
- \regex_nfa:Nw \c_regex_no_match_regex
- \l_regex_max_state_int = \c_one
- \l_regex_capturing_group_int = \c_zero
- \tex_toks:D \c_zero { \s_stop }
- \s_stop
- }
-% \end{macrocode}
-% \end{variable}
-%
-% \begin{macro}{\regex_new:N}
-% As for many data types deriving from the \texttt{tl} data type,
-% creating a new one is simply a matter of checking that it does not
-% exist yet, and setting it equal to a default value.
-% \begin{macrocode}
-\cs_new_protected:Npn \regex_new:N #1
- {
- \chk_if_free_cs:N #1
- \cs_gset_eq:NN #1 \c_regex_no_match_regex
+ \regex_branch:n
+ { \regex_class:NnnnN \c_true_bool { } { 1 } { 0 } \c_true_bool }
}
% \end{macrocode}
-% \end{macro}
-%
-% \begin{variable}{\l_regex_internal_regex}
-% This holds a temporary pre-compiled regular expression
-% when matching a control sequence name.
-% \begin{macrocode}
-\regex_new:N \l_regex_internal_regex
-% \end{macrocode}
-% \end{variable}
-%
-% \subsubsection{Other variables}
-%^^A todo: move?
-%^^A todo: run with the check-declarations option.
-%
-% \begin{variable}{\l_regex_replacement_csnames_int}
-% The behaviour of closing braces inside a replacement text depends on
-% whether a sequences |\c{| or |\u{| has been encountered. The number
-% of \enquote{open} such sequences that should be closed by |}| is
-% stored in \cs{l_regex_replacement_csnames_int}, and decreased by
-% $1$ by each |}|.
-% \begin{macrocode}
-\int_new:N \l_regex_replacement_csnames_int
-% \end{macrocode}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_replacement_int}
-% \begin{macrocode}
-\int_new:N \l_regex_replacement_int
-% \end{macrocode}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_submatch_int}
-% \begin{variable}{\l_regex_submatch_start_int}
-% \begin{macrocode}
-\int_new:N \l_regex_submatch_int
-\int_new:N \l_regex_submatch_start_int
-% \end{macrocode}
-% \end{variable}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_match_count_int}
-% The number of matches found so far is stored
-% in \cs{l_regex_match_count_int}. This is only used
-% in the \cs{regex_count:nnN} functions.
-% \begin{macrocode}
-\int_new:N \l_regex_match_count_int
-% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{regex_begin, regex_end}
-% Those flags are raised to indicate extra begin-group
-% or end-group tokens when extracting submatches.
+% \begin{variable}{\l_regex_balance_int}
+% The first thing we do when matching is to go once through the query
+% token list and store the information for each token as \tn{muskip}
+% and \tn{toks} registers. During this phase, \cs{l_regex_balance_int}
+% counts the balance of begin-group and end-group character tokens
+% which appear before a given point in the token list, and we store it
+% as the shrink component of each \tn{muskip} register. This variable
+% is also used to keep track of the balance in the replacement text.
% \begin{macrocode}
-\flag_new:n { regex_begin }
-\flag_new:n { regex_end }
+\int_new:N \l_regex_balance_int
% \end{macrocode}
% \end{variable}
%
-% \subsection{Helpers}
-%
-% \subsubsection{Toks}
-%
-% Two unrelated sets of functions for manipulating \tn{toks} registers.
-%
-% \begin{macro}[int]{\regex_toks_put_left:Nx}
-% \begin{macro}[int]{\regex_toks_put_right:Nx}
-% During the building phase, every \tn{toks} register starts with
-% \cs{s_stop}, and we wish to add \texttt{x}-expanded material to
-% those registers. The expansion is done \enquote{by hand} for
-% optimization (these operations are used quite a lot). When adding
-% material to the left, we define \cs{regex_tmp:w} to remove the
-% \cs{s_stop} marker and put it back to the left of the new material.
-% \begin{macrocode}
-\cs_new_protected:Npn \regex_toks_put_left:Nx #1#2
- {
- \cs_set_nopar:Npx \regex_tmp:w \s_stop { \s_stop #2 }
- \tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN
- { \exp_after:wN \regex_tmp:w \tex_the:D \tex_toks:D #1 }
- }
-\cs_new_protected:Npn \regex_toks_put_right:Nx #1#2
- {
- \cs_set_nopar:Npx \regex_tmp:w {#2}
- \tex_toks:D #1 \exp_after:wN
- { \tex_the:D \tex_toks:D \exp_after:wN #1 \regex_tmp:w }
- }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-%
-% \begin{macro}[int, rEXP]{\regex_toks_range:nn}
-% \begin{macro}[aux, rEXP]{\regex_toks_range:ww}
-% Some non-expandable functions store pieces of their result in
-% \tn{toks} registers. Those pieces are concatenated using
-% \cs{regex_toks_range:nn}, which expects two integers, \meta{start}
-% and \meta{end}, and expands to the contents of the \tn{toks}
-% registers from \meta{start} (inclusive) to \meta{end}
-% (exclusive). This correctly expands to nothing if \meta{start}
-% $\geq$ \meta{end}.
-% \begin{macrocode}
-\cs_new:Npn \regex_toks_range:nn #1#2
- {
- \exp_after:wN \regex_toks_range:ww
- \int_use:N \int_eval:w #1 \exp_after:wN ;
- \int_use:N \int_eval:w #2 ;
- \prg_break_point:n { }
- }
-\cs_new:Npn \regex_toks_range:ww #1 ; #2 ;
- {
- \if_num:w #1 < #2 \exp_stop_f:
- \else:
- \exp_after:wN \prg_map_break:
- \fi:
- \tex_the:D \tex_toks:D #1 \exp_stop_f:
- \exp_after:wN \regex_toks_range:ww
- \int_use:N \int_eval:w #1 + \c_one ; #2 ;
- }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-%
-% On the one hand, when performing the matching, the \tn{toks} registers
-% hold submatch information, followed by the instruction for a given
-% state of the NFA. The two parts are separated by \cs{s_stop}. On the
-% other hand, we provide functions to unpack the contents from a range
-% of \tn{toks} within an \texttt{x}-expanding assignment.
-%
-% When it is time to extract submatches from the token list, the various
-% tokens are stored in \tn{toks} registers numbered from
-% \cs{l_regex_min_index_int} inclusive to \cs{l_regex_max_index_int}
-% exclusive. Furthermore, the \tn{skip} registers hold\ldots{} ^^A ??
-%
-% \begin{macro}[int]{\regex_query_submatch:nn}
-% \begin{macro}[int]{\regex_query_submatch:w}
-%^^A ??
-% \begin{macrocode}
-\cs_new:Npn \regex_query_submatch:nn #1#2
- {
- \if_num:w #1 < \l_regex_capturing_group_int
- \exp_after:wN \regex_query_submatch:w
- \int_use:N \int_eval:w #1 + #2 ;
- \fi:
- }
-\cs_new:Npn \regex_query_submatch:w #1 ;
- {
- \regex_toks_range:nn
- { \tex_skip:D #1 }
- { \etex_gluestretch:D \tex_skip:D #1 }
- }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-%
-% \subsubsection{Sequences}
-%
-% \begin{macro}[int]{\regex_seq_pop_int:NN}
-% \begin{macro}[int]{\regex_seq_get_int:NN}
-% \begin{macro}[int]{\regex_seq_push_int:NN}
-% When building the regular expression, we keep track of some integers
-% (pointers to various states) without help from \TeX{}'s grouping.
-% Here are variants of \cs{seq_pop:NN} and \cs{seq_get:NN} which
-% assign using \cs{int_set:Nn} rather than \cs{tl_set:Nn}.
-% \begin{macrocode}
-\cs_new_protected:Npn \regex_seq_pop_int:NN #1#2
- {
- \seq_pop:NN #1 \l_regex_internal_a_tl
- \int_set:Nn #2 \l_regex_internal_a_tl
- }
-\cs_new_protected:Npn \regex_seq_get_int:NN #1#2
- {
- \seq_get:NN #1 \l_regex_internal_a_tl
- \int_set:Nn #2 \l_regex_internal_a_tl
- }
-\cs_new_protected:Npn \regex_seq_push_int:NN #1#2
- { \seq_push:No #1 { \int_use:N #2 } }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-%
-% \begin{macro}[int]{\regex_seq_pop_use:N}
-% \begin{macro}[int]{\regex_seq_get_use:N}
-% When building the regular expression, some settings are kept
-% local to capturing groups without any help from \TeX{}'s grouping.
-% This is done \enquote{by hand}, in sequences whose items should
-% be run immediately.
-% \begin{macrocode}
-\cs_new_protected:Npn \regex_seq_pop_use:N #1
- {
- \seq_pop:NN #1 \l_regex_internal_a_tl
- \l_regex_internal_a_tl
- }
-\cs_new_protected:Npn \regex_seq_get_use:N #1
- {
- \seq_get:NN #1 \l_regex_internal_a_tl
- \l_regex_internal_a_tl
- }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-%
-% \subsubsection{Grabbing digits}
-%
-% \begin{macro}[int]{\regex_get_digits:nw}
-% \begin{macro}[aux]{\regex_get_digits_loop:N,\regex_get_digits_end:w}
-% When parsing the \texttt{\{} quantifier, we need a tool to grab
-% digits until reaching the first non-digit. This
-% \cs{regex_get_digits:nw} function places whatever digits it found as
-% a brace group after |#1|.
-% \begin{macrocode}
-\cs_new_protected:Npn \regex_get_digits:nw #1
- {
- \tex_afterassignment:D \regex_tmp:w
- \cs_set_nopar:Npx \regex_tmp:w
- {
- \exp_not:n {#1}
- { \if_false: } } \fi:
- \regex_get_digits_aux:NN
- }
-\cs_new:Npn \regex_get_digits_aux:NN #1#2
- {
- \if_meaning:w \regex_build_raw:N #1
- \if_num:w 9 < 1 \exp_not:N #2 \exp_stop_f:
- #2
- \else:
- \regex_get_digits_end:w #1 #2
- \fi:
- \else:
- \regex_get_digits_end:w #1 #2
- \fi:
- \regex_get_digits_aux:NN
- }
-\cs_new:Npn \regex_get_digits_end:w #1 \fi: #2 \regex_get_digits_aux:NN
- {
- \fi: #2
- \if_false: { { \fi: } }
- #1
- }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-%
% \subsubsection{Testing characters}
%
% \begin{macro}[int]{\regex_break_point:TF}
@@ -1458,9 +898,23 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}[int]{\regex_item_caseful_equal:n}
-% \begin{macro}[int]{\regex_item_caseful_range:nn}
-% \begin{macro}[int]{\regex_item_caseful_geq:n}
+% \begin{macro}[int]{\regex_item_reverse:n}
+% This function makes showing regular expressions easier, and lets us
+% define |\D| in terms of |\d| for instance. There is a subtelty: the
+% end of the query is marked by $-2$, and will thus match |\D| and
+% other negated properties; this case is catched by another part of
+% the code.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_item_reverse:n #1
+ {
+ #1
+ \regex_break_point:TF { } \regex_break_true:w
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[int]
+% {\regex_item_caseful_equal:n, \regex_item_caseful_range:nn}
% Simple comparisons triggering \cs{regex_break_true:w} when true.
% \begin{macrocode}
\cs_new_protected:Npn \regex_item_caseful_equal:n #1
@@ -1477,29 +931,24 @@
\fi:
\fi:
}
-\cs_new_protected:Npn \regex_item_caseful_geq:n #1
- {
- \reverse_if:N \if_num:w #1 > \l_regex_current_char_int
- \exp_after:wN \regex_break_true:w
- \fi:
- }
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}[int]{\regex_item_caseless_equal:n}
-% \begin{macro}[int]{\regex_item_caseless_range:nn}
-% \begin{macro}[int]{\regex_item_caseless_geq:n}
-% For caseless matching, we perform the test both on
-% \cs{l_regex_current_char_int} and on
-% \cs{l_regex_case_changed_char_int}.
+% \begin{macro}[int]
+% {\regex_item_caseless_equal:n, \regex_item_caseless_range:nn}
+% For caseless matching, we perform the test both on the
+% \texttt{current_char} and on the \texttt{case_changed_char}. Before
+% doing the second set of tests, we make sure that
+% \texttt{case_changed_char} has been computed.
% \begin{macrocode}
\cs_new_protected:Npn \regex_item_caseless_equal:n #1
{
\if_num:w #1 = \l_regex_current_char_int
\exp_after:wN \regex_break_true:w
\fi:
+ \if_num:w \l_regex_case_changed_char_int = \c_max_int
+ \regex_compute_case_changed_char:
+ \fi:
\if_num:w #1 = \l_regex_case_changed_char_int
\exp_after:wN \regex_break_true:w
\fi:
@@ -1511,64 +960,57 @@
\exp_after:wN \exp_after:wN \exp_after:wN \regex_break_true:w
\fi:
\fi:
+ \if_num:w \l_regex_case_changed_char_int = \c_max_int
+ \regex_compute_case_changed_char:
+ \fi:
\reverse_if:N \if_num:w #1 > \l_regex_case_changed_char_int
\reverse_if:N \if_num:w #2 < \l_regex_case_changed_char_int
\exp_after:wN \exp_after:wN \exp_after:wN \regex_break_true:w
\fi:
\fi:
}
-\cs_new_protected:Npn \regex_item_caseless_geq:n #1
- {
- \reverse_if:N \if_num:w #1 > \l_regex_current_char_int
- \exp_after:wN \regex_break_true:w
- \fi:
- \reverse_if:N \if_num:w #1 > \l_regex_case_changed_char_int
- \exp_after:wN \regex_break_true:w
- \fi:
- }
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}[int]{\regex_item_equal:n}
-% \begin{macro}[int]{\regex_item_range:nn}
-% \begin{macro}[int]{\regex_item_geq:n}
-% By default, matching takes the letter case into account.
-% Note that those functions are not protected:
-% they will expand at the building phase, hard-coding which
-% states take care of caseless versus caseful matching.
+% \begin{macro}[int]{\regex_compute_case_changed_char:}
+% This function is called when \cs{l_regex_case_changed_char_int} has
+% not yet been computed (or rather, when it is set to the marker value
+% \cs{c_max_int}). If the current character code is in the range
+% $[65,90]$ (upper-case), then add $32$, making it lowercase. If it is
+% in the lower-case letter range $[97,122]$, subtract $32$.
% \begin{macrocode}
-\cs_new:Npn \regex_item_equal:n { \regex_item_caseful_equal:n }
-\cs_new:Npn \regex_item_range:nn { \regex_item_caseful_range:nn }
-\cs_new:Npn \regex_item_geq:n { \regex_item_caseful_geq:n }
+\cs_new_protected_nopar:Npn \regex_compute_case_changed_char:
+ {
+ \int_set_eq:NN \l_regex_case_changed_char_int \l_regex_current_char_int
+ \if_num:w \l_regex_current_char_int < \c_ninety_one
+ \if_num:w \l_regex_current_char_int < \c_sixty_five
+ \else:
+ \int_add:Nn \l_regex_case_changed_char_int { \c_thirty_two }
+ \fi:
+ \else:
+ \if_num:w \l_regex_current_char_int < \c_one_hundred_twenty_three
+ \if_num:w \l_regex_current_char_int < \c_ninety_seven
+ \else:
+ \int_sub:Nn \l_regex_case_changed_char_int { \c_thirty_two }
+ \fi:
+ \fi:
+ \fi:
+ }
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}[int]{\regex_build_caseless:,\regex_build_caseful:}
-% Switch between caseful and caseless matching.
-% This is only done during the building phase.
+% \begin{macro}[int, EXP]{\regex_item_equal:n, \regex_item_range:nn}
+% Those must always be defined to expand to a \texttt{caseful}
+% (default) or \texttt{caseless} version, and not be protected: they
+% must expand when compiling, to hard-code which tests are caseless or
+% caseful.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_caseless:
- {
- \bool_set_true:N \l_regex_caseless_bool
- \cs_set:Npn \regex_item_equal:n { \regex_item_caseless_equal:n }
- \cs_set:Npn \regex_item_range:nn { \regex_item_caseless_range:nn }
- \cs_set:Npn \regex_item_geq:n { \regex_item_caseless_geq:n }
- }
-\cs_new_protected_nopar:Npn \regex_build_caseful:
- {
- \bool_set_false:N \l_regex_caseless_bool
- \cs_set:Npn \regex_item_equal:n { \regex_item_caseful_equal:n }
- \cs_set:Npn \regex_item_range:nn { \regex_item_caseful_range:nn }
- \cs_set:Npn \regex_item_geq:n { \regex_item_caseful_geq:n }
- }
+\cs_new_eq:NN \regex_item_equal:n ?
+\cs_new_eq:NN \regex_item_range:nn ?
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[int]{\regex_item_catcode:nT}
+% \begin{macro}[int]{\regex_item_catcode:nT, \regex_item_catcode_reverse:nT}
% \begin{macro}[aux]{\regex_item_catcode_aux:}
% The argument is a sum of powers of $4$ with exponents given by the
% allowed category codes (between $0$ and $13$). Dividing by a given
@@ -1594,39 +1036,172 @@
\exp_after:wN \use_none:n
\fi:
}
+\cs_new_protected:Npn \regex_item_catcode_reverse:nT #1#2
+ { \regex_item_catcode:nT {#1} { \regex_item_reverse:n {#2} } }
% \end{macrocode}
% \end{macro}
% \end{macro}
%
+% \begin{macro}[int]{\regex_item_exact:nn, \regex_item_exact_cs:c}
+% This matches an exact \meta{category}-\meta{character code} pair, or
+% an exact control sequence.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_item_exact:nn #1#2
+ {
+ \if_num:w #1 = \l_regex_current_catcode_int
+ \if_num:w #2 = \l_regex_current_char_int
+ \exp_after:wN \exp_after:wN \exp_after:wN \regex_break_true:w
+ \fi:
+ \fi:
+ }
+\cs_new_protected:Npn \regex_item_exact_cs:c #1
+ {
+ \int_compare:nNnTF \l_regex_current_catcode_int = \c_zero
+ {
+ \str_if_eq:xxTF
+ {
+ \exp_after:wN \exp_after:wN \exp_after:wN \cs_to_str:N
+ \tex_the:D \tex_toks:D \l_regex_current_pos_int
+ }
+ { #1 }
+ { \regex_break_true:w } { }
+ }
+ { }
+ }
+% \end{macrocode}
+% \end{macro}
+%
% \begin{macro}[int]{\regex_item_cs:n}
-% Match a control sequence (the argument is a pre-compiled regex).
+% Match a control sequence (the argument is a compiled regex).
% First test the catcode of the current token to be zero.
% Then perform the matching test, and break if the csname
% indeed matches. The three \cs{exp_after:wN} expand the contents
-% of \cs{l_regex_current_token_tl} (of the form \cs{exp_not:n}
+% of the \tn{toks}\meta{current position} (of the form \cs{exp_not:n}
% \Arg{control sequence}) to \meta{control sequence}.
% \begin{macrocode}
\cs_new_protected:Npn \regex_item_cs:n #1
{
\int_compare:nNnT \l_regex_current_catcode_int = \c_zero
{
- \tl_set:Nn \l_regex_internal_regex {#1}
- \bool_set_eq:NN \l_regex_saved_success_bool \g_regex_success_bool
- \exp_args:NNx \regex_match:NnTF \l_regex_internal_regex
- {
- \exp_after:wN \exp_after:wN
- \exp_after:wN \cs_to_str:N \l_regex_current_token_tl
- }
- {
- \bool_gset_eq:NN \g_regex_success_bool \l_regex_saved_success_bool
- \regex_break_true:w
- }
- { \bool_gset_eq:NN \g_regex_success_bool \l_regex_saved_success_bool }
+ \group_begin:
+ \regex_single_match:
+ \regex_disable_submatches:
+ \regex_build_for_cs:n {#1}
+ \bool_set_eq:NN \l_regex_saved_success_bool \g_regex_success_bool
+ \exp_args:Nx \regex_match:n
+ {
+ \exp_after:wN \exp_after:wN
+ \exp_after:wN \cs_to_str:N
+ \tex_the:D \tex_toks:D \l_regex_current_pos_int
+ }
+ \if_meaning:w \c_true_bool \g_regex_success_bool
+ \group_insert_after:N \regex_break_true:w
+ \fi:
+ \bool_gset_eq:NN \g_regex_success_bool \l_regex_saved_success_bool
+ \group_end:
}
}
% \end{macrocode}
% \end{macro}
%
+% \subsubsection{Character property tests}
+%
+% \begin{macro}[aux]
+% {
+% \regex_prop_d:, \regex_prop_h:, \regex_prop_s:,
+% \regex_prop_v:, \regex_prop_w:, \regex_prop_N:
+% }
+% Character property tests for |\d|, |\W|, \emph{etc.} These character
+% properties are not affected by the |(?i)| option. The characters
+% recognized by each one are as follows: |\d=[0-9]|,
+% |\w=[0-9A-Z_a-z]|, \verb*+\s=[\ \^^I\^^J\^^L\^^M]+,
+% \verb*+\h=[\ \^^I]+, |\v=[\^^J-\^^M]|, and the upper case
+% counterparts match anything that the lower case does not match. The
+% order in which the various tests appear is optimized for usual
+% mostly lower case letter text.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \regex_prop_d:
+ { \regex_item_caseful_range:nn \c_forty_eight { 57 } } % 0--9
+\cs_new_protected_nopar:Npn \regex_prop_h:
+ {
+ \regex_item_caseful_equal:n \c_thirty_two % space
+ \regex_item_caseful_equal:n \c_nine % tab
+ }
+\cs_new_protected_nopar:Npn \regex_prop_s:
+ {
+ \regex_item_caseful_equal:n \c_thirty_two % space
+ \regex_item_caseful_equal:n \c_nine % tab
+ \regex_item_caseful_equal:n \c_ten % lf
+ \regex_item_caseful_equal:n \c_twelve % ff
+ \regex_item_caseful_equal:n \c_thirteen % cr
+ }
+\cs_new_protected_nopar:Npn \regex_prop_v:
+ { \regex_item_caseful_range:nn \c_ten \c_thirteen } % lf, vtab, ff, cr
+\cs_new_protected_nopar:Npn \regex_prop_w:
+ {
+ \regex_item_caseful_range:nn \c_ninety_seven { 122 } % a--z
+ \regex_item_caseful_range:nn \c_sixty_five { 90 } % A--Z
+ \regex_item_caseful_range:nn \c_forty_eight { 57 } % 0--9
+ \regex_item_caseful_equal:n { 95 } % _
+ }
+\cs_new_protected_nopar:Npn \regex_prop_N:
+ { \regex_item_reverse:n { \regex_item_caseful_equal:n \c_ten } }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]
+% {
+% \regex_posix_alnum:, \regex_posix_alpha:, \regex_posix_ascii:,
+% \regex_posix_blank:, \regex_posix_cntrl:, \regex_posix_digit:,
+% \regex_posix_graph:, \regex_posix_lower:, \regex_posix_print:,
+% \regex_posix_punct:, \regex_posix_space:, \regex_posix_upper:,
+% \regex_posix_word: , \regex_posix_xdigit:
+% }
+% \textsc{posix} properties. No surprise.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \regex_posix_alnum:
+ { \regex_posix_alpha: \regex_posix_digit: }
+\cs_new_protected_nopar:Npn \regex_posix_alpha:
+ { \regex_posix_lower: \regex_posix_upper: }
+\cs_new_protected_nopar:Npn \regex_posix_ascii:
+ { \regex_item_caseful_range:nn \c_zero \c_one_hundred_twenty_seven }
+\cs_new_eq:NN \regex_posix_blank: \regex_prop_h:
+\cs_new_protected_nopar:Npn \regex_posix_cntrl:
+ {
+ \regex_item_caseful_range:nn \c_zero { 31 }
+ \regex_item_caseful_equal:n \c_one_hundred_twenty_seven
+ }
+\cs_new_eq:NN \regex_posix_digit: \regex_prop_d:
+\cs_new_protected_nopar:Npn \regex_posix_graph:
+ { \regex_item_caseful_range:nn { 33 } { 126 } }
+\cs_new_protected_nopar:Npn \regex_posix_lower:
+ { \regex_item_caseful_range:nn \c_ninety_seven { 122 } }
+\cs_new_protected_nopar:Npn \regex_posix_print:
+ { \regex_item_caseful_range:nn \c_thirty_two { 126 } }
+\cs_new_protected_nopar:Npn \regex_posix_punct:
+ {
+ \regex_item_caseful_range:nn { 33 } { 47 }
+ \regex_item_caseful_range:nn { 58 } { 64 }
+ \regex_item_caseful_range:nn { 91 } { 96 }
+ \regex_item_caseful_range:nn { 123 } { 126 }
+ }
+\cs_new_protected_nopar:Npn \regex_posix_space:
+ {
+ \regex_item_caseful_equal:n \c_thirty_two
+ \regex_item_caseful_range:nn \c_nine \c_thirteen
+ }
+\cs_new_protected_nopar:Npn \regex_posix_upper:
+ { \regex_item_caseful_range:nn \c_sixty_five { 90 } }
+\cs_new_eq:NN \regex_posix_word: \regex_prop_w:
+\cs_new_protected_nopar:Npn \regex_posix_xdigit:
+ {
+ \regex_posix_digit:
+ \regex_item_caseful_range:nn \c_sixty_five { 70 }
+ \regex_item_caseful_range:nn \c_ninety_seven { 102 }
+ }
+% \end{macrocode}
+% \end{macro}
+%
% \subsubsection{Simple character escape}
%
% Before actually parsing the regular expression or the replacement
@@ -1644,53 +1219,56 @@
% \meta{inline~3} \Arg{token list} The \meta{token list} is converted to
% a string, then read from left to right, interpreting backslashes as
% escaping the next character. Unescaped characters are fed to the
-% function \meta{fn1}, and escaped characters are fed to the function
-% \meta{fn2} within an \texttt{x}-expansion context (typically those
+% function \meta{inline~1}, and escaped characters are fed to the function
+% \meta{inline~2} within an \texttt{x}-expansion context (typically those
% functions perform some tests on their argument to decide how to output
% them). The escape sequences |\a|, |\e|, |\f|, |\n|, |\r|, |\t| and
% |\x| are recognized, and those are replaced by the corresponding
-% character, then fed to \meta{fn3}. The result is then left in the
-% input stream.
+% character, then fed to \meta{inline~3}. The result is then left in the
+% input stream. Spaces are ignored unless escaped.
%
-% The idea is to feed unescaped characters to one function,
-% escaped characters to another, and feed |\a|, |\e|, |\f|,
-% |\n|, |\r|, |\t| and |\x| converted to the appropriate
-% character to a third function. Spaces are ignored unless
-% escaped.
+% The conversion is mostly done within an \texttt{x}-expanding
+% assignment, except for the |\x| escape sequence, which is not amenable
+% to that in general. For this, we use the general framework of
+% \cs{tl_set_build:Nw}.
%
% \begin{macro}[int]{\regex_escape_use:nnnn}
-% \begin{macro}[aux]{\regex_escape_loop:N}
-% \begin{macro}[aux]+\regex_escape_\:w+
-% Go through |#4| once, applying |#1|, |#2|, or |#3| to each character
-% (after de-escaping it), then leave the result in the input
-% stream. Most of the work is done within an \texttt{x}-expanding
-% assignment, but the |\x| escape sequence cannot be done in that
-% way. Therefore, we interrupt the assignment at each |\x| escape
-% sequence, store the partial result in a \tn{toks} register, and at
-% the end unpack all of the \tn{toks} registers to the left of the
-% last chunk of \cs{l_regex_internal_a_tl}.
+% The result is built in \cs{l_regex_internal_a_tl}, which is then
+% left in the input stream. Go through |#4| once, applying |#1|,
+% |#2|, or |#3| as relevant to each character (after de-escaping
+% it). Note that we cannot replace \cs{tl_se:Nx} and
+% \cs{tl_build_one:o} by a single call to \cs{tl_build_one:x}, because
+% the \texttt{x}-expanding assignment is interrupted by |\x|.
% \begin{macrocode}
\cs_new_protected:Npn \regex_escape_use:nnnn #1#2#3#4
{
- \group_begin:
+%<trace> \trace_push:nnn { regex } { 1 } { regex_escape_use:nnnn }
+ \tl_set_build:Nw \l_regex_internal_a_tl
\cs_set_nopar:Npn \regex_escape_unescaped:N ##1 { #1 }
\cs_set_nopar:Npn \regex_escape_escaped:N ##1 { #2 }
\cs_set_nopar:Npn \regex_escape_raw:N ##1 { #3 }
\int_set:Nn \tex_escapechar:D { 92 }
- \str_gset_other:Nn \g_regex_internal_tl {#4}
- \int_zero:N \l_regex_internal_a_int
- \tl_set:Nx \l_regex_internal_a_tl
+ \str_gset_other:Nn \g_regex_internal_tl { #4 }
+ \tl_set:Nx \l_regex_internal_b_tl
{
\exp_after:wN \regex_escape_loop:N \g_regex_internal_tl
{ break } \prg_break_point:n { }
}
- \use:x
- {
- \group_end:
- \regex_toks_range:nn \c_zero \l_regex_internal_a_int
- \exp_not:o \l_regex_internal_a_tl
- }
+ \tl_build_one:o \l_regex_internal_b_tl
+ \tl_build_end:
+%<trace> \trace_pop:nnn { regex } { 1 } { regex_escape_use:nnnn }
+ \l_regex_internal_a_tl
}
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_escape_loop:N}
+% \begin{macro}[aux]+\regex_escape_\:w+
+% \cs{regex_escape_loop:N} reads one character: if it is special
+% (space, backslash, or end-marker), perform the associated action,
+% otherwise it is simply an unescaped character. After a backslash,
+% the same is done, but unknown characters are \enquote{escaped}.
+% \begin{macrocode}
\cs_new:Npn \regex_escape_loop:N #1
{
\cs_if_exist_use:cF { regex_escape_\token_to_str:N #1:w }
@@ -1707,11 +1285,9 @@
% \end{macrocode}
% \end{macro}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}[aux]{\regex_escape_unescaped:N}
-% \begin{macro}[aux]{\regex_escape_escaped:N}
-% \begin{macro}[aux]{\regex_escape_raw:N}
+% \begin{macro}[aux]
+% {\regex_escape_unescaped:N, \regex_escape_escaped:N, \regex_escape_raw:N}
% Those functions are never called before being given a new meaning,
% so their definitions here don't matter.
% \begin{macrocode}
@@ -1720,20 +1296,18 @@
\cs_new_eq:NN \regex_escape_raw:N ?
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}[aux]+\regex_escape_\q_recursion_tail:w+
-% \begin{macro}[aux]+\regex_escape_/\q_recursion_tail:w+
-% \begin{macro}[aux]+\regex_escape_ :w+
% \begin{macro}[aux]
% {
+% \regex_escape_break:w, \regex_escape_/break:w,
% \regex_escape_/a:w, \regex_escape_/e:w, \regex_escape_/f:w,
% \regex_escape_/n:w, \regex_escape_/r:w, \regex_escape_/t:w
% }
-% The loop is ended upon seeing \cs{q_recursion_tail}.
-% Spaces are ignored, and |\a|, |\e|, |\f|, |\n|, |\r|, |\t| take
-% their meaning here.
+% \begin{macro}[aux]+\regex_escape_ :w+
+% The loop is ended upon seeing the end-marker
+% \enquote{\texttt{break}}, with an error if the string ended in a
+% backslash. Spaces are ignored, and |\a|, |\e|, |\f|, |\n|, |\r|,
+% |\t| take their meaning here.
% \begin{macrocode}
\cs_new_eq:NN \regex_escape_break:w \prg_map_break:
\cs_new_nopar:cpn { regex_escape_/break:w }
@@ -1758,126 +1332,149 @@
% \end{macrocode}
% \end{macro}
% \end{macro}
-% \end{macro}
-% \end{macro}
%
% \begin{macro}[aux]{\regex_escape_/x:w}
-% \begin{macro}[aux]{\regex_escape_x_test:N}
-% \begin{macro}[aux]{\regex_escape_x_unbraced_i:N}
-% \begin{macro}[aux]{\regex_escape_x_unbraced_ii:N}
-% \begin{macro}[aux]{\regex_escape_x_braced_loop:N}
-% \begin{macro}[aux]{\regex_escape_x_braced_end:N}
-% \begin{macro}[aux]{\regex_escape_x_end:}
-% When |\x| is encountered, interrupt the assignment,
-% and distinguish the cases of a braced or unbraced syntax.
-% In the braced case, collect arbitrarily many hexadecimal digits,
-% building the number in \cs{l_regex_current_char_int} (using
-% \cs{str_aux_hexadecimal_use:NTF}), and check that
-% the run of digits was stopped by a closing brace.
-% In the unbraced case, collect up to two hexadecimal digits,
-% possibly less, building the character number in \cs{l_regex_current_char_int}.
-% In both cases, once all digits have been collected, use
-% the \TeX{} primitive \tn{lowercase} to produce that character,
-% and use an \cs{if_false:} trick to restart the assignment.
-% \begin{macrocode}
-\cs_new_nopar:cpn { regex_escape_/x:w } \regex_escape_loop:N
+% \begin{macro}[aux]{\regex_escape_x_end:w, \regex_escape_x_large:n}
+% When |\x| is encountered, \cs{regex_escape_x_test:N} is responsible
+% for grabbing some hexadecimal digits, and feeding the result to
+% \cs{regex_escape_x_end:w}. If the number is $<256$, then it is
+% turned into a byte and fed to \cs{regex_escape_raw:N}. Otherwise,
+% interrupt the assignment, and either produce an error, or use a
+% standard \tn{lowercase} trick depending on the precise value.
+% \begin{macrocode}
+\cs_new:cpn { regex_escape_/x:w } \regex_escape_loop:N
{
- \if_false: { \fi: }
- \tex_toks:D \l_regex_internal_a_int
- \exp_after:wN { \l_regex_internal_a_tl }
- \int_incr:N \l_regex_internal_a_int
- \int_zero:N \l_regex_current_char_int
- \regex_escape_x_test:N
+ \exp_after:wN \regex_escape_x_end:w
+ \int_value:w "0 \regex_escape_x_test:N
}
-\cs_new_protected:Npx \regex_escape_x_test:N #1
+\cs_new:Npn \regex_escape_x_end:w #1 ;
{
- \exp_not:N \token_if_eq_charcode:NNTF \c_space_token #1
- { \exp_not:N \regex_escape_x_test:N }
+ \int_compare:nNnTF {#1} < \c_two_hundred_fifty_six
{
- \l_regex_current_char_int = "0
- \exp_not:N \token_if_eq_charcode:NNTF \c_lbrace_str #1
- { \exp_not:N \regex_escape_x_braced_loop:N }
- { \exp_not:N \regex_escape_x_unbraced_i:N #1 }
+ \exp_last_unbraced:Nf \regex_escape_raw:N
+ { \str_output_byte:n {#1} }
}
+ { \regex_escape_x_large:n {#1} }
}
-\cs_new:Npn \regex_escape_x_unbraced_i:N #1
- {
- \str_aux_hexadecimal_use:NTF #1
- { \regex_escape_x_unbraced_ii:N }
- { \exp_stop_f: \regex_escape_x_end: #1 }
- }
-\cs_new:Npn \regex_escape_x_unbraced_ii:N #1
+\group_begin:
+ \char_set_catcode_other:n { 0 }
+ \cs_new:Npn \regex_escape_x_large:n #1
+ {
+ \if_false: { \fi: }
+ \tl_build_one:o \l_regex_internal_b_tl
+ \int_compare:nNnTF {#1} > \c_max_char_int
+ {
+ \msg_kernel_error:nnx { regex } { x-overflow } {#1}
+ \tl_set:Nx \l_regex_internal_b_tl
+ { \if_false: } \fi: \regex_escape_loop:N
+ }
+ {
+ \char_set_lccode:nn { \c_zero } {#1}
+ \tl_to_lowercase:n
+ {
+ \tl_set:Nx \l_regex_internal_b_tl
+ { \if_false: } \fi:
+ \regex_escape_raw:N ^^@
+ \regex_escape_loop:N
+ }
+ }
+ }
+\group_end:
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_escape_x_test:N}
+% Find out whether the first character is a left brace (allowing any
+% number of hexadecimal digits), or not (allowing up to two
+% hexadecimal digits). We need to check for the end-of-string marker.
+% Eventually, call either \cs{regex_escape_x_loop:N} or
+% \cs{regex_escape_x_ii:N}.
+% \begin{macrocode}
+\cs_new:Npn \regex_escape_x_test:N #1
{
- \token_if_eq_charcode:NNTF \c_space_token #1
- { \regex_escape_x_unbraced_ii:N }
+ \str_if_eq:xxTF {#1} { break } { ; }
{
- \str_aux_hexadecimal_use:NTF #1
- { \exp_stop_f: \regex_escape_x_end: }
- { \exp_stop_f: \regex_escape_x_end: #1 }
+ \if_charcode:w \c_space_token #1
+ \exp_after:wN \regex_escape_x_test:N
+ \else:
+ \exp_after:wN \regex_escape_x_test_ii:N
+ \exp_after:wN #1
+ \fi:
}
}
-\cs_new:Npn \regex_escape_x_braced_loop:N #1
+\cs_new:Npn \regex_escape_x_test_ii:N #1
+ {
+ \if_charcode:w \c_lbrace_str #1
+ \exp_after:wN \regex_escape_x_loop:N
+ \else:
+ \str_aux_hexadecimal_use:NTF #1
+ { \exp_after:wN \regex_escape_x_ii:N }
+ { ; \exp_after:wN \regex_escape_loop:N \exp_after:wN #1 }
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_escape_x_ii:N}
+% This looks for the second digit in the unbraced case.
+% \begin{macrocode}
+\cs_new:Npn \regex_escape_x_ii:N #1
{
- \token_if_eq_charcode:NNTF \c_space_token #1
- { \regex_escape_x_braced_loop:N }
+ \str_if_eq:xxTF {#1} { break } { ; }
{
\str_aux_hexadecimal_use:NTF #1
- { \regex_escape_x_braced_loop:N }
- { \exp_stop_f: \regex_escape_x_braced_end:N #1 }
+ { ; \regex_escape_loop:N }
+ { ; \regex_escape_loop:N #1 }
}
}
-\cs_new_protected:Npx \regex_escape_x_braced_end:N #1
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_escape_x_loop:N}
+% Grab hexadecimal digits, skip spaces, and at the end, check that
+% there is a right brace, otherwise raise an error outside the
+% assignment.
+% \begin{macrocode}
+\cs_new:Npn \regex_escape_x_loop:N #1
{
- \exp_not:N \token_if_eq_charcode:NNTF \c_rbrace_str #1
- { \exp_not:N \regex_escape_x_end: }
+ \str_aux_hexadecimal_use:NTF #1
+ { \regex_escape_x_loop:N }
{
- \msg_kernel_error:nnx { regex } { x-missing-rbrace }
- { \int_use:N \l_regex_current_char_int }
- \exp_not:N \regex_escape_x_end: #1
+ \token_if_eq_charcode:NNTF \c_space_token #1
+ { \regex_escape_x_loop:N }
+ {
+ ;
+ \exp_after:wN \token_if_eq_charcode:NNTF \c_rbrace_str #1
+ { \regex_escape_loop:N }
+ {
+ \if_false: { \fi: }
+ \tl_build_one:o \l_regex_internal_b_tl
+ \msg_kernel_error:nn { regex } { x-missing-rbrace } {#1}
+ \tl_set:Nx \l_regex_internal_b_tl
+ { \if_false: } \fi: \regex_escape_loop:N #1
+ }
+ }
}
}
-\group_begin:
- \char_set_catcode_other:N \^^@
- \cs_new_protected_nopar:Npn \regex_escape_x_end:
- {
- \if_num:w \l_regex_current_char_int > \c_max_char_int
- \msg_kernel_error:nnx { regex } { x-overflow }
- { \int_use:N \l_regex_current_char_int }
- \exp_after:wN \use:n
- \else:
- \tex_lccode:D \c_zero \l_regex_current_char_int
- \exp_after:wN \tl_to_lowercase:n
- \fi:
- {
- \tl_set:Nx \l_regex_internal_a_tl
- { \if_false: } \fi:
- \regex_escape_raw:N ^^@
- \regex_escape_loop:N
- }
- }
-\group_end:
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}[EXP,aux]{\regex_char_if_alphanumeric:NTF}
-% \begin{macro}[EXP,aux]{\regex_char_if_special:NTF}
-% These two tests are used in the first pass when parsing a
-% regular expression. That pass is responsible for finding
-% escaped and non-escaped characters, and recognizing which
-% ones have special meanings and which should be interpreted
-% as \enquote{raw} characters. Namely,
+% \begin{macro}[EXP, aux]
+% {\regex_char_if_alphanumeric:NTF, \regex_char_if_special:NTF}
+% These two tests are used in the first pass when parsing a regular
+% expression. That pass is responsible for finding escaped and
+% non-escaped characters, and recognizing which ones have special
+% meanings and which should be interpreted as \enquote{raw}
+% characters. Namely,
% \begin{itemize}
-% \item alphanumerics are \enquote{raw} if they are not escaped,
-% and may have a special meaning when escaped;
-% \item non-alphanumeric printable ascii characters are \enquote{raw}
-% if they are escaped, and may have a special meaning when not escaped;
-% \item characters other than printable ascii are always \enquote{raw}.
+% \item alphanumerics are \enquote{raw} if they are not escaped, and
+% may have a special meaning when escaped;
+% \item non-alphanumeric printable ascii characters are
+% \enquote{raw} if they are escaped, and may have a special
+% meaning when not escaped;
+% \item characters other than printable ascii are always
+% \enquote{raw}.
% \end{itemize}
% The code is ugly, and highly based on magic numbers and the ascii
% codes of characters. This is mostly unavoidable for performance
@@ -1933,15 +1530,178 @@
}
% \end{macrocode}
% \end{macro}
+%
+% \subsection{Compiling}
+%
+% A regular expression starts its life as a string of characters. In
+% this section, we convert it to internal instructions, resulting in a
+% \enquote{compiled} regular expression. This compiled expression is
+% then turned into states of an automaton in the building
+% phase. Compiled regular expressions consist of the following:
+% \begin{itemize}
+% \item \cs{regex_class:NnnnN} \meta{boolean} \Arg{tests} \Arg{min}
+% \Arg{more} \meta{lazyness}
+% \item \cs{regex_group:nnnN} \Arg{branches} \Arg{min} \Arg{more}
+% \meta{lazyness}, also \cs{regex_group_no_capture:nnnN} and
+% \cs{regex_group_resetting:nnnN} with the same syntax.
+% \item \cs{regex_branch:n} \Arg{contents}
+% \item \cs{regex_command_K:}
+% \item \cs{regex_assertion:Nn} \meta{boolean} \Arg{assertion test},
+% where the \meta{assertion test} is \cs{regex_b_test:} or
+% |{|\cs{regex_anchor:N} \meta{integer}|}|
+% \end{itemize}
+% Tests can be the following:
+% \begin{itemize}
+% \item \cs{regex_item_caseful_equal:n} \Arg{char code}
+% \item \cs{regex_item_caseless_equal:n} \Arg{char code}
+% \item \cs{regex_item_caseful_range:nn} \Arg{min} \Arg{max}
+% \item \cs{regex_item_caseless_range:nn} \Arg{min} \Arg{max}
+% \item \cs{regex_item_catcode:nT} \Arg{catcode bitmap} \Arg{tests}
+% \item \cs{regex_item_catcode_reverse:nT} \Arg{catcode bitmap} \Arg{tests}
+% \item \cs{regex_item_reverse:n} \Arg{tests}
+% \item \cs{regex_item_exact:nn} \Arg{catcode} \Arg{char code}
+% \item \cs{regex_item_exact_cs:c} \Arg{csname}
+% \item \cs{regex_item_cs:n} \Arg{compiled regex}
+% \end{itemize}
+%
+% \subsubsection{Variables used when compiling}
+%
+% \begin{variable}{\l_regex_group_level_int}
+% We make sure to open the same number of groups as we close.
+% \begin{macrocode}
+\int_new:N \l_regex_group_level_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_regex_mode_int}
+% While compiling, ten modes are recognized, labelled $-63$, $-23$,
+% $-6$, $-2$, $0$, $2$, $3$, $6$, $23$, $63$. See
+% section~\ref{sec:regex-modes}.
+% \begin{macrocode}
+\int_new:N \l_regex_mode_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_regex_catcodes_int, \l_regex_default_catcodes_int}
+% \begin{variable}{\l_regex_catcodes_bool}
+% We wish to allow constructions such as |\c[^BE](..\cL[a-z]..)|,
+% where the outer catcode test applies to the whole group, but is
+% superseeded by the inner catcode test. For this to work, we need to
+% keep track of lists of allowed category codes:
+% \cs{l_regex_catcodes_int} and \cs{l_regex_default_catcodes_int} are
+% bitmaps, sums of $4^c$, for all allowed catcodes $c$. The latter is
+% local to each capturing group, and we reset
+% \cs{l_regex_catcodes_int} to that value after each character or
+% class, changing it only when encountering a |\c| escape. The boolean
+% records whether the list of categories of a catcode test has to be
+% inverted: compare |\c[^BE]| and |\c[BE]|.
+% \begin{macrocode}
+\int_new:N \l_regex_catcodes_int
+\int_new:N \l_regex_default_catcodes_int
+\bool_new:N \l_regex_catcodes_bool
+% \end{macrocode}
+% \end{variable}
+% \end{variable}
+%
+% \begin{variable}
+% {
+% \c_regex_catcode_C_int, \c_regex_catcode_B_int, \c_regex_catcode_E_int,
+% \c_regex_catcode_M_int, \c_regex_catcode_T_int, \c_regex_catcode_P_int,
+% \c_regex_catcode_U_int, \c_regex_catcode_D_int, \c_regex_catcode_S_int,
+% \c_regex_catcode_L_int, \c_regex_catcode_O_int, \c_regex_catcode_A_int
+% }
+% \begin{variable}{\c_regex_all_catcodes_int}
+% Constants: $4^c$ for each category, and the sum of all powers of $4$.
+% \begin{macrocode}
+\int_const:Nn \c_regex_catcode_C_int { "1 }
+\int_const:Nn \c_regex_catcode_B_int { "4 }
+\int_const:Nn \c_regex_catcode_E_int { "10 }
+\int_const:Nn \c_regex_catcode_M_int { "40 }
+\int_const:Nn \c_regex_catcode_T_int { "100 }
+\int_const:Nn \c_regex_catcode_P_int { "1000 }
+\int_const:Nn \c_regex_catcode_U_int { "4000 }
+\int_const:Nn \c_regex_catcode_D_int { "10000 }
+\int_const:Nn \c_regex_catcode_S_int { "100000 }
+\int_const:Nn \c_regex_catcode_L_int { "400000 }
+\int_const:Nn \c_regex_catcode_O_int { "1000000 }
+\int_const:Nn \c_regex_catcode_A_int { "4000000 }
+\int_const:Nn \c_regex_all_catcodes_int { "5515155 }
+% \end{macrocode}
+% \end{variable}
+% \end{variable}
+%
+% \begin{variable}{\l_regex_internal_regex}
+% The compilation step stores its result in this variable.
+% \begin{macrocode}
+\cs_new_eq:NN \l_regex_internal_regex \c_regex_no_match_regex
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_regex_show_prefix_seq}
+% This sequence holds the prefix that makes up the line displayed to
+% the user. The various items must be removed from the right, which is
+% tricky with a token list, hence we use a sequence.
+% \begin{macrocode}
+\seq_new:N \l_regex_show_prefix_seq
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_regex_show_lines_int}
+% A hack. To know whether a given class has a single item in it or
+% not, we count the number of lines when showing the class.
+% \begin{macrocode}
+\int_new:N \l_regex_show_lines_int
+% \end{macrocode}
+% \end{variable}
+%
+% \subsubsection{Generic helpers used when compiling}
+%
+% \begin{macro}[int]{\regex_get_digits:NTFw}
+% \begin{macro}[aux, rEXP]{\regex_get_digits_loop:w}
+% If followed by some raw digits, collect them one by one in the
+% integer variable |#1|, and take the \texttt{true} branch. Otherwise,
+% take the \texttt{false} branch.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_get_digits:NTFw #1#2#3#4#5
+ {
+ \regex_if_raw_digit:NNTF #4 #5
+ { #1 = #5 \regex_get_digits_loop:nw {#2} }
+ { #3 #4 #5 }
+ }
+\cs_new:Npn \regex_get_digits_loop:nw #1#2#3
+ {
+ \regex_if_raw_digit:NNTF #2 #3
+ { #3 \regex_get_digits_loop:nw {#1} }
+ { \scan_stop: #1 #2 #3 }
+ }
+% \end{macrocode}
+% \end{macro}
% \end{macro}
%
-% \subsection{Building}
+% \begin{macro}[aux, EXP]{\regex_if_raw_digit:NNTF}
+% Test used when grabbing digits for the |{m,n}| quantifier.
+% It only accepts non-escaped digits.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \regex_if_raw_digit:NN #1#2 { TF }
+ {
+ \if_meaning:w \regex_compile_raw:N #1
+ \if_int_compare:w \c_one < 1 #2 \exp_stop_f:
+ \prg_return_true:
+ \else:
+ \prg_return_false:
+ \fi:
+ \else:
+ \prg_return_false:
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
%
-% \subsubsection{Build mode}
+% \subsubsection{Mode}
% \label{sec:regex-modes}
%
-% When building the NFA corresponding to a given regex, we can be in
-% ten distinct modes, which we label by some magic numbers:
+% When compiling the \textsc{nfa} corresponding to a given regex string,
+% we can be in ten distinct modes, which we label by some magic numbers:
% \begin{itemize}
% \item[-6] |[\c{...}]| control sequence in a class,
% \item[-2] |\c{...}| control sequence,
@@ -1955,8 +1715,8 @@
% \item[63] |[\c[...]]| class inside mode $6$.
% \end{itemize}
% This list is exhaustive, because |\c| escape sequences cannot be
-% nested, and character classes cannot be nested directly. The choice
-% of numbers is such as to optimize the most useful tests, and make
+% nested, and character classes cannot be nested directly. The choice of
+% numbers is such as to optimize the most useful tests, and make
% transitions from one mode to another as simple as possible.
% \begin{itemize}
% \item Even modes mean that we are not directly in a character class.
@@ -1980,15 +1740,15 @@
% any mode.
% \end{itemize}
%
-% \begin{macro}[int, EXP]{\regex_build_if_in_class:TF}
-% Test whether we are currently in a character class (at the
-% inner-most level of nesting). There, many escape sequences are not
-% recognized, and special characters are normal. Also, for every raw
-% character, we must look ahead for a possible raw dash.
+% \begin{macro}[int, EXP]{\regex_if_in_class:TF}
+% Test whether we are directly in a character class (at the innermost
+% level of nesting). There, many escape sequences are not recognized,
+% and special characters are normal. Also, for every raw character, we
+% must look ahead for a possible raw dash.
% \begin{macrocode}
-\cs_new_nopar:Npn \regex_build_if_in_class:TF
+\cs_new_nopar:Npn \regex_if_in_class:TF
{
- \if_int_odd:w \l_regex_build_mode_int
+ \if_int_odd:w \l_regex_mode_int
\exp_after:wN \use_i:nn
\else:
\exp_after:wN \use_ii:nn
@@ -1997,16 +1757,35 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}{\regex_build_if_assertions_forbidden:TF}
+% \begin{macro}[int, EXP]{\regex_if_in_cs:TF}
+% Right braces are special only directly inside control sequences (at
+% the inner-most level of nesting, not counting groups).
+% \begin{macrocode}
+\cs_new_nopar:Npn \regex_if_in_cs:TF
+ {
+ \if_int_odd:w \l_regex_mode_int
+ \exp_after:wN \use_ii:nn
+ \else:
+ \if_int_compare:w \l_regex_mode_int < \c_zero
+ \exp_after:wN \exp_after:wN \exp_after:wN \use_i:nn
+ \else:
+ \exp_after:wN \exp_after:wN \exp_after:wN \use_ii:nn
+ \fi:
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[int, EXP]{\regex_if_in_class_or_catcode:TF}
% Assertions are only allowed in modes $0$, $-2$, and $-6$,
% \emph{i.e.}, even, non-positive modes.
% \begin{macrocode}
-\cs_new_nopar:Npn \regex_build_if_assertions_forbidden:TF
+\cs_new_nopar:Npn \regex_if_in_class_or_catcode:TF
{
- \if_int_odd:w \l_regex_build_mode_int
+ \if_int_odd:w \l_regex_mode_int
\exp_after:wN \use_i:nn
\else:
- \if_int_compare:w \l_regex_build_mode_int > \c_zero
+ \if_int_compare:w \l_regex_mode_int > \c_zero
\exp_after:wN \exp_after:wN \exp_after:wN \use_i:nn
\else:
\exp_after:wN \exp_after:wN \exp_after:wN \use_ii:nn
@@ -2016,295 +1795,457 @@
% \end{macrocode}
% \end{macro}
%
-% \subsubsection{Helpers for building an NFA}
-%
-% \begin{macro}[int]{\regex_build_new_state:}
-% Add a new state to the NFA. At the end of the building phase, we
-% want every \tn{toks} register to start with \cs{s_stop}, hence
-% initialize the new register appropriately. Then update
-% the left, right, and max \meta{states}.
+% \begin{macro}[int, EXP]{\regex_if_within_catcode:TF}
+% This test takes the true branch if we are in a catcode test, either
+% immediately following it (modes $2$ and $6$) or in a class on which
+% it applies (modes $23$ and $63$). This is used to tweak how left
+% brackets behave in modes $2$ and $6$.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_new_state:
+\cs_new_nopar:Npn \regex_if_within_catcode:TF
{
- \tex_toks:D \l_regex_max_state_int { \s_stop }
- \int_set_eq:NN \l_regex_left_state_int \l_regex_right_state_int
- \int_set_eq:NN \l_regex_right_state_int \l_regex_max_state_int
- \int_incr:N \l_regex_max_state_int
+ \if_int_compare:w \l_regex_mode_int > \c_zero
+ \exp_after:wN \use_i:nn
+ \else:
+ \exp_after:wN \use_ii:nn
+ \fi:
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_transition:NN}
-% \begin{macro}[aux]{\regex_build_transitions:NNNN}
-% These functions create a new state, and put one or two transitions
-% starting from the old current state.
+% \begin{macro}[aux]{\regex_chk_c_allowed:T}
+% The |\c| escape sequence is only allowed in modes $0$ and $3$,
+% \emph{i.e.}, not within any other |\c| escape sequence.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_transition:NN #1#2
+\cs_new_protected:Npn \regex_chk_c_allowed:T
{
- \regex_build_new_state:
- \regex_toks_put_right:Nx \l_regex_left_state_int
- { #1 { \int_eval:n { #2 - \l_regex_left_state_int } } }
+ \if_num:w \l_regex_mode_int = \c_zero
+ \exp_after:wN \use:n
+ \else:
+ \if_num:w \l_regex_mode_int = \c_three
+ \exp_after:wN \exp_after:wN \exp_after:wN \use:n
+ \else:
+ \msg_kernel_error:nn { regex } { c-bad-mode }
+ \exp_after:wN \exp_after:wN \exp_after:wN \use_none:n
+ \fi:
+ \fi:
}
-\cs_new_protected:Npn \regex_build_transitions:NNNN #1#2#3#4
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_mode_quit_c:}
+% This function changes the mode as it is needed just after a catcode
+% test.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_mode_quit_c:
{
- \regex_build_new_state:
- \regex_toks_put_right:Nx \l_regex_left_state_int
- {
- #1 { \int_eval:n { #2 - \l_regex_left_state_int } }
- #3 { \int_eval:n { #4 - \l_regex_left_state_int } }
- }
+ \if_num:w \l_regex_mode_int = \c_two
+ \l_regex_mode_int = \c_zero
+ \else:
+ \if_num:w \l_regex_mode_int = \c_six
+ \l_regex_mode_int = \c_three
+ \fi:
+ \fi:
}
% \end{macrocode}
% \end{macro}
-% \end{macro}
%
-% \subsubsection{From regex to NFA: framework}
+% \subsubsection{Framework}
%
-% \begin{macro}[int]{\regex_build:w, \regex_build_end:}
-% \begin{macro}[int]{\regex_build_exit_class:, \regex_build_exit_groups:}
-% There are two situations where we want to convert a regular
-% expression given as a string to an NFA. The first case is when the
-% regex is given directly by the user, as an \texttt{n} argument to
-% one of the \pkg{l3regex} functions, and the second case is when
-% building the regex given as an argument of the |\c{...}| escape
-% sequence within another regular expression. In both cases, some
-% variables must be reset before starting to build the NFA, using
-% \cs{regex_build:w}. Once building has ended (for instance when
-% encountering the closing brace in |\c{...}|), we make sure to close
-% any dangling class or open group, then add
-% \cs{regex_action_success:} to make the match successful if it
-% reaches the last state. The capturing group number is incremented:
-% then its value tells us how many capturing group there are,
-% including $0$.
+% \begin{macro}[int]{\regex_compile:w, \regex_compile_end:}
+% Used when compiling a user regex or a regex for the |\c{...}| escape
+% sequence within another regex. Start building a token list within a
+% group (with \texttt{x}-expansion at the outset), and set a few
+% variables (group level, catcodes), then start the first branch. At
+% the end, make sure there are no dangling classes nor groups, close
+% the last branch: we are done building \cs{l_regex_internal_regex}.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build:w
- {
- \int_set_eq:NN \l_regex_catcodes_default_int \c_regex_catcodes_all_int
- \int_set_eq:NN \l_regex_catcodes_int \l_regex_catcodes_default_int
- \int_set_eq:NN \l_regex_capturing_group_int \c_zero
- \seq_clear:N \l_regex_capturing_group_seq
- \tl_clear:N \l_regex_tests_tl
- \bool_set_true:N \l_regex_tests_bool
- \int_zero:N \l_regex_max_state_int
- \regex_build_new_state:
- }
-\cs_new_protected_nopar:Npn \regex_build_end:
+\cs_new_protected_nopar:Npn \regex_compile:w
{
- \regex_build_exit_class:
- \regex_build_exit_groups:
- \regex_toks_put_right:Nx \l_regex_right_state_int
- { \regex_action_success: }
- \int_incr:N \l_regex_capturing_group_int
+ \tl_set_build_x:Nw \l_regex_internal_regex
+ \int_zero:N \l_regex_group_level_int
+ \int_set_eq:NN \l_regex_default_catcodes_int \c_regex_all_catcodes_int
+ \int_set_eq:NN \l_regex_catcodes_int \l_regex_default_catcodes_int
+ \cs_set_nopar:Npn \regex_item_equal:n { \regex_item_caseful_equal:n }
+ \cs_set_nopar:Npn \regex_item_range:nn { \regex_item_caseful_range:nn }
+ \tl_build_one:n { \regex_branch:n { \if_false: } \fi: }
}
-\cs_new_protected_nopar:Npn \regex_build_exit_class:
+\cs_new_protected_nopar:Npn \regex_compile_end:
{
- \regex_build_if_in_class:TF
- {
- \msg_kernel_error:nn { regex } { missing-rbrack }
- \use:c { regex_build_]: }
- }
- { }
- }
-\cs_new_protected_nopar:Npn \regex_build_exit_groups:
- {
- \seq_if_empty:NF \l_regex_capturing_group_seq
- {
+ \regex_if_in_class:TF
+ {
+ \msg_kernel_error:nn { regex } { missing-rbrack }
+ \use:c { regex_compile_]: }
+ \prg_do_nothing: \prg_do_nothing:
+ }
+ { }
+ \if_num:w \l_regex_group_level_int > \c_zero
\msg_kernel_error:nnx { regex } { missing-rparen }
- { \seq_length:N \l_regex_capturing_group_seq }
+ { \int_use:N \l_regex_group_level_int }
\prg_replicate:nn
- { \seq_length:N \l_regex_capturing_group_seq }
- { \regex_build_close_aux: \regex_build_group_: }
- }
+ { \l_regex_group_level_int }
+ {
+ \tl_build_one:n
+ {
+ \if_false: { \fi: }
+ \if_false: { \fi: } { 1 } { 0 } \c_true_bool
+ }
+ \tl_build_end:
+ \tl_build_one:o \l_regex_internal_regex
+ }
+ \fi:
+ \tl_build_one:n { \if_false: { \fi: } }
+ \tl_build_end:
}
% \end{macrocode}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}[int]{\regex_build:n}
-% Setup some variables with \cs{regex_build:w}, and additionally set
-% the build mode to $0$: we are neither in a class nor a |\c|
-% construction. Initiate the regular expression by a wildcard: the
-% search is unanchored and should be tried at every index in the token
-% list. We surround the regular expression by parentheses with
-% \cs{regex_build_open_aux:} and \cs{regex_build_close_aux:}, so that
-% any alternative appears within a group (\emph{e.g.}, \verb"ab|cd"
-% will now be within the external group); this also forms a capturing
-% group (labelled $0$), which gives us the whole match as the $0$-th
-% submatch. The regular expression itself is parsed by using the
-% generic framework of \cs{regex_escape_use:nnnn} to recognize special
-% characters, escaped ones, and escape sequences such as |\n| or
-% |\x{02f}|. This results in successive \cs{regex_build_raw:N},
-% \cs{regex_build_escaped:N}, \cs{regex_build_special:N} followed by
-% their arguments. The trailing \cs{prg_do_nothing:} ensure that the
-% look-ahead done by some of the operations is harmless. Finally,
-% \cs{regex_build_end:} adds the finishing code (checking that
-% parentheses are properly nested, for instance).
+% \begin{macro}[int]{\regex_compile:n}
+% The compilation is done between \cs{regex_compile:w} and
+% \cs{regex_compile_end:}, starting in mode~$0$. Then
+% \cs{regex_escape_use:nnnn} distinguishes special characters, escaped
+% alphanumerics, and raw characters, interpreting |\a|, |\x| and other
+% sequences. The $4$ trailing \cs{prg_do_nothing:} are needed because
+% some functions defined later look up to $4$ tokens ahead. Before
+% ending, make sure that any |\c{...}| is properly closed.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build:n #1
+\cs_new_protected:Npn \regex_compile:n #1
{
- \regex_build:w
- \int_set_eq:NN \l_regex_build_mode_int \c_zero
- \regex_build_new_state:
- \regex_toks_put_right:Nx \l_regex_left_state_int
- { \regex_action_start_wildcard: }
- \regex_build_open_aux:
- \regex_escape_use:nnnn
- {
- \regex_char_if_special:NTF ##1
- \regex_build_special:N \regex_build_raw:N ##1
- }
- {
- \regex_char_if_alphanumeric:NTF ##1
- \regex_build_escaped:N \regex_build_raw:N ##1
- }
- { \regex_build_raw:N ##1 }
- { #1 }
- \prg_do_nothing: \prg_do_nothing:
- \prg_do_nothing: \prg_do_nothing:
- \int_compare:nNnT \l_regex_build_mode_int < \c_zero
+ \regex_compile:w
+ \int_set:Nn \tex_escapechar:D { 92 }
+ \int_set_eq:NN \l_regex_mode_int \c_zero
+ \regex_escape_use:nnnn
+ {
+ \regex_char_if_special:NTF ##1
+ \regex_compile_special:N \regex_compile_raw:N ##1
+ }
+ {
+ \regex_char_if_alphanumeric:NTF ##1
+ \regex_compile_escaped:N \regex_compile_raw:N ##1
+ }
+ { \regex_compile_raw:N ##1 }
+ { #1 }
+ \prg_do_nothing: \prg_do_nothing:
+ \prg_do_nothing: \prg_do_nothing:
+ \int_compare:nNnT \l_regex_mode_int < \c_zero
{
\msg_kernel_error:nn { regex } { c-missing-rbrace }
- \exp_after:wN \regex_build_special:N \c_rbrace_str
+ \regex_compile_end:
+ \regex_compile_one:x
+ { \regex_item_cs:n { \exp_not:o \l_regex_internal_regex } }
+ \prg_do_nothing: \prg_do_nothing:
+ \prg_do_nothing: \prg_do_nothing:
}
- \seq_put_right:Nn \l_regex_capturing_group_seq {0}
- \regex_build_close_aux: \regex_build_group_:
- \regex_build_end:
+ \regex_compile_end:
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_special:N}
-% \begin{macro}[aux]{\regex_build_escaped:N}
+% \begin{macro}[int]{\regex_compile_escaped:N, \regex_compile_special:N}
% If the special character or escaped alphanumeric has a particular
% meaning in regexes, the corresponding function is used. Otherwise,
% it is interpreted as a raw character. We distinguish special
% characters from escaped alphanumeric characters because they behave
% differently when appearing as an end-point of a range.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_special:N #1
+\cs_new_protected:Npn \regex_compile_special:N #1
{
- \cs_if_exist_use:cF { regex_build_#1: }
- { \regex_build_raw:N #1 }
+ \cs_if_exist_use:cF { regex_compile_#1: }
+ { \regex_compile_raw:N #1 }
}
-\cs_new_protected:Npn \regex_build_escaped:N #1
+\cs_new_protected:Npn \regex_compile_escaped:N #1
{
- \cs_if_exist_use:cF { regex_build_/#1: }
- { \regex_build_raw:N #1 }
+ \cs_if_exist_use:cF { regex_compile_/#1: }
+ { \regex_compile_raw:N #1 }
}
% \end{macrocode}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}[int]{\regex_build_one:n, \regex_build_one:x}
-% In a class, add the argument to the current class. Outside a class,
-% this argument is the only test, and we look for quantifiers.
+% \begin{macro}[int]{\regex_compile_one:x}
+% This is used after finding one \enquote{test}, such as |\d|, or a
+% raw character. If that followed a catcode test (\emph{e.g.}, |\cL|),
+% then restore the mode. If we are not in a class, then the test is
+% \enquote{standalone}, and we need to add \cs{regex_class:NnnnN} and
+% search for quantifiers. In any case, insert the test, possibly
+% together with a catcode test if appropriate.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_one:n #1
- { \regex_build_one:x { \exp_not:n {#1} } }
-\cs_new_protected:Npn \regex_build_one:x #1
+\cs_new_protected:Npn \regex_compile_one:x #1
{
- \tl_put_right:Nx \l_regex_tests_tl
+ \regex_mode_quit_c:
+ \regex_if_in_class:TF { }
+ {
+ \tl_build_one:n
+ { \regex_class:NnnnN \c_true_bool { \if_false: } \fi: }
+ }
+ \tl_build_one:x
{
- \if_num:w \l_regex_catcodes_int < \c_regex_catcodes_all_int
+ \if_num:w \l_regex_catcodes_int < \c_regex_all_catcodes_int
\regex_item_catcode:nT { \int_use:N \l_regex_catcodes_int }
+ { \exp_not:N \exp_not:n {#1} }
\else:
- \exp_after:wN \use:n
+ \exp_not:N \exp_not:n {#1}
\fi:
- {#1}
}
- \int_set_eq:NN \l_regex_catcodes_int \l_regex_catcodes_default_int
- \if_num:w \l_regex_build_mode_int = \c_two
- \l_regex_build_mode_int = \c_zero
- \else:
- \if_num:w \l_regex_build_mode_int = \c_six
- \l_regex_build_mode_int = \c_three
- \fi:
- \fi:
- \if_int_odd:w \l_regex_build_mode_int \else:
- \exp_after:wN \regex_build_one_quantifier:
- \fi:
+ \int_set_eq:NN \l_regex_catcodes_int \l_regex_default_catcodes_int
+ \regex_if_in_class:TF { } { \regex_compile_quantifier:w }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[int]
+% {\regex_compile_abort_tokens:n, \regex_compile_abort_tokens:x}
+% This function places the collected tokens back in the input stream,
+% each as a raw character. Spaces are not preserved.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_compile_abort_tokens:n #1
+ {
+ \use:x
+ {
+ \exp_args:No \tl_map_function:nN { \tl_to_str:n {#1} }
+ \regex_compile_raw:N
+ }
+ }
+\cs_generate_variant:Nn \regex_compile_abort_tokens:n { x }
+% \end{macrocode}
+% \end{macro}
+%
+% \subsubsection{Quantifers}
+%
+% \begin{macro}[int]{\regex_compile_quantifier:w}
+% This looks ahead and finds any quantifier (special character equal
+% to either of \texttt{?+*\{}).
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_compile_quantifier:w #1#2
+ {
+ \token_if_eq_meaning:NNTF #1 \regex_compile_special:N
+ {
+ \cs_if_exist_use:cF { regex_compile_quantifier_#2:w }
+ { \regex_compile_quantifier_none: #1 #2 }
+ }
+ { \regex_compile_quantifier_none: #1 #2 }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_compile_quantifier_none:}
+% \begin{macro}[aux]{\regex_compile_quantifier_abort:xNN}
+% Those functions are called whenever there is no quantifier, or a
+% braced construction is invalid (equivalent to no quantifier, and
+% whatever characters were grabbed are left raw).
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_compile_quantifier_none:
+ { \tl_build_one:n { \if_false: { \fi: } { 1 } { 0 } \c_false_bool } }
+\cs_new_protected:Npn \regex_compile_quantifier_abort:xNN #1#2#3
+ {
+ \regex_compile_quantifier_none:
+ \msg_kernel_warning:nnxx { regex } { invalid-quantifier } {#1} {#3}
+ \regex_compile_abort_tokens:x {#1}
+ #2 #3
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_compile_quantifier_lazyness:nnNN}
+% Once the \enquote{main} quantifier (\texttt{?}, \texttt{*},
+% \texttt{+} or a braced construction) is found, we check whether it
+% is lazy (followed by a question mark). We then add to the compiled
+% regex a closing brace (ending \cs{regex_class:NnnnN} and friends),
+% the start-point of the range, its end-point, and a boolean,
+% \texttt{true} for lazy and \texttt{false} for greedy operators.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_compile_quantifier_lazyness:nnNN #1#2#3#4
+ {
+ \str_if_eq:nnTF { #3 #4 } { \regex_compile_special:N ? }
+ { \tl_build_one:n { \if_false: { \fi: } { #1 } { #2 } \c_true_bool } }
+ {
+ \tl_build_one:n { \if_false: { \fi: } { #1 } { #2 } \c_false_bool }
+ #3 #4
+ }
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}{\regex_tests_action_cost:n}
-% The argument is the target state if the test succeeds.
+% \begin{macro}[aux]
+% {
+% \regex_compile_quantifier_?:w,
+% \regex_compile_quantifier_*:w,
+% \regex_compile_quantifier_+:w
+% }
+% For each \enquote{basic} quantifier, |?|, |*|, |+|, feed the correct
+% arguments to \cs{regex_compile_quantifier_lazyness:nnNN}, $-1$ means
+% that there is no upper bound on the number of repetitions.
+% \begin{macrocode}
+\cs_new_protected_nopar:cpn { regex_compile_quantifier_?:w }
+ { \regex_compile_quantifier_lazyness:nnNN { 0 } { 1 } }
+\cs_new_protected_nopar:cpn { regex_compile_quantifier_*:w }
+ { \regex_compile_quantifier_lazyness:nnNN { 0 } { -1 } }
+\cs_new_protected_nopar:cpn { regex_compile_quantifier_+:w }
+ { \regex_compile_quantifier_lazyness:nnNN { 1 } { -1 } }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]+\regex_compile_quantifier_{:w+ ^^A}
+% \begin{macro}[aux]
+% {
+% \regex_compile_quantifier_braced_i:w,
+% \regex_compile_quantifier_braced_ii:w,
+% \regex_compile_quantifier_braced_iii:w,
+% }
+% Three possible syntaxes: \texttt{\{\meta{int}\}},
+% \texttt{\{\meta{int},\}}, or \texttt{\{\meta{int},\meta{int}\}}. Any
+% other syntax causes us to abort and put whatever we collected back
+% in the input stream, as \texttt{raw} characters, including the
+% opening brace. Grab a number into \cs{l_regex_internal_a_int}. If
+% the number is followed by a right brace, the range is $[a,a]$. If
+% followed by a comma, grab one more number, and call the \texttt{_ii}
+% or \texttt{_iii} auxiliary. Those auxiliaries check for a closing
+% brace, leading to the range $[a,\infty]$ or $[a,b]$, encoded as
+% $\{a\}\{-1\}$ and $\{a\}\{b-a\}$.
% \begin{macrocode}
-\cs_new:Npn \regex_tests_action_cost:n #1
+\cs_new_protected:cpn { regex_compile_quantifier_ \c_lbrace_str :w }
+ {
+ \regex_get_digits:NTFw \l_regex_internal_a_int
+ { \regex_compile_quantifier_braced_i:w }
+ { \regex_compile_quantifier_abort:xNN { \c_lbrace_str } }
+ }
+\cs_new_protected:Npn \regex_compile_quantifier_braced_i:w #1#2
+ {
+ \prg_case_str:xxn { #1 #2 }
+ {
+ { \regex_compile_special:N \c_rbrace_str }
+ {
+ \exp_args:No \regex_compile_quantifier_lazyness:nnNN
+ { \int_use:N \l_regex_internal_a_int } { 0 }
+ }
+ { \regex_compile_special:N , }
+ {
+ \regex_get_digits:NTFw \l_regex_internal_b_int
+ { \regex_compile_quantifier_braced_iii:w }
+ { \regex_compile_quantifier_braced_ii:w }
+ }
+ }
+ {
+ \regex_compile_quantifier_abort:xNN
+ { \c_lbrace_str \int_use:N \l_regex_internal_a_int }
+ #1 #2
+ }
+ }
+\cs_new_protected:Npn \regex_compile_quantifier_braced_ii:w #1#2
+ {
+ \str_if_eq:xxTF
+ { #1 #2 } { \regex_compile_special:N \c_rbrace_str }
+ {
+ \exp_args:No \regex_compile_quantifier_lazyness:nnNN
+ { \int_use:N \l_regex_internal_a_int } { -1 }
+ }
+ {
+ \regex_compile_quantifier_abort:xNN
+ { \c_lbrace_str \int_use:N \l_regex_internal_a_int , }
+ #1 #2
+ }
+ }
+\cs_new_protected:Npn \regex_compile_quantifier_braced_iii:w #1#2
{
- \exp_not:o \l_regex_tests_tl
- \bool_if:NTF \l_regex_tests_bool
- { \regex_break_point:TF { \regex_action_cost:n {#1} } { } }
- { \regex_break_point:TF { } { \regex_action_cost:n {#1} } }
+ \str_if_eq:xxTF
+ { #1 #2 } { \regex_compile_special:N \c_rbrace_str }
+ {
+ \if_num:w \l_regex_internal_a_int > \l_regex_internal_b_int
+ \msg_kernel_error:nnxx { regex } { backwards-quantifier }
+ { \int_use:N \l_regex_internal_a_int }
+ { \int_use:N \l_regex_internal_b_int }
+ \int_zero:N \l_regex_internal_b_int
+ \else:
+ \int_sub:Nn \l_regex_internal_b_int \l_regex_internal_a_int
+ \fi:
+ \exp_args:Noo \regex_compile_quantifier_lazyness:nnNN
+ { \int_use:N \l_regex_internal_a_int }
+ { \int_use:N \l_regex_internal_b_int }
+ }
+ {
+ \regex_compile_quantifier_abort:xNN
+ {
+ \c_lbrace_str
+ \int_use:N \l_regex_internal_a_int ,
+ \int_use:N \l_regex_internal_b_int
+ }
+ #1 #2
+ }
}
% \end{macrocode}
% \end{macro}
+% \end{macro}
%
% \subsubsection{Raw characters}
%
-% \begin{macro}[int]{\regex_build_raw_error:N}
-% Within character classes, some escaped alphanumeric sequences such
-% as |\b| do not have any meaning. They are replaced by a raw
-% character, after spitting out an error.
+% \begin{macro}[int]{\regex_compile_raw_error:N}
+% Within character classes, and following catcode tests, some escaped
+% alphanumeric sequences such as |\b| do not have any meaning. They
+% are replaced by a raw character, after spitting out an error.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_raw_error:N #1
+\cs_new_protected:Npn \regex_compile_raw_error:N #1
{
- \if_int_odd:w \l_regex_build_mode_int
- \msg_kernel_error:nnx { regex } { class-bad-escape } {#1}
- \else:
- \msg_kernel_error:nnx { regex } { catcode-bad-escape } {#1}
- \fi:
- \regex_build_raw:N #1
+ \msg_kernel_error:nnx { regex } { bad-escape } {#1}
+ \regex_compile_raw:N #1
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[int]{\regex_build_raw:N}
+% \begin{macro}[int]{\regex_compile_raw:N}
% If we are in a character class and the next character is an
% unescaped dash, this denotes a range. Otherwise, the current
% character |#1| matches itself.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_raw:N #1#2#3
+\cs_new_protected:Npn \regex_compile_raw:N #1#2#3
{
- \regex_build_if_in_class:TF
- { \str_if_eq:nnTF {#2#3} { \regex_build_special:N - } }
- { \use_ii:nn }
- { \regex_class_range:Nw #1 }
+ \regex_if_in_class:TF
{
- \regex_build_one:x { \regex_item_equal:n { \int_value:w `#1 ~ } }
+ \str_if_eq:nnTF {#2#3} { \regex_compile_special:N - }
+ { \regex_compile_range:Nw #1 }
+ {
+ \regex_compile_one:x
+ { \regex_item_equal:n { \int_value:w `#1 ~ } }
+ #2 #3
+ }
+ }
+ {
+ \regex_compile_one:x
+ { \regex_item_equal:n { \int_value:w `#1 ~ } }
#2 #3
}
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_class_range:Nw}
+% \begin{macro}[aux]{\regex_compile_range:Nw, \regex_if_end_range:NNTF}
% We have just read a raw character followed by a dash; this should be
-% followed by an end-point for the range. If the following character
-% is an escaped alphanumeric, or if it is an unescaped right bracket,
-% then we have an error, so we put the initial character and the dash
-% back, as raw characters. Otherwise, build the range, checking that
-% it is in the right order, and optimizing for equal end-points.
+% followed by an end-point for the range. Valid end-points are: any
+% raw character; any special character, except a right bracket. In
+% particular, escaped characters are forbidden.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_class_range:Nw #1#2#3
+\prg_new_protected_conditional:Npnn \regex_if_end_range:NN #1#2 { TF }
{
- \token_if_eq_meaning:NNTF #2 \regex_build_escaped:N % [
- { \use_i:nn } { \str_if_eq:nnTF { #2#3 } { \regex_build_special:N ] } }
- {
- \msg_kernel_warning:nnxx { regex } { range-missing-end } % [
- {#1} { \token_if_eq_charcode:NNF #3 ] { \c_backslash_str } #3 }
- \tl_put_right:Nx \l_regex_tests_tl
- {
- \regex_item_equal:n { \int_value:w `#1 ~ }
- \regex_item_equal:n { \int_value:w `- ~ }
- }
- #2#3
- }
+ \if_meaning:w \regex_compile_raw:N #1
+ \prg_return_true:
+ \else:
+ \if_meaning:w \regex_compile_special:N #1
+ \if_charcode:w ] #2
+ \prg_return_false:
+ \else:
+ \prg_return_true:
+ \fi:
+ \else:
+ \prg_return_false:
+ \fi:
+ \fi:
+ }
+\cs_new_protected:Npn \regex_compile_range:Nw #1#2#3
+ {
+ \regex_if_end_range:NNTF #2 #3
{
\if_num:w `#1 > `#3 \exp_stop_f:
\msg_kernel_error:nnxx { regex } { range-backwards } {#1} {#3}
\else:
- \tl_put_right:Nx \l_regex_tests_tl
+ \tl_build_one:x
{
\if_num:w `#1 = `#3 \exp_stop_f:
\regex_item_equal:n
@@ -2315,1796 +2256,2155 @@
}
\fi:
}
+ {
+ \msg_kernel_warning:nnxx { regex } { range-missing-end }
+ {#1} { \c_backslash_str #3 }
+ \tl_build_one:x
+ {
+ \regex_item_equal:n { \int_value:w `#1 ~ }
+ \regex_item_equal:n { \int_value:w `- ~ }
+ }
+ #2#3
+ }
}
% \end{macrocode}
% \end{macro}
%
% \subsubsection{Character properties}
%
-% \begin{macro}[aux]{\regex_build_.:}
+% \begin{macro}[aux]{\regex_compile_.:, \regex_prop_.:}
% In a class, the dot has no special meaning. Outside, insert
-% \cs{c_regex_._tl}, which matches any character or control
-% sequence, and refuses $-2$, which marks the end of the token list.
+% \cs{regex_prop_.:}, which matches any character or control
+% sequence, and refuses $-2$ (end-marker).
% \begin{macrocode}
-\cs_new_protected_nopar:cpx { regex_build_.: }
+\cs_new_protected_nopar:cpx { regex_compile_.: }
{
- \exp_not:N \regex_build_if_in_class:TF
- { \regex_build_raw:N . }
- { \regex_build_one:n \exp_not:c { c_regex_._tl } }
+ \exp_not:N \regex_if_in_class:TF
+ { \regex_compile_raw:N . }
+ { \regex_compile_one:x \exp_not:c { regex_prop_.: } }
+ }
+\cs_new_protected_nopar:cpn { regex_prop_.: }
+ {
+ \if_num:w \l_regex_current_char_int > - \c_two
+ \exp_after:wN \regex_break_true:w
+ \fi:
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_/d:,\regex_build_/D:}
-% \begin{macro}[aux]{\regex_build_/h:,\regex_build_/H:}
-% \begin{macro}[aux]{\regex_build_/s:,\regex_build_/S:}
-% \begin{macro}[aux]{\regex_build_/v:,\regex_build_/V:}
-% \begin{macro}[aux]{\regex_build_/w:,\regex_build_/W:}
-% \begin{macro}[aux]{\regex_build_/N:}
-% The constants \cs{c_regex_d_tl}, \emph{etc.} hold
+% \begin{macro}[aux]
+% {
+% \regex_compile_/d:, \regex_compile_/D:,
+% \regex_compile_/h:, \regex_compile_/H:,
+% \regex_compile_/s:, \regex_compile_/S:,
+% \regex_compile_/v:, \regex_compile_/V:,
+% \regex_compile_/w:, \regex_compile_/W:,
+% \regex_compile_/N:,
+% }
+% The constants \cs{regex_prop_d:}, \emph{etc.} hold
% a list of tests which match the corresponding character
% class, and jump to the \cs{regex_break_point:TF} marker.
% As for a normal character, we check for quantifiers.
% \begin{macrocode}
-\tl_map_inline:nn { dDhHsSvVwWN }
+\cs_set_protected:Npn \regex_tmp:w #1#2
{
- \cs_new_protected_nopar:cpx { regex_build_/#1: }
- { \regex_build_one:n \exp_not:c { c_regex_#1_tl } }
+ \cs_new_protected_nopar:cpx { regex_compile_/#1: }
+ { \regex_compile_one:x \exp_not:c { regex_prop_#1: } }
+ \cs_new_protected_nopar:cpx { regex_compile_/#2: }
+ {
+ \regex_compile_one:x
+ { \regex_item_reverse:n \exp_not:c { regex_prop_#1: } }
+ }
}
+\regex_tmp:w d D
+\regex_tmp:w h H
+\regex_tmp:w s S
+\regex_tmp:w v V
+\regex_tmp:w w W
+\cs_new_protected_nopar:cpn { regex_compile_/N: }
+ { \regex_compile_one:x \regex_prop_N: }
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
%
% \subsubsection{Anchoring and simple assertions}
%
-% \begin{macro}[aux]{\regex_build_simple_assertion:nn}
-% Assertions are only allowed in modes $0$, $-2$ and $-6$. In
-% character classes, or immediately following a category code test,
-% escaped letters cause errors, while the meaning of special
-% characters is ignored: |#1| is used instead. In modes where the
-% assertion is allowed, we insert the test |#2|; if the assertion is
-% successful, move from the \texttt{left} state to the \texttt{right}
-% state.
+% \begin{macro}[aux]{\regex_compile_anchor:NF}
+% \begin{macro}[aux]+\regex_compile_^:+
+% \begin{macro}[aux]{\regex_compile_/A:, \regex_compile_/G:}
+% \begin{macro}[aux]+\regex_compile_$:+
+% \begin{macro}[aux]{\regex_compile_/Z:, \regex_compile_/z:}
+% In modes where assertions are allowed, anchor to the start of the
+% query, the start of the match, or the end of the query, depending on
+% the integer |#1|. In other modes, |#2| treats the character as raw,
+% with an error for escaped letters (|$| is valid in a class, but |\A|
+% is definitely a mistake on the user's part).
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_simple_assertion:nn #1#2
+\cs_new_protected:Npn \regex_compile_anchor:NF #1#2
{
- \regex_build_if_assertions_forbidden:TF {#1}
+ \regex_if_in_class_or_catcode:TF {#2}
{
- \regex_build_new_state:
- \regex_toks_put_right:Nx \l_regex_left_state_int
- {
- \exp_not:n {#2}
- {
- \regex_action_free:n
- {
- \int_eval:n
- { \l_regex_right_state_int - \l_regex_left_state_int }
- }
- }
- }
- %^^A todo: add \regex_assertion_quantifier:
+ \tl_build_one:n
+ { \regex_assertion:Nn \c_true_bool { \regex_anchor:N #1 } }
}
}
-% \end{macrocode}
-% \end{macro}
-%
-% \begin{macro}[aux]+\regex_build_^:+
-% \begin{macro}[aux]{\regex_build_/A:}
-% \begin{macro}[aux]{\regex_build_/G:}
-% \begin{macro}[aux]+\regex_build_$:+
-% \begin{macro}[aux]{\regex_build_/Z:}
-% \begin{macro}[aux]{\regex_build_/z:}
-% Anchoring at the start corresponds to checking that the current
-% character is the first in the token list. Anchoring to the beginning
-% of the match attempt uses \cs{l_regex_start_index_int} instead of
-% \cs{c_zero}. End anchors match the end of the token list, marked by
-% a character code of $-2$.
-% \begin{macrocode}
-\cs_new_protected_nopar:cpn { regex_build_^: }
- {
- \regex_build_simple_assertion:nn { \regex_build_raw:N ^ }
- { \int_compare:nNnT \l_regex_min_index_int = \l_regex_current_index_int }
- }
-\cs_new_protected_nopar:cpn { regex_build_/A: }
- {
- \regex_build_simple_assertion:nn { \regex_build_raw_error:N A }
- { \int_compare:nNnT \l_regex_min_index_int = \l_regex_current_index_int }
- }
-\cs_new_protected_nopar:cpn { regex_build_/G: }
- {
- \regex_build_simple_assertion:nn { \regex_build_raw_error:N G }
- { \int_compare:nNnT \l_regex_start_index_int = \l_regex_current_index_int }
- }
-\group_begin:
- \char_set_catcode_other:N \$
- \cs_new_protected_nopar:cpn { regex_build_$: } % $
- {
- \regex_build_simple_assertion:nn { \regex_build_raw:N $ } % $
- { \int_compare:nNnT \l_regex_current_char_int < \c_minus_one }
- }
-\group_end:
-\cs_new_protected_nopar:cpn { regex_build_/Z: }
+\cs_set_protected:Npn \regex_tmp:w #1#2
{
- \regex_build_simple_assertion:nn { \regex_build_raw_error:N Z }
- { \int_compare:nNnT \l_regex_current_char_int < \c_minus_one }
+ \cs_new_protected_nopar:cpn { regex_compile_/#1: }
+ { \regex_compile_anchor:NF #2 { \regex_compile_raw_error:N #1 } }
}
-\cs_new_protected_nopar:cpn { regex_build_/z: }
+\regex_tmp:w A \l_regex_min_pos_int
+\regex_tmp:w G \l_regex_start_pos_int
+\regex_tmp:w Z \l_regex_max_pos_int
+\regex_tmp:w z \l_regex_max_pos_int
+\cs_set_protected:Npn \regex_tmp:w #1#2
{
- \regex_build_simple_assertion:nn { \regex_build_raw_error:N z }
- { \int_compare:nNnT \l_regex_current_char_int < \c_minus_one }
+ \cs_new_protected_nopar:cpn { regex_compile_#1: }
+ { \regex_compile_anchor:NF #2 { \regex_compile_raw:N #1 } }
}
+\exp_args:Nx \regex_tmp:w { \iow_char:N \^ } \l_regex_min_pos_int
+\exp_args:Nx \regex_tmp:w { \iow_char:N \$ } \l_regex_max_pos_int
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}[int]{\regex_build_/b:}
-% \begin{macro}[int]{\regex_build_/B:}
-% \begin{macro}[aux]{\regex_if_word_boundary:TF}
-% Contrarily to |^| and |$|, which could be implemented without
-% really knowing what precedes in the token list, this requires
-% more information, namely, the knowledge of the last character
-% code. Case sensitivity does not change word boundaries.
+% \begin{macro}[aux]{\regex_compile_/b:, \regex_compile_/B:}
+% Contrarily to |^| and |$|, which could be implemented without really
+% knowing what precedes in the token list, this requires more
+% information, namely, the knowledge of the last character code.
% \begin{macrocode}
-\cs_new_protected_nopar:cpn { regex_build_/b: }
- {
- \regex_build_simple_assertion:nn { \regex_build_raw_error:N b }
- { \regex_if_word_boundary:T }
- }
-\cs_new_protected_nopar:cpn { regex_build_/B: }
+\cs_new_protected_nopar:cpn { regex_compile_/b: }
{
- \regex_build_simple_assertion:nn { \regex_build_raw_error:N B }
- { \regex_if_word_boundary:TF { } }
+ \regex_if_in_class_or_catcode:TF
+ { \regex_compile_raw_error:N b }
+ {
+ \tl_build_one:n
+ { \regex_assertion:Nn \c_true_bool { \regex_b_test: } }
+ }
}
-\cs_new_protected_nopar:Npn \regex_if_word_boundary:TF
+\cs_new_protected_nopar:cpn { regex_compile_/B: }
{
- \group_begin:
- \int_set_eq:NN \l_regex_current_char_int \l_regex_last_char_int
- \c_regex_w_tl
- \regex_break_point:TF
- { \group_end: \c_regex_W_tl \regex_item_equal:n { -2 } }
- { \group_end: \c_regex_w_tl }
- \regex_break_point:TF
+ \regex_if_in_class_or_catcode:TF
+ { \regex_compile_raw_error:N B }
+ {
+ \tl_build_one:n
+ { \regex_assertion:Nn \c_false_bool { \regex_b_test: } }
+ }
}
-\cs_new_protected_nopar:Npn \regex_if_word_boundary:T
- { \regex_if_word_boundary:TF \use:n \use_none:n }
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \subsubsection{Entering and exiting character classes}
-%
-% \begin{macro}[aux]{\regex_build_[:}
-% In a class, left brackets mean nothing. Outside a class, this starts
-% a class, whose first characters may have a special meaning.
-% \begin{macrocode}
-\cs_new_protected_nopar:cpn { regex_build_[: }
- {
- \regex_build_if_in_class:TF
- { \regex_build_raw:N [ }
- { \regex_class_first:NNNN }
- }
-% \end{macrocode}
-% \end{macro}
+% \subsubsection{Character classes}
%
-% \begin{macro}[aux]{\regex_build_]:}
+% \begin{macro}[aux]{\regex_compile_]:}
% Outside a class, right brackets have no meaning. In a class, change
% the mode ($m\to (m-15)/13$, truncated) to reflect the fact that we
-% are leaving the class. If we are still in a class after leaving one,
-% then this is the case |[...\cL[...]...]|, and we insert the relevant
-% closing material in \cs{l_regex_tests_tl}. Otherwise look for
+% are leaving the class. Look for quantifiers, unless we are still in
+% a class after leaving one (the case of |[...\cL[...]...]|).
% quantifiers.
% \begin{macrocode}
-\cs_new_protected:cpn { regex_build_]: }
+\cs_new_protected:cpn { regex_compile_]: }
{
- \regex_build_if_in_class:TF
+ \regex_if_in_class:TF
{
- \if_num:w \l_regex_build_mode_int > \c_sixteen
- \tl_set:Nx \l_regex_tests_tl
- {
- \exp_not:o \l_regex_tests_saved_tl
- \if_num:w \l_regex_catcodes_saved_int < \c_regex_catcodes_all_int
- \regex_item_catcode:nT
- { \int_use:N \l_regex_catcodes_saved_int }
- \else:
- \exp_after:wN \use:n
- \fi:
- {
- \exp_not:o \l_regex_tests_tl
- \bool_if:NF \l_regex_tests_bool
- { \regex_break_point:TF { } \regex_break_true:w }
- }
- }
- \bool_set_eq:NN \l_regex_tests_bool \l_regex_tests_saved_bool
+ \if_num:w \l_regex_mode_int > \c_sixteen
+ \tl_build_one:n { \if_false: { \fi: } }
\fi:
- \tex_advance:D \l_regex_build_mode_int - \c_fifteen
- \tex_divide:D \l_regex_build_mode_int \c_thirteen
- \if_int_odd:w \l_regex_build_mode_int \else:
- \exp_after:wN \regex_build_one_quantifier:
+ \tex_advance:D \l_regex_mode_int - \c_fifteen
+ \tex_divide:D \l_regex_mode_int \c_thirteen
+ \if_int_odd:w \l_regex_mode_int \else:
+ \exp_after:wN \regex_compile_quantifier:w
\fi:
}
- { \regex_build_raw:N ] }
+ { \regex_compile_raw:N ] }
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_class_first:NNNN}
-% This starts a class. Change the mode by appending $3$ to it, and
-% reset the variables \cs{l_regex_tests_tl} and
-% \cs{l_regex_bool_tl}. In the special case of mode $63$
-% (|[\c[...]]|), we open a group, to avoid overriding the setting of
-% \cs{l_regex_tests_bool} and \cs{l_regex_tests_tl}; the group ends at
-% the matching right bracket. If the first character is |^|, then the
-% class is inverted. We keep track of this in
-% \cs{l_regex_tests_bool}. If the next character is a right bracket,
-% then it should be changed to a raw one (dirty hack here; the |F|
-% argument of \cs{str_if_eq:nnTF} is the trailing |#3|).
+% \begin{macro}[aux]{\regex_compile_[:}
+% In a class, left brackets might introduce a \textsc{posix} character
+% class, or mean nothing. Immediately following |\c|\meta{category},
+% we must insert the apropriate catcode test, then parse the class; we
+% pre-expand the catcode as an optimization. Otherwise (modes $0$,
+% $-2$ and $-6$) just parse the class. The mode is updated later.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_class_first:NNNN #1#2#3#4
+\cs_new_protected_nopar:cpn { regex_compile_[: }
{
- \l_regex_build_mode_int = \int_value:w \l_regex_build_mode_int 3 ~ %
- \if_num:w \l_regex_build_mode_int > \c_sixteen
- \tl_set_eq:NN \l_regex_tests_saved_tl \l_regex_tests_tl
- \bool_set_eq:NN \l_regex_tests_saved_bool \l_regex_tests_bool
- \int_set_eq:NN \l_regex_catcodes_saved_int \l_regex_catcodes_int
- \int_set_eq:NN \l_regex_catcodes_int \c_regex_catcodes_all_int
- \fi:
- \token_if_eq_meaning:NNTF #1 \regex_build_special:N
+ \regex_if_in_class:TF
+ { \regex_compile_class_posix_test:w }
{
- \token_if_eq_charcode:NNTF #2 ^
+ \regex_if_within_catcode:TF
{
- \bool_set_false:N \l_regex_tests_bool % [
- \str_if_eq:nnTF {#3#4} { \regex_build_special:N ] }
- { \regex_build_raw:N }
- }
- { % [
- \token_if_eq_charcode:NNTF #2 ]
- { \regex_build_raw:N #2 }
- { #1 #2 }
+ \exp_after:wN \regex_compile_class_catcode:w
+ \int_use:N \l_regex_catcodes_int ;
}
+ { \regex_compile_class_normal:w }
}
- { #1 #2 }
- #3 #4
}
% \end{macrocode}
% \end{macro}
%
-% \subsubsection{Catcodes and csnames}
-%
-% \begin{macro}[aux]{\regex_build_/c:}
-% The |\c| escape sequence is only allowed in modes $0$ and $3$,
-% \emph{i.e.}, not within any other |\c| escape sequence.
+% \begin{macro}[aux]{\regex_compile_class_normal:w}
+% In the \enquote{normal} case, we will insert \cs{regex_class:NnnnN}
+% \meta{boolean} in the compiled code. The \meta{boolean} is true for
+% positive classes, and false for negative classes, characterized by a
+% leading |^|. The auxiliary \cs{regex_compile_class:TFNN} also
+% checks for a leading |]| which has a special meaning.
% \begin{macrocode}
-\cs_new_protected:cpn { regex_build_/c: }
+\cs_new_protected_nopar:Npn \regex_compile_class_normal:w
{
- \if_case:w \l_regex_build_mode_int
- \exp_after:wN \regex_build_c_aux:wNN
- \or: \or: \or: \exp_after:wN \regex_build_c_aux:wNN
- \fi:
- \msg_kernel_error:nn { regex } { c-bad-mode }
- \s_stop
+ \regex_compile_class:TFNN
+ { \regex_class:NnnnN \c_true_bool }
+ { \regex_class:NnnnN \c_false_bool }
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_c_aux:wNN}
-% The |\c| escape sequence can be followed by a capital letter
-% representing a character category, by a left bracket which starts a
-% list of categories, or by a brace group holding a regular expression
-% for a control sequence name. Otherwise, raise an error.
+% \begin{macro}[aux]{\regex_compile_class_catcode:w}
+% This function is called for a left bracket in modes $2$ or $6$
+% (catcode test, and catcode test within a class). In mode $2$ the
+% whole construction needs to be put in a class (like single
+% character). Then determine if the class is positive or negative,
+% inserting \cs{regex_item_catcode:nT} or the \texttt{reverse} variant
+% as appropriate, each with the current catcodes bitmap |#1| as an
+% argument, and reset the catcodes.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_c_aux:wNN #1 \s_stop #2#3
+\cs_new_protected:Npn \regex_compile_class_catcode:w #1;
{
- \token_if_eq_meaning:NNTF #2 \regex_build_raw:N
- {
- \cs_if_exist:cTF { c_regex_catcode_#3_int }
- {
- \int_set_eq:Nc \l_regex_catcodes_int { c_regex_catcode_#3_int }
- \l_regex_build_mode_int
- = \if_case:w \l_regex_build_mode_int \c_two \else: \c_six \fi:
- }
- }
- { \cs_if_exist_use:cF { regex_build_c_#3: } }
- {
- \msg_kernel_error:nnx { regex } { c-missing-category } {#3}
- #2 #3
- }
+ \if_int_compare:w \l_regex_mode_int = \c_two
+ \tl_build_one:n
+ { \regex_class:NnnnN \c_true_bool { \if_false: } \fi: }
+ \fi:
+ \int_set_eq:NN \l_regex_catcodes_int \l_regex_default_catcodes_int
+ \regex_compile_class:TFNN
+ { \regex_item_catcode:nT {#1} }
+ { \regex_item_catcode_reverse:nT {#1} }
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}+\regex_build_c_{:+ ^^A }
-% The case of a left brace is easy, based on what we have done so far:
-% in a group, build the regular expression, after changing the mode to
-% forbid nesting |\c|.
+% \begin{macro}[aux]
+% {\regex_compile_class:TFNN, \regex_compile_class_ii:NN}
+% If the first character is |^|, then the class is negative (use
+% |#2|), otherwise it is positive (use |#1|). If the next character
+% is a right bracket, then it should be changed to a raw one.
% \begin{macrocode}
-\cs_new_protected:cpn { regex_build_c_ \c_lbrace_str : }
+\cs_new_protected:Npn \regex_compile_class:TFNN #1#2#3#4
{
- \group_begin:
- \l_regex_build_mode_int
- = - \if_case:w \l_regex_build_mode_int \c_two \else: \c_six \fi:
- \regex_build:w
+ \l_regex_mode_int = \int_value:w \l_regex_mode_int 3 \exp_stop_f:
+ \str_if_eq:nnTF { #3 #4 } { \regex_compile_special:N ^ }
+ {
+ \tl_build_one:n { #2 { \if_false: } \fi: }
+ \regex_compile_class_ii:NN
+ }
+ {
+ \tl_build_one:n { #1 { \if_false: } \fi: }
+ \regex_compile_class_ii:NN #3 #4
+ }
+ }
+\cs_new_protected:Npn \regex_compile_class_ii:NN #1#2
+ {
+ \token_if_eq_charcode:NNTF #2 ]
+ { \regex_compile_raw:N #2 }
+ { #1 #2 }
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_c_[:}
-% \begin{macro}[aux]{\regex_build_c_lbrack_loop:NN}
-% \begin{macro}[aux]{\regex_build_c_lbrack_end:}
-% \begin{macro}[aux]{\regex_build_add:N}
-% When encountering |\c[|, the task is to collect uppercase letters
-% representing character categories.
+% \begin{macro}[aux]
+% {
+% \regex_compile_class_posix_test:w,
+% \regex_compile_class_posix:NNNNw,
+% \regex_compile_class_posix_loop:w,
+% \regex_compile_class_posix_end:w
+% }
+% Here we check for a syntax such as |[:alpha:]|. We also detect |[=|
+% and |[.| which have a meaning in \textsc{posix} regular expressions,
+% but are not implemented in \pkg{l3regex}. In case we see |[:|, grab
+% raw characters until hopefully reaching |:]|. If that's missing, or
+% the \textsc{posix} class is unknown, abort. If all is right, add the
+% test to the current class, with an extra \cs{regex_item_reverse:n}
+% for negative classes.
% \begin{macrocode}
-\cs_new_protected:cpn { regex_build_c_[: } #1#2
+\cs_new_protected:Npn \regex_compile_class_posix_test:w #1#2
{
- \int_zero:N \l_regex_catcodes_int
- \str_if_eq:nnTF { #1 #2 } { \regex_build_special:N ^ }
- {
- \bool_set_false:N \l_regex_catcodes_bool
- \regex_build_c_lbrack_loop:NN
- }
+ \token_if_eq_meaning:NNT \regex_compile_special:N #1
{
- \bool_set_true:N \l_regex_catcodes_bool
- \regex_build_c_lbrack_loop:NN
- #1#2
+ \prg_case_str:nnn { #2 }
+ {
+ : { \regex_compile_class_posix:NNNNw }
+ = { \msg_kernel_warning:nnx { regex } { posix-unsupported } { = } }
+ . { \msg_kernel_warning:nnx { regex } { posix-unsupported } { . } }
+ }
+ { }
}
+ \regex_compile_raw:N [ #1 #2
}
-\cs_new_protected:Npn \regex_build_c_lbrack_loop:NN #1#2
+\cs_new_protected:Npn \regex_compile_class_posix:NNNNw #1#2#3#4#5#6
{
- \token_if_eq_meaning:NNTF #1 \regex_build_raw:N
+ \str_if_eq:nnTF { #5 #6 } { \regex_compile_special:N ^ }
{
- \cs_if_exist:cTF { c_regex_catcode_#2_int }
- {
- \exp_args:Nc \regex_build_c_lbrack_add:N
- { c_regex_catcode_#2_int }
- \regex_build_c_lbrack_loop:NN
- }
+ \bool_set_false:N \l_regex_internal_bool
+ \tl_set:Nx \l_regex_internal_a_tl { \if_false: } \fi:
+ \regex_compile_class_posix_loop:w
}
- { % [
- \token_if_eq_charcode:NNTF #2 ]
- { \regex_build_c_lbrack_end: }
+ {
+ \bool_set_true:N \l_regex_internal_bool
+ \tl_set:Nx \l_regex_internal_a_tl { \if_false: } \fi:
+ \regex_compile_class_posix_loop:w #5 #6
}
- {
- \msg_kernel_error:nnx { regex } { c-missing-rbrack } {#2}
- \regex_build_c_lbrack_end:
- }
- }
-\cs_new_protected_nopar:Npn \regex_build_c_lbrack_end:
- {
- \l_regex_build_mode_int
- = \if_case:w \l_regex_build_mode_int \c_two \else: \c_six \fi:
- \if_meaning:w \c_false_bool \l_regex_catcodes_bool
- \int_set:Nn \l_regex_catcodes_int
- { \c_regex_catcodes_all_int - \l_regex_catcodes_int }
- \fi:
}
-\cs_new_protected:Npn \regex_build_c_lbrack_add:N #1
+\cs_new:Npn \regex_compile_class_posix_loop:w #1#2
{
- \if_int_odd:w \int_eval:w \l_regex_catcodes_int / #1 \int_eval_end:
- \else:
- \tex_advance:D \l_regex_catcodes_int #1
- \fi:
+ \token_if_eq_meaning:NNTF \regex_compile_raw:N #1
+ { #2 \regex_compile_class_posix_loop:w }
+ { \if_false: { \fi: } \regex_compile_class_posix_end:w #1 #2 }
}
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-%
-% ^^A{
-% \begin{macro}+\regex_build_}:+
-% Non-escaped right braces are only special if they appear when
-% building the regular expression for a csname. In that case, make
-% sure that any character class and group is closed, anchor the match
-% at the end, add the code for making the match successful, and expand
-% the contents of the regular expression we just built. Otherwise,
-% replace the brace with an escaped brace.
-% \begin{macrocode}
-\cs_new_protected:cpn { regex_build_ \c_rbrace_str : }
+\cs_new_protected:Npn \regex_compile_class_posix_end:w #1#2#3#4
{
- \int_compare:nNnTF \l_regex_build_mode_int < \c_zero
+ \str_if_eq:nnTF { #1 #2 #3 #4 }
+ { \regex_compile_special:N : \regex_compile_special:N ] }
{
- \regex_build_exit_class:
- \regex_build_exit_groups:
- \regex_build_escaped:N Z
- \regex_build_end:
- \use:x
+ \cs_if_exist:cTF { regex_posix_ \l_regex_internal_a_tl : }
{
- \group_end:
- \regex_build_one:n
- {
- \regex_item_cs:n
- { \regex_set_aux:N \l_regex_internal_regex }
- }
+ \regex_compile_one:x
+ {
+ \bool_if:NF \l_regex_internal_bool \regex_item_reverse:n
+ \exp_not:c { regex_posix_ \l_regex_internal_a_tl : }
+ }
}
- }
- { \exp_after:wN \regex_build_raw:N \c_rbrace_str }
- }
-% \end{macrocode}
-% \end{macro}
-%
-% \subsubsection{Quantifiers}
-%
-% \begin{macro}[int]{\regex_build_quantifier:w}
-% This looks ahead and finds any quantifier (control character
-% equal to either of |?+*{|). ^^A}
-% When all characters for the quantifier are found, the corresponding
-% function is called.
-% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_quantifier:w #1#2
- {
- \token_if_eq_meaning:NNTF #1 \regex_build_special:N
- {
- \cs_if_exist_use:cF { regex_build_quantifier_#2:w }
{
- \regex_build_quantifier_end:nn { } { }
- #1 #2
+ \msg_kernel_warning:nnx { regex } { posix-unknown }
+ { \l_regex_internal_a_tl }
+ \regex_compile_abort_tokens:x
+ {
+ [: \bool_if:NF \l_regex_internal_bool { ^ }
+ \l_regex_internal_a_tl :]
+ }
}
}
{
- \regex_build_quantifier_end:nn { } { }
- #1 #2
+ \msg_kernel_error:nnxx { regex } { posix-missing-close }
+ { [: \l_regex_internal_a_tl } { #2 #4 }
+ \regex_compile_abort_tokens:x { [: \l_regex_internal_a_tl }
+ #1 #2 #3 #4
}
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_quantifier_?:w}
-% \begin{macro}[aux]{\regex_build_quantifier_*:w}
-% \begin{macro}[aux]{\regex_build_quantifier_+:w}
-% For each \enquote{basic} quantifier, |?|, |*|, |+|, feed the correct
-% arguments to \cs{regex_build_quantifier_aux:nnNN}.
-% \begin{macrocode}
-\cs_new_protected_nopar:cpn { regex_build_quantifier_?:w }
- { \regex_build_quantifier_aux:nnNN { } { ? } }
-\cs_new_protected_nopar:cpn { regex_build_quantifier_*:w }
- { \regex_build_quantifier_aux:nnNN { } { * } }
-\cs_new_protected_nopar:cpn { regex_build_quantifier_+:w }
- { \regex_build_quantifier_aux:nnNN { } { + } }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-% \end{macro}
+% \subsubsection{Groups and alternations}
%
-% \begin{macro}[aux]{\regex_build_quantifier_aux:nnNN}
-% Once the \enquote{main} quantifier (\texttt{?}, \texttt{*},
-% \texttt{+} or a braced construction) is found, we check
-% whether it is lazy (followed by a question mark),
-% and calls the appropriate function. Here |#1| holds some extra
-% arguments that the final function needs in the case of braced
-% constructions, and is empty otherwise.
+% \begin{macro}[aux]{\regex_compile_group_begin:N, \regex_compile_group_end:}
+% The contents of a regex group are turned into compiled code in
+% \cs{l_regex_internal_regex}, which ends up with items of the form
+% \cs{regex_branch:n} \Arg{concatenation}. This construction is done
+% using \pkg{l3tl-build} within a \TeX{} group, which automatically
+% makes sure that options (case-sensitivity and default catcode) are
+% reset at the end of the group. The argument |#1| is
+% \cs{regex_group:nnnN} or a variant thereof. A small subtelty to
+% support |\cL(abc)| as a shorthand for |(\cLa\cLb\cLc)|: exit any
+% pending catcode test, save the category code at the start of the
+% group as the default catcode for that group, and make sure that the
+% catcode is restored to the default outside the group.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_quantifier_aux:nnNN #1#2#3#4
+\cs_new_protected:Npn \regex_compile_group_begin:N #1
{
- \str_if_eq:nnTF { #3 #4 } { \regex_build_special:N ? }
- { \regex_build_quantifier_end:nn { #2 #4 } {#1} }
- {
- \regex_build_quantifier_end:nn { #2 } {#1}
- #3 #4
- }
+ \tl_build_one:n { #1 { \if_false: } \fi: }
+ \regex_mode_quit_c:
+ \tl_set_build:Nw \l_regex_internal_regex
+ \int_set_eq:NN \l_regex_default_catcodes_int \l_regex_catcodes_int
+ \int_incr:N \l_regex_group_level_int
+ \tl_build_one:n { \regex_branch:n { \if_false: } \fi: }
+ }
+\cs_new_protected:Npn \regex_compile_group_end:
+ {
+ \if_num:w \l_regex_group_level_int > \c_zero
+ \tl_build_one:n { \if_false: { \fi: } }
+ \tl_build_end:
+ \int_set_eq:NN \l_regex_catcodes_int \l_regex_default_catcodes_int
+ \tl_build_one:o \l_regex_internal_regex
+ \exp_after:wN \regex_compile_quantifier:w
+ \else:
+ \msg_kernel_warning:nn { regex } { extra-rparen }
+ \exp_after:wN \regex_compile_raw:N \exp_after:wN )
+ \fi:
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]+\regex_build_quantifier_{:w+ ^^A}
-% \begin{macro}[aux]{\regex_build_quantifier_lbrace:n}
-% \begin{macro}[aux]{\regex_build_quantifier_lbrace:nw}
-% \begin{macro}[aux]{\regex_build_quantifier_lbrace:nnw}
-% Three possible syntaxes: \texttt{\{\meta{int}\}},
-% \texttt{\{\meta{int},\}}, or \texttt{\{\meta{int},\meta{int}\}}.
+% \begin{macro}[aux]{\regex_compile_(:}
+% In a class, parentheses are not special. Outside, check for a |?|,
+% denoting special groups, and run the code for the corresponding
+% special group.
% \begin{macrocode}
-\cs_new_protected_nopar:cpn { regex_build_quantifier_ \c_lbrace_str :w }
- { \regex_get_digits:nw { \regex_build_quantifier_lbrace:n } }
-\cs_new_protected:Npn \regex_build_quantifier_lbrace:n #1
+\cs_new_protected_nopar:cpn { regex_compile_(: }
{
- \tl_if_empty:nTF {#1}
- {
- \regex_build_quantifier_end:nn { } { }
- \exp_after:wN \regex_build_raw:N \c_lbrace_str
- }
- { \regex_build_quantifier_lbrace:nw {#1} }
+ \regex_if_in_class:TF { \regex_compile_raw:N ( }
+ { \regex_compile_lparen:w }
}
-\cs_new_protected:Npx \regex_build_quantifier_lbrace:nw #1#2#3
+\cs_new_protected:Npn \regex_compile_lparen:w #1#2#3#4
{
- \exp_not:N \prg_case_str:nnn { #2 #3 }
+ \str_if_eq:nnTF { #1 #2 } { \regex_compile_special:N ? }
{
- { \exp_not:N \regex_build_special:N , }
+ \cs_if_exist_use:cF
+ { regex_compile_special_group_\token_to_str:N #4 :w }
{
- \exp_not:N \regex_get_digits:nw
- { \exp_not:N \regex_build_quantifier_lbrace:nnw {#1} }
+ \msg_kernel_warning:nnx { regex } { special-group-unknown }
+ { (? \token_to_str:N #4 }
+ \regex_compile_group_begin:N \regex_group:nnnN
+ \regex_compile_raw:N ? #3 #4
}
- { \exp_not:N \regex_build_special:N \c_rbrace_str }
- { \exp_not:N \regex_build_quantifier_end:nn {n} { {#1} } }
}
{
- \exp_not:N \regex_build_quantifier_end:nn { } { }
- \exp_not:N \regex_build_raw:N \c_lbrace_str #1#2
- }
- }
-\cs_new_protected:Npn \regex_build_quantifier_lbrace:nnw #1#2#3#4
- {
- \str_if_eq:xxTF
- { \exp_not:n {#3#4} }
- { \exp_not:N \regex_build_special:N \c_rbrace_str }
- {
- \tl_if_empty:nTF {#2}
- { \regex_build_quantifier_aux:nnNN { {#1} } { n* } }
- { \regex_build_quantifier_aux:nnNN { {#1} {#2} } { nn } }
- }
- {
- \regex_build_quantifier_end:nn { } { }
- \use:x
- {
- \exp_args:No \tl_map_function:nN
- { \c_lbrace_str #1 , #2 }
- \regex_build_raw:N
- }
- #3 #4
+ \regex_compile_group_begin:N \regex_group:nnnN
+ #1 #2 #3 #4
}
}
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_quantifier_end:nn}
-% When all quantifiers are found, we will call the relevant
-% \cs{regex_build_one/group_\meta{quantifiers}:} function.
+% \begin{macro}[aux]+\regex_compile_|:+
+% In a class, the pipe is not special. Otherwise, end the current
+% branch and open another one.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_quantifier_end:nn #1#2
+\cs_new_protected_nopar:cpn { regex_compile_|: }
{
- \use:c { regex_build_ \l_regex_one_or_group_tl _ #1 : } #2
- \tl_clear:N \l_regex_tests_tl
- \bool_set_true:N \l_regex_tests_bool
+ \regex_if_in_class:TF { \regex_compile_raw:N | }
+ {
+ \tl_build_one:n
+ { \if_false: { \fi: } \regex_branch:n { \if_false: } \fi: }
+ }
}
% \end{macrocode}
% \end{macro}
%
-% \subsubsection{Quantifiers for one character or character class}
-%
-% \begin{macro}[aux]{\regex_build_one_quantifier:}
-% Used for one single character, or a character class.
-% Contrarily to \cs{regex_build_group_quantifier:},
-% we don't need to keep track of submatches, and no thread
-% can be created within one repetition, so things are relatively easy.
+% \begin{macro}[aux]{\regex_compile_):}
+% Within a class, parentheses are not special. Outside, close a group.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_one_quantifier:
+\cs_new_protected_nopar:cpn { regex_compile_): }
{
- \tl_set:Nx \l_regex_one_or_group_tl { one }
- \regex_build_quantifier:w
+ \regex_if_in_class:TF { \regex_compile_raw:N ) }
+ { \regex_compile_group_end: }
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_one_n:, \regex_build_one_n?:}
-% \begin{macro}[aux]{\regex_build_one_:}
-% This function is called in case the syntax is
-% \texttt{\{\meta{int}\}}. Greedy and lazy operators are identical,
-% since the number of repetitions is fixed. Build one state for each
-% repetition, with a transition controlled by the tests that we have
-% collected. The case where we find no quantifier is the special case
-% $n=1$.
+% \begin{macro}[aux]{\regex_compile_special_group_::w}
+% \begin{macro}[aux]+\regex_compile_special_group_|:w+
+% Non-capturing, and resetting groups are easy to take care of during
+% compilation; for those groups, the harder parts will come when
+% building.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_one_n: #1
- {
- \prg_replicate:nn {#1}
- {
- \regex_build_transition:NN
- \regex_tests_action_cost:n \l_regex_right_state_int
- }
- }
-\cs_new_eq:cN { regex_build_one_n?: } \regex_build_one_n:
-\cs_new_protected_nopar:Npn \regex_build_one_:
- { \regex_build_one_n: \c_one }
+\cs_new_protected_nopar:cpn { regex_compile_special_group_::w }
+ { \regex_compile_group_begin:N \regex_group_no_capture:nnnN }
+\cs_new_protected_nopar:cpn { regex_compile_special_group_|:w }
+ { \regex_compile_group_begin:N \regex_group_resetting:nnnN }
% \end{macrocode}
% \end{macro}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_one_n*:, \regex_build_one_n*?:}
-% \begin{macro}[aux]{\regex_build_one_*:, \regex_build_one_*?:}
-% \begin{macro}[aux]{\regex_build_one_+:, \regex_build_one_+?:}
-% Those functions are called in case the syntax is
-% \texttt{\{\meta{int},\}}. The |*| and |+| quantifiers (greedy or
-% lazy) are special cases for $n=0$ or $n=1$. In the case $n=0$,
-% build a costly transition going from the current state to itself,
-% and a free transition moving to a new state. In the case $n\geq 1$,
-% reuse \cs{regex_build_one_n:} to match $n$ repetition, then add free
-% transitions from the last state to the previous one, and from the
-% last state to a new one. The greedy and lazy versions only differ in
-% the order of transitions.
+% \begin{macro}[aux]
+% {\regex_compile_special_group_i:w, \regex_compile_special_group_-:w}
+% The match can be made case-insensitive by setting the option with
+% \texttt{(?i)}; the original behaviour is restored by \texttt{(?-i)}.
+% This is the only supported option.
% \begin{macrocode}
-\cs_new_protected:cpn { regex_build_one_n*: } #1
+\cs_new_protected:Npn \regex_compile_special_group_i:w #1#2
{
- \int_compare:nNnTF {#1} = \c_zero
+ \str_if_eq:nnTF { #1 #2 } { \regex_compile_special:N ) }
{
- \regex_build_transitions:NNNN
- \regex_tests_action_cost:n \l_regex_left_state_int
- \regex_action_free:n \l_regex_right_state_int
+ \cs_set_nopar:Npn \regex_item_equal:n { \regex_item_caseless_equal:n }
+ \cs_set_nopar:Npn \regex_item_range:nn { \regex_item_caseless_range:nn }
}
{
- \regex_build_one_n: {#1}
- \int_set_eq:NN \l_regex_internal_a_int \l_regex_left_state_int
- \regex_build_transitions:NNNN
- \regex_action_free:n \l_regex_internal_a_int
- \regex_action_free:n \l_regex_right_state_int
+ \msg_kernel_warning:nnx { regex } { unknown-option } { (?i #2 }
+ \regex_compile_raw:N (
+ \regex_compile_raw:N ?
+ \regex_compile_raw:N i
+ #1 #2
}
}
-\cs_new_protected:cpn { regex_build_one_n*?: } #1
+\cs_new_protected_nopar:cpn { regex_compile_special_group_-:w } #1#2#3#4
{
- \int_compare:nNnTF {#1} = \c_zero
+ \str_if_eq:nnTF { #1 #2 #3 #4 }
+ { \regex_compile_raw:N i \regex_compile_special:N ) }
{
- \regex_build_transitions:NNNN
- \regex_action_free:n \l_regex_right_state_int
- \regex_tests_action_cost:n \l_regex_left_state_int
+ \cs_set_nopar:Npn \regex_item_equal:n { \regex_item_caseful_equal:n }
+ \cs_set_nopar:Npn \regex_item_range:nn { \regex_item_caseful_range:nn }
}
{
- \regex_build_one_n: {#1}
- \int_set_eq:NN \l_regex_internal_a_int \l_regex_left_state_int
- \regex_build_transitions:NNNN
- \regex_action_free:n \l_regex_right_state_int
- \regex_action_free:n \l_regex_internal_a_int
+ \msg_kernel_warning:nnx { regex } { unknown-option } { (?-#2#4 }
+ \regex_compile_raw:N (
+ \regex_compile_raw:N ?
+ \regex_compile_raw:N -
+ #1 #2 #3 #4
}
}
-\cs_new_protected_nopar:cpx { regex_build_one_*: }
- { \exp_not:c { regex_build_one_n*: } \c_zero }
-\cs_new_protected_nopar:cpx { regex_build_one_*?: }
- { \exp_not:c { regex_build_one_n*?: } \c_zero }
-\cs_new_protected_nopar:cpx { regex_build_one_+: }
- { \exp_not:c { regex_build_one_n*: } \c_one }
-\cs_new_protected_nopar:cpx { regex_build_one_+?: }
- { \exp_not:c { regex_build_one_n*?: } \c_one }
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_one_nn:, \regex_build_one_nn?:}
-% \begin{macro}[aux]{\regex_build_one_?:, \regex_build_one_??:}
-% \begin{macro}[aux]{\regex_build_one_nn_aux:nnnn}
-% Those functions are called when the syntax is
-% \texttt{\{\meta{int},\meta{int}\}}, or for the |?| quantifier, with
-% $m=0$ and $n=1$.
+% \subsubsection{Catcodes and csnames}
+%
+% \begin{macro}[aux]{\regex_compile_/c:, \regex_compile_c_test:NN}
+% The |\c| escape sequence can be followed by a capital letter
+% representing a character category, by a left bracket which starts a
+% list of categories, or by a brace group holding a regular expression
+% for a control sequence name. Otherwise, raise an error.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_one_nn:
- {
- \regex_build_one_nn_aux:nnnn { }
- {
- \regex_build_transitions:NNNN
- \regex_tests_action_cost:n \l_regex_right_state_int
- \regex_action_free:n \l_regex_internal_a_int
- }
- }
-\cs_new_protected_nopar:cpn { regex_build_one_nn?: }
+\cs_new_protected:cpn { regex_compile_/c: }
+ { \regex_chk_c_allowed:T { \regex_compile_c_test:NN } }
+\cs_new_protected:Npn \regex_compile_c_test:NN #1#2
{
- \regex_build_one_nn_aux:nnnn {?}
+ \token_if_eq_meaning:NNTF #1 \regex_compile_raw:N
{
- \regex_build_transitions:NNNN
- \regex_action_free:n \l_regex_internal_a_int
- \regex_tests_action_cost:n \l_regex_right_state_int
+ \cs_if_exist:cTF { c_regex_catcode_#2_int }
+ {
+ \int_set_eq:Nc \l_regex_catcodes_int { c_regex_catcode_#2_int }
+ \l_regex_mode_int
+ = \if_case:w \l_regex_mode_int \c_two \else: \c_six \fi:
+ }
}
+ { \cs_if_exist_use:cF { regex_compile_c_#2:w } }
+ {
+ \msg_kernel_error:nnx { regex } { c-missing-category } {#2}
+ #1 #2
+ }
}
-\cs_new_protected:Npn \regex_build_one_nn_aux:nnnn #1#2#3#4
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]
+% {
+% \regex_compile_c_[:w,
+% \regex_compile_c_lbrack_loop:NN,
+% \regex_compile_c_lbrack_add:N,
+% \regex_compile_c_lbrack_end:,
+% }
+% When encountering |\c[|, the task is to collect uppercase letters
+% representing character categories. First check for |^| which negates
+% the list of category codes.
+% \begin{macrocode}
+\cs_new_protected:cpn { regex_compile_c_[:w } #1#2
{
- \regex_build_one_n: {#3}
- \int_compare:nNnTF {#3} > {#4}
+ \l_regex_mode_int
+ = \if_case:w \l_regex_mode_int \c_two \else: \c_six \fi:
+ \int_zero:N \l_regex_catcodes_int
+ \str_if_eq:nnTF { #1 #2 } { \regex_compile_special:N ^ }
{
- \msg_kernel_error:nnxxx
- { regex } { quantifier-backwards } {#3} {#4} {#1}
+ \bool_set_false:N \l_regex_catcodes_bool
+ \regex_compile_c_lbrack_loop:NN
}
{
- \int_set:Nn \l_regex_internal_a_int
- { \l_regex_right_state_int + #4 - #3 }
- \prg_replicate:nn { #4 - #3 } { #2 }
+ \bool_set_true:N \l_regex_catcodes_bool
+ \regex_compile_c_lbrack_loop:NN
+ #1 #2
}
}
-\cs_new_protected_nopar:cpn { regex_build_one_?: }
- { \regex_build_one_nn: \c_zero \c_one }
-\cs_new_protected_nopar:cpx { regex_build_one_??: }
- { \exp_not:c { regex_build_one_nn?: } \c_zero \c_one }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-%
-% \subsubsection{Groups and alternation}
-%
-% We support the syntax \texttt{\meta{expr1}|\ldots{}%^^A
-% |\meta{expr$\sb{n}$}} for alternations.
-%
-% \begin{macro}[aux]{\regex_build_(:, \regex_build_):}
-% \begin{macro}[aux]{\regex_build_open_aux:}
-% \begin{macro}[aux]+\regex_build_|:+
-% \begin{macro}[aux]{\regex_build_begin_alternation:,
-% \regex_build_end_alternation:}
-% Grouping and alternation go together.
-% \begin{itemize}
-% \item Allocate the next available number for the end vertex
-% of the alternation/group and store it on a stack (so that nested
-% alternations work).
-% \item Put free transitions to separate all cases of the alternation.
-% \item Build each branch separately, and merge them to the common
-% end-node.
-% \item Test for a quantifier, and if needed, transfer the initial
-% vertex to a new vertex.
-% \end{itemize}
-% \begin{macrocode}
-\cs_new_protected:cpn { regex_build_(: } #1#2
+\cs_new_protected:Npn \regex_compile_c_lbrack_loop:NN #1#2
{
- \regex_build_if_in_class:TF
+ \token_if_eq_meaning:NNTF #1 \regex_compile_raw:N
{
- \regex_build_raw:N (
- #1 #2
+ \cs_if_exist:cTF { c_regex_catcode_#2_int }
+ {
+ \exp_args:Nc \regex_compile_c_lbrack_add:N
+ { c_regex_catcode_#2_int }
+ \regex_compile_c_lbrack_loop:NN
+ }
}
{
- \str_if_eq:nnTF { #1 #2 } { \regex_build_special:N ? }
- { \regex_build_special_group:NN }
+ \token_if_eq_charcode:NNTF #2 ]
+ { \regex_compile_c_lbrack_end: }
+ }
{
- \int_incr:N \l_regex_capturing_group_int
- \regex_seq_push_int:NN
- \l_regex_capturing_group_seq \l_regex_capturing_group_int
- \regex_build_open_aux:
+ \msg_kernel_error:nnx { regex } { c-missing-rbrack } {#2}
+ \regex_compile_c_lbrack_end:
#1 #2
}
- }
}
-\cs_new_protected_nopar:Npn \regex_build_open_aux:
+\cs_new_protected:Npn \regex_compile_c_lbrack_add:N #1
{
- \regex_build_new_state:
- \regex_seq_push_int:NN \l_regex_left_state_seq \l_regex_left_state_int
- \regex_seq_push_int:NN \l_regex_right_state_seq \l_regex_right_state_int
- \bool_if:NTF \l_regex_caseless_bool
- { \seq_push:Nn \l_regex_end_group_seq \regex_build_caseless: }
- { \seq_push:Nn \l_regex_end_group_seq \regex_build_caseful: }
- \seq_push:Nn \l_regex_end_alternation_seq { }
- \regex_build_begin_alternation:
+ \if_int_odd:w \int_eval:w \l_regex_catcodes_int / #1 \int_eval_end:
+ \else:
+ \tex_advance:D \l_regex_catcodes_int #1
+ \fi:
}
-\cs_new_protected_nopar:cpn { regex_build_|: }
+\cs_new_protected_nopar:Npn \regex_compile_c_lbrack_end:
{
- \regex_build_if_in_class:TF { \regex_build_raw:N | }
- {
- \regex_build_end_alternation:
- \regex_build_begin_alternation:
- }
+ \if_meaning:w \c_false_bool \l_regex_catcodes_bool
+ \int_set:Nn \l_regex_catcodes_int
+ { \c_regex_all_catcodes_int - \l_regex_catcodes_int }
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}+\regex_compile_c_{:+
+% The case of a left brace is easy, based on what we have done so far:
+% in a group, compile the regular expression, after changing the mode
+% to forbid nesting |\c|. Additionally, disable submatch tracking
+% since groups don't escape the scope of |\c{...}|.
+% \begin{macrocode}
+\cs_new_protected_nopar:cpn { regex_compile_c_ \c_lbrace_str :w }
+ {
+ \regex_compile:w
+ \regex_disable_submatches:
+ \l_regex_mode_int
+ = - \if_case:w \l_regex_mode_int \c_two \else: \c_six \fi:
}
-\cs_new_protected_nopar:cpn { regex_build_): }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}+\regex_compile_}:+
+% Non-escaped right braces are only special if they appear when
+% compiling the regular expression for a csname, but not within a
+% class: |\c{[}{]}| matches the control sequences |\}| and
+% |\{|\ldots{} Admittedly, that would be better done as
+% |\c{[{}]}|. So, end compiling the inner regex (this closes any
+% dangling class or group). Then insert the corresponding test in the
+% outer regex.
+% \begin{macrocode}
+\cs_new_protected:cpn { regex_compile_ \c_rbrace_str : }
{
- \regex_build_if_in_class:TF { \regex_build_raw:N ) }
+ \regex_if_in_cs:TF
{
- \seq_if_empty:NTF \l_regex_capturing_group_seq
- { \msg_kernel_error:nn { regex } { extra-rparen } }
- {
- \regex_build_close_aux:
- \regex_build_group_quantifier:
- }
+ \regex_compile_end:
+ \regex_compile_one:x
+ { \regex_item_cs:n { \exp_not:o \l_regex_internal_regex } }
}
- }
-\cs_new_protected_nopar:Npn \regex_build_close_aux:
- {
- \regex_build_end_alternation:
- \regex_seq_pop_int:NN \l_regex_left_state_seq \l_regex_left_state_int
- \regex_seq_pop_int:NN \l_regex_right_state_seq \l_regex_right_state_int
- \regex_seq_pop_use:N \l_regex_end_group_seq
- \seq_pop:NN \l_regex_end_alternation_seq \l_regex_internal_a_tl
+ { \exp_after:wN \regex_compile_raw:N \c_rbrace_str }
}
% \end{macrocode}
-% Building each branch.
+% \end{macro}
+%
+% \subsubsection{Raw token lists with \cs{u}}
+%
+% \begin{macro}[aux]{\regex_compile_/u:}
+% \begin{macro}[aux, EXP]{\regex_compile_u_loop:NN}
+% The |\u| escape is invalid in classes and directly following a
+% catcode test. Otherwise, it must be followed by a left brace. We
+% then collect the characters for the argument of |\u| within an
+% \texttt{x}-expanding assignment. In principle we could just wait to
+% encounter a right brace, but this is unsafe: if the right brace is
+% missing, then we will reach the end-markers of the regex, and
+% continue, leading to obscure fatal errors. Instead, we only allow
+% raw and special characters, and stop when encountering a special
+% right brace, any escaped character, or the end-marker.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_begin_alternation:
+\cs_new_protected:cpn { regex_compile_/u: } #1#2
{
- \regex_build_new_state:
- \regex_seq_get_int:NN \l_regex_left_state_seq \l_regex_left_state_int
- \regex_toks_put_right:Nx \l_regex_left_state_int
+ \regex_if_in_class_or_catcode:TF
+ { \regex_compile_raw_error:N u #1 #2 }
{
- \regex_action_free:n
+ \str_if_eq:xxTF {#1#2} { \regex_compile_special:N \c_lbrace_str }
{
- \int_eval:n
- { \l_regex_right_state_int - \l_regex_left_state_int }
+ \tl_set:Nx \l_regex_internal_a_tl { \if_false: } \fi:
+ \regex_compile_u_loop:NN
+ }
+ {
+ \msg_kernel_error:nn { regex } { u-missing-lbrace }
+ \regex_compile_raw:N u #1 #2
}
}
}
-\cs_new_protected_nopar:Npn \regex_build_end_alternation:
+\cs_new:Npn \regex_compile_u_loop:NN #1#2
{
- \int_set_eq:NN \l_regex_left_state_int \l_regex_right_state_int
- \regex_seq_get_int:NN \l_regex_right_state_seq \l_regex_right_state_int
- \regex_toks_put_right:Nx \l_regex_left_state_int
+ \token_if_eq_meaning:NNTF #1 \regex_compile_raw:N
+ { #2 \regex_compile_u_loop:NN }
{
- \regex_action_free:n
+ \token_if_eq_meaning:NNTF #1 \regex_compile_special:N
+ {
+ \exp_after:wN \token_if_eq_charcode:NNTF \c_rbrace_str #2
+ { \if_false: { \fi: } \regex_compile_u_end: }
+ { #2 \regex_compile_u_loop:NN }
+ }
{
- \int_eval:n
- { \l_regex_right_state_int - \l_regex_left_state_int }
+ \if_false: { \fi: }
+ \msg_kernel_error:nnx { regex } { u-missing-rbrace } {#2}
+ \regex_compile_u_end:
+ #1 #2
}
}
- \regex_seq_get_use:N \l_regex_end_alternation_seq
}
% \end{macrocode}
% \end{macro}
% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}{\regex_build_special_group:NN}
-% Same method as elsewhere: if the combination |(?#1| ^^A )
-% is known, then use that. Otherwise, treat the question mark
-% as if it had been escaped.
+% \begin{macro}[aux]{\regex_compile_u_end:}
+% Once we have extracted the variable's name, we store the contents of
+% that variable in \cs{l_regex_internal_a_tl}. The behaviour of |\u|
+% then depends on whether we are within a |\c{...}| escape (in this
+% case, the variable is turned to a string), or not.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_special_group:NN #1#2
+\cs_new_protected:Npn \regex_compile_u_end:
{
- \cs_if_exist_use:cF { regex_build_special_group_\token_to_str:N #2 : }
- {
- \msg_kernel_error:nnx { regex } { special-group-unknown }
- { (? \token_to_str:N #2 } %)
- \regex_build_special:N ( % )
- \regex_build_raw:N ?
- #1 #2
- }
+ \tl_set:Nv \l_regex_internal_a_tl { \l_regex_internal_a_tl }
+ \if_num:w \l_regex_mode_int = \c_zero
+ \regex_compile_u_not_cs:
+ \else:
+ \regex_compile_u_in_cs:
+ \fi:
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}{\regex_build_special_group_::}
-% Non-capturing groups are like capturing groups, except that
-% we set the group id to \texttt{-1}, which will then inhibit
-% submatching in \cs{regex_build_group_submatches:NN}.
-% The group number is not increased.
+% \begin{macro}[aux]{\regex_compile_u_in_cs:}
+% When |\u| appears within a control sequence, we convert the variable
+% to a string with escaped spaces. Then for each character insert a
+% class matching exactly that character, once.
% \begin{macrocode}
-\cs_new_protected_nopar:cpn { regex_build_special_group_:: }
+\cs_new_protected:Npn \regex_compile_u_in_cs:
{
- \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_minus_one
- \regex_build_open_aux:
+ \exp_args:NNo \str_gset_other:Nn \g_regex_internal_tl
+ { \l_regex_internal_a_tl }
+ \tl_build_one:x
+ {
+ \tl_map_function:NN \g_regex_internal_tl
+ \regex_compile_u_in_cs_aux:n
+ }
+ }
+\cs_new:Npn \regex_compile_u_in_cs_aux:n #1
+ {
+ \regex_class:NnnnN \c_true_bool
+ { \regex_item_caseful_equal:n { \int_value:w `#1 } }
+ { 1 } { 0 } \c_false_bool
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}+\regex_build_special_group_|:+
-% The special group \verb"(?|..|..)" is non-capturing
-% (hence we set the |capturing_group| to $-1$), and resets
-% the group number in each branch of the alternation.
-% We use a variant of \cs{regex_build_open_aux:}, adding
-% some code to be performed at every alternation, and at
-% the end of the group. Namely, we keep track of
-% the maximal value that \cs{l_regex_capturing_group_int}
-% takes, and restore that value when the group end,
-% and in every branch, we reset the capturing group number.
+% \begin{macro}[aux]{\regex_compile_u_not_cs:}
+% In mode $0$, the |\u| escape adds one state to the NFA for each
+% token in \cs{l_regex_internal_a_tl}. If a given \meta{token} is a
+% control sequence, then insert a string comparison test, otherwise,
+% \cs{regex_item_exact:nn} which compares catcode and character code.
% \begin{macrocode}
-\cs_new_protected_nopar:cpn { regex_build_special_group_|: }
+\cs_new_protected:Npn \regex_compile_u_not_cs:
{
- \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_minus_one
- \regex_build_new_state:
- \regex_seq_push_int:NN \l_regex_left_state_seq \l_regex_left_state_int
- \regex_seq_push_int:NN \l_regex_right_state_seq \l_regex_right_state_int
- \seq_push:Nx \l_regex_end_alternation_seq
+ \exp_args:No \tl_analysis_map_inline:nn { \l_regex_internal_a_tl }
{
- \exp_not:N \int_compare:nNnT
- \l_regex_capturing_group_int
- > \l_regex_capturing_group_max_int
+ \tl_build_one:n
{
- \int_set_eq:NN
- \l_regex_capturing_group_max_int
- \l_regex_capturing_group_int
+ \regex_class:NnnnN \c_true_bool
+ {
+ \if_num:w "##2 = \c_zero
+ \regex_item_exact_cs:c { \exp_after:wN \cs_to_str:N ##1 }
+ \else:
+ \regex_item_exact:nn { \int_value:w "##2 } { ##3 }
+ \fi:
+ }
+ { 1 } { 0 } \c_false_bool
}
- \int_set:Nn \l_regex_capturing_group_int
- { \int_use:N \l_regex_capturing_group_int }
}
- \seq_push:Nx \l_regex_end_group_seq
- {
- \bool_if:NTF \l_regex_caseless_bool
- \regex_build_caseless:
- \regex_build_caseful:
- \int_set_eq:NN
- \l_regex_capturing_group_int
- \l_regex_capturing_group_max_int
- }
- \regex_build_begin_alternation:
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}{\regex_build_special_group_i:}
-% \begin{macro}{\regex_build_special_group_-:}
-% \begin{macro}[aux]{\regex_build_options:NNN}
-% \begin{macro}[aux]{\regex_build_option_+i:}
-% \begin{macro}[aux]{\regex_build_option_-i:}
-% The match can be made case-insensitive by setting the option
-% with \texttt{(?i)}.
+% \subsubsection{Other}
+%
+% \begin{macro}[aux]{\regex_compile_/K:}
+% The |\K| control sequence is currently the only \enquote{command},
+% which performs some action, rather than matching something. It is
+% allowed in the same contexts as |\b|. At the compilation stage, we
+% leave it as a single control sequence, defined later.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_special_group_i:
+\cs_new_protected_nopar:cpn { regex_compile_/K: }
{
- \regex_build_options:NNN +
- \regex_build_raw:N i
+ \int_compare:nNnTF \l_regex_mode_int = \c_zero
+ { \tl_build_one:n { \regex_command_K: } }
+ { \regex_compile_raw_error:N K }
}
-\cs_new_protected_nopar:cpn { regex_build_special_group_-: }
- {
- \regex_build_options:NNN -
- }
-\cs_new_protected:Npn \regex_build_options:NNN #1#2#3
+% \end{macrocode}
+% \end{macro}
+%
+% \subsubsection{Showing regexes}
+%
+% \begin{macro}[aux]{\regex_show_aux:Nx}
+% Within a \cs{tl_set_build:Nw} \ldots{} \cs{tl_build_end:} group, we
+% redefine all the function that can appear in a compiled regex, then
+% run the regex. The result is then shown.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_show_aux:Nx #1#2
{
- \token_if_eq_meaning:NNTF \regex_build_raw:N #2
- {
- \cs_if_exist_use:cF { regex_build_option_#1#3: }
- { \msg_kernel_error:nnx { regex } { unknown-option } { #3 } }
- \regex_build_options:NNN #1
- }
- {
- \prg_case_str:nnn { #3 }
- { % (
- { ) } { }
- { - } { \regex_build_options:NNN - }
- }
- { \msg_kernel_error:nnx { regex } { invalid-in-option } { #3 } }
- }
+ \tl_set_build:Nw \l_regex_internal_a_tl
+ \cs_set_protected_nopar:Npn \regex_branch:n
+ {
+ \seq_pop_right:NN \l_regex_show_prefix_seq \l_regex_internal_a_tl
+ \regex_show_one:n { +-branch }
+ \seq_put_right:No \l_regex_show_prefix_seq \l_regex_internal_a_tl
+ \use:n
+ }
+ \cs_set_protected_nopar:Npn \regex_group:nnnN
+ { \regex_show_group_aux:nnnnN { } }
+ \cs_set_protected_nopar:Npn \regex_group_no_capture:nnnN
+ { \regex_show_group_aux:nnnnN { ~(no~capture) } }
+ \cs_set_protected_nopar:Npn \regex_group_resetting:nnnN
+ { \regex_show_group_aux:nnnnN { ~(resetting) } }
+ \cs_set_eq:NN \regex_class:NnnnN \regex_show_class:NnnnN
+ \cs_set_protected_nopar:Npn \regex_command_K:
+ { \regex_show_one:n { reset~match~start~(\iow_char:N\\K) } }
+ \cs_set_protected:Npn \regex_assertion:Nn ##1##2
+ { \regex_show_one:n { \bool_if:NF ##1 { negative~ } assertion:~##2 } }
+ \cs_set_nopar:Npn \regex_b_test: { word~boundary }
+ \cs_set_eq:NN \regex_anchor:N \regex_show_anchor_to_str:N
+ \cs_set_protected:Npn \regex_item_caseful_equal:n ##1
+ { \regex_show_one:n { char~code~\int_eval:n{##1} } }
+ \cs_set_protected:Npn \regex_item_caseful_range:nn ##1##2
+ { \regex_show_one:n { range~[\int_eval:n{##1}, \int_eval:n{##2}] } }
+ \cs_set_protected:Npn \regex_item_caseless_equal:n ##1
+ { \regex_show_one:n { char~code~\int_eval:n{##1}~(caseless) } }
+ \cs_set_protected:Npn \regex_item_caseless_range:nn ##1##2
+ {
+ \regex_show_one:n
+ { Range~[\int_eval:n{##1}, \int_eval:n{##2}]~(caseless) }
+ }
+ \cs_set_protected:Npn \regex_item_catcode:nT
+ { \regex_show_item_catcode:NnT \c_true_bool }
+ \cs_set_protected:Npn \regex_item_catcode_reverse:nT
+ { \regex_show_item_catcode:NnT \c_false_bool }
+ \cs_set_protected:Npn \regex_item_reverse:n
+ { \regex_show_scope:nn { Reversed~match } }
+ \cs_set_protected:Npn \regex_item_exact:nn ##1##2
+ { \regex_show_one:n { char~##2,~catcode~##1 } }
+ \cs_set_protected:Npn \regex_item_exact_cs:c ##1
+ { \regex_show_one:n { control~sequence~\iow_char:N\\##1 } }
+ \cs_set_protected:Npn \regex_item_cs:n
+ { \regex_show_scope:nn { control~sequence } }
+ \cs_set:cpn { regex_prop_.: } { \regex_show_one:n { any~token } }
+ \seq_clear:N \l_regex_show_prefix_seq
+ \regex_show_push:n { ~ }
+ #1
+ \tl_build_end:
+ \msg_aux_show:x { > Compiled~regex~#2: \l_regex_internal_a_tl }
}
-\cs_new_protected_nopar:cpn { regex_build_option_+i: }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_show_one:n}
+% Every part of the final message go through this function, which adds
+% one line to the output, with the apropriate prefix.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_show_one:n #1
{
- \regex_build_caseless:
- \cs_set_eq:NN \regex_match_loop_case_hook:
- \regex_match_loop_caseless_hook:
+ \int_incr:N \l_regex_show_lines_int
+ \tl_build_one:x
+ { \iow_newline: \seq_use:N \l_regex_show_prefix_seq #1 }
}
-\cs_new_protected_nopar:cpn { regex_build_option_-i: }
- { \regex_build_caseful: }
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \subsubsection{Quantifiers for groups}
-%
-% \begin{macro}[aux]{\regex_build_group_quantifier:}
-% Used for one group. We need to keep track of submatches,
-% threads can be created within one repetition, so things are hard.
-% The code for the group that was just built starts
-% at \cs{l_regex_left_state_int} and ends at
-% \cs{l_regex_right_state_int}.
+% \begin{macro}[aux]
+% {\regex_show_push:n, \regex_show_pop:, \regex_show_scope:nn}
+% Enter and exit levels of nesting. The \texttt{scope} function prints
+% its first argument as an \enquote{introduction}, then performs its
+% second argument in a deeper level of nesting.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_group_quantifier:
+\cs_new_protected:Npn \regex_show_push:n #1
+ { \seq_put_right:Nx \l_regex_show_prefix_seq { #1 ~ } }
+\cs_new_protected:Npn \regex_show_pop:
+ { \seq_pop_right:NN \l_regex_show_prefix_seq \l_regex_internal_a_tl }
+\cs_new_protected:Npn \regex_show_scope:nn #1#2
{
- \tl_set:Nn \l_regex_one_or_group_tl { group }
- \regex_build_quantifier:w
+ \regex_show_one:n {#1}
+ \regex_show_push:n { ~ }
+ #2
+ \regex_show_pop:
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_group_submatches:NN}
-% Once the quantifier is found by \cs{regex_build_quantifier:w},
-% we insert the code for tracking submatches.
+% \begin{macro}[aux]{\regex_show_group_aux:nnnnN}
+% We display all groups in the same way, simply adding a message,
+% \texttt{(no capture)} or \texttt{(resetting)}, to special groups.
+% The odd \cs{use_ii:nn} avoids printing a spurious \texttt{+-branch}
+% for the first branch.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_group_submatches:NN #1#2
+\cs_new_protected:Npn \regex_show_group_aux:nnnnN #1#2#3#4#5
{
- \seq_pop:NN \l_regex_capturing_group_seq \l_regex_internal_a_tl
- \int_compare:nNnF { \l_regex_internal_a_tl } < \c_zero
- {
- \regex_toks_put_left:Nx #1
- { \regex_action_submatch:n { \l_regex_internal_a_tl < } }
- \regex_toks_put_left:Nx #2
- { \regex_action_submatch:n { \l_regex_internal_a_tl > } }
- }
+ \regex_show_one:n { ,-group~begin #1 }
+ \regex_show_push:n { | }
+ \use_ii:nn #2
+ \regex_show_pop:
+ \regex_show_one:n
+ { `-group~end \regex_msg_repeated:nnN {#3} {#4} #5 }
}
% \end{macrocode}
% \end{macro}
%
-% Most quantifiers require to add an extra state before the group.
-% This is done by shifting the current contents of the \cs{tex_toks:D}
-% \cs{l_regex_internal_a_int} to a new state.
-%
-% \begin{macro}[aux]{\regex_build_group_qs_aux:NN}
-% \begin{macro}[aux]{\regex_build_group_?:}
-% \begin{macro}[aux]{\regex_build_group_??:}
-% Shift the state at which the group begins using
-% \cs{regex_build_group_shift:N}, then add two transitions.
-% The first transition is taken once the group has been
-% traversed: in the case of \texttt{?} and \texttt{??},
-% we should exit by going to \cs{l_regex_right_state_int},
-% while for \texttt{*} and \texttt{*?} we loop by going to
-% \cs{l_regex_internal_a_int}.
-% The second transition corresponds to skipping the group;
-% it has lower priority (\texttt{put_right}) for greedy
-% operators, and higher priority (\texttt{put_left}) for
-% lazy operators.
-% We shift the left state to avoid interactions with
-% \texttt{action_submatch}.
+% \begin{macro}[aux]{\regex_show_class:NnnnN}
+% I'm entirely unhappy about this function: I couldn't find a way to
+% test if a class is a single test. Instead, collect the
+% representation of the tests in the class. If that had more than one
+% line, write \texttt{Match} or \texttt{Don't match} on its own line,
+% with the repeating information if any. Then the various tests on
+% lines of their own, and finally a line. Otherwise, we need to
+% evaluate the representation of the tests again (since the prefix is
+% incorrect). That's clunky, but not too expensive, since it's only
+% one test.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_group_qs_aux:
+\cs_set:Npn \regex_show_class:NnnnN #1#2#3#4#5
{
- \int_set_eq:NN \l_regex_internal_a_int \l_regex_left_state_int
- \regex_build_new_state:
- \tex_toks:D \l_regex_right_state_int = \tex_toks:D \l_regex_internal_a_int
- \regex_toks_put_left:Nx \l_regex_right_state_int
+ \tl_set_build:Nw \l_regex_internal_a_tl
+ \int_zero:N \l_regex_show_lines_int
+ \regex_show_push:n {~}
+ #2
+ \exp_last_unbraced:Nf
+ \prg_case_int:nnn { \l_regex_show_lines_int }
{
- \int_sub:Nn \l_regex_current_state_int
+ {0}
{
- \int_eval:n % ^^A here we lie!
- { \l_regex_right_state_int - \l_regex_internal_a_int }
+ \tl_build_end:
+ \regex_show_one:n { \bool_if:NTF #1 { Fail } { Pass } }
}
- }
- \use:x
- {
- \tex_toks:D \l_regex_internal_a_int
+ {1}
{
- \s_stop
- \regex_action_free:n
+ \tl_build_end:
+ \bool_if:NTF #1
{
- \int_eval:n
- { \l_regex_right_state_int - \l_regex_internal_a_int }
+ #2
+ \tl_build_one:n { \regex_msg_repeated:nnN {#3} {#4} #5 }
+ }
+ {
+ \regex_show_one:n
+ { Don't~match~\regex_msg_repeated:nnN {#3} {#4} #5 }
+ \tl_build_one:o \l_regex_internal_a_tl
}
}
}
- \regex_build_group_submatches:NN
- \l_regex_right_state_int \l_regex_left_state_int
- }
-\cs_new_protected:Npn \regex_build_group_qs_aux:NN #1#2
- {
- \regex_build_group_qs_aux:
- \int_set_eq:NN \l_regex_right_state_int \l_regex_left_state_int
- \regex_build_transition:NN \regex_action_free:n #1
- #2 \l_regex_internal_a_int
{
- \regex_action_free:n
- { \int_eval:n { \l_regex_right_state_int - \l_regex_internal_a_int } }
+ \tl_build_end:
+ \regex_show_one:n
+ {
+ \bool_if:NTF #1 { M } { Don't~m } atch
+ \regex_msg_repeated:nnN {#3} {#4} #5
+ }
+ \tl_build_one:o \l_regex_internal_a_tl
}
}
-\cs_new_protected_nopar:cpn { regex_build_group_?: }
- {
- \regex_build_group_qs_aux:NN
- \l_regex_right_state_int \regex_toks_put_right:Nx
- }
-\cs_new_protected_nopar:cpn { regex_build_group_??: }
- {
- \regex_build_group_qs_aux:NN
- \l_regex_right_state_int \regex_toks_put_left:Nx
- }
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_group_n_aux:n}
-% \begin{macro}[aux]{\regex_build_group_transition_right_max:}
-% The braced quantifiers rely on replicating the states corresponding
-% to the group that has just been built, and joining the right state
-% of each copy to the left state of the next copy. Once this function
-% has been run, \cs{l_regex_internal_a_int} points to the last copy of
-% the initial left-most state, \cs{l_regex_left/right_state_int} have
-% their initial values. Furthermore, \cs{l_regex_max_state_int} is set
-% appropriately to the largest allocated \tn{toks} register plus $1$.
+% \begin{macro}[aux, rEXP]{\regex_show_anchor_to_str:N}
+% The argument is an integer telling us where the anchor is. We
+% convert that to the relevant info.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_group_transition_right_max:
+\cs_new:Npn \regex_show_anchor_to_str:N #1
{
- \regex_toks_put_right:Nx \l_regex_right_state_int
+ anchor~at~
+ \prg_case_str:nnn { #1 }
{
- \regex_action_free:n
- { \int_eval:n { \l_regex_max_state_int - \l_regex_right_state_int } }
+ { \l_regex_min_pos_int } { start~(\iow_char:N\\A) }
+ { \l_regex_start_pos_int } { start~of~match~(\iow_char:N\\G) }
+ { \l_regex_max_pos_int } { end~(\iow_char:N\\Z) }
}
+ { <error:~'#1'~not~recognized> }
}
-\cs_new_protected:Npn \regex_build_group_n_aux:n #1
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_show_item_catcode:NnT}
+% Produce a sequence of categories which the catcode bitmap |#2|
+% contains, and show it, indenting the tests on which this catcode
+% constraint applies.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_show_item_catcode:NnT #1#2
{
- \int_set_eq:NN \l_regex_internal_a_int \l_regex_left_state_int
- \int_set_eq:NN \l_regex_internal_b_int \l_regex_max_state_int
- \int_add:Nn \l_regex_right_state_int
- { ( #1 - 1 ) * ( \l_regex_max_state_int - \l_regex_left_state_int ) }
- \int_set:Nn \l_regex_max_state_int
- {
- \l_regex_left_state_int
- + (#1) * ( \l_regex_max_state_int - \l_regex_left_state_int )
- }
- \int_while_do:nNnn \l_regex_internal_b_int < \l_regex_max_state_int
+ \seq_set_split:Nnn \l_regex_internal_seq { } { CBEMTPUDSLOA }
+ \seq_set_filter:NNn \l_regex_internal_seq \l_regex_internal_seq
+ { \int_if_odd_p:n { #2 / \int_use:c { c_regex_catcode_##1_int } } }
+ \regex_show_scope:nn
{
- \tex_toks:D \l_regex_internal_b_int
- = \tex_toks:D \l_regex_internal_a_int
- \int_incr:N \l_regex_internal_a_int
- \int_incr:N \l_regex_internal_b_int
+ categories~\seq_use:N \l_regex_internal_seq, ~
+ \bool_if:NF #1 { negative~ } class
}
}
% \end{macrocode}
% \end{macro}
+%
+% \subsection{Building}
+%
+% \subsubsection{Variables used while building}
+%
+% \begin{variable}{\l_regex_min_state_int, \l_regex_max_state_int}
+% The last state that was allocated is $\cs{l_regex_max_state_int}-1$,
+% so that \cs{l_regex_max_state_int} always points to a free state.
+% The \texttt{min_state} variable is always $0$, but is included to
+% avoid hard-coding this value.
+% \begin{macrocode}
+\int_new:N \l_regex_min_state_int
+\int_new:N \l_regex_max_state_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_regex_left_state_int, \l_regex_right_state_int}
+% \begin{variable}{\l_regex_left_state_seq, \l_regex_right_state_seq}
+% Alternatives are implemented by branching from a \texttt{left} state
+% into the various choices, then merging those into a \texttt{right}
+% state. We store information about those states in two sequences.
+% Those states are also used to implement group quantifiers. Most
+% often, the left and right pointers only differ by~$1$.
+% \begin{macrocode}
+\int_new:N \l_regex_left_state_int
+\int_new:N \l_regex_right_state_int
+\seq_new:N \l_regex_left_state_seq
+\seq_new:N \l_regex_right_state_seq
+% \end{macrocode}
+% \end{variable}
+% \end{variable}
+%
+% \begin{variable}{\l_regex_capturing_group_int}
+% \cs{l_regex_capturing_group_int} is the \textsc{id} number that will
+% be assigned to a capturing group if one was opened now. This starts
+% at $0$ for the group enclosing the full regular expression, and
+% groups are counted in the order of their left parenthesis, except
+% when encountering \texttt{resetting} groups.
+% \begin{macrocode}
+\int_new:N \l_regex_capturing_group_int
+% \end{macrocode}
+% \end{variable}
+%
+% \subsubsection{Framework}
+%
+% This phase is about going from a compiled regex to an \textsc{nfa}.
+% Each state of the \textsc{nfa} is stored in a \tn{toks}. The
+% operations which can appear in the \tn{toks} are
+% \begin{itemize}
+% \item \cs{regex_action_start_wildcard:} inserted at the start
+% of the regular expression to make it unanchored.
+% \item \cs{regex_action_success:} marks the exit state of the
+% \textsc{nfa}.
+% \item \cs{regex_action_cost:n} \Arg{shift} is a transition from the
+% current \meta{state} to $\meta{state}+\meta{shift}$, which
+% consumes the current character: the target state is saved and will
+% be considered again when matching at the next position.
+% \item \cs{regex_action_free:n} \Arg{shift}, and
+% \cs{regex_action_free_group:n} \Arg{shift} are free transitions,
+% which immediately perform the actions for the state
+% $\meta{state}+\meta{shift}$ of the \textsc{nfa}. They differ in
+% how they detect and avoid infinite loops. For now, we just need to
+% know that the \texttt{group} variant must be used for transitions
+% back to the start of a group.
+% \item \cs{regex_action_submatch:n} \Arg{key} where the \meta{key} is
+% a group number followed by |<| or |>| for the beginning or end of
+% group. This causes the current position in the query to be stored
+% as the \meta{key} submatch boundary.
+% \end{itemize}
+%
+% We strive to preserve the following properties while building.
+% \begin{itemize}
+% \item The current capturing group is
+% $\text{\texttt{capturing_group}}-1$, and if a group is opened now,
+% it will be labelled \texttt{capturing_group}.
+% \item The last allocated state is $\text{\texttt{max_state}}-1$, so
+% \texttt{max_state} is a free state.
+% \item The \texttt{left_state} points to a state to the left of the
+% current group or of the last class.
+% \item The \texttt{right_state} points to a newly created,
+% empty state, with some transitions leading to it.
+% \item The \texttt{left/right} sequences hold a list of the
+% corresponding end-points of nested groups.
+% \end{itemize}
+%
+% \begin{macro}[int]{\regex_build:n, \regex_build:N}
+% The \texttt{n}-type function first compiles its argument. Reset some
+% variables. Allocate two states, and put a wildcard in state $0$
+% (transitions to state $1$ and $0$ state). Then build the regex
+% within a (capturing) group, which will be numbered $0$ (current
+% value of \texttt{capturing_group}). Finally, if the match reaches the
+% last state, it is successful.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_build:n #1
+ {
+ \regex_compile:n {#1}
+ \regex_build:N \l_regex_internal_regex
+ }
+\cs_new_protected:Npn \regex_build:N #1
+ {
+%<trace> \trace_push:nnn { regex } { 1 } { regex_build }
+ \int_set:Nn \tex_escapechar:D { 92 }
+ \int_zero:N \l_regex_capturing_group_int
+ \int_set_eq:NN \l_regex_max_state_int \l_regex_min_state_int
+ \regex_build_new_state:
+ \regex_build_new_state:
+ \regex_toks_put_right:Nn \l_regex_left_state_int
+ { \regex_action_start_wildcard: }
+ \regex_group:nnnN {#1} { 1 } { 0 } \c_false_bool
+ \regex_toks_put_right:Nn \l_regex_right_state_int
+ { \regex_action_success: }
+%<trace> \regex_trace_states:n { 2 }
+%<trace> \trace_pop:nnn { regex } { 1 } { regex_build }
+ }
+% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_group_n:, \regex_build_group_n?:}
-% \begin{macro}[aux]{\regex_build_group_:}
-% These functions are called in case the syntax is
-% \texttt{\{\meta{int}\}}, or in the absence of quantifier (with
-% $n=1$). Greedy and lazy operators are identical, since the number of
-% repetitions is fixed. We only record the submatch information at
-% the last repetition; and of course, in the case $n=0$, we record
-% nothing.
+% \begin{macro}[int]{\regex_build_for_cs:n}
+% When using a regex to match a cs, we don't insert a wildcard, we
+% anchor at the end, and since we ignore submatches, there is no need
+% to surround the expression with a group. However, for branches to
+% work properly at the outer level, we need to put the appropriate
+% \texttt{left} and \texttt{right} states in their sequence.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_group_n: #1
+\cs_new_protected:Npn \regex_build_for_cs:n #1
{
- \regex_build_group_transition_right_max:
- \regex_build_group_n_aux:n {#1}
+%<trace> \trace_push:nnn { regex } { 1 } { regex_build_for_cs }
+ \int_set_eq:NN \l_regex_max_state_int \l_regex_min_state_int
+ \regex_build_new_state:
\regex_build_new_state:
- \int_compare:nNnF {#1} = \c_zero
+ \regex_push_lr_states:
+ #1
+ \regex_pop_lr_states:
+ \regex_toks_put_right:Nn \l_regex_right_state_int
{
- \regex_build_group_submatches:NN
- \l_regex_internal_a_int \l_regex_left_state_int
+ \if_num:w \l_regex_current_pos_int = \l_regex_max_pos_int
+ \exp_after:wN \regex_action_success:
+ \fi:
}
+%<trace> \regex_trace_states:n { 2 }
+%<trace> \trace_pop:nnn { regex } { 1 } { regex_build_for_cs }
}
-\cs_new_eq:cN { regex_build_group_n?: } \regex_build_group_n:
-\cs_new_protected_nopar:Npn \regex_build_group_:
- { \regex_build_group_n: \c_one }
% \end{macrocode}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_group_n*:, \regex_build_group_n*?:}
-% \begin{macro}[aux]{\regex_build_group_*:, \regex_build_group_*?:}
-% \begin{macro}[aux]{\regex_build_group_+:, \regex_build_group_+?:}
-% \begin{macro}[aux]{\regex_build_group_n_star_aux:NNn}
-% These functions are called in case the syntax is
-% \texttt{\{\meta{int},\}}. They are somewhat hybrid between
-% the \texttt{\{\meta{int}\}} and the \texttt{*} quantifiers.
-% Contrarily to the \texttt{*} quantifier, for which we had
-% to be careful not to overwrite the submatch information in
-% case no iteration was made, here, we know that the submatch
-% information is overwritten in any case.
+% \subsubsection{Helpers for building an \textsc{nfa}}
+%
+% \begin{macro}[int]{\regex_push_lr_states:, \regex_pop_lr_states:}
+% When building the regular expression, we keep track of pointers to
+% the left-end and right-end of each group without help from \TeX{}'s
+% grouping.
% \begin{macrocode}
-\cs_new_protected_nopar:cpn { regex_build_group_n*: }
+\cs_new_protected_nopar:Npn \regex_push_lr_states:
{
- \regex_build_group_n_star_aux:NNn
- \regex_toks_put_right:Nx \regex_toks_put_left:Nx
+ \seq_push:No \l_regex_left_state_seq
+ { \int_use:N \l_regex_left_state_int }
+ \seq_push:No \l_regex_right_state_seq
+ { \int_use:N \l_regex_right_state_int }
}
-\cs_new_protected_nopar:cpn { regex_build_group_n*?: }
+\cs_new_protected_nopar:Npn \regex_pop_lr_states:
{
- \regex_build_group_n_star_aux:NNn
- \regex_toks_put_left:Nx \regex_toks_put_right:Nx
+ \seq_pop:NN \l_regex_left_state_seq \l_regex_internal_a_tl
+ \int_set:Nn \l_regex_left_state_int \l_regex_internal_a_tl
+ \seq_pop:NN \l_regex_right_state_seq \l_regex_internal_a_tl
+ \int_set:Nn \l_regex_right_state_int \l_regex_internal_a_tl
}
-\cs_new_protected:Npn \regex_build_group_n_star_aux:NNn #1#2#3
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[int]{\regex_toks_put_left:Nx}
+% \begin{macro}[int]{\regex_toks_put_right:Nx, \regex_toks_put_right:Nn}
+% During the building phase we wish to add \texttt{x}-expanded
+% material to \tn{toks}, either to the left or to the right. The
+% expansion is done \enquote{by hand} for optimization (these
+% operations are used quite a lot). The \texttt{Nn} version of
+% \cs{regex_toks_put_right:Nx} is provided because it is more
+% efficient than \texttt{x}-expanding with \cs{exp_not:n}.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_toks_put_left:Nx #1#2
{
- \int_compare:nNnTF {#3} = \c_zero
- { \regex_build_group_qs_aux:NN \l_regex_internal_a_int #1 }
- {
- \regex_build_group_transition_right_max:
- \regex_build_group_n_aux:n {#3}
- \regex_build_new_state:
- #2 \l_regex_left_state_int
- {
- \regex_action_free:n
- {
- \int_eval:n
- { \l_regex_internal_a_int - \l_regex_left_state_int }
- }
- }
- \regex_build_group_submatches:NN
- \l_regex_internal_a_int \l_regex_left_state_int
- }
+ \cs_set_nopar:Npx \regex_tmp:w { #2 }
+ \tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN
+ { \exp_after:wN \regex_tmp:w \tex_the:D \tex_toks:D #1 }
}
-\cs_new_protected_nopar:cpx { regex_build_group_*: }
- { \exp_not:c { regex_build_group_n*: } \c_zero }
-\cs_new_protected_nopar:cpx { regex_build_group_*?: }
- { \exp_not:c { regex_build_group_n*?: } \c_zero }
-\cs_new_protected_nopar:cpx { regex_build_group_+: }
- { \exp_not:c { regex_build_group_n*: } \c_one }
-\cs_new_protected_nopar:cpx { regex_build_group_+?: }
- { \exp_not:c { regex_build_group_n*?: } \c_one }
+\cs_new_protected:Npn \regex_toks_put_right:Nx #1#2
+ {
+ \cs_set_nopar:Npx \regex_tmp:w {#2}
+ \tex_toks:D #1 \exp_after:wN
+ { \tex_the:D \tex_toks:D \exp_after:wN #1 \regex_tmp:w }
+ }
+\cs_new_protected:Npn \regex_toks_put_right:Nn #1#2
+ { \tex_toks:D #1 \exp_after:wN { \tex_the:D \tex_toks:D #1 #2 } }
% \end{macrocode}
% \end{macro}
% \end{macro}
-% \end{macro}
+%
+% \begin{macro}[int]
+% {
+% \regex_build_transition_left:NNN,
+% \regex_build_transition_right:nNn
+% }
+% Add a transition from |#2| to |#3| using the function |#1|. The
+% \texttt{left} function is used for higher priority transitions, and
+% the \texttt{right} function for lower priority transitions (which
+% should be performed later). The signatures differ to reflect the
+% differing usage later on. Both functions could be optimized.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_build_transition_left:NNN #1#2#3
+ { \regex_toks_put_left:Nx #2 { #1 { \int_eval:n { #3 - #2 } } } }
+\cs_new_protected:Npn \regex_build_transition_right:nNn #1#2#3
+ { \regex_toks_put_right:Nx #2 { #1 { \int_eval:n { #3 - #2 } } } }
+% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_group_nn:, \regex_build_group_nn?:}
-% \begin{macro}[aux]{\regex_build_group_nn_aux:nnnn}
-% \begin{macro}[aux, rEXP]{\regex_build_group_nn_aux_ii:w}
-% These functions are called when the syntax is either
-% \texttt{\{\meta{int},\}} or \texttt{\{\meta{int},\meta{int}\}}.
+% \begin{macro}[int]{\regex_build_new_state:}
+% Add a new empty state to the \textsc{nfa}. Then update the
+% \texttt{left}, \texttt{right}, and \texttt{max} states, so that the
+% \texttt{right} state is the new empty state, and the \texttt{left}
+% state points to the previously \enquote{current} state.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_group_nn:
- {
- \regex_build_group_nn_aux:nnnn { }
- \regex_build_group_transition_right_max:
- }
-\cs_new_protected_nopar:cpn { regex_build_group_nn?: }
+\cs_new_protected_nopar:Npn \regex_build_new_state:
{
- \regex_build_group_nn_aux:nnnn { }
+%<*trace>
+ \trace:nnx { regex } { 2 }
{
- \use:x
- {
- \tex_toks:D \l_regex_right_state_int
- {
- \exp_after:wN \regex_build_group_nn_aux_ii:w
- \tex_the:D \tex_toks:D \l_regex_right_state_int
- }
- }
+ regex~new~state~
+ L=\int_use:N \l_regex_left_state_int ~ -> ~
+ R=\int_use:N \l_regex_right_state_int ~ -> ~
+ M=\int_use:N \l_regex_max_state_int ~ -> ~
+ \int_eval:n { \l_regex_max_state_int + \c_one }
}
+%</trace>
+ \tex_toks:D \l_regex_max_state_int { }
+ \int_set_eq:NN \l_regex_left_state_int \l_regex_right_state_int
+ \int_set_eq:NN \l_regex_right_state_int \l_regex_max_state_int
+ \int_incr:N \l_regex_max_state_int
}
-\cs_new_protected:Npn \regex_build_group_nn_aux:nnnn #1#2#3#4
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[int]{\regex_build_transitions_lazyness:NNNNN}
+% This function creates a new state, and puts two transitions starting
+% from the old current state. The order of the transitions is
+% controlled by |#1|, true for lazy quantifiers, and false for greedy
+% quantifiers.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_build_transitions_lazyness:NNNNN #1#2#3#4#5
{
- \int_compare:nNnTF {#3} > {#4}
- {
- \msg_kernel_error:nnxxx
- { regex } { quantifier-backwards } {#3} {#4} {#1}
- }
+ \regex_build_new_state:
+ \regex_toks_put_right:Nx \l_regex_left_state_int
{
- \int_compare:nNnTF {#3} = \c_zero
- {
- \regex_build_group_qs_aux:
- \int_set_eq:NN \l_regex_right_state_int \l_regex_left_state_int
- \int_set_eq:NN \l_regex_left_state_int \l_regex_internal_a_int
- \regex_build_group_transition_right_max:
- \regex_build_group_n_aux:n { #4 - #3 }
- \int_set_eq:NN \l_regex_internal_c_int \l_regex_right_state_int
- \int_set_eq:NN \l_regex_right_state_int \l_regex_left_state_int
- \prg_replicate:nn { #4 - #3 }
- {
- #2
- \int_add:Nn \l_regex_right_state_int
- { \l_regex_internal_b_int - \l_regex_internal_a_int }
- }
- \int_set_eq:NN \l_regex_right_state_int \l_regex_internal_c_int
- \int_set_eq:NN \l_regex_left_state_int \l_regex_internal_a_int
- \regex_build_new_state:
- }
- {
- \regex_build_group_transition_right_max:
- \regex_build_group_n_aux:n {#3}
- \int_set_eq:NN \l_regex_left_state_int \l_regex_internal_a_int
- \regex_build_group_submatches:NN
- \l_regex_left_state_int \l_regex_right_state_int
- \regex_build_group_n_aux:n { #4 - #3 + \c_one }
- \int_sub:Nn \l_regex_right_state_int
- { \l_regex_internal_a_int - \l_regex_left_state_int }
- \prg_replicate:nn { #4 - #3 }
- {
- #2
- \int_add:Nn \l_regex_right_state_int
- { \l_regex_internal_b_int - \l_regex_internal_a_int }
- }
- \int_set_eq:NN \l_regex_left_state_int \l_regex_internal_a_int
- \regex_build_new_state:
- }
+ \if_meaning:w \c_true_bool #1
+ #2 { \int_eval:n { #3 - \l_regex_left_state_int } }
+ #4 { \int_eval:n { #5 - \l_regex_left_state_int } }
+ \else:
+ #4 { \int_eval:n { #5 - \l_regex_left_state_int } }
+ #2 { \int_eval:n { #3 - \l_regex_left_state_int } }
+ \fi:
}
}
-\cs_new_protected:Npn \regex_build_group_nn_aux_ii:w #1 \regex_action_free:n
+% \end{macrocode}
+% \end{macro}
+%
+% \subsubsection{Building classes}
+%
+% \begin{macro}[int]{\regex_class:NnnnN}
+% \begin{macro}[int, rEXP]{\regex_tests_action_cost:n}
+% The arguments are: \meta{boolean} \Arg{tests} \Arg{min} \Arg{more}
+% \meta{lazyness}. First store the tests with a trailing
+% \cs{regex_action_cost:n}, in the true branch of
+% \cs{regex_break_point:TF} for positive classes, or the false branch
+% for negative classes. The integer \meta{more} is $0$ for fixed
+% repetitions, $-1$ for unbounded repetitions, and
+% $\meta{max}-\meta{min}$ for a range of repetitions.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_class:NnnnN #1#2#3#4#5
{
- #1
- \regex_action_free:n
+ \cs_set_nopar:Npx \regex_tests_action_cost:n ##1
{
- \int_eval:n
- { \l_regex_max_state_int - \l_regex_right_state_int }
+ \exp_not:n { \exp_not:n {#2} }
+ \bool_if:NTF #1
+ { \regex_break_point:TF { \regex_action_cost:n {##1} } { } }
+ { \regex_break_point:TF { } { \regex_action_cost:n {##1} } }
}
- \regex_action_free:n
+ \if_case:w - #4 \exp_stop_f:
+ \regex_class_repeat:n {#3}
+ \or: \regex_class_repeat:nN {#3} #5
+ \else: \regex_class_repeat:nnN {#3} {#4} #5
+ \fi:
}
+\cs_new:Npn \regex_tests_action_cost:n { \regex_action_cost:n }
% \end{macrocode}
% \end{macro}
% \end{macro}
-% \end{macro}
-%
-% \subsubsection{Matching raw token lists with \cs{u}}
-%
-%^^A todo: document interactions between caselessness and |\u{...}|
-%^^A todo: clarify when l_regex_catcodes_int is reset.
%
-% \begin{macro}[aux]{\regex_build_/u:}
-% \begin{macro}[aux]{\regex_build_u_loop:NN}
-% The |\u| escape is invalid in classes nad directly following a
-% catcode test. Otherwise, it must be followed by a left brace. We
-% then collect the characters for the argument of |\u| within an
-% \texttt{x}-expanding assignment. In principle we could just wait to
-% encounter a right brace, but this is unsafe: if the right brace is
-% missing, then we will reach the end-markers of the regex, and
-% continue, leading to obscure fatal errors. Instead, we only allow
-% raw and special characters, and stop when encountering a special
-% right brace.
+% \begin{macro}[aux]{\regex_class_repeat:n}
+% This is used for a fixed number of repetitions. Build one state for
+% each repetition, with a transition controlled by the tests that we
+% have collected. That works just fine for |#1|${}=0$ repetitions:
+% nothing is built.
% \begin{macrocode}
-\cs_new_protected:cpn { regex_build_/u: } #1#2
+\cs_new_protected:Npn \regex_class_repeat:n #1
{
- \regex_build_if_assertions_forbidden:TF
- { \regex_build_raw_error:N u #1#2 }
+ \prg_replicate:nn {#1}
{
- \str_if_eq:xxTF {#1#2} { \regex_build_special:N \c_lbrace_str }
- {
- \tl_set:Nx \l_regex_internal_a_tl { \if_false: } \fi:
- \regex_build_u_loop:NN
- }
- {
- \msg_kernel_error:nn { regex } { u-missing-lbrace }
- #1#2
- }
+ \regex_build_new_state:
+ \regex_build_transition_right:nNn \regex_tests_action_cost:n
+ \l_regex_left_state_int \l_regex_right_state_int
}
}
-\cs_new:Npn \regex_build_u_loop:NN #1#2
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_class_repeat:nN}
+% This implements unbounded repetitions of a single class (\emph{e.g.}
+% the |*| and |+| quantifiers). If the minimum number |#1| of
+% repetitions is $0$, then build a transition from the current state
+% to itself governed by the tests, and a free transition to a new
+% state (hence skipping the tests). Otherwise, call
+% \cs{regex_class_repeat:n} for the code to match |#1| repetitions,
+% and add free transitions from the last state to the previous one,
+% and to a new one. In both cases, the order of transitions is
+% controlled by the lazyness boolean |#2|.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_class_repeat:nN #1#2
+ {
+ \if_num:w #1 = \c_zero
+ \regex_build_transitions_lazyness:NNNNN #2
+ \regex_action_free:n \l_regex_right_state_int
+ \regex_tests_action_cost:n \l_regex_left_state_int
+ \else:
+ \regex_class_repeat:n {#1}
+ \int_set_eq:NN \l_regex_internal_a_int \l_regex_left_state_int
+ \regex_build_transitions_lazyness:NNNNN #2
+ \regex_action_free:n \l_regex_right_state_int
+ \regex_action_free:n \l_regex_internal_a_int
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_class_repeat:nnN}
+% We want to build the code to match from |#1| to $|#1|+|#2|$
+% repetitions. Match |#1| repetitions (can be $0$). Compute the final
+% state of the next construction as \texttt{a}. Build $|#2|>0$ states,
+% each with a transition to the next state governed by the tests, and
+% a transition to the final state \texttt{a}. The computation of
+% \texttt{a} is safe because states are allocated in order, starting
+% from \texttt{max_state}.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_class_repeat:nnN #1#2#3
{
- \token_if_eq_meaning:NNTF #1 \regex_build_raw:N
- { #2 \regex_build_u_loop:NN }
+ \regex_class_repeat:n {#1}
+ \int_set:Nn \l_regex_internal_a_int
+ { \l_regex_max_state_int + #2 - \c_one }
+ \prg_replicate:nn { #2 }
{
- \token_if_eq_meaning:NNTF #1 \regex_build_special:N
- {
- \exp_after:wN \token_if_eq_charcode:NNTF \c_rbrace_str #2
- { \if_false: { \fi: } \regex_build_u_end: }
- { #2 \regex_build_u_loop:NN }
- }
- {
- \if_false: { \fi: }
- \msg_kernel_error:nnx { regex } { u-missing-rbrace } {#2}
- \regex_build_u_end:
- #1#2
- }
+ \regex_build_transitions_lazyness:NNNNN #3
+ \regex_action_free:n \l_regex_internal_a_int
+ \regex_tests_action_cost:n \l_regex_right_state_int
}
}
% \end{macrocode}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_u_end:}
-% Once we have extracted the variable's name, we store the contents of
-% that variable in \cs{l_regex_internal_a_tl}. The behaviour of |\u|
-% then depends on whether we are within a |\c{...}| escape (in this
-% case, the variable is turned to a string), or not. Afterwards, clean
-% up the value of \cs{l_regex_tests_tl}, but not
-% \cs{l_regex_tests_bool}, because this is guaranteed to be
-% \texttt{true} at the start, and is not affected by anything we do
-% here.
+% \subsubsection{Building groups}
+%
+% \begin{macro}[aux]{\regex_group_aux:nnnnN}
+% Arguments: \Arg{label} \Arg{contents} \Arg{min} \Arg{more}
+% \meta{lazyness}. If \meta{min} is $0$, we need to add a state before
+% building the group, so that the thread which skips the group does
+% not also set the start-point of the submatch. After adding one more
+% state, the \texttt{left_state} is the left end of the group, from
+% which all branches will stem, and the \texttt{right_state} is the
+% right end of the group, and all branches end their course in that
+% state. We store those two integers to be queried for each branch, we
+% build the \textsc{nfa} states for the contents |#2| of the group,
+% and we forget about the two integers. Once this is done, perform the
+% repetition: either exactly |#3| times, or |#3| or more times, or
+% between |#3| and $|#3|+|#4|$ times, with lazyness |#5|. The
+% \meta{label} |#1| is used for submatch tracking. Each of the three
+% auxiliaries expects \texttt{left_state} and \texttt{right_state} to
+% be set properly.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_u_end:
+\cs_new_protected:Npn \regex_group_aux:nnnnN #1#2#3#4#5
{
- \tl_set:Nv \l_regex_internal_a_tl \l_regex_internal_a_tl
- \if_num:w \l_regex_build_mode_int = \c_zero
- \regex_build_u_not_cs:
- \else:
- \regex_build_u_in_cs:
- \fi:
- \tl_clear:N \l_regex_tests_tl
+%<trace> \trace_push:nnn { regex } { 1 } { regex_group }
+ \if_num:w #3 = \c_zero
+ \regex_build_new_state:
+%<assert>\assert_int:n { \l_regex_max_state_int = \l_regex_right_state_int + 1 }
+ \regex_build_transition_right:nNn \regex_action_free_group:n
+ \l_regex_left_state_int \l_regex_right_state_int
+ \fi:
+ \regex_build_new_state:
+ \regex_push_lr_states:
+ #2
+ \regex_pop_lr_states:
+ \if_case:w - #4 \exp_stop_f:
+ \regex_group_repeat:nn {#1} {#3}
+ \or: \regex_group_repeat:nnN {#1} {#3} #5
+ \else: \regex_group_repeat:nnnN {#1} {#3} {#4} #5
+ \fi:
+%<trace> \trace_pop:nnn { regex } { 1 } { regex_group }
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_u_in_cs:}
-% When |\u| appears within a control sequence, the variable is
-% converted to a string, and we add to the NFA one state per
-% character. We use \cs{regex_build_one_:} directly, as this avoids
-% having to search for a quantifier.
+% \begin{macro}[int]{\regex_group:nnnN, \regex_group_no_capture:nnnN}
+% Hand to \cs{regex_group_aux:nnnnnN} the label of that group
+% (expanded), and the group itself, with some extra commands to
+% perform.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_u_in_cs:
+\cs_new_protected:Npn \regex_group:nnnN #1
{
- \exp_args:NNo \str_gset_other:Nn \g_regex_internal_tl
- { \l_regex_internal_a_tl }
- \tl_map_inline:Nn \g_regex_internal_tl
+ \exp_args:No \regex_group_aux:nnnnN
+ { \int_use:N \l_regex_capturing_group_int }
{
- \tl_set:Nx \l_regex_tests_tl { \regex_item_equal:n { `##1 } }
- \regex_build_one_:
+ \int_incr:N \l_regex_capturing_group_int
+ #1
}
}
+\cs_new_protected_nopar:Npn \regex_group_no_capture:nnnN
+ { \regex_group_aux:nnnnN { -1 } }
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_u_not_cs:}
-% In mode $0$, the |\u| escape adds one state to the NFA for each
-% token in \cs{l_regex_internal_a_tl}. The tests corresponding to a
-% given \meta{token} are: first test the category, then, if the
-% \meta{token} is a control sequence, compare the string
-% representations, otherwise compare the character codes. We could
-% compare the string representations in all cases, but that doesn't
-% interact well with caseless matching.
+% \begin{macro}[int]{\regex_group_resetting:nnnN}
+% \begin{macro}[aux]{\regex_group_resetting_loop:nnNn}
+% Again, hand the label $-1$ to \cs{regex_group_aux:nnnnN}, but this
+% time we work a little bit harder to keep track of the maximum group
+% label at the end of any branch, and to reset the group number at
+% each branch. This relies on the fact that a compiled regex always is
+% a sequence of items of the form \cs{regex_branch:n} \Arg{branch}.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_u_not_cs:
+\cs_new_protected:Npn \regex_group_resetting:nnnN #1
{
- \exp_args:No \tl_analysis_map_inline:nn { \l_regex_internal_a_tl }
+ \regex_group_aux:nnnnN { -1 }
{
- \tl_set:Nx \l_regex_tests_tl
- {
- \exp_not:N \int_compare:nNnT
- \l_regex_current_catcode_int = { \int_value:w "##2 }
- {
- \int_compare:nNnTF { "##2 } = \c_zero
- {
- \exp_not:n
- { \str_if_eq:xxT { \l_regex_current_token_tl } {##1} }
- { \regex_break_true:w }
- }
- { \regex_item_equal:n {##3} }
- }
- }
- \regex_build_one_:
+ \exp_args:Noo \regex_group_resetting_loop:nnNn
+ { \int_use:N \l_regex_capturing_group_int }
+ { \int_use:N \l_regex_capturing_group_int }
+ #1
+ { ?? \prg_map_break:n } { }
+ \prg_break_point:n { }
}
}
+\cs_new_protected:Npn \regex_group_resetting_loop:nnNn #1#2#3#4
+ {
+ \use_none:nn #3 { \int_set:Nn \l_regex_capturing_group_int {#1} }
+ \int_set:Nn \l_regex_capturing_group_int {#2}
+ #3 {#4}
+ \exp_args:Nf \regex_group_resetting_loop:nnNn
+ { \int_max:nn {#1} { \l_regex_capturing_group_int } }
+ {#2}
+ }
% \end{macrocode}
% \end{macro}
+% \end{macro}
%
-% \subsection{Matching}
-%
-% At every index, we unpack that array of active states and
-% empty it. Then loop over all active states, and perform the
-% instruction at that state of the NFA. This can involve
-% \enquote{free} transitions to other states, or transitions
-% which \enquote{consume} the current character. For free
-% transitions, the instruction at the new state of the NFA is
-% performed. When a transition consumes a character, the new
-% state is put in the array of \tn{skip} registers: it will
-% be active again when the next character is read.
-%
-% If two paths through the NFA \enquote{collide} in the sense
-% that they reach the same state when reading a given
-% character, then any future execution will be identical for
-% both. Hence, it is indeed enough to keep track of which
-% states are active. [In the presence of back-references, the
-% future execution is affected by how the previous match took
-% place; this is why we cannot support those non-regular
-% features.]
-%
-% Many of the functions require extracting the submatches for
-% the \enquote{best} match. Execution paths through the NFA
-% are ordered by precedence: for instance, the regular
-% expression \texttt{a?} creates two paths, matching either
-% an empty token list or a single \texttt{a}; the path matching
-% an \texttt{a} has higher precedence. When two paths collide,
-% the path with the highest precedence is kept, and the other
-% one is discarded. The submatch information for a given path
-% is stored at the start of the \tn{toks} register which
-% holds the state at which that path currently is.
-%
-% Deciding to store the submatch information in \tn{toks}
-% registers alongside with states of the NFA unfortunately
-% implies some shuffling around. The two other options are to
-% store the submatch information in one control sequence per
-% path, which wastes csnames, or to store all of the submatch
-% information in one property list, which turns out to be too
-% slow. A tricky aspect of submatch tracking is to know when
-% to get rid of submatch information. This naturally happens
-% when submatch information is stored in \tn{toks} registers:
-% if the information is not moved, it will be overwritten
-% later.
-%
-% The presence of $\epsilon$-transitions (transitions which
-% consume no character) leads to potential infinite loops;
-% for instance the regular expression |(a??)*| could lead to
-% an infinite recursion, where |a??| matches no character, |*|
-% loops back to the start of the group, and |a??| matches no
-% character again. Therefore, we need to keep track of the
-% states of the NFA visited at the current step. More
-% precisely, a state is marked as \enquote{visited} if the
-% instructions for that state have been inserted in the input
-% stream, by setting the corresponding \tn{dimen} register to
-% a value which uniquely identifies at which step it was last
-% inserted.
-%
-% \subsubsection{Helpers for running the NFA}
-%
-% \begin{macro}[aux]{\regex_store_state:n}
-% Put the given state in the array of \tn{skip} registers.
-% This is done by increasing the pointer
-% \cs{l_regex_max_active_int}, and converting the integer
-% to a dimension (suitable for a \tn{skip} assignment) in
-% scaled points.
+% \begin{macro}[int]{\regex_branch:n}
+% Add a free transition from the left state of the current group to a
+% brand new state, starting point of this branch. Once the branch is
+% built, add a transition from its last state to the right state of
+% the group. The left and right states of the group are extracted from
+% the relevant sequences.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_store_state:n #1
+\cs_new_protected:Npn \regex_branch:n #1
{
- \int_incr:N \l_regex_max_active_int
- \tex_skip:D \l_regex_max_active_int #1 sp \scan_stop:
- \regex_store_submatches:n {#1}
+%<trace> \trace_push:nnn { regex } { 1 } { regex_branch }
+ \regex_build_new_state:
+ \seq_get:NN \l_regex_left_state_seq \l_regex_internal_a_tl
+ \int_set:Nn \l_regex_left_state_int \l_regex_internal_a_tl
+ \regex_build_transition_right:nNn \regex_action_free:n
+ \l_regex_left_state_int \l_regex_right_state_int
+ #1
+ \seq_get:NN \l_regex_right_state_seq \l_regex_internal_a_tl
+ \regex_build_transition_right:nNn \regex_action_free:n
+ \l_regex_right_state_int \l_regex_internal_a_tl
+%<trace> \trace_pop:nnn { regex } { 1 } { regex_branch }
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[int]{\regex_state_use:}
-% \begin{macro}[int]{\regex_state_use_with_submatches:}
-% \begin{macro}[aux]{\regex_state_use_aux_ii:w}
-% \begin{macro}[aux]{\regex_state_use_aux:n}
-% Use a given program instruction, unless it has already been
-% executed at this step. The \tn{toks} registers begin with
-% some submatch information, ignored by \cs{regex_state_use:},
-% but not by \cs{regex_state_use_with_submatches:}.
-% A state is free if it is not marker as taken, namely
-% if the corresponding \tn{dimen} register is not
-% \cs{l_regex_step_int} in \texttt{sp}.
-% The primitive conditional is ended before unpacking
-% the \tn{toks} register.
+% \begin{macro}[int]{\regex_group_repeat:nn}
+% This function is called to repeat a group a fixed number of times
+% |#2|; if this is $0$ we remove the group altogether (but don't reset
+% the \texttt{capturing_group} label). Otherwise, the auxiliary
+% \cs{regex_group_repeat_aux:n} copies |#2| times the \tn{toks} for
+% the group, and leaves \texttt{internal_a} pointing to the left end
+% of the last repetition. We only record the submatch information at
+% the last repetition. Finally, add a state at the end (the transition
+% to it has been taken care of by the replicating auxiliary.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_state_use_with_submatches:
- { \regex_state_use_aux:n { } }
-\cs_new_protected_nopar:Npn \regex_state_use:
- { \regex_state_use_aux:n { \exp_after:wN \use_none_delimit_by_s_stop:w } }
-\cs_new_protected:Npn \regex_state_use_aux:n #1
+\cs_new_protected:Npn \regex_group_repeat:nn #1#2
{
- \if_num:w \tex_dimen:D \l_regex_current_state_int < \l_regex_step_int
- \tex_dimen:D \l_regex_current_state_int
- = \l_regex_step_int sp \scan_stop:
- #1 \tex_the:D \tex_toks:D \exp_after:wN \l_regex_current_state_int
+ \if_num:w #2 = \c_zero
+ \int_set:Nn \l_regex_max_state_int
+ { \l_regex_left_state_int - \c_one }
+ \regex_build_new_state:
+ \else:
+ \regex_group_repeat_aux:n {#2}
+ \regex_group_submatches:nNN {#1}
+ \l_regex_internal_a_int \l_regex_right_state_int
+ \regex_build_new_state:
\fi:
- \scan_stop:
}
% \end{macrocode}
% \end{macro}
+%
+% \begin{macro}[aux]{\regex_group_submatches:nNN}
+% This inserts in states |#2| and |#3| the code for tracking
+% submatches of the group |#1|, unless inhibited by a label of $-1$.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_group_submatches:nNN #1#2#3
+ {
+ \if_num:w #1 > \c_minus_one
+ \regex_toks_put_left:Nx #2 { \regex_action_submatch:n { #1 < } }
+ \regex_toks_put_left:Nx #3 { \regex_action_submatch:n { #1 > } }
+ \fi:
+ }
+% \end{macrocode}
% \end{macro}
-% \end{macro}
+%
+% \begin{macro}[aux]{\regex_group_repeat_aux:n}
+% Here we repeat \tn{toks} ranging from \texttt{left_state} to
+% \texttt{max_state}, $|#1|>0$ times. First add a transition so that
+% the copies will \enquote{chain} properly. Compute the shift
+% \texttt{c} between the original copy and the last copy we
+% want. Shift the \texttt{right_state} and \texttt{max_state} to their
+% final values. We then want to perform \texttt{c} copy operations. At
+% the end, \texttt{b} is equal to the \texttt{max_state}, and
+% \texttt{a} points to the left of the last copy of the group.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_group_repeat_aux:n #1
+ {
+ \regex_build_transition_right:nNn \regex_action_free:n
+ \l_regex_right_state_int \l_regex_max_state_int
+ \int_set_eq:NN \l_regex_internal_a_int \l_regex_left_state_int
+ \int_set_eq:NN \l_regex_internal_b_int \l_regex_max_state_int
+ \if_num:w \int_eval:w #1 > \c_one
+ \int_set:Nn \l_regex_internal_c_int
+ {
+ ( #1 - \c_one )
+ * ( \l_regex_internal_b_int - \l_regex_internal_a_int )
+ }
+ \tex_advance:D \l_regex_right_state_int \l_regex_internal_c_int
+ \tex_advance:D \l_regex_max_state_int \l_regex_internal_c_int
+ \prg_replicate:nn \l_regex_internal_c_int
+ {
+ \tex_toks:D \l_regex_internal_b_int
+ = \tex_toks:D \l_regex_internal_a_int
+ \tex_advance:D \l_regex_internal_a_int \c_one
+ \tex_advance:D \l_regex_internal_b_int \c_one
+ }
+ \fi:
+ }
+% \end{macrocode}
% \end{macro}
%
-% \subsubsection{Submatch tracking when running the NFA}
+% \begin{macro}[aux]{\regex_group_repeat:nnN}
+% This function is called to repeat a group at least $n$ times; the
+% case $n=0$ is very different from $n>0$. Assume first that $n=0$.
+% Insert submatch tracking information at the start and end of the
+% group, add a free transition from the right end to the
+% \enquote{true} left state \texttt{a} (remember: in this case we had
+% added an extra state before the left state). This forms the loop,
+% which we break away from by adding a free transition from \texttt{a}
+% to a new state.
%
-% \begin{macro}[int]{\regex_disable_submatches:}
-% Some user functions don't require tracking submatches.
-% We get a performance improvement by simply defining the
-% relevant functions to remove their argument and do nothing
-% with it.
+% Now consider the case $n>0$. Repeat the group $n$ times, chaining
+% various copies with a free transition. Add submatch tracking only to
+% the last copy, then add a free transition from the right end back to
+% the left end of the last copy, either before or after the transition
+% to move on towards the rest of the \textsc{nfa}. This transition can
+% end up before submatch tracking, but that is irrelevant since it
+% only does so when going again through the group, recording new
+% matches. Finally, add a state; we already have a transition pointing
+% to it from \cs{regex_group_repeat_aux:n}.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_disable_submatches:
+\cs_new_protected:Npn \regex_group_repeat:nnN #1#2#3
{
- \cs_set_eq:NN \regex_state_use_with_submatches: \regex_state_use:
- \cs_set_eq:NN \regex_store_submatches:n
- \regex_protected_use_none:n
- \cs_set_eq:NN \regex_action_submatches:n
- \regex_protected_use_none:n
+ \if_num:w #2 = \c_zero
+ \regex_group_submatches:nNN {#1}
+ \l_regex_left_state_int \l_regex_right_state_int
+ \int_set:Nn \l_regex_internal_a_int
+ { \l_regex_left_state_int - \c_one }
+ \regex_build_transition_right:nNn \regex_action_free:n
+ \l_regex_right_state_int \l_regex_internal_a_int
+ \regex_build_new_state:
+ \if_meaning:w \c_true_bool #3
+ \regex_build_transition_left:NNN \regex_action_free:n
+ \l_regex_internal_a_int \l_regex_right_state_int
+ \else:
+ \regex_build_transition_right:nNn \regex_action_free:n
+ \l_regex_internal_a_int \l_regex_right_state_int
+ \fi:
+ \else:
+ \regex_group_repeat_aux:n {#2}
+ \regex_group_submatches:nNN {#1}
+ \l_regex_internal_a_int \l_regex_right_state_int
+ \if_meaning:w \c_true_bool #3
+ \regex_build_transition_right:nNn \regex_action_free_group:n
+ \l_regex_right_state_int \l_regex_internal_a_int
+ \else:
+ \regex_build_transition_left:NNN \regex_action_free_group:n
+ \l_regex_right_state_int \l_regex_internal_a_int
+ \fi:
+ \regex_build_new_state:
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_group_repeat:nnnN}
+% We wish to repeat the group between |#2| and $|#2|+|#3|$ times, with
+% a lazyness controlled by |#4|. We insert submatch tracking up front:
+% in principle, we could avoid recording submatches for the first |#2|
+% copies of the group, but that forces us to treat specially the case
+% $|#2|=0$. Repeat that group with submatch tracking $|#2|+|#3|$ times
+% (the maximum number of repetitions). Then our goal is to add |#3|
+% transitions from the end of the |#2|-th group, and each subsequent
+% groups, to the end. For a lazy quantifier, we add those transitions
+% to the left states, before submatch tracking. For the greedy case,
+% we add the transitions to the right states, after submatch tracking
+% and the transitions which go on with more repetitions. In the greedy
+% case with $|#2|=0$, the transition which skips over all copies of
+% the group must be added separately, because its starting state does
+% not follow the normal pattern: we had to add it \enquote{by hand}
+% earlier.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_group_repeat:nnnN #1#2#3#4
+ {
+ \regex_group_submatches:nNN {#1}
+ \l_regex_left_state_int \l_regex_right_state_int
+ \regex_group_repeat_aux:n { #2 + #3 }
+ \if_meaning:w \c_true_bool #4
+ \int_set_eq:NN \l_regex_left_state_int \l_regex_max_state_int
+ \prg_replicate:nn { #3 }
+ {
+ \int_sub:Nn \l_regex_left_state_int
+ { \l_regex_internal_b_int - \l_regex_internal_a_int }
+ \regex_build_transition_left:NNN \regex_action_free:n
+ \l_regex_left_state_int \l_regex_max_state_int
+ }
+ \else:
+ \prg_replicate:nn { #3 - \c_one }
+ {
+ \int_sub:Nn \l_regex_right_state_int
+ { \l_regex_internal_b_int - \l_regex_internal_a_int }
+ \regex_build_transition_right:nNn \regex_action_free:n
+ \l_regex_right_state_int \l_regex_max_state_int
+ }
+ \if_num:w #2 = \c_zero
+ \int_set:Nn \l_regex_right_state_int
+ { \l_regex_left_state_int - \c_one }
+ \else:
+ \int_sub:Nn \l_regex_right_state_int
+ { \l_regex_internal_b_int - \l_regex_internal_a_int }
+ \fi:
+ \regex_build_transition_right:nNn \regex_action_free:n
+ \l_regex_right_state_int \l_regex_max_state_int
+ \fi:
+ \regex_build_new_state:
}
-\cs_new_protected:Npn \regex_protected_use_none:n #1 { }
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[int]{\regex_store_submatches:n}
-% \begin{macro}[aux]{\regex_store_submatches_aux:w}
-% \begin{macro}[aux]{\regex_store_submatches_aux_ii:Nnnw}
-% The submatch information pertaining to one given thread is moved
-% from state to state as we execute the NFA.
-% We make sure that most of the \tn{toks} register is not read
-% before being assigned again to that same register.
+% \subsubsection{Others}
+%
+% \begin{macro}[int]{\regex_assertion:Nn, \regex_b_test:, \regex_anchor:N}
+% Usage: \cs{regex_assertion:Nn} \meta{boolean} \Arg{test}, where the
+% \meta{test} is either of the two other functions. Add a free
+% transition to a new state, conditionally to the assertion test. The
+% \cs{regex_b_test:} test is used by the |\b| and |\B| escape: check
+% if the last character was a word character or not, and do the same
+% to the current character. The boundary-markers of the string are
+% non-word characters for this purpose. Anchors at the start or end
+% of match use \cs{regex_anchor:N}, with a position controlled by the
+% integer |#1|.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_store_submatches:n #1
+\cs_new_protected:Npn \regex_assertion:Nn #1#2
{
- \tex_toks:D #1 \exp_after:wN
+ \regex_build_new_state:
+ \regex_toks_put_right:Nx \l_regex_left_state_int
{
- \tex_romannumeral:D
- \exp_after:wN \regex_store_submatches_aux:w
- \tex_the:D \tex_toks:D #1
+ \exp_not:n {#2}
+ \regex_break_point:TF
+ \bool_if:NF #1 { { } }
+ {
+ \regex_action_free:n
+ {
+ \int_eval:n
+ { \l_regex_right_state_int - \l_regex_left_state_int }
+ }
+ }
+ \bool_if:NT #1 { { } }
}
}
-\cs_new_protected:Npn \regex_store_submatches_aux:w #1 \s_stop
+\cs_new_protected:Npn \regex_anchor:N #1
{
- \regex_store_submatches_aux_ii:Nnnw
- #1
- \regex_state_submatches:nn \c_minus_one \q_prop
- \s_stop
+ \if_num:w #1 = \l_regex_current_pos_int
+ \exp_after:wN \regex_break_true:w
+ \fi:
}
-\cs_new_protected:Npn \regex_store_submatches_aux_ii:Nnnw
- \regex_state_submatches:nn #1 #2 #3 \s_stop
+\cs_new_protected_nopar:Npn \regex_b_test:
{
- \exp_after:wN \c_zero
- \exp_after:wN \regex_state_submatches:nn \exp_after:wN
- { \int_value:w \int_eval:w \l_regex_step_int + \c_one \exp_after:wN }
- \exp_after:wN { \l_regex_current_submatches_prop }
- \regex_state_submatches:nn {#1} {#2}
- \s_stop
+ \group_begin:
+ \int_set_eq:NN \l_regex_current_char_int \l_regex_last_char_int
+ \regex_prop_w:
+ \regex_break_point:TF
+ { \group_end: \regex_item_reverse:n \regex_prop_w: }
+ { \group_end: \regex_prop_w: }
}
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}[aux]{\regex_state_submatches:nn}
-% This function is inserted by \cs{regex_store_submatches:n}
-% in the \tn{toks} register holding a given state, and it is
-% performed when the state is used.
+% \begin{macro}[int]{\regex_command_K:}
+% Change the starting point of the $0$-th submatch (full match), and
+% transition to a new state, pretending that this is a fresh thread.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_state_submatches:nn #1#2
+\cs_new_protected_nopar:Npn \regex_command_K:
{
- \if_num:w #1 = \l_regex_step_int
- \tl_set:Nn \l_regex_current_submatches_prop { #2 }
- \fi:
+ \regex_build_new_state:
+ \regex_toks_put_right:Nx \l_regex_left_state_int
+ {
+ \regex_action_submatch:n { 0< }
+ \bool_set_true:N \l_regex_fresh_thread_bool
+ \regex_action_free:n
+ { \int_eval:n { \l_regex_right_state_int - \l_regex_left_state_int } }
+ \bool_set_false:N \l_regex_fresh_thread_bool
+ }
}
% \end{macrocode}
% \end{macro}
%
+% \subsection{Matching}
+%
+% We search for matches by running all the execution threads through the
+% \textsc{nfa} in parallel, reading one token of the query at each step.
+% The \textsc{nfa} contains \enquote{free} transitions to other states,
+% and transitions which \enquote{consume} the current token. For free
+% transitions, the instruction at the new state of the \textsc{nfa} is
+% performed immediately. When a transition consumes a character, the
+% new state is appended to a list of \enquote{active states}, stored in
+% \tn{skip} registers: this thread will be active again when the next
+% token is read from the query. At every step (for each token in the
+% query), we unpack that list of active states and the corresponding
+% submatch props, and empty those.
+%
+% If two paths through the \textsc{nfa} \enquote{collide} in the sense
+% that they reach the same state after reading a given token, then they
+% only differ in how they previously matched, and the future execution
+% will be identical for both. (Note that this would be wrong in the
+% presence of back-references.) Hence, we only need to keep one of the
+% two threads: the thread with the highest priority. Our \textsc{nfa} is
+% built in such a way that higher priority actions always come before
+% lower priority actions, which makes things work.
+%
+% The explanation in the previous paragraph may make us think that we
+% simply need to keep track of which states were visited at a given
+% step: after all, the loop generated when matching |(a?)*| against |a|
+% is broken, isn't it? No. The group first matches |a|, as it should,
+% then repeats; it attempts to match |a| again but fails; it skips |a|,
+% and finds out that this state has already been seen at this position
+% in the query: the match stops. The capturing group is (wrongly) |a|.
+% What went wrong is that a thread collided with itself, and the later
+% version, which has gone through the group one more times with an empty
+% match, should have a higher priority than not going through the group.
+%
+% We solve this by distinguishing \enquote{normal} free transitions
+% \cs{regex_action_free:n} from transitions
+% \cs{regex_action_free_group:n} which go back to the start of the
+% group. The former will keep threads unless they have been visited by a
+% \enquote{completed} thread, while the latter kind of transition also
+% prevents going back to a state visited by the current thread.
+%
+% \subsubsection{Variables used when matching}
+%
+% \begin{variable}
+% {
+% \l_regex_min_pos_int,
+% \l_regex_max_pos_int,
+% \l_regex_current_pos_int,
+% \l_regex_start_pos_int,
+% \l_regex_success_pos_int,
+% }
+% The tokens in the query are indexed from \texttt{min_pos} for the
+% first to $\texttt{max_pos}-1$ for the last, and their information is
+% stored in \tn{muskip} and \tn{toks} registers with those numbers. We
+% don't start from $0$ because the \tn{toks} registers with low
+% numbers are used to hold the states of the \textsc{nfa}. We match
+% without backtracking, keeping all threads in lockstep at the
+% \texttt{current_pos} in the query. The starting point of the current
+% match attempt is \texttt{start_pos}, and \texttt{success_pos},
+% updated whenever a thread succeeds, is used as the next starting
+% position.
+% \begin{macrocode}
+\int_new:N \l_regex_min_pos_int
+\int_new:N \l_regex_max_pos_int
+\int_new:N \l_regex_current_pos_int
+\int_new:N \l_regex_start_pos_int
+\int_new:N \l_regex_success_pos_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}
+% {
+% \l_regex_current_char_int,
+% \l_regex_current_catcode_int,
+% \l_regex_last_char_int,
+% \l_regex_case_changed_char_int
+% }
+% The character and category codes of the token at the current
+% position; the character code of the token at the previous position;
+% and the character code of the result of changing the case of the
+% current token (|A-Z|$\leftrightarrow$|a-z|). This last integer is
+% only computed when necessary, and is otherwise \cs{c_max_int}. The
+% \texttt{current_char} variable is also used in various other phases
+% to hold a character code.
+% \begin{macrocode}
+\int_new:N \l_regex_current_char_int
+\int_new:N \l_regex_current_catcode_int
+\int_new:N \l_regex_last_char_int
+\int_new:N \l_regex_case_changed_char_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_regex_current_state_int}
+% For every character in the token list, each of the active states is
+% considered in turn. The variable \cs{l_regex_current_state_int}
+% holds the state of the \textsc{nfa} which is currently considered:
+% transitions are then given as shifts relative to the current state.
+% \begin{macrocode}
+\int_new:N \l_regex_current_state_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}
+% {\l_regex_current_submatches_prop, \l_regex_success_submatches_prop}
+% The submatches for the thread which is currently active are stored
+% in the \texttt{current_submatches} property list variable. This
+% property list is stored by \cs{regex_action_cost:n} into the
+% \tn{toks} register for the target state of the transition, to be
+% retrieved when matching at the next position. When a thread
+% succeeds, this property list is copied to
+% \cs{l_regex_success_submatches_prop}: only the last sucessful thread
+% will remain there.
+% \begin{macrocode}
+\prop_new:N \l_regex_current_submatches_prop
+\prop_new:N \l_regex_success_submatches_prop
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_regex_step_int}
+% This integer, always even, is increased every time a character in
+% the query is read, and not reset when doing multiple matches. For
+% each \meta{state} in the \textsc{nfa} we store in
+% \tn{dimen}\meta{state} the last step in which this state was
+% encountered. This lets us break infinite loops by not visiting the
+% same state twice in the same step. In fact, \tn{dimen}\meta{state}
+% is equal \texttt{step} when we have started performing the
+% operations of \tn{toks}\meta{state}, but not finished yet. However,
+% once we finish, we set \tn{dimen}\meta{state} to
+% $\text{\texttt{step}}+1$. This is needed to track submatches
+% properly (see building phase). The \texttt{step} is also used to
+% attach each set of submatch information to a given iteration (and
+% automatically discard it when it corresponds to a past step).
+% \begin{macrocode}
+\int_new:N \l_regex_step_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_regex_min_active_int, \l_regex_max_active_int}
+% All the currently active states are kept in order of precedence in
+% the \tn{skip} registers, and the corresponding submatches in the
+% \tn{toks}. For our purposes, those serve as an array, indexed from
+% \texttt{min_active} (inclusive) to \texttt{max_active} (excluded).
+% At the start of every step, the whole array is unpacked, so that the
+% space can immediately be reused, and \texttt{max_active} is reset to
+% \texttt{min_active}, effectively clearing the array.
+% \begin{macrocode}
+\int_new:N \l_regex_min_active_int
+\int_new:N \l_regex_max_active_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_regex_every_match_tl}
+% Every time a match is found, this token list is used. For single
+% matching, the token list is empty. For multiple matching, the token
+% list is set to repeat the matching, after performing some operation
+% which depends on the user function. See \cs{regex_single_match:} and
+% \cs{regex_multi_match:n}.
+% \begin{macrocode}
+\tl_new:N \l_regex_every_match_tl
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_regex_fresh_thread_bool, \l_regex_empty_success_bool}
+% \begin{macro}[aux]{\regex_if_two_empty_matches:F}
+% When doing multiple matches, we need to avoid infinite loops where
+% each iteration matches the same empty token list. When an empty
+% token list is matched, the next successful match of the same empty
+% token list is suppressed. We detect empty matches by setting
+% \cs{l_regex_fresh_thread_bool} to \texttt{true} for threads which
+% directly come from the start of the regex or from the |\K| command,
+% and testing that boolean whenever a thread succeeds. The function
+% \cs{regex_if_two_empty_matches:F} is redefined at every match
+% attempt, depending on whether the previous match was empty or not:
+% if it was, then the function must cancel a purported success if it
+% is empty and at the same spot as the previous match; otherwise, we
+% definitely don't have two identical empty matches, so the function
+% is \cs{use:n}.
+% \begin{macrocode}
+\bool_new:N \l_regex_fresh_thread_bool
+\bool_new:N \l_regex_empty_success_bool
+\cs_new_eq:NN \regex_if_two_empty_matches:F \use:n
+% \end{macrocode}
+% \end{macro}
+% \end{variable}
+%
+% \begin{variable}
+% {
+% \g_regex_success_bool,
+% \l_regex_saved_success_bool,
+% \l_regex_match_success_bool
+% }
+% The boolean \cs{l_regex_match_success_bool} is true if the current
+% match attempt was successful, and \cs{g_regex_success_bool} is true
+% if there was at least one successful match. This is the only global
+% variable in this whole module, but we would need it to be local when
+% matching a control sequence with |\c{...}|. This is done by saving
+% the global variable into \cs{l_regex_saved_success_bool}, which is
+% local, hence not affected by the changes due to inner regex
+% functions.
+% \begin{macrocode}
+\bool_new:N \g_regex_success_bool
+\bool_new:N \l_regex_saved_success_bool
+\bool_new:N \l_regex_match_success_bool
+% \end{macrocode}
+% \end{variable}
+%
% \subsubsection{Matching: framework}
%
% \begin{macro}[int]{\regex_match:n}
-%^^A todo: update doc.
-% Then reset a few variables which should be set only once,
-% before the first match, even in the case of multiple matches.
-% Then run the NFA (\cs{regex_match_once:} matches multiple times
-% when appropriate).
+% First store the query into \tn{toks} and \tn{muskip} registers (see
+% \cs{regex_query_set:nnn}). Then initialize the variables that should
+% be set once for each user function (even for multiple
+% matches). Namely, the overall matching is not yet sucessful; none of
+% the states should be marked as visited (\tn{dimen} registers), and
+% we start at step $0$; we pretend that there was a previous match
+% ending at the start of the query, which was not empty (to avoid
+% smothering an empty match at the start). Once all this is set up, we
+% are ready for the ride. Find the first match.
% \begin{macrocode}
\cs_new_protected:Npn \regex_match:n #1
{
- \int_zero:N \l_regex_nesting_int
- \int_set_eq:NN \l_regex_current_index_int \l_regex_max_state_int
+%<trace> \trace_push:nnx { regex } { 1 } { regex_match }
+%<trace> \trace:nnx { regex } { 1 } { analyzing~query~token~list }
+ \int_zero:N \l_regex_balance_int
+ \int_set:Nn \l_regex_current_pos_int { \c_two * \l_regex_max_state_int }
\regex_query_set:nnn { } { -1 } { -2 }
- \int_set_eq:NN \l_regex_min_index_int \l_regex_current_index_int
+ \int_set_eq:NN \l_regex_min_pos_int \l_regex_current_pos_int
\tl_analysis_map_inline:nn {#1}
- {
- \regex_query_set:nnn {##1} {"##2} {##3}
- \if_case:w "##2 \exp_stop_f:
- \or: \int_incr:N \l_regex_nesting_int
- \or: \int_decr:N \l_regex_nesting_int
- \fi:
- }
- \int_set_eq:NN \l_regex_max_index_int \l_regex_current_index_int
+ { \regex_query_set:nnn {##1} {"##2} {##3} }
+ \int_set_eq:NN \l_regex_max_pos_int \l_regex_current_pos_int
\regex_query_set:nnn { } { -1 } { -2 }
- \regex_match_initial_setup:
+%<trace> \trace:nnx { regex } { 1 } { initializing }
+ \bool_gset_false:N \g_regex_success_bool
+ \prg_stepwise_inline:nnnn
+ \l_regex_min_state_int \c_one { \l_regex_max_state_int - \c_one }
+ { \tex_dimen:D ##1 \c_one sp \scan_stop: }
+ \int_set_eq:NN \l_regex_min_active_int \l_regex_max_state_int
+ \int_set_eq:NN \l_regex_step_int \c_zero
+ \int_set_eq:NN \l_regex_success_pos_int \l_regex_min_pos_int
+ \int_set:Nn \l_regex_submatch_int
+ { \c_two * \l_regex_max_state_int }
+ \bool_set_false:N \l_regex_empty_success_bool
\regex_match_once:
- }
-\cs_new_protected:Npn \regex_query_set:nnn #1#2#3
- {
- \tex_muskip:D \l_regex_current_index_int
- = \etex_gluetomu:D
- #3 sp
- plus #2 sp
- minus \l_regex_nesting_int sp
- \scan_stop:
- \tex_toks:D \l_regex_current_index_int {#1}
- \int_incr:N \l_regex_current_index_int
- }
-\cs_new_protected_nopar:Npn \regex_query_get:
- {
- \tl_set:Nx \l_regex_current_token_tl
- { \tex_the:D \tex_toks:D \l_regex_current_index_int }
- \l_regex_current_char_int
- = \etex_mutoglue:D \tex_muskip:D \l_regex_current_index_int
- \l_regex_current_catcode_int = \etex_gluestretch:D
- \etex_mutoglue:D \tex_muskip:D \l_regex_current_index_int
+%<trace> \trace_pop:nnx { regex } { 1 } { regex_match }
}
% \end{macrocode}
% \end{macro}
%
% \begin{macro}[int]{\regex_match_once:}
-% Set up more variables in \cs{regex_match_setup:}.
-% If there was a match, use the token list \cs{l_regex_every_match_tl},
-% which may call \cs{regex_match_once:} again to achieve multiple matches.
+% This function finds one match, then does some action defined by the
+% \texttt{every_match} token list, which may recursively call
+% \cs{regex_match_once:}. First initialize some variables: set the
+% conditional which detects identical empty matches; this match
+% attempt starts at the previous \texttt{success_pos}, is not yet
+% sucessful, and has no submatches yet; clear the array of active
+% threads, and put the starting state $0$ in it. We are then almost
+% ready to read our first token in the query, but we actually start
+% one position earlier than the start, and \texttt{get} that token, so
+% that the \texttt{last_char} will be set properly for word
+% boundaries. Then call \cs{regex_match_loop:}, which runs through the
+% query until the end or until a successful match breaks early.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \regex_match_once:
{
- \regex_match_setup:
+ \if_meaning:w \c_true_bool \l_regex_empty_success_bool
+ \cs_set_nopar:Npn \regex_if_two_empty_matches:F
+ { \int_compare:nNnF \l_regex_start_pos_int = \l_regex_current_pos_int }
+ \else:
+ \cs_set_eq:NN \regex_if_two_empty_matches:F \use:n
+ \fi:
+ \int_set_eq:NN \l_regex_start_pos_int \l_regex_success_pos_int
+ \bool_set_false:N \l_regex_match_success_bool
+ \prop_clear:N \l_regex_current_submatches_prop
+ \int_set_eq:NN \l_regex_max_active_int \l_regex_min_active_int
+ \regex_store_state:n { \l_regex_min_state_int }
+ \int_set:Nn \l_regex_current_pos_int
+ { \l_regex_start_pos_int - \c_one }
\regex_query_get:
\regex_match_loop:
- \prg_break_point:n { }
- \bool_if:NT \l_regex_success_match_bool
- {
- \bool_gset_true:N \g_regex_success_bool
- \l_regex_every_match_tl
- }
+ \l_regex_every_match_tl
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_match_initial_setup:}
-% This function holds the setup that should be done
-% only once for one given pattern matching.
-% It is called only once for the whole token list.
-% On the other hand, \cs{regex_match_setup:}
-% is called for every match in the token list in case of
-% repeated matches.
+% \begin{macro}[int]{\regex_single_match:, \regex_multi_match:n}
+% For a single match, the overall success is determined by whether the
+% only match attempt is a success. When doing multiple matches, the
+% overall matching is successful as soon as any match
+% succeeds. Perform the action |#1|, then find the next match.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_match_initial_setup:
+\cs_new_protected_nopar:Npn \regex_single_match:
{
- \int_set_eq:NN \l_regex_step_int \c_minus_one
- \prg_stepwise_inline:nnnn
- \c_zero \c_one { \l_regex_max_state_int - \c_one }
- { \tex_dimen:D ##1 \l_regex_step_int sp }
- \int_set:Nn \l_regex_start_index_int
- { \l_regex_min_index_int - \c_one }
- \int_set_eq:NN \l_regex_current_index_int \l_regex_min_index_int
- \int_set_eq:NN \l_regex_success_index_int \l_regex_min_index_int
- \int_set_eq:NN \l_regex_submatch_int \l_regex_max_state_int
- \bool_set_false:N \l_regex_success_empty_bool
- \bool_gset_false:N \g_regex_success_bool
+ \tl_set:Nn \l_regex_every_match_tl
+ { \bool_gset_eq:NN \g_regex_success_bool \l_regex_match_success_bool }
}
-% \end{macrocode}
-% \end{macro}
-%
-% \begin{macro}[aux]{\regex_match_setup:}
-% Every time a match starts, \cs{regex_match_setup:} resets
-% a few variables.
-% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_match_setup:
+\cs_new_protected:Npn \regex_multi_match:n #1
{
- \prop_clear:N \l_regex_current_submatches_prop
- \bool_if:NTF \l_regex_success_empty_bool
- { \cs_set_eq:NN \regex_if_two_empty_matches:F \regex_if_match_empty:F }
- { \cs_set_eq:NN \regex_if_two_empty_matches:F \use:n }
- \int_set_eq:NN \l_regex_start_index_int \l_regex_success_index_int
- \int_set:Nn \l_regex_current_index_int
- { \l_regex_start_index_int - \c_one }
- \bool_set_false:N \l_regex_success_match_bool
- \int_zero:N \l_regex_max_active_int
- \regex_store_state:n {0} %^^A _state_int!
+ \tl_set:Nn \l_regex_every_match_tl
+ {
+ \if_meaning:w \c_true_bool \l_regex_match_success_bool
+ \bool_gset_true:N \g_regex_success_bool
+ #1
+ \exp_after:wN \regex_match_once:
+ \fi:
+ }
}
% \end{macrocode}
% \end{macro}
%
% \begin{macro}[aux]{\regex_match_loop:}
-% \begin{macro}[aux]{\regex_match_one_active:n}
-% \begin{macro}[aux]{\regex_match_one_active_aux:n}
-% Setup what needs to be reset at every character,
-% then set \cs{l_regex_current_char_int} to the
-% character code of the token that is read
-% (and $-1$ for the end of the token list), and loop
-% over the elements of the \tn{skip} array. Then repeat.
-% There are a couple of tests to stop reading the token list
-% when no active state is left, or when the end is reached.
-% At every step in reading the token list, we store the character
-% code of the current character in \cs{l_regex_current_char_int},
-% unless the end was reached: then we store $-1$.
+% \begin{macro}[aux, rEXP]{\regex_match_one_active:w}
+% At each new position, set some variables and get the new character
+% and category from the query. Then unpack the array of active
+% threads, and clear it by resetting its length
+% (\texttt{max_active}). This results in a sequence of
+% \cs{regex_use_state_and_submatches:nn} \Arg{state} \Arg{prop}, and
+% we consider those states one by one in order. As soon as a thread
+% succeeds, exit the step, and, if there are threads to consider at the
+% next position, and we have not reached the end of the string,
+% repeat the loop. Otherwise, the last thread that succeeded is what
+% \cs{regex_match_once:} matches. We explain the \texttt{fresh_thread}
+% business when describing \cs{regex_action_wildcard:}.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \regex_match_loop:
{
- \int_incr:N \l_regex_current_index_int
- \int_incr:N \l_regex_step_int
- \bool_set_false:N \l_regex_fresh_thread_bool
+ \tex_advance:D \l_regex_step_int \c_two
+ \int_incr:N \l_regex_current_pos_int
\int_set_eq:NN \l_regex_last_char_int \l_regex_current_char_int
+ \int_set_eq:NN \l_regex_case_changed_char_int \c_max_int
\regex_query_get:
- \regex_match_loop_case_hook:
\use:x
{
- \int_zero:N \l_regex_max_active_int
- \regex_match_one_active:w 1 ; \prg_break_point:n { }
- \exp_not:N \prg_break_point:n { }
+ \int_set_eq:NN \l_regex_max_active_int \l_regex_min_active_int
+ \exp_after:wN \regex_match_one_active:w
+ \int_use:N \l_regex_min_active_int ;
}
- \if_num:w \l_regex_current_char_int < \c_minus_one
- \exp_after:wN \prg_map_break:
- \fi:
- \if_num:w \l_regex_max_active_int = \c_zero
- \exp_after:wN \prg_map_break:
+ \prg_break_point:n { \bool_set_false:N \l_regex_fresh_thread_bool }
+ \if_num:w \l_regex_max_active_int > \l_regex_min_active_int
+ \if_num:w \l_regex_current_pos_int < \l_regex_max_pos_int
+ \exp_after:wN \exp_after:wN \exp_after:wN \regex_match_loop:
+ \fi:
\fi:
- \regex_match_loop:
}
\cs_new:Npn \regex_match_one_active:w #1;
{
- \if_num:w #1 > \l_regex_max_active_int
- \exp_after:wN \prg_map_break:
+ \if_num:w #1 < \l_regex_max_active_int
+ \regex_use_state_and_submatches:nn
+ { \int_value:w \tex_skip:D #1 }
+ { \tex_the:D \tex_toks:D #1 }
+ \exp_after:wN \regex_match_one_active:w
+ \int_use:N \int_eval:w #1 + \c_one \exp_after:wN ;
\fi:
- \regex_match_one_active_aux:n
- { \int_value:w \tex_skip:D #1 }
- \exp_after:wN \regex_match_one_active:w
- \int_use:N \int_eval:w #1 + \c_one ;
}
-\cs_new_protected:Npn \regex_match_one_active_aux:n #1
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_query_set:nnn}
+% The arguments are: tokens that \texttt{o} and \texttt{x} expand to
+% one token of the query, the catcode, and the character code. Store
+% those, and the current brace balance (used later to check for
+% overall brace balance) in a \tn{muskip} register and a \tn{toks},
+% then update the \texttt{balance}.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_query_set:nnn #1#2#3
{
- \int_set:Nn \l_regex_current_state_int {#1}
- \prop_clear:N \l_regex_current_submatches_prop
- \regex_state_use_with_submatches:
+ \tex_muskip:D \l_regex_current_pos_int
+ = \etex_gluetomu:D
+ #3 sp
+ plus #2 sp
+ minus \l_regex_balance_int sp
+ \scan_stop:
+ \tex_toks:D \l_regex_current_pos_int {#1}
+ \int_incr:N \l_regex_current_pos_int
+ \if_case:w #2 \exp_stop_f:
+ \or: \int_incr:N \l_regex_balance_int
+ \or: \int_decr:N \l_regex_balance_int
+ \fi:
}
% \end{macrocode}
% \end{macro}
+%
+% \begin{macro}[aux]{\regex_query_get:}
+% Extract the current character and category codes from the
+% \tn{muskip} register of the current position: those are the main and
+% the stretch components, and we need a conversion to avoid \TeX{}'s
+% \enquote{incompatible glue units} error.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \regex_query_get:
+ {
+ \l_regex_current_char_int
+ = \etex_mutoglue:D \tex_muskip:D \l_regex_current_pos_int
+ \l_regex_current_catcode_int = \etex_gluestretch:D
+ \etex_mutoglue:D \tex_muskip:D \l_regex_current_pos_int
+ }
+% \end{macrocode}
% \end{macro}
+%
+% \subsubsection{Using states of the \textsc{nfa}}
+%
+% \begin{macro}[int]{\regex_use_state:}
+% Use the current \textsc{nfa} instruction. The state is initially
+% marked as belonging to the current \texttt{step}: this allows normal
+% free transition to repeat, but group-repeating transitions
+% won't. Once we are done exploring all the branches it spawned, the
+% state is marked as $\texttt{step}+1$: any thread hitting it at that
+% point will be terminated.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \regex_use_state:
+ {
+%<*trace>
+ \trace:nnx { regex } { 2 } { state~\int_use:N \l_regex_current_state_int }
+%</trace>
+ \tex_dimen:D \l_regex_current_state_int
+ = \l_regex_step_int sp \scan_stop:
+ \tex_the:D \tex_toks:D \l_regex_current_state_int
+ \tex_dimen:D \l_regex_current_state_int
+ = \int_eval:w \l_regex_step_int + \c_one \int_eval_end: sp \scan_stop:
+ }
+% \end{macrocode}
% \end{macro}
%
-% \begin{macro}{\regex_match_loop_case_hook:}
-% \begin{macro}{\regex_match_loop_caseless_hook:}
-% In the case where the regular expression contains caseless matching,
-% the \cs{regex_match_loop_case_hook:} (normally empty) is redefined
-% to set \cs{l_regex_case_changed_char_int} properly.
+% \begin{macro}[int]{\regex_use_state_and_submatches:nn}
+% This function is called as one item in the array of active threads
+% after that array has been unpacked for a new step. Update the
+% \texttt{current_state} and \texttt{current_submatches} and use the
+% state if it has not yet been encountered at this step.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_match_loop_case_hook: { }
-\cs_new_protected_nopar:Npn \regex_match_loop_caseless_hook:
+\cs_new_protected:Npn \regex_use_state_and_submatches:nn #1 #2
{
- \int_set_eq:NN \l_regex_case_changed_char_int \l_regex_current_char_int
- \if_num:w \l_regex_current_char_int < \c_ninety_one
- \if_num:w \l_regex_current_char_int < \c_sixty_five
- \else:
- \int_add:Nn \l_regex_case_changed_char_int { \c_thirty_two }
- \fi:
- \else:
- \if_num:w \l_regex_current_char_int < \c_one_hundred_twenty_three
- \if_num:w \l_regex_current_char_int < \c_ninety_seven
- \else:
- \int_sub:Nn \l_regex_case_changed_char_int { \c_thirty_two }
- \fi:
- \fi:
+ \int_set:Nn \l_regex_current_state_int {#1}
+ \if_num:w \tex_dimen:D \l_regex_current_state_int < \l_regex_step_int
+ \tl_set:Nn \l_regex_current_submatches_prop {#2}
+ \exp_after:wN \regex_use_state:
\fi:
+ \scan_stop:
}
% \end{macrocode}
% \end{macro}
-% \end{macro}
%
% \subsubsection{Actions when matching}
%
% \begin{macro}[int]{\regex_action_start_wildcard:}
-%: the first state has a free transition to the second
-% state, where the regular expression really begins, and a costly
-% transition to itself, to try again at the next character. ^^A ??
-% The search is made unanchored at the start by putting
-% a free transition to the real start of the NFA, and a
-% costly transition to the same state, waiting for the
-% next token in the query. This combination
-% could be reused (with some changes). We sometimes need
-% to know that the match for a given thread starts at
-% this character. For that, we use the boolean
-% \cs{l_regex_fresh_thread_bool}.
+% For an unanchored match, state $0$ has a free transition to the next
+% and a costly one to itself, to repeat at the next position. To catch
+% repeated identical empty matches, we need to know if a successful
+% thread corresponds to an empty match. The instruction resetting
+% \cs{l_regex_fresh_thread_bool} may be skipped by a successful
+% thread, hence we had to add it to \cs{regex_match_loop:} too.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \regex_action_start_wildcard:
{
@@ -4116,176 +4416,348 @@
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}[int]{\regex_action_free:n, \regex_action_free_group:n}
+% \begin{macro}[aux]{\regex_action_free_aux:nn}
+% These functions copy a thread after checking that the \textsc{nfa}
+% state has not already been used at this position. If not, store
+% submatches in the new state, and insert the instructions for that
+% state in the input stream. Then restore the old value of
+% \cs{l_regex_current_state_int} and of the current submatches. The
+% two types of free transitions differ by how they test that the state
+% has not been encountered yet: the \texttt{group} version is
+% stricter, and will not use a state if it was used earlier in the
+% current thread, hence forcefully breaking the loop, while the
+% \enquote{normal} version will revisit a state when within the thread
+% itself.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \regex_action_free:n
+ { \regex_action_free_aux:nn { > \l_regex_step_int \else: } }
+\cs_new_protected_nopar:Npn \regex_action_free_group:n
+ { \regex_action_free_aux:nn { < \l_regex_step_int } }
+\cs_new_protected:Npn \regex_action_free_aux:nn #1#2
+ {
+ \use:x
+ {
+ \int_add:Nn \l_regex_current_state_int {#2}
+ \exp_not:n
+ {
+ \if_num:w \tex_dimen:D \l_regex_current_state_int #1
+ \exp_after:wN \regex_use_state:
+ \fi:
+ }
+ \int_set:Nn \l_regex_current_state_int
+ { \int_use:N \l_regex_current_state_int }
+ \tl_set:Nn \exp_not:N \l_regex_current_submatches_prop
+ { \exp_not:o \l_regex_current_submatches_prop }
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
% \begin{macro}[int]{\regex_action_cost:n}
-% A transition which consumes the current character and moves
-% to state |#1|.
+% A transition which consumes the current character and shifts the
+% state by |#1|. The resulting state is stored in the \tn{skip} array
+% for use at the next position, and we also store the current
+% submatches.
% \begin{macrocode}
\cs_new_protected:Npn \regex_action_cost:n #1
{
- \exp_args:Nf \regex_store_state:n
- { \int_eval:n { \l_regex_current_state_int + #1 } }
+ \exp_args:No \regex_store_state:n
+ { \int_use:N \int_eval:w \l_regex_current_state_int + #1 }
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[int]{\regex_action_success:}
-% There is a successful match when an execution path reaches
-% the end of the regular expression. Then store the current
-% step and submatches. The current step is then interrupted
-% with \cs{prg_map_break:},
-% and only paths with higher precedence are pursued further.
-% The values stored here may be overwritten by a later success
-% of a path with higher precedence.
+% \begin{macro}[int]{\regex_store_state:n}
+% \begin{macro}[aux]{\regex_store_submatches:}
+% Put the given state in the array of \tn{skip} registers (converted
+% to a dimension in scaled points), and increment the length of the
+% array. Then store the current submatch in the This is done by
+% increasing the pointer \cs{l_regex_max_active_int}, and converting
+% the integer to a dimension (suitable for a \tn{skip} assignment) in
+% scaled points.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_action_success:
+\cs_new_protected:Npn \regex_store_state:n #1
{
- \regex_if_two_empty_matches:F
- {
- \bool_set_true:N \l_regex_success_match_bool
- \bool_set_eq:NN \l_regex_success_empty_bool
- \l_regex_fresh_thread_bool
- \int_set_eq:NN \l_regex_success_index_int \l_regex_current_index_int
- \prop_set_eq:NN \l_regex_success_submatches_prop
- \l_regex_current_submatches_prop
- \prg_map_break:
- }
+ \regex_store_submatches:
+ \tex_skip:D \l_regex_max_active_int = #1 sp \scan_stop:
+ \int_incr:N \l_regex_max_active_int
+ }
+\cs_new_protected_nopar:Npn \regex_store_submatches:
+ {
+ \tex_toks:D \l_regex_max_active_int \exp_after:wN
+ { \l_regex_current_submatches_prop }
}
% \end{macrocode}
% \end{macro}
+% \end{macro}
%
-% \begin{macro}[int]{\regex_action_free:n}
-% To copy a thread, check whether the program state has already
-% been used at this character. If not, store submatches in the
-% new state, and insert the instructions for that state in the
-% input stream.
-% Then restore the old value of \cs{l_regex_current_state_int}
-% and of the current submatches.
+% \begin{macro}[int]{\regex_disable_submatches:}
+% Some user functions don't require tracking submatches.
+% We get a performance improvement by simply defining the
+% relevant functions to remove their argument and do nothing
+% with it.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_action_free:n #1
+\cs_new_protected_nopar:Npn \regex_disable_submatches:
{
- \use:x
- {
- \int_add:Nn \l_regex_current_state_int {#1}
- \regex_state_use:
- \int_set:Nn \l_regex_current_state_int
- { \int_use:N \l_regex_current_state_int }
- \tl_set:Nn \exp_not:N \l_regex_current_submatches_prop
- { \exp_not:o \l_regex_current_submatches_prop }
- }
+ \cs_set_protected_nopar:Npn \regex_store_submatches: { }
+ \cs_set_protected:Npn \regex_action_submatch:n ##1 { }
}
% \end{macrocode}
% \end{macro}
%
% \begin{macro}[int]{\regex_action_submatch:n}
-% Update the current submatches with the information
-% from the current index.
+% Update the current submatches with the information from the current
+% position. Maybe a bottleneck.
% \begin{macrocode}
\cs_new_protected:Npn \regex_action_submatch:n #1
{
\prop_put:Nno \l_regex_current_submatches_prop {#1}
- { \int_use:N \l_regex_current_index_int }
+ { \int_use:N \l_regex_current_pos_int }
}
% \end{macrocode}
% \end{macro}
%
-% \subsubsection{??}%^^A todo: title
-%
-% \begin{macro}{\regex_if_match_empty:T, \regex_if_match_empty:F}
-%
+% \begin{macro}[int]{\regex_action_success:}
+% There is a successful match when an execution path reaches the last
+% state in the \textsc{nfa}, unless this marks a second identical
+% empty match. Then mark that there was a successful match; it is
+% empty if it is \enquote{fresh}; and we store the current position
+% and submatches. The current step is then interrupted with
+% \cs{prg_map_break:}, and only paths with higher precedence are
+% pursued further. The values stored here may be overwritten by a
+% later success of a path with higher precedence.
% \begin{macrocode}
-\cs_new:Npn \regex_if_match_empty:T
- { \int_compare:nNnT \l_regex_start_index_int = \l_regex_current_index_int }
-\cs_new:Npn \regex_if_match_empty:F
- { \int_compare:nNnF \l_regex_start_index_int = \l_regex_current_index_int }
+\cs_new_protected_nopar:Npn \regex_action_success:
+ {
+ \regex_if_two_empty_matches:F
+ {
+ \bool_set_true:N \l_regex_match_success_bool
+ \bool_set_eq:NN \l_regex_empty_success_bool
+ \l_regex_fresh_thread_bool
+ \int_set_eq:NN \l_regex_success_pos_int \l_regex_current_pos_int
+ \prop_set_eq:NN \l_regex_success_submatches_prop
+ \l_regex_current_submatches_prop
+ \prg_map_break:
+ }
+ }
% \end{macrocode}
% \end{macro}
%
% \subsection{Replacement}
%
-% \begin{macro}[rEXP]{\regex_submatch_nesting_aux:n}
+% \subsubsection{Variables and helpers used in replacement}
+%
+% \begin{variable}{\l_regex_replacement_csnames_int}
+% The behaviour of closing braces inside a replacement text depends on
+% whether a sequences |\c{| or |\u{| has been encountered. The number
+% of \enquote{open} such sequences that should be closed by |}| is
+% stored in \cs{l_regex_replacement_csnames_int}, and decreased by
+% $1$ by each |}|.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_submatch_nesting_aux:n #1
- {
- + \etex_glueshrink:D \etex_mutoglue:D \etex_muexpr:D
- \tex_muskip:D \etex_gluestretch:D \tex_skip:D #1
- - \tex_muskip:D \tex_skip:D #1
- \scan_stop:
+\int_new:N \l_regex_replacement_csnames_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_regex_balance_tl}
+% This token list holds the replacement text for
+% \cs{regex_replacement_balance_one_match:n} while it is being built
+% incrementally.
+% \begin{macrocode}
+\tl_new:N \l_regex_balance_tl
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{macro}[aux, rEXP]{\regex_replacement_balance_one_match:n}
+% This expects as an argument the first index of a range of \tn{skip}
+% registers which hold the submatch information for a given match. It
+% can be used within an integer expression to obtain the brace balance
+% incurred by performing the replacement on that match. This combines
+% the braces lost by removing the match, braces added by all the
+% submatches appearing in the replacement, and braces appearing
+% explicitly in the replacement. Even though it is always redefined
+% before use, we initialize it as for an empty replacement. An
+% important property is that concatenating several calls to that
+% function must result in a valid integer expression (hence a leading
+% |+| in the actual definition).
+% \begin{macrocode}
+\cs_new:Npn \regex_replacement_balance_one_match:n #1
+ { - \regex_submatch_balance:n {#1} }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux, rEXP]{\regex_replacement_do_one_match:n}
+% The input is the same as \cs{regex_replacement_balance_one_match:n}.
+% This function is redefined to expand to the part of the token list
+% from the end of the previous match to a given match, followed by the
+% replacement text. Hence concatenating the result of this function
+% with all possible arguments (one call for each match), as well as
+% the range from the end of the last match to the end of the string,
+% will produce the fully replaced token list. The initialization does
+% not matter, but we set it as for an empty replacement.
+% \begin{macrocode}
+\cs_new:Npn \regex_replacement_do_one_match:n #1
+ {
+ \regex_query_range:nn
+ { \etex_glueshrink:D \tex_skip:D #1 }
+ { \tex_skip:D #1 }
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[int]{\regex_replacement:n}
-% Our goal here is to analyse the replacement text. First take care
-% of detecting escaped and non-escaped characters using
-% \cs{regex_escape_use:nnnn} with three protected arguments.
-% This inserts in the input stream a token list of the form
-% \meta{fn 1} \meta{char 1} \ldots{} \meta{fn $N$} \meta{char $N$},
-% where \meta{fn $i$} is one of the three functions, and \meta{char $i$}
-% a character in the string |#1|.
-% \begin{macrocode}
-\cs_new:Npn \regex_nesting:n #1 { } %^^A move. Rename?
-\tl_new:N \l_regex_nesting_tl
-\cs_new_protected:Npn \regex_replacement:n #1
+% \begin{macro}[aux]{\regex_replacement_exp_not:N}
+% This function lets us navigate around the fact that the primitive
+% \cs{exp_not:n} requires a braced argument. As far as I can tell, it
+% is only needed if the user tries to include in the replacement text
+% a control sequence set equal to a macro parameter character, such as
+% \cs{c_parameter_token}. Indeed, within an \texttt{x}-expanding
+% assignment, \cs{exp_not:N}~|#| behaves as a single |#|, whereas
+% \cs{exp_not:n}~|{#}| behaves as a doubled |##|.
+% \begin{macrocode}
+\cs_new:Npn \regex_replacement_exp_not:N #1 { \exp_not:n {#1} }
+% \end{macrocode}
+% \end{macro}
+%
+% \subsubsection{Query and brace balance}
+%
+% \begin{macro}[int, rEXP]{\regex_query_range:nn}
+% \begin{macro}[aux, rEXP]{\regex_query_range_loop:ww}
+% When it is time to extract submatches from the token list, the
+% various tokens are stored in \tn{toks} registers numbered from
+% \cs{l_regex_min_pos_int} inclusive to \cs{l_regex_max_pos_int}
+% exclusive. The function \cs{regex_query_range:nn} \Arg{min}
+% \Arg{max} unpacks registers from the position \meta{min} to the
+% position $\meta{max}-1$ included. Once this is expanded, a second
+% \texttt{x}-expansion will result in the actual tokens from the
+% query. That second expansion is only done by user functions at the
+% very end of their operation, after checking (and correcting) the
+% brace balance first.
+% \begin{macrocode}
+\cs_new:Npn \regex_query_range:nn #1#2
{
- \int_zero:N \l_regex_replacement_int
- \int_zero:N \l_regex_nesting_int
- \tl_clear:N \l_regex_nesting_tl
- \regex_escape_use:nnnn
- { \regex_replacement_unescaped:N ##1 }
- { \regex_replacement_escaped:N ##1 }
- { \regex_replacement_raw:N ##1 }
- {#1}
- \prg_do_nothing: \prg_do_nothing:
- \cs_set:Npx \regex_nesting:n ##1
- {
- + \int_use:N \l_regex_nesting_int
- \l_regex_nesting_tl
- - \regex_submatch_nesting_aux:n {##1}
- }
- \use:x
- {
- \exp_not:n { \cs_set:Npn \regex_replacement_tl:n ##1 }
- { \regex_toks_range:nn \c_zero \l_regex_replacement_int }
- }
- % ^^A rename!
+ \exp_after:wN \regex_query_range_loop:ww
+ \int_use:N \int_eval:w #1 \exp_after:wN ;
+ \int_use:N \int_eval:w #2 ;
+ \prg_break_point:n { }
+ }
+\cs_new:Npn \regex_query_range_loop:ww #1 ; #2 ;
+ {
+ \if_num:w #1 < #2 \exp_stop_f:
+ \else:
+ \exp_after:wN \prg_map_break:
+ \fi:
+ \tex_the:D \tex_toks:D #1 \exp_stop_f:
+ \exp_after:wN \regex_query_range_loop:ww
+ \int_use:N \int_eval:w #1 + \c_one ; #2 ;
}
% \end{macrocode}
% \end{macro}
+% \end{macro}
%
-% \begin{macro}[aux]{\regex_replacement_raw:N}
+% \begin{macro}[int]{\regex_query_submatch:n}
+% When this function is called, \tn{skip}$i$ holds the start and end
+% positions for the $i$-th overall submatch as its main and stretch
+% components. In the case of repeated matches, submatches from all the
+% matches are put one after the other in blocks of
+% \cs{l_regex_capturing_group_int} \tn{skip} registers.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_replacement_raw:N #1
- { \regex_replacement_put:n {#1} }
+\cs_new:Npn \regex_query_submatch:n #1
+ {
+ \regex_query_range:nn
+ { \tex_skip:D \int_eval:w #1 }
+ { \etex_gluestretch:D \tex_skip:D \int_eval:w #1 }
+ }
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_replacement_put:n}
-% Raw characters are stored in a toks register.
+% \begin{macro}[rEXP]{\regex_submatch_balance:n}
+% Every user function must result in a balanced token list (unbalanced
+% token lists cannot be stored by TeX). When we unpacked the query, we
+% kept track of the brace balance as the shrink component of
+% \tn{muskip} registers, hence the contribution from a given range is
+% the difference between the shrink components of
+% \tn{muskip}\meta{max~pos} and \tn{muskip}\meta{min~pos}. For the
+% $i$-th submatch, the end-points of the range are the main and
+% stretch components of \tn{skip}$i$. The trailing \cs{scan_stop:} is
+% gobbled by \cs{etex_muexpr:D}, and the whole expression can be cast
+% safely to an integer (no trailing expansion).
% \begin{macrocode}
-\cs_new_protected:Npn \regex_replacement_put:n #1
+\cs_new_protected:Npn \regex_submatch_balance:n #1
{
- \tex_toks:D \l_regex_replacement_int {#1}
- \int_incr:N \l_regex_replacement_int
+ \etex_glueshrink:D \etex_mutoglue:D \etex_muexpr:D
+ \tex_muskip:D \etex_gluestretch:D \tex_skip:D #1
+ - \tex_muskip:D \tex_skip:D #1
+ \scan_stop:
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_replacement_escaped:N}
-% \begin{macro}[aux]{\regex_replacement_submatch:w}
-% \begin{macro}[aux]{\regex_replacement_submatch_aux:nN}
+% \subsubsection{Framework}
+%
+% \begin{macro}[int]{\regex_replacement:n}
+% \begin{macro}[aux]{\regex_replacement_aux:n}
+% The replacement text is built incrementally by abusing \tn{toks}
+% within a group (see \pkg{l3tl-build}). We keep track in
+% \cs{l_regex_balance_int} of the balance of explicit begin- and
+% end-group tokens and \cs{l_regex_balance_tl} will consist of some
+% code to compute the brace balance from submatches (see its
+% description). Detect unescaped right braces, and escaped characters,
+% with trailing \cs{prg_do_nothing:} because some of the later
+% function look-ahead. Once the whole replacement text has been
+% parsed, make sure that there is no open csname. Finally, define the
+% \texttt{balance_one_match} and \texttt{do_one_match} functions.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_replacement_unescaped:N #1
+\cs_new_protected:Npn \regex_replacement:n #1
{
- \if_charcode:w \c_rbrace_str #1
- \if_num:w \l_regex_replacement_csnames_int > \c_zero
- \regex_replacement_put:n \cs_end:
- \else:
- \regex_replacement_put:n #1
+%<trace> \trace_push:nnn { regex } { 1 } { regex_replacement:n }
+ \tl_set_build:Nw \l_regex_internal_a_tl
+ \int_zero:N \l_regex_balance_int
+ \tl_clear:N \l_regex_balance_tl
+ \regex_escape_use:nnnn
+ {
+ \if_charcode:w \c_rbrace_str ##1
+ \regex_replacement_rbrace:N \else: \tl_build_one:n \fi: ##1
+ }
+ { \regex_replacement_escaped:N ##1 }
+ { \tl_build_one:n ##1 }
+ {#1}
+ \prg_do_nothing: \prg_do_nothing:
+ \if_int_compare:w \l_regex_replacement_csnames_int > \c_zero
+ \msg_kernel_error:nnx { regex } { replacement-missing-rbrace }
+ { \int_use:N \l_regex_replacement_csnames_int }
+ \tl_build_one:x
+ { \prg_replicate:nn \l_regex_replacement_csnames_int \cs_end: }
\fi:
- \else:
- \regex_replacement_put:n #1
- \fi:
+ \cs_gset:Npx \regex_replacement_balance_one_match:n ##1
+ {
+ + \int_use:N \l_regex_balance_int
+ \l_regex_balance_tl
+ - \regex_submatch_balance:n {##1}
+ }
+ \tl_build_end:
+ \exp_args:No \regex_replacement_aux:n \l_regex_internal_a_tl
+%<trace> \trace_pop:nnn { regex } { 1 } { regex_replacement:n }
}
+\cs_new_protected:Npn \regex_replacement_aux:n #1
+ {
+ \cs_set:Npn \regex_replacement_do_one_match:n ##1
+ {
+ \regex_query_range:nn
+ { \etex_glueshrink:D \tex_skip:D ##1 }
+ { \tex_skip:D ##1 }
+ #1
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_replacement_escaped:N}
+% As in parsing a regular expression, we use an auxiliary built from
+% |#1| if defined. Otherwise, check for escaped digits (standing from
+% submatches from $0$ to $9$): anything else is a raw character.
+% \begin{macrocode}
\cs_new_protected:Npn \regex_replacement_escaped:N #1
{
\cs_if_exist_use:cF { regex_replacement_#1:w }
@@ -4293,34 +4765,45 @@
\if_num:w \c_one < 1#1 \exp_stop_f:
\regex_replacement_put_submatch:n {#1}
\else:
- \regex_replacement_put:n #1
+ \tl_build_one:n #1
\fi:
}
}
+% \end{macrocode}
+% \end{macro}
+%
+% \subsubsection{Submatches}
+%
+% \begin{macro}[aux]{\regex_replacement_put_submatch:n}
+% Insert a submatch in the replacement text. This is dropped if the
+% submatch number is larger than the number of capturing groups.
+% Unless the submatch appears inside a |\c{...}| or |\u{...}|
+% construction, it must be taken into account in the brace balance.
+% Here, |##1| will receive a pointer to the $0$-th submatch for a
+% given match. We cannot use \cs{int_eval:n} because it is
+% expandable, and would be expanded too early (short of adding
+% \cs{exp_not:N}, making the code messy again).
+% \begin{macrocode}
\cs_new_protected:Npn \regex_replacement_put_submatch:n #1
{
- \regex_replacement_put:n
- { \regex_query_submatch:nn {#1} {##1} }
- \if_num:w \l_regex_replacement_csnames_int = \c_zero
- \tl_put_right:Nn \l_regex_nesting_tl
- {
- \exp_not:N \if_num:w #1 < \l_regex_capturing_group_int
- \regex_submatch_nesting_aux:n { \int_eval:w #1+##1 \int_eval_end: }
- \exp_not:N \fi:
- }
+ \if_num:w #1 < \l_regex_capturing_group_int
+ \tl_build_one:n { \regex_query_submatch:n { #1 + ##1 } }
+ \if_num:w \l_regex_replacement_csnames_int = \c_zero
+ \tl_put_right:Nn \l_regex_balance_tl
+ { + \regex_submatch_balance:n { \int_eval:w #1+##1 \int_eval_end: } }
+ \fi:
\fi:
}
-\cs_new_protected:Npn \regex_replacement_error:NNN #1#2#3
- {
- \msg_kernel_error:nnxx { regex } { #1-command }
- { replacement~text } {#3}
- #2 #3
- }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_replacement_g:w, \regex_replacement_g_digits:NN}
+% An ugly method to grab digits for the |\g| escape sequence. At the
+% end of the run of digits, check that it ends with a right brace.
+% \begin{macrocode}
\cs_new_protected:Npn \regex_replacement_g:w #1#2
{
- \str_if_eq:xxTF
- { \exp_not:n { #1#2 } }
- { \regex_replacement_unescaped:N \c_lbrace_str }
+ \str_if_eq:xxTF { #1#2 } { \tl_build_one:n \c_lbrace_str }
{
\int_zero:N \l_regex_internal_a_int
\regex_replacement_g_digits:NN
@@ -4329,40 +4812,48 @@
}
\cs_new_protected:Npn \regex_replacement_g_digits:NN #1#2
{
- \token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N
+ \token_if_eq_meaning:NNTF #1 \tl_build_one:n
{
\if_num:w \c_one < 1#2 \exp_stop_f:
- \int_set:Nn \l_regex_internal_a_int { \c_ten * \l_regex_internal_a_int + #2 }
+ \int_set:Nn \l_regex_internal_a_int
+ { \c_ten * \l_regex_internal_a_int + #2 }
\exp_after:wN \use_i:nnn
\exp_after:wN \regex_replacement_g_digits:NN
\else:
- \if_charcode:w \c_rbrace_str #2
- \exp_args:No \regex_replacement_put_submatch:n
- { \int_use:N \l_regex_internal_a_int }
- \exp_after:wN \exp_after:wN \exp_after:wN \use_none:nn
- \else:
- \exp_after:wN \exp_after:wN
- \exp_after:wN \regex_replacement_error:NNN
- \exp_after:wN \exp_after:wN \exp_after:wN g
- \fi:
+ \exp_after:wN \regex_replacement_error:NNN
+ \exp_after:wN g
\fi:
}
- { \regex_replacement_error:NNN g }
- #1 #2
- }
-\cs_new_protected:Npn \regex_replacement_u:w #1#2
- {
- \str_if_eq:xxTF { #1#2 } { \regex_replacement_unescaped:N \c_lbrace_str }
{
- \int_incr:N \l_regex_replacement_csnames_int
- \regex_replacement_put:n
- { \exp_not:n { \exp_after:wN \exp_not:V \cs:w } }
+ \if_meaning:w \regex_replacement_rbrace:N #1
+ \exp_args:No \regex_replacement_put_submatch:n
+ { \int_use:N \l_regex_internal_a_int }
+ \exp_after:wN \use_none:nn
+ \else:
+ \exp_after:wN \regex_replacement_error:NNN
+ \exp_after:wN g
+ \fi:
}
- { \regex_replacement_error:NNN u #1#2 }
+ #1 #2
}
+% \end{macrocode}
+% \end{macro}
+%
+% \subsubsection{Csnames in replacement}
+%
+% \begin{macro}[aux]{\regex_replacement_c:w}
+% \begin{macro}[aux]+\regex_replacement_c_{:w+
+% |\c| can be followed by a left brace, or by a letter for which we
+% have defined a way to produce that category of characters. The
+% appropriate definitions for catcodes are introduced later. For
+% control sequences, if we are within a control sequence, convert
+% the token list to a string, otherwise simply prevent expansion,
+% with a weird cross-over between \cs{exp_not:n} and \cs{exp_not:N}
+% (see this helper's description for an explanation).
+% \begin{macrocode}
\cs_new_protected:Npn \regex_replacement_c:w #1#2
{
- \token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N
+ \token_if_eq_meaning:NNTF #1 \tl_build_one:n
{
\cs_if_exist_use:cF { regex_replacement_c_#2:w }
{ \regex_replacement_error:NNN c #1#2 }
@@ -4371,245 +4862,341 @@
}
\cs_new_protected_nopar:cpn { regex_replacement_c_ \c_lbrace_str :w }
{
+ \if_case:w \l_regex_replacement_csnames_int
+ \tl_build_one:n
+ { \exp_not:n { \exp_after:wN \regex_replacement_exp_not:N \cs:w } }
+ \else:
+ \tl_build_one:n { \exp_not:n { \exp_after:wN \tl_to_str:N \cs:w } }
+ \fi:
\int_incr:N \l_regex_replacement_csnames_int
- \regex_replacement_put:n
- { \exp_not:n { \exp_after:wN \regex_replacement_exp_not:N \cs:w } }
}
-\cs_new:Npn \regex_replacement_exp_not:N #1 { \exp_not:n {#1} }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_replacement_u:w}
+% Check that |\u| is followed by a left brace. If so, start a control
+% sequence with \cs{cs:w}, which is then unpacked either with
+% \cs{exp_not:V} or \cs{tl_to_str:V} depending on the current context.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_replacement_u:w #1#2
+ {
+ \str_if_eq:xxTF { #1#2 } { \tl_build_one:n \c_lbrace_str }
+ {
+ \if_case:w \l_regex_replacement_csnames_int
+ \tl_build_one:n { \exp_not:n { \exp_after:wN \exp_not:V \cs:w } }
+ \else:
+ \tl_build_one:n { \exp_not:n { \exp_after:wN \tl_to_str:V \cs:w } }
+ \fi:
+ \int_incr:N \l_regex_replacement_csnames_int
+ }
+ { \regex_replacement_error:NNN u #1#2 }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_replacement_rbrace:N}
+% Within a |\c{...}| or |\u{...}| construction, end the control
+% sequence, and decrease the brace count. Otherwise, this is a raw
+% right brace.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_replacement_rbrace:N #1
+ {
+ \if_int_compare:w \l_regex_replacement_csnames_int > \c_zero
+ \tl_build_one:n \cs_end:
+ \int_decr:N \l_regex_replacement_csnames_int
+ \else:
+ \tl_build_one:n #1
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \subsubsection{Characters in replacement}
+%
+% We will need to change the category code of the null character many
+% times, hence work in a group. The catcode-specific macros below are
+% defined in alphabetical order; if you are trying to understand the
+% code, start from the end of the alphabet as those categories are
+% simpler than active or begin-group.
+% \begin{macrocode}
\group_begin:
-
- \char_set_catcode_math_superscript:N \^^@
- \cs_new_protected_nopar:Npn \regex_replacement_c_U:w
- { \regex_replacement_char:nNN { ^^@ } }
-
- \char_set_catcode_alignment:N \^^@
- \cs_new_protected_nopar:Npn \regex_replacement_c_T:w
- { \regex_replacement_char:nNN { ^^@ } }
-
- \cs_new_protected:Npn \regex_replacement_c_S:w #1#2
+% \end{macrocode}
+%
+% \begin{macro}[aux]{\regex_replacement_char:nNN}
+% The only way to produce an arbitrary character--catcode pair is to
+% use the \tn{lowercase} or \tn{uppercase} primitives. This is a
+% wrapper for our purposes. The first argument is the null character
+% with various catcodes. The second and third arguments are grabbed
+% from the input stream: |#3| is the character whose character code to
+% reproduce.
+% \begin{macrocode}
+ \cs_new_protected:Npn \regex_replacement_char:nNN #1#2#3
{
- \int_compare:nNnTF { `#2 } = \c_zero
- { \regex_replacement_error:NNN c #1#2 }
- {
- \char_set_lccode:nn {32} { `#2 }
- \tl_to_lowercase:n { \regex_replacement_put:n {~} }
- }
+ \if_meaning:w \prg_do_nothing: #3
+ \msg_kernel_error:nn { regex } { replacement-catcode-end }
+ \else:
+ \tex_lccode:D \c_zero = `#3 \scan_stop:
+ \tl_to_lowercase:n { \tl_build_one:n {#1} }
+ \fi:
}
-
- \char_set_catcode_parameter:N \^^@
- \cs_new_protected_nopar:Npn \regex_replacement_c_P:w
- { \regex_replacement_char:nNN { ^^@^^@^^@^^@^^@^^@^^@^^@ } }
-
- \char_set_catcode_other:N \^^@
- \cs_new_protected_nopar:Npn \regex_replacement_c_O:w
- { \regex_replacement_char:nNN { ^^@ } }
-
- \char_set_catcode_math_toggle:N \^^@
- \cs_new_protected_nopar:Npn \regex_replacement_c_M:w
- { \regex_replacement_char:nNN { ^^@ } }
-
- \char_set_catcode_letter:N \^^@
- \cs_new_protected_nopar:Npn \regex_replacement_c_L:w
- { \regex_replacement_char:nNN { ^^@ } }
-
- \char_set_catcode_group_end:N \^^@
- \cs_new_protected_nopar:Npn \regex_replacement_c_E:w
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_replacement_c_A:w}
+% For an active character, expansion must be avoided, twice because we
+% later do two \texttt{x}-expansions, to unpack \tn{toks} for the
+% query, and to expand their contents to tokens of the query.
+% \begin{macrocode}
+ \char_set_catcode_active:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_A:w
+ { \regex_replacement_char:nNN { \exp_not:n { \exp_not:N ^^@ } } }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_replacement_c_B:w}
+% An explicit begin-group token increases the balance, unless within a
+% |\c{...}| or |\u{...}| construction. Add the desired begin-group
+% character, using the standard \cs{if_false:} trick. We eventually
+% \texttt{x}-expand twice. The first time must yield a balanced token
+% list, and the second one gives the bare begin-group token. The
+% \cs{exp_after:wN} is not strictly needed, but is more consistent
+% with \pkg{l3tl-analysis}.
+% \begin{macrocode}
+ \char_set_catcode_group_begin:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_B:w
{
- \int_decr:N \l_regex_nesting_int
- \regex_replacement_char:nNN { \if_false: { \fi: ^^@ }
+ \if_num:w \l_regex_replacement_csnames_int = \c_zero
+ \int_incr:N \l_regex_balance_int
+ \fi:
+ \regex_replacement_char:nNN
+ { \exp_not:n { \exp_after:wN ^^@ \if_false: } \fi: } }
}
-
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_replacement_c_C:w}
+% This is not quite catcode-related: when the user requests a
+% character with category \enquote{control sequence}, the
+% one-character control symbol is returned. As for the active
+% character, we prepare for two \texttt{x}-expansions.
+% \begin{macrocode}
+ \cs_new_protected:Npn \regex_replacement_c_C:w #1#2
+ { \tl_build_one:n { \exp_not:N \exp_not:N \exp_not:c {#2} } }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_replacement_c_D:w}
+% Subscripts fit the mould: \tn{lowercase} the null byte with the
+% correct category.
+% \begin{macrocode}
\char_set_catcode_math_subscript:N \^^@
\cs_new_protected_nopar:Npn \regex_replacement_c_D:w
{ \regex_replacement_char:nNN { ^^@ } }
-
- \char_set_catcode_group_begin:N \^^@
- \cs_new_protected_nopar:Npn \regex_replacement_c_B:w
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_replacement_c_E:w}
+% Similar to the begin-group case, the second \texttt{x}-expansion
+% produces the bare end-group token.
+% \begin{macrocode}
+ \char_set_catcode_group_end:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_E:w
{
- \int_incr:N \l_regex_nesting_int
- \regex_replacement_char:nNN { \exp_after:wN ^^@ \if_false: } \fi: }
+ \if_num:w \l_regex_replacement_csnames_int = \c_zero
+ \int_decr:N \l_regex_balance_int
+ \fi:
+ \regex_replacement_char:nNN
+ { \exp_not:n { \if_false: { \fi: ^^@ } }
}
-
- \char_set_catcode_active:N \^^@
- \cs_new_protected_nopar:Npn \regex_replacement_c_A:w
- { \regex_replacement_char:nNN { \exp_not:N ^^@ } }
-
-\group_end:
-
-\cs_new_protected:Npn \regex_replacement_char:nNN #1#2#3
- {
- \char_set_lccode:nn \c_zero { `#3 }
- \tl_to_lowercase:n
- { \regex_replacement_put:n { \exp_not:n {#1} } }
- }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-%
-% \subsection{User commands}
-%
-% \subsubsection{Precompiled pattern}
-%
-% A given pattern is often reused to match many different query token lists.
-% We thus give a means of storing the NFA corresponding to a given
-% pattern in a token list variable of the form
-% \begin{quote}
-% \cs{regex_nfa:Nw} \meta{variable~name} \\
-% \meta{assignments} \\
-% \cs{tex_toks:D} 0 \{ \meta{instruction0} \} \\
-% \ldots{} \\
-% \cs{tex_toks:D} $n$ \{ \meta{instruction$\sb{n}$} \} \\
-% \cs{s_stop}
-% \end{quote}
-% where $n$ is the number of states in the NFA,
-% and the various \meta{instruction$\sb{i}$} control
-% how the NFA behaves in state $i$. The \cs{regex_nfa:Nw}
-% function removes the whole NFA from the input stream
-% and produces an error: the \meta{nfa var} should only be
-% accessed through dedicated functions. This rather drastic
-% approach is taken because assignments triggered by the
-% contents of \meta{nfa var} may overwrite data which is used
-% elsewhere, unless everything is done carefully in a group.
-%
-% \begin{macro}{\regex_gset:Nn}
-% \begin{macro}{\regex_const:Nn}
-% \begin{macro}{\regex_set:Nn}
-% \begin{macro}[aux]{\regex_set_aux:NNn}
-% The three user functions only differ with which function is used
-% to assign the pre-compiled regular expression to the user's variable.
-% Internally, they all first build the NFA corresponding to the regex,
-% then store the contents of all the necessary \tn{toks} registers
-% in the user's variable.
-% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_set:Nn
- { \regex_set_aux:NNn \tl_set:Nn }
-\cs_new_protected_nopar:Npn \regex_gset:Nn
- { \regex_set_aux:NNn \tl_gset:Nn }
-\cs_new_protected_nopar:Npn \regex_const:Nn
- { \regex_set_aux:NNn \tl_const:Nn }
-\cs_new_protected:Npn \regex_set_aux:NNn #1#2#3
- {
- \group_begin:
- \regex_build:n {#3}
- \use:x
- {
- \group_end:
- #1 \exp_not:N #2 { \regex_set_aux:N #2 }
- }
- }
% \end{macrocode}
% \end{macro}
+%
+% \begin{macro}[aux]{\regex_replacement_c_L:w}
+% Simply \tn{lowercase} a letter null byte to produce an arbitrary letter.
+% \begin{macrocode}
+ \char_set_catcode_letter:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_L:w
+ { \regex_replacement_char:nNN { ^^@ } }
+% \end{macrocode}
% \end{macro}
+%
+% \begin{macro}[aux]{\regex_replacement_c_M:w}
+% No surprise here, we lowercase the null math toggle.
+% \begin{macrocode}
+ \char_set_catcode_math_toggle:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_M:w
+ { \regex_replacement_char:nNN { ^^@ } }
+% \end{macrocode}
% \end{macro}
+%
+% \begin{macro}[aux]{\regex_replacement_c_O:w}
+% Lowercase an other null byte.
+% \begin{macrocode}
+ \char_set_catcode_other:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_O:w
+ { \regex_replacement_char:nNN { ^^@ } }
+% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[int, rEXP]{\regex_set_aux:N}
-% \begin{macro}[aux, rEXP]{\regex_set_aux:n}
-% \begin{macro}[aux, EXP]{\regex_nfa:Nw}
-% Within a group, build the NFA corresponding to the given regular
-% expression, with submatch tracking. Then save the contents of all
-% relevant \tn{toks} registers into the variable outside the group.
-% The auxiliary \cs{regex_nfa:Nw} is not protected: this ensures that
-% the NFA will properly be replaced by an error message in expansion
-% contexts.
+% \begin{macro}[aux]{\regex_replacement_c_P:w}
+% For macro parameters, expansion is a tricky issue. We need to
+% prepare for two \texttt{x}-expansions and passing through various
+% macro definitions. Note that we cannot replace one \cs{exp_not:n} by
+% doubling the macro parameter characters because this would misbehave
+% if a mischievous user asks for |\c{\cP\#}|, since that macro
+% parameter character would be doubled.
% \begin{macrocode}
-\cs_new:Npn \regex_set_aux:N #1
- {
- \exp_not:n { \regex_nfa:Nw #1 }
- \l_regex_max_state_int
- = \int_use:N \l_regex_max_state_int
- \l_regex_capturing_group_int
- = \int_use:N \l_regex_capturing_group_int
- \token_if_eq_meaning:NNT
- \regex_match_loop_case_hook:
- \regex_match_loop_caseless_hook:
- {
- \cs_set_eq:NN \regex_match_loop_case_hook:
- \regex_match_loop_caseless_hook:
- }
- \prg_stepwise_function:nnnN
- \c_zero \c_one {\l_regex_max_state_int - \c_one }
- \regex_set_aux:n
- \s_stop
- }
-\cs_new:Npn \regex_set_aux:n #1
- { \tex_toks:D #1 { \tex_the:D \tex_toks:D #1 } }
-\cs_new:Npn \regex_nfa:Nw #1
- {
- \msg_expandable_kernel_error:nnn { regex } { nfa-misused } {#1}
- \use_none_delimit_by_s_stop:w
- }
+ \char_set_catcode_parameter:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_P:w
+ {
+ \regex_replacement_char:nNN
+ { \exp_not:n { \exp_not:n { ^^@^^@^^@^^@ } } }
+ }
% \end{macrocode}
% \end{macro}
+%
+% \begin{macro}[aux]{\regex_replacement_c_S:w}
+% Spaces are normalized on input by \TeX{} to have character code
+% $32$. It is in fact impossible to get a token with character code
+% $0$ and category code $10$. Hence we use $32$ instead of $0$ as our
+% base character.
+% \begin{macrocode}
+ \cs_new_protected:Npn \regex_replacement_c_S:w #1#2
+ {
+ \if_meaning:w \prg_do_nothing: #2
+ \msg_kernel_error:nn { regex } { replacement-catcode-end }
+ \else:
+ \if_num:w `#2 = \c_zero
+ \msg_kernel_error:nn { regex } { replacement-null-space }
+ \fi:
+ \tex_lccode:D 32 = `#2 \scan_stop:
+ \tl_to_lowercase:n { \tl_build_one:n {~} }
+ \fi:
+ }
+% \end{macrocode}
% \end{macro}
+%
+% \begin{macro}[aux]{\regex_replacement_c_T:w}
+% No surprise for alignment tabs here. Those are surrounded by the
+% appropriate braces whenever necessary, hence they don't cause
+% trouble in alignment settings.
+% \begin{macrocode}
+ \char_set_catcode_alignment:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_T:w
+ { \regex_replacement_char:nNN { ^^@ } }
+% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[int]{\regex_use:N}
-% No error-checking.
+% \begin{macro}[aux]{\regex_replacement_c_U:w}
+% Simple call to \cs{regex_replacement_char:nNN} which lowercases the
+% math superscript |^^@|.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_use:N
- { \exp_last_unbraced:No \use_none:nn }
+ \char_set_catcode_math_superscript:N \^^@
+ \cs_new_protected_nopar:Npn \regex_replacement_c_U:w
+ { \regex_replacement_char:nNN { ^^@ } }
% \end{macrocode}
% \end{macro}
%
-% \subsubsection{Generic auxiliary functions}
+% Restore the catcode of the null byte.
+% \begin{macrocode}
+\group_end:
+% \end{macrocode}
%
-% Most of \pkg{l3regex}'s work is done within a group.
+% \subsubsection{An error}
%
-% \begin{macro}[aux]{\regex_aux_return:}
-% This function triggers either \cs{prg_return_false:}
-% or \cs{prg_return_true:} as appropriate to whether a
-% match was found or not.
+% \begin{macro}[aux]{\regex_replacement_error:NNN}
+% Simple error reporting by calling one of the messages
+% \texttt{replacement-c}, \texttt{replacement-g}, or
+% \texttt{replacement-u}.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_aux_return:
+\cs_new_protected:Npn \regex_replacement_error:NNN #1#2#3
{
- \if_meaning:w \c_true_bool \g_regex_success_bool
- \prg_return_true:
- \else:
- \prg_return_false:
- \fi:
+ \msg_kernel_error:nnx { regex } { replacement-#1 } {#3}
+ #2 #3
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \subsection{User functions}
+%
+% \begin{macro}{\regex_new:N}
+% Before being assigned a sensible value, a regex variable matches
+% nothing.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_new:N #1
+ { \cs_new_eq:NN #1 \c_regex_no_match_regex }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\regex_set:Nn, \regex_gset:Nn, \regex_const:Nn}
+% Compile, then store the result in the user variable with the
+% appropriate assignment function.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \regex_set:Nn #1#2
+ {
+ \regex_compile:n {#2}
+ \tl_set_eq:NN #1 \l_regex_internal_regex
+ }
+\cs_new_protected_nopar:Npn \regex_gset:Nn #1#2
+ {
+ \regex_compile:n {#2}
+ \tl_gset_eq:NN #1 \l_regex_internal_regex
+ }
+\cs_new_protected_nopar:Npn \regex_const:Nn #1#2
+ {
+ \regex_compile:n {#2}
+ \tl_const:Nx #1 { \exp_not:o \l_regex_internal_regex }
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}{\regex_aux_build_match:nn}
-% This auxiliary is used by user functions whose \meta{regex} argument
-% is given as an explicit regular expression within braces. In that
-% case, we need to build the automaton corresponding to that regular
-% expression, then perform the matching on the given token list |#2|.
+% \begin{macro}{\regex_show:N, \regex_show:n}
+% User functions: the \texttt{n} variant requires compilation first.
+% Then show the variable with some appropriate text. The auxiliary
+% \cs{regex_show_aux:Nx} is defined in a different section.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_aux_build_match:nn #1#2
+\cs_new_protected:Npn \regex_show:n #1
{
- \regex_build:n {#1}
- \regex_match:n {#2}
+ \regex_compile:n {#1}
+ \regex_show_aux:Nx \l_regex_internal_regex
+ { { \tl_to_str:n {#1} } }
}
+\cs_new_protected:Npn \regex_show:N #1
+ { \regex_show_aux:Nx #1 { variable~\token_to_str:N #1 } }
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}{\regex_aux_use_match:Nn}
-% This auxiliary is used by user functions whose \meta{regex} argument
-% is given as a pre-compiled regex variable. We make sure that
-% the token list variable indeed is an automaton (by testing the
-% first token). If not, the match is deemed unsuccessful, after
-% raising an error. If we have an automaton, \enquote{use} it,
-% then perform the matching on the given token list |#2|.
+% \begin{macro}[TF]{\regex_match:nn, \regex_match:Nn}
+% Those conditionals are based on a common auxiliary defined
+% later. Its first argument builds the \textsc{nfa} corresponding to
+% the regex, and the second argument is the query token list. Once we
+% have performed the match, convert the resulting boolean to
+% \cs{prg_return_true:} or \texttt{false}.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_aux_use_match:Nn #1#2
+\prg_new_protected_conditional:Npnn \regex_match:nn #1#2 { T , F , TF }
{
- \exp_args:No \tl_if_head_eq_meaning:nNTF {#1} \regex_nfa:Nw
- {
- \regex_use:N #1
- \regex_match:n {#2}
- }
- {
- \msg_kernel_error:nnx { regex } { not-nfa } { \token_to_str:N #1 }
- \bool_gset_false:N \g_regex_success_bool
- }
+ \regex_match_aux:nn { \regex_build:n {#1} } {#2}
+ \regex_aux_return:
+ }
+\prg_new_protected_conditional:Npnn \regex_match:Nn #1#2 { T , F , TF }
+ {
+ \regex_match_aux:nn { \regex_build:N #1 } {#2}
+ \regex_aux_return:
}
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}{\regex_count:nnN, \regex_count:NnN}
+% Again, use an auxiliary whose first argument builds the \textsc{nfa}.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_count:nnN #1
+ { \regex_count_aux:nnN { \regex_build:n {#1} } }
+\cs_new_protected:Npn \regex_count:NnN #1
+ { \regex_count_aux:nnN { \regex_build:N #1 } }
+% \end{macrocode}
+% \end{macro}
+%
% \begin{macro}
% {
% \regex_extract_once:nnN, \regex_extract_once:NnN,
@@ -4626,92 +5213,206 @@
% \regex_replace_all:nnN, \regex_replace_all:NnN,
% \regex_split:nnN, \regex_split:NnN
% }
-% We define here $40$ user functions, following a common pattern
-% in terms of an auxiliary such as \cs{regex_extract_once_aux:NnnN}
-% (those auxiliaries are defined in the coming sections).
-% The arguments handed to the auxiliary are
-% \cs{regex_aux_build_match:nn} or \cs{regex_aux_use_match:Nn},
-% followed by the three arguments of the user function.
-% The conditionals call \cs{regex_aux_return:} to return
-% either \texttt{true} or \texttt{false} once matching
-% has been performed.
+% We define here $40$ user functions, following a common pattern in
+% terms of \texttt{_aux:nnN} auxiliaries, defined in the coming
+% subsections. The auxiliary is handed \cs{regex_build:n} or
+% \cs{regex_build:N} with the appropriate regex argument, then all
+% other necessary arguments (replacement text, token list, \emph{etc.}
+% The conditionals call \cs{regex_aux_return:} to return either
+% \texttt{true} or \texttt{false} once matching has been performed.
% \begin{macrocode}
\cs_set_protected:Npn \regex_tmp:w #1#2#3
{
- \cs_new_protected_nopar:Npn #1
- { #3 \regex_aux_build_match:nn }
- \cs_new_protected_nopar:Npn #2
- { #3 \regex_aux_use_match:Nn }
- \prg_new_protected_conditional:Npnn #1 ##1##2##3 { T , F , TF }
- {
- #3 \regex_aux_build_match:nn {##1} {##2} ##3
- \regex_aux_return:
- }
+ \cs_new_protected:Npn #2 ##1 { #1 { \regex_build:n {##1} } }
+ \cs_new_protected:Npn #3 ##1 { #1 { \regex_build:N ##1 } }
\prg_new_protected_conditional:Npnn #2 ##1##2##3 { T , F , TF }
- {
- #3 \regex_aux_use_match:Nn {##1} {##2} ##3
- \regex_aux_return:
- }
- }
-\tl_map_inline:nn
- {
- { extract_once } { extract_all }
- { replace_once } { replace_all }
- { split }
- }
- {
- \exp_args:Nccc \regex_tmp:w
- { regex_#1:nnN } { regex_#1:NnN } { regex_#1_aux:NnnN }
+ { #1 { \regex_build:n {##1} } {##2} ##3 \regex_aux_return: }
+ \prg_new_protected_conditional:Npnn #3 ##1##2##3 { T , F , TF }
+ { #1 { \regex_build:N ##1 } {##2} ##3 \regex_aux_return: }
}
+\regex_tmp:w \regex_extract_once_aux:nnN
+ \regex_extract_once:nnN \regex_extract_once:NnN
+\regex_tmp:w \regex_extract_all_aux:nnN
+ \regex_extract_all:nnN \regex_extract_all:NnN
+\regex_tmp:w \regex_replace_once_aux:nnN
+ \regex_replace_once:nnN \regex_replace_once:NnN
+\regex_tmp:w \regex_replace_all_aux:nnN
+ \regex_replace_all:nnN \regex_replace_all:NnN
+\regex_tmp:w \regex_split_aux:nnN \regex_split:nnN \regex_split:NnN
% \end{macrocode}
% \end{macro}
% \end{macro}
%
-% \subsubsection{Submatches, once the correct match is found}
+% \subsubsection{Variables and helpers for user functions}
%
-% \begin{macro}[int]{\regex_extract:}
-% \begin{macro}[aux]{\regex_extract_aux_b:wn}
-% \begin{macro}[aux]{\regex_extract_aux_e:wn}
+% \begin{variable}{\l_regex_match_count_int}
+% The number of matches found so far is stored
+% in \cs{l_regex_match_count_int}. This is only used
+% in the \cs{regex_count:nnN} functions.
+% \begin{macrocode}
+\int_new:N \l_regex_match_count_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{regex_begin, regex_end}
+% Those flags are raised to indicate extra begin-group
+% or end-group tokens when extracting submatches.
+% \begin{macrocode}
+\flag_new:n { regex_begin }
+\flag_new:n { regex_end }
+% \end{macrocode}
+% \end{variable}
%
+% \begin{variable}{\l_regex_submatch_int, \l_regex_zeroth_submatch_int}
+% The end-points of each submatch are stored as main and stretch
+% components of \tn{skip}\meta{submatch}, where \meta{submatch} ranges
+% from \cs{l_regex_max_state_int} (inclusive) to
+% \cs{l_regex_submatch_int} (exclusive). Each successful match comes
+% with a $0$-th submatch (the full match), and one match for each
+% capturing group: submatches corresponding to the last successful
+% match are labelled starting at
+% \texttt{zeroth_submatch}. Additionally, the shrink component of this
+% $0$-th submatch is the position at which that match attempt started:
+% this is used for splitting and replacements.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_extract:
+\int_new:N \l_regex_submatch_int
+\int_new:N \l_regex_zeroth_submatch_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{macro}[aux]{\regex_aux_return:}
+% This function triggers either \cs{prg_return_false:} or
+% \cs{prg_return_true:} as appropriate to whether a match was found or
+% not. It is used by all user conditionals.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \regex_aux_return:
{
- \int_set_eq:NN \l_regex_submatch_start_int \l_regex_submatch_int
\if_meaning:w \c_true_bool \g_regex_success_bool
- \prg_replicate:nn \l_regex_capturing_group_int
- {
- \tex_skip:D \l_regex_submatch_int \c_zero sp \scan_stop:
- \int_incr:N \l_regex_submatch_int
- }
- \prop_map_inline:Nn \l_regex_success_submatches_prop
- {
- \if_num:w ##1 \c_max_int
- \exp_after:wN \regex_extract_aux_b:wn \int_use:N
- \else:
- \exp_after:wN \regex_extract_aux_e:wn \int_use:N
- \fi:
- \int_eval:w \l_regex_submatch_start_int + ##1 {##2}
- }
+ \prg_return_true:
+ \else:
+ \prg_return_false:
\fi:
}
-\cs_new_protected:Npn \regex_extract_aux_b:wn #1 < #2
+% \end{macrocode}
+% \end{macro}
+%
+% \subsubsection{Matching}
+%
+% \begin{macro}[aux]{\regex_match_aux:nn}
+% We don't track submatches, and stop after a single match. Build the
+% \textsc{nfa} with |#1|, and perform the match on the query |#2|.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_match_aux:nn #1#2
{
- \tex_skip:D #1 = #2 sp
- plus \etex_gluestretch:D \tex_skip:D #1 \scan_stop:
+ \group_begin:
+ \regex_disable_submatches:
+ \regex_single_match:
+ #1
+ \regex_match:n {#2}
+ \group_end:
}
-\cs_new_protected:Npn \regex_extract_aux_e:wn #1 > #2
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_count_aux:nnN}
+% Again, we don't care about submatches. Instead of aborting after the
+% first \enquote{longest match} is found, we search for multiple
+% matches, incrementing \cs{l_regex_match_count_int} every time to
+% record the number of matches. Build the \textsc{nfa} and match. At
+% the end, store the result in the user's variable.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_count_aux:nnN #1#2#3
{
- \tex_skip:D #1
- = 1 \tex_skip:D #1 plus #2 sp \scan_stop:
+ \group_begin:
+ \regex_disable_submatches:
+ \int_zero:N \l_regex_match_count_int
+ \regex_multi_match:n { \int_incr:N \l_regex_match_count_int }
+ #1
+ \regex_match:n {#2}
+ \exp_args:NNNo
+ \group_end:
+ \int_set:Nn #3 { \int_use:N \l_regex_match_count_int }
}
% \end{macrocode}
% \end{macro}
+%
+% \subsubsection{Extracting submatches}
+%
+% \begin{macro}[aux]{\regex_extract_once_aux:nnN, \regex_extract_all_aux:nnN}
+% Match once or multiple times. After each match (or after the only
+% match), extract the submatches using \cs{regex_extract:}. At the
+% end, store the sequence containing all the submatches into the user
+% variable |#3| after closing the group.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_extract_once_aux:nnN #1#2#3
+ {
+ \group_begin:
+ \regex_single_match:
+ #1
+ \regex_match:n {#2}
+ \regex_extract:
+ \regex_group_end_extract_seq:N #3
+ }
+\cs_new_protected:Npn \regex_extract_all_aux:nnN #1#2#3
+ {
+ \group_begin:
+ \regex_multi_match:n { \regex_extract: }
+ #1
+ \regex_match:n {#2}
+ \regex_group_end_extract_seq:N #3
+ }
+% \end{macrocode}
% \end{macro}
+%
+% \begin{macro}[aux]{\regex_split_aux:nnN}
+% Splitting at submatches is a bit more tricky. For each match,
+% extract all submatches, and replace the zeroth submatch by the part
+% of the query between the start of the match attempt and the start of
+% the zeroth submatch. This is inhibited if the delimiter matched an
+% empty token list at the start of this match attempt. After the last
+% match, store the last part of the token list, which ranges from the
+% start of the match attempt to the end of the query. This step is
+% inhibited if the last match was empty and at the very end: decrement
+% \cs{l_regex_submatch_int}, which controls which \tn{skip} registers
+% will be used.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_split_aux:nnN #1#2#3
+ {
+ \group_begin:
+ \regex_multi_match:n
+ {
+ \if_num:w \l_regex_start_pos_int < \l_regex_success_pos_int
+ \regex_extract:
+ \tex_skip:D \l_regex_zeroth_submatch_int
+ = \l_regex_start_pos_int sp
+ plus \tex_skip:D \l_regex_zeroth_submatch_int \scan_stop:
+ \fi:
+ }
+ #1
+ \regex_match:n {#2}
+%<assert>\assert_int:n { \l_regex_current_pos_int = \l_regex_max_pos_int }
+ \tex_skip:D \l_regex_submatch_int
+ = \l_regex_start_pos_int sp plus \l_regex_max_pos_int sp \scan_stop:
+ \int_incr:N \l_regex_submatch_int
+ \if_meaning:w \c_true_bool \l_regex_empty_success_bool
+ \if_num:w \l_regex_start_pos_int = \l_regex_max_pos_int
+ \int_decr:N \l_regex_submatch_int
+ \fi:
+ \fi:
+ \regex_group_end_extract_seq:N #3
+ }
+% \end{macrocode}
% \end{macro}
%
-% \begin{macro}{\regex_group_end_extract_seq:N}
-% \begin{macro}[aux]{\regex_extract_seq_aux:n}
-% \begin{macro}[aux]{\regex_extract_seq_aux:ww}
+% \begin{macro}[aux]{\regex_group_end_extract_seq:N}
+% The end-points of submatches are stored as the main and stretch
+% components of \tn{skip} registers from \cs{l_regex_max_state_int} to
+% \cs{l_regex_submatch_int} (exclusive). Extract the relevant ranges
+% into \cs{l_regex_internal_a_tl}. We detect unbalanced results using
+% the two flags \texttt{regex_begin} and \texttt{regex_end}, raised
+% whenever we see too many begin-group or end-group tokens in a
+% submatch. We disable \cs{seq_item:n} to prevent two
+% \texttt{x}-expansions.
% \begin{macrocode}
\cs_new_protected:Npn \regex_group_end_extract_seq:N #1
{
@@ -4721,7 +5422,7 @@
\tl_set:Nx \l_regex_internal_a_tl
{
\prg_stepwise_function:nnnN
- \l_regex_max_state_int
+ { \c_two * \l_regex_max_state_int }
\c_one
{ \l_regex_submatch_int - \c_one }
\regex_extract_seq_aux:n
@@ -4729,17 +5430,33 @@
\int_compare:nNnF
{ \flag_height:n { regex_begin } + \flag_height:n { regex_end } }
= \c_zero
- { \msg_kernel_error:nn { regex } { sequence-unbalanced } }
- \tl_set:Nx \l_regex_internal_a_tl { \l_regex_internal_a_tl }
- \exp_args:NNNo \group_end:
- \tl_set:Nn #1 \l_regex_internal_a_tl
+ {
+ \msg_kernel_error:nnxxx { regex } { result-unbalanced }
+ { splitting~or~extracting~submatches }
+ { \flag_height:n { regex_end } }
+ { \flag_height:n { regex_begin } }
+ }
+ \use:x
+ {
+ \group_end:
+ \tl_set:Nn \exp_not:N #1 { \l_regex_internal_a_tl }
+ }
}
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux, EXP]{\regex_extract_seq_aux:n, \regex_extract_seq_aux:ww}
+% The \texttt{:n} auxiliary builds one item of the sequence of
+% submatches. First compute the brace balance of the submatch, then
+% extract the submatch from the query, adding the appropriate braces
+% and raising a flag if the submatch is not balanced.
+% \begin{macrocode}
\cs_new:Npn \regex_extract_seq_aux:n #1
{
\seq_item:n
{
\exp_after:wN \regex_extract_seq_aux:ww
- \int_value:w \regex_submatch_nesting_aux:n {#1} ; #1;
+ \int_value:w \regex_submatch_balance:n {#1} ; #1;
}
}
\cs_new:Npn \regex_extract_seq_aux:ww #1; #2;
@@ -4748,7 +5465,7 @@
\flag_raise:n { regex_end }
\prg_replicate:nn {-#1} { \exp_not:n { { \if_false: } \fi: } }
\fi:
- \regex_query_submatch:w #2;
+ \regex_query_submatch:n {#2}
\if_num:w #1 > \c_zero
\flag_raise:n { regex_begin }
\prg_replicate:nn {#1} { \exp_not:n { \if_false: { \fi: } } }
@@ -4756,278 +5473,182 @@
}
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
-%
-% \subsubsection{Matching}
-%
-% \begin{macro}[TF]{\regex_match:nn}
-% \begin{macro}[TF]{\regex_match:Nn}
-% \begin{macro}{\regex_match_aux:n}
-% We don't track submatches. Then either build the NFA corresponding
-% to the regular expression, or use a precompiled pattern. Then match,
-% using the internal \cs{regex_match:n}. Finally return the result
-% after closing the group.
-% \begin{macrocode}
-\prg_new_protected_conditional:Npnn \regex_match:nn #1#2 { T , F , TF }
- {
- \regex_match_aux:n
- { \regex_aux_build_match:nn {#1} {#2} }
- }
-\prg_new_protected_conditional:Npnn \regex_match:Nn #1#2 { T , F , TF }
- {
- \regex_match_aux:n
- { \regex_aux_use_match:Nn #1 {#2} }
- }
-\cs_new_protected:Npn \regex_match_aux:n #1
- {
- \group_begin:
- \tl_clear:N \l_regex_every_match_tl
- \regex_disable_submatches:
- #1
- \group_end:
- \regex_aux_return:
- }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}{\regex_count:nnN}
-% \begin{macro}{\regex_count:NnN}
-% \begin{macro}{\regex_count_aux:n}
-% Instead of aborting once the first \enquote{longest match} is found,
-% we repeat the search. The code is such that the search will not
-% start on the same character, hence avoiding infinite loops.
+% \begin{macro}[aux]
+% {\regex_extract:, \regex_extract_aux_b:wn, \regex_extract_aux_e:wn}
+% Our task here is to extract from the property list
+% \cs{l_regex_success_submatches_prop} the list of end-points of
+% submatches, and store them in \tn{skip} registers, from
+% \cs{l_regex_zeroth_submatch_int} upwards. We begin by emptying those
+% \tn{skip} registers. Then for each \meta{key}--\meta{value} pair in
+% the property list update the appropriate \tn{skip} component. This
+% is somewhat a hack: the \meta{key} is a non-negative integer
+% followed by |<| or |>|, which we use in a comparison to $-1$. At the
+% end, store the information about the position at which the match
+% attempt started, as a shrink component.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_count:nnN
- { \regex_count_aux:NnnN \regex_aux_build_match:nn }
-\cs_new_protected_nopar:Npn \regex_count:NnN
- { \regex_count_aux:NnnN \regex_aux_use_match:Nn }
-\cs_new_protected:Npn \regex_count_aux:NnnN #1#2#3#4
+\cs_new_protected_nopar:Npn \regex_extract:
{
- \group_begin:
- \regex_disable_submatches:
- \int_zero:N \l_regex_match_count_int
- \tl_set:Nn \l_regex_every_match_tl
+ \if_meaning:w \c_true_bool \g_regex_success_bool
+ \int_set_eq:NN \l_regex_zeroth_submatch_int \l_regex_submatch_int
+ \prg_replicate:nn \l_regex_capturing_group_int
{
- \int_incr:N \l_regex_match_count_int
- \regex_match_once:
+ \tex_skip:D \l_regex_submatch_int \c_zero sp \scan_stop:
+ \int_incr:N \l_regex_submatch_int
}
- #1 {#2} {#3}
- \exp_args:NNNo
- \group_end:
- \int_set:Nn #4 { \int_use:N \l_regex_match_count_int }
- }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-%
-% \subsubsection{Submatch extraction}
-%
-% \begin{macro}[aux]{\regex_extract_once_aux:NnnN}
-% As announced, here comes the auxiliary for extracting one match.
-% Since we only want one match, \cs{l_regex_every_match_tl}
-% is empty, and does not trigger the matching code again.
-% After matching, \cs{regex_extract:} extracts submatches
-% into various \tn{skip} registers, and those are then concatenated
-% into a sequence by \cs{regex_group_end_extract_seq:N}. That function
-% is also responsible for closing the group.
-% \begin{macrocode}
-\cs_new_protected:Npn \regex_extract_once_aux:NnnN #1#2#3#4
- {
- \group_begin:
- \tl_set:Nn \l_regex_every_match_tl { \regex_extract: }
- #1 {#2} {#3}
- \regex_group_end_extract_seq:N #4
+ \prop_map_inline:Nn \l_regex_success_submatches_prop
+ {
+ \if_num:w ##1 \c_minus_one
+ \exp_after:wN \regex_extract_aux_e:wn \int_value:w
+ \else:
+ \exp_after:wN \regex_extract_aux_b:wn \int_value:w
+ \fi:
+ \int_eval:w \l_regex_zeroth_submatch_int + ##1 {##2}
+ }
+ \tex_skip:D \l_regex_zeroth_submatch_int
+ = \tex_the:D \tex_skip:D \l_regex_zeroth_submatch_int
+ minus \l_regex_start_pos_int sp \scan_stop:
+ \fi:
}
-% \end{macrocode}
-% \end{macro}
-%
-% \begin{macro}[aux]{\regex_extract_all_aux:NnnN}
-%^^A todo: update doc
-% The set of submatches will be built progressively
-% in \cs{l_regex_result_seq}. For each match, extract
-% the submatches, and concatenate that to the right of
-% the result sequence, then start matching again.
-% Finally, copy the result in the user's sequence variable.
-% \begin{macrocode}
-\cs_new_protected:Npn \regex_extract_all_aux:NnnN #1#2#3#4
+\cs_new_protected:Npn \regex_extract_aux_b:wn #1 < #2
{
- \group_begin:
- \tl_set:Nn \l_regex_every_match_tl
- { \regex_extract: \regex_match_once: }
- #1 {#2} {#3}
- \regex_group_end_extract_seq:N #4
+ \tex_skip:D #1 = #2 sp
+ plus \etex_gluestretch:D \tex_skip:D #1 \scan_stop:
}
-% \end{macrocode}
-% \end{macro}
-%
-% \subsubsection{Splitting a token list by matches of a regex}
-%
-% \begin{macro}[aux]{\regex_split_aux:NnnN}
-% Recurse through the matches, and for each, do the following.
-% Extract the submatches into various \tn{skip} registers,
-% then replace the match |\0|, which should not be kept in the
-% final result, and replace it by the part of the token list
-% before the match.
-% This process must be inhibited to avoid creating empty items
-% if the regex matched an empty token list at the place where
-% the match attempt started.
-% After the last successful match, we need to add to the result
-% the part of the token list after the last match, unless the
-% last match was empty and at the very end.
-% Finally, \cs{regex_group_end_extract_seq:N} builds a sequence
-% from all the \tn{skip} registers, and assigns it to |#4|
-% after closing the group.
-% \begin{macrocode}
-\cs_new_protected:Npn \regex_split_aux:NnnN #1#2#3#4
+\cs_new_protected:Npn \regex_extract_aux_e:wn #1 > #2
{
- \group_begin:
- \tl_set:Nn \l_regex_every_match_tl
- {
- \if_num:w \l_regex_start_index_int < \l_regex_success_index_int
- \regex_extract:
- \tex_skip:D \l_regex_submatch_start_int
- = \l_regex_start_index_int sp
- plus \tex_skip:D \l_regex_submatch_start_int \scan_stop:
- \fi:
- \regex_match_once:
- }
- #1 {#2} {#3}
- \tex_skip:D \l_regex_submatch_int
- = \l_regex_start_index_int sp
- plus \l_regex_current_index_int sp \scan_stop:
- \int_incr:N \l_regex_submatch_int
- \if_num:w \l_regex_start_index_int = \l_regex_current_index_int
- \if_meaning:w \c_true_bool \l_regex_success_empty_bool
- \int_decr:N \l_regex_submatch_int
- \fi:
- \fi:
- \regex_group_end_extract_seq:N #4
+ \tex_skip:D #1
+ = 1 \tex_skip:D #1 plus #2 sp \scan_stop:
}
% \end{macrocode}
% \end{macro}
%
% \subsubsection{Replacement}
%
-% \begin{macro}[aux]{\regex_replace_once_aux:NnnN}
-% The replacement text is analysed by \cs{regex_replacement:n},
-% which defines \cs{regex_replacement_tl:n} to expand to the
-% replaced token list, assuming that submatches are stored in
-% various \tn{skip} registers, as done by \cs{regex_extract:}.
-% If there is a match, we grab
-% the parts before and after it, and get the result
-% by \texttt{x}-expanding twice.
-% \begin{macrocode}
-\cs_new_protected:Npn \regex_replace_once_aux:NnnN #1#2#3#4
+% \begin{macro}[aux]{\regex_replace_once_aux:nnN}
+% Build the \textsc{nfa} and the replacement functions, then find a
+% single match. If the match failed, simply exit the
+% group. Otherwise, we do the replacement. Extract submatches. Compute
+% the brace balance corresponding to replacing this match by the
+% replacement (this depends on submatches). Prepare the replaced token
+% list: the replacement function produces the tokens from the start of
+% the query to the start of the match and the replacement text for
+% this match; we need to add the tokens from the end of the match to
+% the end of the query. Finally, store the result in the user's
+% variable after closing the group: this step involves an additional
+% \texttt{x}-expansion, and checks that braces are balanced in the
+% final result.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_replace_once_aux:nnN #1#2#3
{
\group_begin:
- \tl_clear:N \l_regex_every_match_tl
- \regex_replacement:n {#3}
- \exp_args:Nno #1 {#2} #4
- \if_meaning:w \c_true_bool \g_regex_success_bool
+ \regex_single_match:
+ #1
+ \regex_replacement:n {#2}
+ \exp_args:No \regex_match:n { #3 }
+ \if_meaning:w \c_false_bool \g_regex_success_bool
+ \group_end:
+ \else:
\regex_extract:
- \int_set:Nn \l_regex_internal_a_int
- { \regex_nesting:n { \l_regex_submatch_start_int } }
- \if_num:w \l_regex_internal_a_int = \c_zero
- \else:
- \msg_kernel_error:nnx { regex } { replace-unbalanced }
- { \l_regex_internal_a_int }
- \fi:
+ \int_set:Nn \l_regex_balance_int
+ {
+ \regex_replacement_balance_one_match:n
+ { \l_regex_zeroth_submatch_int }
+ }
\tl_set:Nx \l_regex_internal_a_tl
{
- \if_num:w \l_regex_internal_a_int < \c_zero
- \prg_replicate:nn { - \l_regex_internal_a_int }
- { \exp_not:n { { \if_false: } \fi: } }
- \fi:
- \regex_toks_range:nn
- { \l_regex_min_index_int }
- { \tex_skip:D \l_regex_submatch_start_int }
- \regex_replacement_tl:n { \l_regex_submatch_start_int }
- \regex_toks_range:nn
- { \etex_gluestretch:D \tex_skip:D \l_regex_submatch_start_int }
- { \l_regex_max_index_int }
- \if_num:w \l_regex_internal_a_int > \c_zero
- \prg_replicate:nn { \l_regex_internal_a_int }
- { \exp_not:n { \if_false: { \fi: } } }
- \fi:
+ \regex_replacement_do_one_match:n { \l_regex_zeroth_submatch_int }
+ \regex_query_range:nn
+ { \etex_gluestretch:D \tex_skip:D \l_regex_zeroth_submatch_int }
+ { \l_regex_max_pos_int }
}
- \tl_set:Nx \l_regex_internal_a_tl { \l_regex_internal_a_tl }
- \exp_args:NNNo \group_end:
- \tl_set:Nn #4 \l_regex_internal_a_tl
- \else:
- \group_end:
+ \regex_group_end_replace:N #3
\fi:
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_replace_all_aux:NnnN}
-% For every match, extract submatches, and add the part before
-% the beginning of the match, as well as the replacement,
-% to the result. After the last match, extract the end
-% of the token list, and add it to the replaced token list.
+% \begin{macro}[aux]{\regex_replace_all_aux:nnN}
+% Match multiple times, and for every match, extract submatches and
+% additionally store the position at which the match attempt started
+% (as the shrink component of a \tn{skip} register). The \tn{skip}
+% registers from \cs{l_regex_max_state_int} to
+% \cs{l_regex_submatch_int} hold information about submatches of every
+% match in order; each match corresponds to
+% \cs{l_regex_capturing_group_int} consecutive \tn{skip} registers.
+% Compute the brace balance corresponding to doing all the
+% replacements: this is the sum of brace balances for replacing each
+% match. Join together the replacement texts for each match (including
+% the part of the query before the match), and the end of the query.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_replace_all_aux:NnnN #1#2#3#4
+\cs_new_protected:Npn \regex_replace_all_aux:nnN #1#2#3
{
\group_begin:
- \tl_set:Nn \l_regex_every_match_tl
- {
- \regex_extract:
- \tex_skip:D \l_regex_submatch_start_int
- = \tex_the:D \tex_skip:D \l_regex_submatch_start_int
- minus \l_regex_start_index_int sp \scan_stop:
- \regex_match_once:
- }
- \regex_replacement:n {#3}
- \exp_args:Nno #1 {#2} #4
- \int_set:Nn \l_regex_internal_a_int
+ \regex_multi_match:n { \regex_extract: }
+ #1
+ \regex_replacement:n {#2}
+ \exp_args:No \regex_match:n {#3}
+ \int_set:Nn \l_regex_balance_int
{
0
\prg_stepwise_function:nnnN
- \l_regex_max_state_int
+ { \c_two * \l_regex_max_state_int }
\l_regex_capturing_group_int
{ \l_regex_submatch_int - \c_one }
- \regex_nesting:n
+ \regex_replacement_balance_one_match:n
}
- \if_num:w \l_regex_internal_a_int = \c_zero
- \else:
- \msg_kernel_error:nnx { regex } { replace-unbalanced }
- { \l_regex_internal_a_int }
- \fi:
\tl_set:Nx \l_regex_internal_a_tl
{
- \if_num:w \l_regex_internal_a_int < \c_zero
- \prg_replicate:nn { - \l_regex_internal_a_int }
- { \exp_not:n { { \if_false: } \fi: } }
- \fi:
\prg_stepwise_function:nnnN
- \l_regex_max_state_int
+ { \c_two * \l_regex_max_state_int }
\l_regex_capturing_group_int
{ \l_regex_submatch_int - \c_one }
- \regex_replace_all_aux:n
- \regex_toks_range:nn
- \l_regex_start_index_int \l_regex_max_index_int
- \if_num:w \l_regex_internal_a_int > \c_zero
- \prg_replicate:nn { \l_regex_internal_a_int }
- { \exp_not:n { \if_false: { \fi: } } }
- \fi:
+ \regex_replacement_do_one_match:n
+ \regex_query_range:nn
+ \l_regex_start_pos_int \l_regex_max_pos_int
}
- \tl_set:Nx \l_regex_internal_a_tl { \l_regex_internal_a_tl }
- \exp_args:NNNo \group_end:
- \tl_set:Nn #4 \l_regex_internal_a_tl
+ \regex_group_end_replace:N #3
}
-\cs_new:Npn \regex_replace_all_aux:n #1
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_group_end_replace:N}
+% If the brace balance is not $0$, raise an error. Then set the user's
+% variable |#1| to the \texttt{x}-expansion of
+% \cs{l_regex_internal_a_tl}, adding the appropriate braces to produce
+% a balanced result. And end the group.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \regex_group_end_replace:N #1
{
- \regex_toks_range:nn
- { \etex_glueshrink:D \tex_skip:D #1 } { \tex_skip:D #1 }
- \regex_replacement_tl:n {#1}
+ \if_num:w \l_regex_balance_int = \c_zero
+ \else:
+ \msg_kernel_error:nnxxx { regex } { result-unbalanced }
+ { replacing }
+ { \int_max:nn { - \l_regex_balance_int } { \c_zero } }
+ { \int_max:nn { \l_regex_balance_int } { \c_zero } }
+ \fi:
+ \use:x
+ {
+ \group_end:
+ \tl_set:Nn \exp_not:N #1
+ {
+ \if_int_compare:w \l_regex_balance_int < \c_zero
+ \prg_replicate:nn { - \l_regex_balance_int }
+ { { \if_false: } \fi: }
+ \fi:
+ \l_regex_internal_a_tl
+ \if_int_compare:w \l_regex_balance_int > \c_zero
+ \prg_replicate:nn { \l_regex_balance_int }
+ { \if_false: { \fi: } }
+ \fi:
+ }
+ }
}
% \end{macrocode}
% \end{macro}
%
+% \subsubsection{Storing and showing compiled patterns}
+%
% \subsection{Messages}
%
% Messages for the preparsing phase.
@@ -5054,6 +5675,17 @@
}
% \end{macrocode}
%
+% Invalid quantifier.
+% \begin{macrocode}
+\msg_kernel_new:nnnn { regex } { invalid-quantifier }
+ { Braced~quantifier~'#1'~may~not~be~followed~by~'#2'. }
+ {
+ The~character~'#2'~is~invalid~in~the~braced~quantifier~'#1'.~
+ The~only~valid~quantifiers~are~'*',~'?',~'+',~'{<int>}',~
+ '{<min>,}'~and~'{<min>,<max>}',~followed~or~not~by~'?'.
+ }
+% \end{macrocode}
+%
% Messages for missing or extra closing brackets and parentheses, with
% some fancy singular/plural handling for the case of parentheses.
% \begin{macrocode}
@@ -5081,21 +5713,31 @@
}
% \end{macrocode}
%
-% Sometimes escaped alphanumerics are not allowed everywhere.
+% Some escaped alphanumerics are not allowed everywhere.
% \begin{macrocode}
-\msg_kernel_new:nnnn { regex } { class-bad-escape }
- { Invalid~escape~\c_backslash_str #1~in~character~class. }
+\msg_kernel_new:nnnn { regex } { bad-escape }
{
- The~escape~sequence~\iow_char:N\\#1~may~not~appear~within~
- a~character~class.~For~instance,~assertions~(which~match~zero~
- characters)~would~not~make~sense~there.
+ Invalid~escape~\c_backslash_str #1~
+ \regex_if_in_cs:TF { within~a~control~sequence. }
+ {
+ \regex_if_in_class:TF
+ { in~a~character~class. }
+ { following~a~category~test. }
+ }
}
-\msg_kernel_new:nnnn { regex } { catcode-bad-escape }
- { Invalid~escape~\c_backslash_str #1~following~category~test. }
{
- The~escape~sequence~\iow_char:N\\#1~may~not~appear~following~
- a~category~test~such~as~\iow_char:N\\cL~or~
- \iow_char:N\\c[\iow_char:N\^BE].
+ The~escape~sequence~\iow_char:N\\#1~may~not~appear~
+ \regex_if_in_cs:TF
+ {
+ within~a~control~sequence~test~introduced~by~
+ \iow_char:N\\c\iow_char:N\{.
+ }
+ {
+ \regex_if_in_class:TF
+ { within~a~character~class~ }
+ { following~a~category~test~such~as~\iow_char:N\\cL~ }
+ because~it~does~not~match~exactly~one~character.
+ }
}
% \end{macrocode}
%
@@ -5141,7 +5783,7 @@
regular~expression,~but~the~closing~']'~is~not~present.
}
\msg_kernel_new:nnnn { regex } { c-missing-category }
- { Invalid~character~'#1'~following~\iow_char\\c~escape. }
+ { Invalid~character~'#1'~following~\iow_char:N\\c~escape. }
{
In~regular~expressions,~the~\iow_char:N\\c~escape~sequence~
may~only~be~followed~by~a~left~brace,~a~left~bracket,~or~a~
@@ -5165,91 +5807,155 @@
}
% \end{macrocode}
%
-% In various cases, the result of a \pkg{l3regex} operation can leave us
-% with an unbalanced token list, which we must re-balance by adding
-% begin-group or end-group character tokens.
+% Errors when encountering the \textsc{posix} syntax |[:...:]|.
% \begin{macrocode}
-\msg_kernel_new:nnnn { regex } { sequence-unbalanced }
+\msg_kernel_new:nnnn { regex } { posix-unsupported }
+ { POSIX~collating~element~'[#1 ~ #1]'~not~supported. }
{
- Missing~
- \flag_if_raised:nTF { regex_end }
- {
- left~
- \flag_if_raised:nTF { regex_begin }
- { and~right~braces } { brace }
- }
- { right~brace }
- \ inserted~in~extracted~match.
+ The~[.foo.]~and~[=bar=]~syntaxes~have~a~special~meaning~in~POSIX~
+ regular~expressions.~This~is~not~supported~by~LaTeX.~Maybe~you~
+ forgot~to~escape~a~left~bracket~in~a~character~class?
}
+\msg_kernel_new:nnnn { regex } { posix-unknown }
+ { POSIX~class~[:#1:]~unknown. }
{
- LaTeX~was~asked~to~extract~submatches~or~split~a~token~list~
- according~to~a~given~regular~expression,~but~some~of~the~resulting~
- items~were~not~balanced.
+ [:#1:]~is~not~among~the~known~POSIX~classes~
+ [:alnum:],~[:alpha:],~[:ascii:],~[:blank:],~
+ [:cntrl:],~[:digit:],~[:graph:],~[:lower:],~
+ [:print:],~[:punct:],~[:space:],~[:upper:],~
+ [:word:],~and~[:xdigit:].
}
-\msg_kernel_new:nnnn { regex } { replace-unbalanced }
- { The~result~of~a~replacement~does~not~have~balanced~braces. }
+\msg_kernel_new:nnnn { regex } { posix-missing-close }
+ { Missing~closing~':]'~for~POSIX~class. }
+ { The~POSIX~syntax~'#1'~must~be~followed~by~':]',~not~'#2'. }
+% \end{macrocode}
+%
+% In various cases, the result of a \pkg{l3regex} operation can leave us
+% with an unbalanced token list, which we must re-balance by adding
+% begin-group or end-group character tokens.
+% \begin{macrocode}
+\msg_kernel_new:nnnn { regex } { result-unbalanced }
+ { Missing~brace~inserted~when~#1. }
{
- LaTeX~was~asked~to~do~some~regular~expression~replacement,~
+ LaTeX~was~asked~to~do~some~regular~expression~operation,~
and~the~resulting~token~list~would~not~have~the~same~number~
- of~begin-group~and~end-group~tokens. \\ \\
- \ \ \ \
- \prg_case_int:nnn {#1}
- {
- { -1 } { A~left~brace~was }
- { 1 } { A~right~brace~was }
- }
- {
- \int_abs:n {#1} ~
- \int_compare:nNnTF {#1} < \c_zero { left } { right } ~
- braces ~ were
- }
- \ inserted.
+ of~begin-group~and~end-group~tokens.~Braces~were~inserted:~
+ #2~left,~#3~right.
}
% \end{macrocode}
%
-% Messages related to NFA variables.
+% Error message for unknown options.
% \begin{macrocode}
-\msg_kernel_new:nnnn { regex } { not-nfa }
- { This~is~not~a~regular~expression~variable. }
+\msg_kernel_new:nnnn { regex } { unknown-option }
+ { Unknown~option~`#1'~for~regular~expressions. }
{
- LaTeX~was~expecting~`#1'~to~be~a~regular~expression~variable.\\
- This~control~sequence~is~not~a~regex~variable.~It's~current~meaning~
- is~\\\\
- \ \ \ \ \token_to_str:N #1 = \token_to_meaning:N #1 .
+ The~only~available~option~is~'case-insensitive',~toggled~by~
+ '(?i)'~and~'(?-i)'.
}
-\msg_kernel_new:nnn { regex } { nfa-misused }
- { Automaton~#1 used~incorrectly. }
% \end{macrocode}
%
+% Errors in the replacement text.
% \begin{macrocode}
-\msg_kernel_new:nnnn { regex } { c-command }
- { Misused~\iow_char:N\\c~or\iow_char:N\\C~command~in~a~#1. }
+\msg_kernel_new:nnnn { regex } { replacement-c }
+ { Misused~\iow_char:N\\c~command~in~a~replacement~text. }
{
- In~a~#1,~the~\iow_char:N\\C~escape~sequence~
+ In~a~replacement~text,~the~\iow_char:N\\c~escape~sequence~
can~be~followed~by~one~of~the~letters~ABCDELMOPSTU~
- or~a~brace~group,~not~by~'#2'.
- }
-\msg_kernel_new:nnnn { regex } { unknown-option }
- { Unknown~option~`#1'~for~regular~expressions. }
- {
- LaTeX~came~across~something~like~`(?#1)'~in~a~regular~expression,~
- but~the~option~`#1'~is~not~known.~It~will~be~ignored.
+ or~a~brace~group,~not~by~'#1'.
}
-\msg_kernel_new:nnnn { regex } { invalid-in-option }
- { Invalid~character~in~option~of~a~regular~expression. }
+\msg_kernel_new:nnnn { regex } { replacement-u }
+ { Misused~\iow_char:N\\u~command~in~a~replacement~text. }
{
- The~character~or~escape~sequence~`#1'~is~not~defined~
- as~an~option~within~regular~expressions.
+ In~a~replacement~text,~the~\iow_char:N\\u~escape~sequence~
+ must~be~~followed~by~a~brace~group~holding~the~name~of~the~
+ variable~to~use.
}
-\msg_kernel_new:nnnn { regex } { g-command }
+\msg_kernel_new:nnnn { regex } { replacement-g }
{ Missing~brace~for~the~\iow_char:N\\g~construction~in~a~replacement~text. }
{
In~the~replacement~text~for~a~regular~expression~search,~
submatches~are~represented~either~as~\iow_char:N \\g{dd..d},~
or~\\d,~where~`d'~are~single~digits.~Here,~a~brace~is~missing.
}
+\msg_kernel_new:nnnn { regex } { replacement-catcode-end }
+ {
+ Missing~character~for~the~\iow_char:N\\c<category><character>~
+ construction~in~a~replacement~text.
+ }
+ {
+ In~a~replacement~text,~the~\iow_char:N\\c~escape~sequence~
+ can~be~followed~by~one~of~the~letters~ABCDELMOPSTU~representing~
+ the~character~category.~Then,~a~character~must~follow.~LaTeX~
+ reached~the~end~of~the~replacement~when~looking~for~that.
+ }
+\msg_kernel_new:nnnn { regex } { replacement-null-space }
+ { TeX~cannot~build~a~space~token~with~character~code~0. }
+ {
+ You~asked~for~a~character~token~with~category~'space',~
+ and~character~code~0,~for~instance~through~
+ '\iow_char:N\\cS\iow_char:N\\x00'.~
+ This~specific~case~is~impossible~and~will~be~replaced~
+ by~a~normal~space.
+ }
+\msg_kernel_new:nnnn { regex } { replacement-missing-rbrace }
+ { Missing~right~brace~inserted~in~replacement~text. }
+ {
+ There~were~\int_use:N \l_regex_replacement_csnames_int \
+ missing~right~braces.
+ }
% \end{macrocode}
%
+% \begin{macro}[aux]{\regex_msg_repeated:nnN}
+% This is not technically a message, but seems related enough to go
+% there. The arguments are: |#1| is the minimum number of repetitions;
+% |#2| is the number of allowed extra repetitions ($-1$ for infinite
+% number), and |#3| tells us aboud lazyness.
+% \begin{macrocode}
+\cs_new:Npn \regex_msg_repeated:nnN #1#2#3
+ {
+ \str_if_eq:xxF { #1 #2 } { 1 0 }
+ {
+ , ~ repeated ~
+ \prg_case_int:nnn {#2}
+ {
+ { -1 } { #1~or~more~times,~\bool_if:NTF #3 { lazy } { greedy } }
+ { 0 } { #1~times }
+ }
+ {
+ between~#1~and~\int_eval:n {#1+#2}~times,~
+ \bool_if:NTF #3 { lazy } { greedy }
+ }
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \subsection{Code for tracing}
+%
+% The tracing code is still very experimental, and is meant to be used
+% with the \pkg{l3trace} package, currently in \texttt{l3trial}.
+%
+% \begin{macro}[int]{\regex_trace_states:n}
+% This function lists the contents of all states of the \textsc{nfa},
+% stored in \tn{toks} from $0$ to \cs{l_regex_max_state_int}
+% (excluded).
+% \begin{macrocode}
+%<*trace>
+\cs_new_protected:Npn \regex_trace_states:n #1
+ {
+ \prg_stepwise_inline:nnnn
+ \l_regex_min_state_int
+ \c_one
+ { \l_regex_max_state_int - 1 }
+ {
+ \trace:nnx { regex } { #1 }
+ { \iow_char:N \\toks ##1 = { \tex_the:D \tex_toks:D ##1 } }
+ }
+ }
+%</trace>
+% \end{macrocode}
+% \end{macro}
+%
% \begin{macrocode}
%</package>
% \end{macrocode}
@@ -5257,84 +5963,27 @@
% \end{implementation}
%
% \endinput
-%^^A
-%^^A \msg_new:nnn { regex } { 4 }
-%^^A { Numbers~out~of~order~in~\iow_char:N\{\iow_char\}~quantifier. }
-%^^A \msg_new:nnn { regex } { 44 } { Invalid~UTF-8~string }
-%^^A \msg_new:nnn { regex } { 46 }
-%^^A { Malformed~\iow_char:N\\P~or\iow_char:N\\p~sequence }
-%^^A \msg_new:nnn { regex } { 47 }
-%^^A { Unknown~property~after~\iow_char:N\\P~or\iow_char:N\\p }
-%^^A
%^^A NOT IMPLEMENTED
-%^^A \cx "control-x", where x is any ASCII character
-%^^A \C one byte, even in UTF-8 mode (best avoided)
%^^A \p{xx} a character with the xx property
%^^A \P{xx} a character without the xx property
+%^^A [[:xxx:]] positive POSIX named set
+%^^A [[:^xxx:]] negative POSIX named set
+%^^A (?=...) positive look ahead
+%^^A (?!...) negative look ahead
+%^^A (?<=...) positive look behind
+%^^A (?<!...) negative look behind
+%^^A (?<name>...) or (?'name'...) or (?P<name>...)
+%^^A named capturing group
%^^A \R a newline sequence
%^^A \X an extended Unicode sequence
-%^^A [[:xxx:]] positive POSIX named set
-%^^A [[:^xxx:]] negative POSIX named set
-%^^A ?+ 0 or 1, possessive
-%^^A *+ 0 or more, possessive
-%^^A ++ 1 or more, possessive
-%^^A {n,m}+ at least n, no more than m, possessive
-%^^A {n,}+ n or more, possessive
-%^^A \K reset start of match
-%^^A (?<name>...) named capturing group (Perl)
-%^^A (?'name'...) named capturing group (Perl)
-%^^A (?P<name>...) named capturing group (Python)
-%^^A (?:...) non-capturing group
-%^^A (?|...) non-capturing group; reset group numbers for
-%^^A capturing groups in each alternative
-%^^A (?>...) atomic, non-capturing group
-%^^A (?#....) comment (not nestable)
-%^^A (?i) caseless
-%^^A (?J) allow duplicate names
-%^^A (?m) multiline
-%^^A (?s) single line (dotall)
-%^^A (?U) default ungreedy (lazy)
-%^^A (?x) extended (ignore white space)
-%^^A (?-...) unset option(s)
-%^^A (*NO_START_OPT) no start-match optimization (PCRE_NO_START_OPTIMIZE)
-%^^A (*UTF8) set UTF-8 mode (PCRE_UTF8)
-%^^A (*UCP) set PCRE_UCP (use Unicode properties for \d etc)
-%^^A (?=...) positive look ahead
-%^^A (?!...) negative look ahead
-%^^A (?<=...) positive look behind
-%^^A (?<!...) negative look behind
-%^^A \n reference by number (can be ambiguous)
-%^^A \gn reference by number
-%^^A \g{n} reference by number
-%^^A \g{-n} relative reference by number
-%^^A \k<name> reference by name (Perl)
-%^^A \k'name' reference by name (Perl)
-%^^A \g{name} reference by name (Perl)
-%^^A \k{name} reference by name (.NET)
-%^^A (?P=name) reference by name (Python)
+%^^A (?C) or (?Cn) callout with data n
%^^A (?R) recurse whole pattern
-%^^A (?n) call subpattern by absolute number
-%^^A (?+n) call subpattern by relative number
-%^^A (?-n) call subpattern by relative number
-%^^A (?&name) call subpattern by name (Perl)
-%^^A (?P>name) call subpattern by name (Python)
-%^^A \g<name> call subpattern by name (Oniguruma)
-%^^A \g'name' call subpattern by name (Oniguruma)
-%^^A \g<n> call subpattern by absolute number (Oniguruma)
-%^^A \g'n' call subpattern by absolute number (Oniguruma)
-%^^A \g<+n> call subpattern by relative number (PCRE extension)
-%^^A \g'+n' call subpattern by relative number (PCRE extension)
-%^^A \g<-n> call subpattern by relative number (PCRE extension)
-%^^A \g'-n' call subpattern by relative number (PCRE extension)
-%^^A (?(n)... absolute reference condition
-%^^A (?(+n)... relative reference condition
-%^^A (?(-n)... relative reference condition
-%^^A (?(<name>)... named reference condition (Perl)
-%^^A (?('name')... named reference condition (Perl)
-%^^A (?(name)... named reference condition (PCRE)
-%^^A (?(R)... overall recursion condition
-%^^A (?(Rn)... specific group recursion condition
-%^^A (?(R&name)... specific recursion condition
+%^^A (?[+-]n) or \g<[+-]n> or (?&name) or (?P>name) or \g<name>
+%^^A call subpattern
+%^^A (?([+-]n)... or (?(<name>)...
+%^^A reference condition
+%^^A (?(R)... or (?(Rn)... or (?(R&name)...
+%^^A recursion condition
%^^A (?(DEFINE)... define subpattern for reference
%^^A (?(assert)... assertion condition
%^^A (*ACCEPT) force successful match
@@ -5343,12 +5992,21 @@
%^^A (*PRUNE) advance to next starting character
%^^A (*SKIP) advance start to current matching position
%^^A (*THEN) local failure, backtrack to next alternation
-%^^A (*CR) carriage return only
-%^^A (*LF) linefeed only
-%^^A (*CRLF) carriage return followed by linefeed
-%^^A (*ANYCRLF) all three of the above
-%^^A (*ANY) any Unicode newline sequence
-%^^A (*BSR_ANYCRLF) CR, LF, or CRLF
-%^^A (*BSR_UNICODE) any Unicode newline sequence
-%^^A (?C) callout
-%^^A (?Cn) callout with data n \ No newline at end of file
+%^^A (*CR) or (*LF) or (*CRLF) or (*ANYCRLF) or (*ANY)
+%^^A newline convention
+%^^A (*BSR_ANYCRLF) or (*BSR_UNICODE)
+%^^A change what \R matches.
+%^^A
+%^^A \cx "control-x", where x is any ASCII character
+%^^A \C one byte, even in UTF-8 mode (best avoided)
+%^^A + possessive quantifiers
+%^^A (?>...) atomic, non-capturing group
+%^^A (?#....) comment (not nestable)
+%^^A (?JmsUx) options (duplicate names; multiline; single line;
+%^^A ungreedy; extended)
+%^^A (*NO_START_OPT) no start-match optimization (PCRE_NO_START_OPTIMIZE)
+%^^A (*UTF8) set UTF-8 mode (PCRE_UTF8)
+%^^A (*UCP) set PCRE_UCP (use Unicode properties for \d etc)
+%^^A \n or \gn or \g{[-]n} or \g{name} or (?P=name)
+%^^A or \k<name> or \k'name' or \k{name}
+%^^A back-references
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx
index 3d73ed10b8f..ef2bc9b5460 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx
@@ -35,7 +35,7 @@
%
%<*driver|package>
\RequirePackage{expl3}
-\GetIdInfo$Id: l3str.dtx 3257 2012-01-29 13:37:03Z bruno $
+\GetIdInfo$Id: l3str.dtx 3460 2012-02-26 12:16:41Z will $
{L3 Experimental Strings}
%</driver|package>
%<*driver>
@@ -189,7 +189,7 @@
% var} are not automatically converted to a string.
% \end{function}
%
-% \section{Characters given by their position}
+% \section{Accessing the contents of a string}
%
% \begin{function}[EXP]
% {\str_length:N, \str_length:n, \str_length_ignore_spaces:n}
@@ -291,8 +291,8 @@
% \str_if_eq:nV, \str_if_eq:VV, \str_if_eq:xx
% }
% \begin{syntax}
-% \cs{str_if_eq_p:nn} \Arg{tl1} \Arg{tl2}
-% \cs{str_if_eq:nnTF} \Arg{tl1} \Arg{tl2} \Arg{true code} \Arg{false code}
+% \cs{str_if_eq_p:nn} \Arg{tl_1} \Arg{tl_2}
+% \cs{str_if_eq:nnTF} \Arg{tl_1} \Arg{tl_2} \Arg{true code} \Arg{false code}
% \end{syntax}
% Compares the two \meta{token lists} on a character by character
% basis, and is \texttt{true} if the two lists contain the same
@@ -305,6 +305,15 @@
% expansion).
% \end{function}
%
+% \section{Viewing strings}
+%
+% \begin{function}{\str_show:N, \str_show:c, \str_show:n}
+% \begin{syntax}
+% \cs{str_show:N} \meta{tl~var}
+% \end{syntax}
+% Displays the content of the \meta{str~var} on the terminal.
+% \end{function}
+%
% \section{Encoding functions}
%
% Traditionally, string encodings only specify how strings of characters
@@ -571,7 +580,7 @@
% \begin{macrocode}
\ProvidesExplPackage
{\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
-\RequirePackage{l3tl-analysis,l3flag}
+\RequirePackage{l3tl-analysis,l3tl-build,l3flag}
% \end{macrocode}
%
% The following string-related functions are currently defined in
@@ -601,6 +610,14 @@
%
% \subsubsection{Assigning strings}
%
+% \begin{macro}{\str_new:N, \str_new:c}
+% A string is simply a token list.
+% \begin{macrocode}
+\cs_new_eq:NN \str_new:N \tl_new:N
+\cs_generate_variant:Nn \str_new:N { c }
+% \end{macrocode}
+% \end{macro}
+%
% \begin{macro}
% {
% \str_set:Nn, \str_set:Nx,
@@ -1337,31 +1354,38 @@
% \end{macrocode}
% \end{macro}
%
+% \subsection{Viewing strings}
+%
+% \begin{macro}{\str_show:n, \str_show:N, \str_show:c}
+% Displays a string on the terminal.
+% \begin{macrocode}
+\cs_new_eq:NN \str_show:n \tl_show:n
+\cs_new_eq:NN \str_show:N \tl_show:N
+\cs_generate_variant:Nn \str_show:N { c }
+% \end{macrocode}
+% \end{macro}
+%
% \subsection{Conversions}
%
% \subsubsection{Producing one byte or character}
%
% \begin{variable}{\c_str_byte_0_tl, \c_str_byte_1_tl, \c_str_byte_255_tl}
% \begin{variable}{\c_str_byte_-1_tl}
-% \begin{variable}{\c_str_positive_bytes_tl}
% For each integer $N$ in the range $[0,255]$, we create a constant
% token list which holds three character tokens with category code
% other: the character with character code $N$, followed by the
% representation of $N$ as two hexadecimal digits. The value $-1$ is
% given a default token list which ensures that later functions give
-% an empty result for the input $-1$. Simultaneously, we build a list
-% of all bytes (from which we remove the null byte) in
-% \cs{c_str_positive_bytes_tl}.
+% an empty result for the input $-1$.
% \begin{macrocode}
\group_begin:
- \tl_clear:N \l_str_internal_tl
\char_set_catcode_other:n { \c_zero }
- \tl_gset:Nx \g_str_result_tl { \tl_to_str:n { 0123456789ABCDEF } }
- \exp_args:No \tl_map_inline:nn { \g_str_result_tl " }
+ \tl_set:Nx \l_str_internal_tl { \tl_to_str:n { 0123456789ABCDEF } }
+ \exp_args:No \tl_map_inline:nn { \l_str_internal_tl " }
{ \char_set_lccode:nn {`#1} { \c_zero } }
- \tl_map_inline:Nn \g_str_result_tl
+ \tl_map_inline:Nn \l_str_internal_tl
{
- \tl_map_inline:Nn \g_str_result_tl
+ \tl_map_inline:Nn \l_str_internal_tl
{
\char_set_lccode:nn { \c_zero } {"#1##1}
\tl_to_lowercase:n
@@ -1369,18 +1393,14 @@
\tl_const:cx
{ c_str_byte_ \int_eval:n {"#1##1} _tl }
{ ^^@ #1 ##1 }
- \tl_put_right:Nn \l_str_internal_tl { ^^@ }
}
}
}
- \tl_const:Nx \c_str_positive_bytes_tl
- { \exp_after:wN \use_none:n \l_str_internal_tl }
\group_end:
\tl_const:cn { c_str_byte_-1_tl } { { } \use_none:n { } }
% \end{macrocode}
% \end{variable}
% \end{variable}
-% \end{variable}
%
% \begin{macro}[int, EXP]{\str_output_byte:n}
% \begin{macro}[int, EXP]{\str_output_byte:w}
@@ -1943,68 +1963,34 @@
% \begin{macro}[aux, rEXP]{\str_encode_native_filter:N}
% In Unicode-aware engines, since building particular characters
% cannot be done expandably in \TeX{}, we cannot hope to get a
-% linear-time function. However, we get quite close by building the
-% result one block of $255$ characters at a time. Set the lowercase
-% code of the \textsc{ascii} character $1$ to the first character code
-% in the block, \emph{etc.}, then lowercase the contents of
-% \cs{c_str_positive_bytes_tl}. Unfortunately, this is complicated by
-% the fact that \TeX{} won't lowercase a character to the null
-% character, thus the need for filtering: whenever the lowercase code
-% of a given character is zero, it is replaced \enquote{by hand} by
-% the null character. Unicode-aware engines will never incur an
-% overflow because the internal string is guaranteed to only contain
-% code points in $[0,\hexnum{10FFFF}]$.
+% linear-time function. However, we get quite close using the
+% \pkg{l3tl-build} module, which abuses \tn{toks} to reach an almost
+% linear time. Use the standard lowercase trick to produce an
+% arbitrary character from the null character, and add that character
+% to the end of the token list being built. At the end of the loop,
+% put the token list together with \cs{tl_build_end:}. Note that we
+% use an \texttt{x}-expanding assignment because it is slightly
+% faster. Unicode-aware engines will never incur an overflow because
+% the internal string is guaranteed to only contain code points in
+% $[0,\hexnum{10FFFF}]$.
% \begin{macrocode}
{
\cs_new_protected_nopar:Npn \str_convert_encode_:
{
- \tex_lccode:D \c_zero \c_zero
- \int_zero:N \l_str_internal_int
- \tl_set_eq:NN \l_str_internal_tl \g_str_result_tl
- \tl_gclear:N \g_str_result_tl
- \exp_after:wN \str_encode_native_loop:w
- \l_str_internal_tl \s_tl { \q_stop \prg_map_break: } \s_tl
- \prg_break_point:n { \str_encode_native_flush: }
+ \int_zero:N \l_tl_build_offset_int
+ \tl_gset_build_x:Nw \g_str_result_tl
+ \exp_after:wN \str_encode_native_loop:w
+ \g_str_result_tl \s_tl { \q_stop \prg_map_break: } \s_tl
+ \prg_break_point:n { }
+ \tl_build_end:
}
\cs_new_protected:Npn \str_encode_native_loop:w #1 \s_tl #2 \s_tl
{
\use_none_delimit_by_q_stop:w #2 \q_stop
- \if_num:w \l_str_internal_int = \c_two_hundred_fifty_five
- \str_encode_native_flush:
- \l_str_internal_int \c_zero
- \fi:
- \tex_advance:D \l_str_internal_int \c_one
\tex_lccode:D \l_str_internal_int \int_eval:w #2 \int_eval_end:
+ \tl_to_lowercase:n { \tl_build_one:n { ^^@ } }
\str_encode_native_loop:w
}
- \cs_new_protected_nopar:Npn \str_encode_native_flush:
- {
- \use:x
- {
- \tl_to_lowercase:n
- {
- \tl_gput_right:Nx \exp_not:N \g_str_result_tl
- {
- \exp_after:wN \str_encode_native_filter:N
- \c_str_positive_bytes_tl
- { ? = \c_zero \fi: \prg_map_break: }
- \prg_break_point:n { }
- }
- }
- }
- }
- \cs_new:Npn \str_encode_native_filter:N #1
- {
- \if_num:w `#1 > \l_str_internal_int
- \exp_after:wN \prg_map_break:
- \fi:
- \if_num:w \tex_lccode:D `#1 = \c_zero
- ^^@
- \else:
- #1
- \fi:
- \str_encode_native_filter:N
- }
}
% \end{macrocode}
% \end{macro}
@@ -3044,7 +3030,8 @@
% character;
% \item $[\hexnum{E000}, \hexnum{FFFF}]$: converted to two bytes;
% \item $[\hexnum{10000}, \hexnum{10FFFF}]$: converted to a pair of
-% surrogates, each two bytes.
+% surrogates, each two bytes. The magic \hexnum{D7C0} is
+% $\hexnum{D800}-\hexnum{10000}/\hexnum{400}$.
% \end{itemize}
% For the duration of this operation, \cs{str_tmp:w} is defined as a
% function to convert a number in the range $[0, \hexnum{FFFF}]$ to a
@@ -3083,7 +3070,7 @@
\str_tmp:w {#1}
\fi:
\else:
- \exp_args:Nf \str_tmp:w { \int_div_truncate:nn {#1} {"400} + "D800 }
+ \exp_args:Nf \str_tmp:w { \int_div_truncate:nn {#1} {"400} + "D7C0 }
\exp_args:Nf \str_tmp:w { \int_mod:nn {#1} {"400} + "DC00 }
\fi:
\fi:
@@ -3257,6 +3244,10 @@
% looping with \cs{str_decode_utf_xvi_pair:NN}. Otherwise, of course,
% complain about the missing surrogate.
%
+% The magic number \hexnum{D7F7} is such that
+% $\hexnum{D7F7}*\hexnum{400} = \hexnum{D800}*\hexnum{400} +
+% \hexnum{DC00} - \hexnum{10000}$.
+%
% Every time we read a pair of bytes, we test for the end-marker
% \cs{q_nil}. When reaching the end, we additionally check that the
% string had an even length. Also, if the end is reached when
@@ -3292,8 +3283,8 @@
#1 #2 #4 #5 \s_tl
\int_eval:n
{
- ( "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 - "D800 ) * "400
- + "100 * \str_tmp:w #4#5 + \str_tmp:w #5#4 - "DC00
+ ( "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 - "D7F7 ) * "400
+ + "100 * \str_tmp:w #4#5 + \str_tmp:w #5#4
}
\s_tl
\exp_after:wN \use_i:nnn
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins
index 676a39bea7b..7dc0c00357d 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins
@@ -39,7 +39,8 @@ Do not distribute a modified version of this file.
\generate{\file{l3str.sty} {\from{l3str.dtx} {package}}}
\generate{\file{l3regex.sty} {\from{l3regex.dtx} {package}}}
\generate{\file{l3tl-analysis.sty} {\from{l3tl-analysis.dtx} {package}}}
-%\generate{\file{l3regex-trace.sty} {\from{l3regex.dtx} {package,trace}}}
+\generate{\file{l3tl-build.sty} {\from{l3tl-build.dtx} {package}}}
+\generate{\file{l3regex-trace.sty} {\from{l3regex.dtx} {package,trace}}}
% Escapings.
\generate{%
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx
index f9294dbb87c..89ce9c22f3d 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx
@@ -82,8 +82,8 @@
% \end{syntax}
% Displays to the terminal the detailed decomposition of the
% \meta{token list} into tokens, showing the category code of each
-% character token, and the meaning of control sequences and active
-% characters.
+% character token, the meaning of control sequences and active
+% characters, and the value of registers.
% \end{function}
%
% \subsection{Internal functions}
@@ -203,6 +203,14 @@
% \end{macrocode}
% \end{variable}
%
+% \begin{variable}{\l_tl_analysis_internal_tl}
+% This token list variable is used to hand the argument of
+% \cs{tl_show_analysis:n} to \cs{tl_show_analysis:N}.
+% \begin{macrocode}
+\tl_new:N \l_tl_analysis_internal_tl
+% \end{macrocode}
+% \end{variable}
+%
% \begin{variable}{\l_tl_analysis_token}
% \begin{variable}{\l_tl_analysis_char_token}
% The tokens in the token list are probed with the \TeX{} primitive
@@ -953,8 +961,8 @@
}
\cs_new_protected:Npn \tl_show_analysis:n #1
{
- \tl_set:Nn \l_tl_tmpa_tl {#1}
- \tl_show_analysis:N \l_tl_tmpa_tl
+ \tl_set:Nn \l_tl_analysis_internal_tl {#1}
+ \tl_show_analysis:N \l_tl_analysis_internal_tl
}
% \end{macrocode}
% \end{macro}
@@ -1004,10 +1012,31 @@
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}[EXP]{\tl_show_analysis_value:N}
+% This expands to the value of |#1| if it has any.
+% \begin{macrocode}
+\cs_new:Npn \tl_show_analysis_value:N #1
+ {
+ \token_if_expandable:NF #1
+ {
+ \token_if_chardef:NTF #1 \prg_map_break: { }
+ \token_if_mathchardef:NTF #1 \prg_map_break: { }
+ \token_if_dim_register:NTF #1 \prg_map_break: { }
+ \token_if_int_register:NTF #1 \prg_map_break: { }
+ \token_if_skip_register:NTF #1 \prg_map_break: { }
+ \token_if_toks_register:NTF #1 \prg_map_break: { }
+ \use_none:nnnn
+ \prg_break_point:n { }
+ \use:n { = \tex_the:D #1 }
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+%
% \begin{macro}[aux, rEXP]{\tl_show_analysis_cs:n}
% \begin{macro}[aux, rEXP]{\tl_show_analysis_active:n}
% \begin{macro}[aux, rEXP]{\tl_show_analysis_long:nn}
-% \begin{macro}[aux, rEXP]{\tl_show_analysis_long_aux:nnn}
+% \begin{macro}[aux, rEXP]{\tl_show_analysis_long_aux:nnnn}
% Control sequences and active characters are printed in the same way,
% making sure not to go beyond the \cs{l_iow_line_length_int}. In case
% of an overflow, we replace the last characters by
@@ -1019,25 +1048,27 @@
{ \exp_args:No \tl_show_analysis_long:nn {#1} { active~character= } }
\cs_new:Npn \tl_show_analysis_long:nn #1
{
- \exp_args:Noo \tl_show_analysis_long_aux:nnn
+ \tl_show_analysis_long_aux:oofn
{ \token_to_str:N #1 }
{ \token_to_meaning:N #1 }
+ { \tl_show_analysis_value:N #1 }
}
-\cs_new:Npn \tl_show_analysis_long_aux:nnn #1#2#3
+\cs_new:Npn \tl_show_analysis_long_aux:nnnn #1#2#3#4
{
\int_compare:nNnTF
- { \str_length:n { #1 ~ ( #3 #2 ) } }
+ { \str_length:n { #1 ~ ( #4 #2 #3 ) } }
> { \l_iow_line_length_int - \c_three }
{
- \str_substr:nnn { #1 ~ ( #3 #2 ) } \c_zero
+ \str_substr:nnn { #1 ~ ( #4 #2 #3 ) } \c_zero
{
\l_iow_line_length_int - \c_three
- \str_length:N \c_tl_show_analysis_etc_str
}
\c_tl_show_analysis_etc_str
}
- { #1 ~ ( #3 #2 ) }
+ { #1 ~ ( #4 #2 #3 ) }
}
+\cs_generate_variant:Nn \tl_show_analysis_long_aux:nnnn { oof }
% \end{macrocode}
% \end{macro}
% \end{macro}
@@ -1060,7 +1091,8 @@
\msg_kernel_new:nnn { tl-analysis } { show }
{
The~token~list~
- \str_if_eq:nnF {#1} { \l_tl_tmpa_tl } { \token_to_str:N #1 ~ }
+ \str_if_eq:nnF {#1} { \l_tl_analysis_internal_tl }
+ { \token_to_str:N #1 ~ }
\tl_if_empty:NTF #1
{ is~empty }
{ contains~the~tokens: }
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-build.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-build.dtx
new file mode 100644
index 00000000000..d3fa2d4e366
--- /dev/null
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-build.dtx
@@ -0,0 +1,301 @@
+% \iffalse meta-comment
+%
+%% File: l3tl-build.dtx Copyright (C) 2011-2012 The LaTeX3 Project
+%%
+%% It may be distributed and/or modified under the conditions of the
+%% LaTeX Project Public License (LPPL), either version 1.3c of this
+%% license or (at your option) any later version. The latest version
+%% of this license is in the file
+%%
+%% http://www.latex-project.org/lppl.txt
+%%
+%% This file is part of the "l3experimental bundle" (The Work in LPPL)
+%% and all files in that bundle must be distributed together.
+%%
+%% The released version of this bundle is available from CTAN.
+%%
+%% -----------------------------------------------------------------------
+%%
+%% The development version of the bundle can be found at
+%%
+%% http://www.latex-project.org/svnroot/experimental/trunk/
+%%
+%% for those people who are interested.
+%%
+%%%%%%%%%%%
+%% NOTE: %%
+%%%%%%%%%%%
+%%
+%% Snapshots taken from the repository represent work in progress and may
+%% not work or may contain conflicting material! We therefore ask
+%% people _not_ to put them into distributions, archives, etc. without
+%% prior consultation with the LaTeX3 Project.
+%%
+%% -----------------------------------------------------------------------
+%
+%<*driver|package>
+\RequirePackage{expl3}
+\GetIdInfo$Id: l3tl-build.dtx 3039 2011-12-08 09:22:35Z bruno $
+ {L3 Experimental Build of Token lists}
+%</driver|package>
+%<*driver>
+\documentclass[full]{l3doc}
+\usepackage{amsmath}
+\begin{document}
+ \DocInput{\jobname.dtx}
+\end{document}
+%</driver>
+% \fi
+%
+%
+% \title{^^A
+% The \textsf{l3tl-build} package: building token lists^^A
+% \thanks{This file describes v\ExplFileVersion,
+% last revised \ExplFileDate.}^^A
+% }
+%
+% \author{^^A
+% The \LaTeX3 Project\thanks
+% {^^A
+% E-mail:
+% \href{mailto:latex-team@latex-project.org}
+% {latex-team@latex-project.org}^^A
+% }^^A
+% }
+%
+% \date{Released \ExplFileDate}
+%
+% \maketitle
+%
+% \begin{documentation}
+%
+% \section{\pkg{l3tl-build} documentation}
+%
+% This module provides no user function.
+%
+% There are two main ways of building token lists from individual
+% tokens. Either in one go within an \texttt{x}-expanding assignment, or
+% by repeatedly using \cs{tl_put_right:Nn}. The first method takes a
+% linear time, but only allows expandable operations. The second method
+% takes a time quadratic in the length of the token list, but allows
+% expandable and non-expandable operations.
+%
+% The goal of this module is to provide functions to build a token list
+% piece by piece in linear time, while allowing non-expandable
+% operations. This is achieved by abusing \tn{toks}: adding some tokens
+% to the token list is done by storing them in a free token register
+% (time $O(1)$ for each such operation). Those token registers are only
+% put together at the end, within an \texttt{x}-expanding assignment,
+% which takes a linear time.\footnote{If we run out of token registers,
+% then the currently filled-up \tn{toks} are put together in a
+% temporary token list, and cleared, and we ultimately use
+% \cs{tl_put_right:Nx} to put those chunks together. Hence the true
+% asymptotic is quadratic, with a very small constant.} Of course,
+% all this must be done in a group: we can't go and clobber the values
+% of legitimate \tn{toks} used by \LaTeXe{}.
+%
+% Since none of the current applications need the ability to insert
+% material on the left of the token list, I have not implemented
+% that. This could be done for instance by using odd-numbered \tn{toks}
+% for the left part, and even-numbered \tn{toks} for the right part.
+%
+% \subsection{Internal functions}
+%
+% \begin{function}
+% {
+% \tl_set_build:Nw, \tl_gset_build:Nw,
+% \tl_set_build_x:Nw, \tl_gset_build_x:Nw
+% }
+% \begin{syntax}
+% \cs{tl_set_build:Nw} \meta{tl~var} \texttt{\ldots{}}
+% \cs{tl_build_one:n} \Arg{tokens_1} \texttt{\ldots{}}
+% \cs{tl_build_one:n} \Arg{tokens_2} \texttt{\ldots{}}
+% \ldots{}
+% \cs{tl_build_end:}
+% \end{syntax}
+% Defines the \meta{tl~var} to contain the contens of \meta{tokens1}
+% followed by \meta{tokens2}, \emph{etc.} This is built in such a way
+% to be more efficient than repeatedly using \cs{tl_put_right:Nn}. The
+% code in \enquote{\texttt{\ldots{}}} does not need to be
+% expandable. The commands \cs{tl_set_build:Nw} and \cs{tl_build_end:}
+% start and end a group. The assignment to the \meta{tl~var} occurs
+% just after the end of that group, using \cs{tl_set:Nn},
+% \cs{tl_gset:Nn}, \cs{tl_set:Nx}, or \cs{tl_gset:Nx}.
+% \end{function}
+%
+% \begin{function}{\tl_build_one:n, \tl_build_one:o, \tl_build_one:x}
+% \begin{syntax}
+% \cs{tl_build_one:n} \Arg{tokens}
+% \end{syntax}
+% This function may only be used within the scope of a
+% \cs{tl_set_build:Nw} function. It adds the \meta{tokens} on the
+% right of the current token list.
+% \end{function}
+%
+% \begin{function}{\tl_build_end:}
+% Ends the scope started by \cs{tl_set_build:Nw}, and performs the
+% relevant assignment.
+% \end{function}
+%
+% \end{documentation}
+%
+% \begin{implementation}
+%
+% \section{\pkg{l3tl-build} implementation}
+%
+% \begin{macrocode}
+%<*initex|package>
+% \end{macrocode}
+%
+% \begin{macrocode}
+\ProvidesExplPackage
+ {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
+% \end{macrocode}
+%
+% \subsection{Variables and helper functions}
+%
+% \begin{variable}{\l_tl_build_start_index_int, \l_tl_build_index_int}
+% Integers pointing to the starting index (currently always starts at
+% zero), and the current index. The corresponding \tn{toks} are
+% accessed directly by number.
+% \begin{macrocode}
+\int_new:N \l_tl_build_start_index_int
+\int_new:N \l_tl_build_index_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_tl_build_result_tl}
+% The resulting token list is normally built in one go by unpacking
+% all \tn{toks} in some range. In the rare cases where there are too
+% many \cs{tl_build_one:n} commands, leading to the depletion of
+% registers, the contents of the current set of \tn{toks} is unpacked
+% into \cs{l_tl_build_result_tl}. This prevents overflow from
+% affecting the end-user (beyond an obvious performance hit).
+% \begin{macrocode}
+\tl_new:N \l_tl_build_result_tl
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{macro}{\tl_build_unpack:}
+% \begin{macro}[aux, EXP]{\tl_build_unpack_loop:w}
+% The various pieces of the token list are built in \tn{toks} from the
+% \texttt{start_index} (inclusive) to the (current) \texttt{index}
+% (excluded). Those \tn{toks} are unpacked and stored in order in the
+% \texttt{result} token list. Optimizations would be possible here,
+% for instance, unpacking $10$ \tn{toks} at a time with a macro
+% expanding to |\the\toks#10...\the\toks#19|, but this should be kept
+% for much later.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \tl_build_unpack:
+ {
+ \tl_put_right:Nx \l_tl_build_result_tl
+ {
+ \exp_after:wN \tl_build_unpack_loop:w
+ \int_use:N \l_tl_build_start_index_int ;
+ \prg_break_point:n { }
+ }
+ }
+\cs_new:Npn \tl_build_unpack_loop:w #1 ;
+ {
+ \if_num:w #1 = \l_tl_build_index_int
+ \exp_after:wN \prg_map_break:
+ \fi:
+ \tex_the:D \tex_toks:D #1 \exp_stop_f:
+ \exp_after:wN \tl_build_unpack_loop:w
+ \int_use:N \int_eval:w #1 + \c_one ;
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \subsection{Building the token list}
+%
+% \begin{macro}
+% {
+% \tl_set_build:Nw , \tl_set_build_x:Nw ,
+% \tl_gset_build:Nw , \tl_gset_build_x:Nw
+% }
+% \begin{macro}[aux]{\tl_set_build_aux:NNw}
+% Similar to what is done for coffins: redefine some command, here
+% \cs{tl_build_end_aux:n} to hold the relevant assignment (see
+% \cs{tl_build_end:} for details). Then initialize the start index and
+% the current index at zero, and empty the \texttt{result} token list.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \tl_set_build:Nw
+ { \tl_set_build_aux:NNw \tl_set:Nn }
+\cs_new_protected_nopar:Npn \tl_set_build_x:Nw
+ { \tl_set_build_aux:NNw \tl_set:Nx }
+\cs_new_protected_nopar:Npn \tl_gset_build:Nw
+ { \tl_set_build_aux:NNw \tl_gset:Nn }
+\cs_new_protected_nopar:Npn \tl_gset_build_x:Nw
+ { \tl_set_build_aux:NNw \tl_gset:Nx }
+\cs_new_protected:Npn \tl_set_build_aux:NNw #1#2
+ {
+ \group_begin:
+ \cs_set_nopar:Npn \tl_build_end_assignment:n
+ { \group_end: #1 #2 }
+ \int_zero:N \l_tl_build_start_index_int
+ \int_zero:N \l_tl_build_index_int
+ \tl_clear:N \l_tl_build_result_tl
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}{\tl_build_end:}
+% \begin{macro}[aux]{\tl_build_end_assignment:n}
+% When we are done building a token list, unpack all \tn{toks} into
+% the \texttt{result} token list, and expand this list before closing
+% the group. The \cs{tl_build_end_assignment:n} function is defined by
+% \cs{tl_set_build_aux:NNw} to end the group and hold the relevant
+% assignment. Its value outside is irrelevant, but just in case, we
+% set it to a function which would clean up the contents of
+% \cs{l_tl_build_result_tl}.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \tl_build_end:
+ {
+ \tl_build_unpack:
+ \exp_args:No
+ \tl_build_end_assignment:n \l_tl_build_result_tl
+ }
+\cs_new_eq:NN \tl_build_end_assignment:n \use_none:n
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}{\tl_build_one:n, \tl_build_one:o, \tl_build_one:x}
+% Store the tokens in a free \tn{toks}, then move the pointer to the
+% next one. If we overflow, unpack the current \tn{toks}, and reset
+% the current index, preparing to fill more \tn{toks}. This could be
+% optimized by avoiding to read |#1|, using \tn{afterassignment}.
+% \begin{macrocode}
+\cs_new_protected:Npn \tl_build_one:n #1
+ {
+ \tex_toks:D \l_tl_build_index_int {#1}
+ \tex_advance:D \l_tl_build_index_int \c_one
+ \if_num:w \l_tl_build_index_int > \c_max_register_int
+ \tl_build_unpack:
+ \l_tl_build_index_int \l_tl_build_start_index_int
+ \fi:
+ }
+\cs_new_protected:Npn \tl_build_one:o #1
+ {
+ \tex_toks:D \l_tl_build_index_int \exp_after:wN {#1}
+ \tex_advance:D \l_tl_build_index_int \c_one
+ \if_num:w \l_tl_build_index_int > \c_max_register_int
+ \tl_build_unpack:
+ \l_tl_build_index_int \l_tl_build_start_index_int
+ \fi:
+ }
+\cs_new_protected:Npn \tl_build_one:x #1
+ { \use:x { \tl_build_one:n {#1} } }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macrocode}
+%</initex|package>
+% \end{macrocode}
+%
+% \end{implementation}
+%
+% \PrintIndex
diff --git a/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins.dtx b/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins.dtx
index 30a58c18817..be9e4e24f75 100644
--- a/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins.dtx
@@ -1,5 +1,5 @@
% \iffalse meta-comment
-%
+%
%% File: xcoffins.dtx Copyright(C) 2010,2011 The LaTeX3 Project
%%
%% It may be distributed and/or modified under the conditions of the
@@ -36,7 +36,7 @@
%
%<*driver|package>
\RequirePackage{xparse}
-\GetIdInfo$Id: xcoffins.dtx 2963 2011-11-15 22:06:41Z joseph $
+\GetIdInfo$Id: xcoffins.dtx 3385 2012-02-12 20:37:09Z bruno $
{L3 Experimental design-level coffins}
%</driver|package>
%<*driver>
@@ -56,13 +56,13 @@
\end{document}
%</driver>
% \fi
-%
+%
% \title{^^A
% The \textsf{xcoffins} package\\ Design-level coffins^^A
% \thanks{This file describes v\ExplFileVersion,
% last revised \ExplFileDate.}^^A
% }
-%
+%
% \author{^^A
% The \LaTeX3 Project\thanks
% {^^A
@@ -89,7 +89,7 @@
% positions, for example the top or horizontal centre. The points
% where these poles intersect are called \enquote{handles}. Two
% coffins can then be aligned by describing the relationship between
-% a handle on one coffin with a handle on the second. In words, an
+% a handle on one coffin with a handle on the second. In words, an
% example might then read
% \begin{quote}
% Align the top-left handle of coffin A with the bottom-right
@@ -97,20 +97,20 @@
% \end{quote}
%
% The locations of coffin handles are much easier to understand
-% visually. Figure~\ref{fgr:handles} shows the standard handle
-% positions for a coffin typeset in horizontal mode (left) and in
+% visually. Figure~\ref{fgr:handles} shows the standard handle
+% positions for a coffin typeset in horizontal mode (left) and in
% vertical mode (right). Notice that the later case results in a greater
% number of handles being available. As illustrated, each handle
% results from the intersection of two poles. For example, the centre
% of the coffin is marked |(hc,vc)|, \emph{i.e.}~it is the
-% point of intersection of the horizontal centre pole with the
+% point of intersection of the horizontal centre pole with the
% vertical centre pole. New handles are generated automatically when
% poles are added to a coffin: handles are \enquote{dynamic} entities.
%
% \begin{figure}
% \hfil
% \begin{minipage}{0.4\textwidth}
-% \SetHorizontalCoffin\ExampleCoffin
+% \SetHorizontalCoffin\ExampleCoffin
% {\color{black!10!white}\rule{1 in}{1 in}}
% \DisplayCoffinHandles\ExampleCoffin{blue}
% \end{minipage}
@@ -123,9 +123,9 @@
% \hfil
% \caption{Standard coffin handles: left, horizontal coffin; right,
% vertical coffin}
-% \label{fgr:handles}
+% \label{fgr:handles}
% \end{figure}
-%
+%
%\section{Creating and setting coffins}
%
% Before any alignment can take place, coffins must be created and
@@ -139,7 +139,7 @@
% \begin{syntax}
% \cs{NewCoffin} \meta{coffin}
% \end{syntax}
-% Before a \meta{coffin} can be used, it must be allocated using
+% Before a \meta{coffin} can be used, it must be allocated using
% \cs{NewCoffin}. The name of the \meta{coffin} should be a
% control sequence (starting with the escape character, usually
% |\|), for example
@@ -175,7 +175,7 @@
% \end{syntax}
% Typesets the \meta{material} in vertical mode constrained to the
% given \meta{width} and stores the result in the \meta{coffin}. The
-% standard poles for the \meta{coffin} are then set up based on the
+% standard poles for the \meta{coffin} are then set up based on the
% size of the typeset material.
% \end{function}
%
@@ -194,7 +194,7 @@
% box of the coffin;
% \item[b] a pole running along the bottom edge of the bounding
% box of the coffin;
-% \item[vc] a pole running horizontally through the centre of the
+% \item[vc] a pole running horizontally through the centre of the
% coffin half-way between the bottom and top edges of the bounding
% box (\emph{i.e.}~the \enquote{vertical centre});
% \item[t] a pole running along the top edge of the bounding
@@ -202,7 +202,7 @@
% \item[H] a pole running along the baseline of the typeset material
% contained in the coffin.
% \end{itemize}
-% In addition, coffins containing vertical-mode material also
+% In addition, coffins containing vertical-mode material also
% feature poles which reflect the richer nature of these systems:
% \begin{itemize}
% \item[B] a pole running along the baseline of the material at the
@@ -219,10 +219,10 @@
% The \meta{pole} will be located at the \meta{offset} from the
% bottom edge of the bounding box of the \meta{coffin}. The
% \meta{offset} should be given as a dimension expression; this may
-% include the terms \cs{TotalHeight}, \cs{Height}, \cs{Depth} and
-% \cs{Width}, which will evaluate to the appropriate dimensions of
-% the \meta{coffin}. For example, to create a pole running
-% horizontally through the coffin at one third of the distance from
+% include the terms \cs{TotalHeight}, \cs{Height}, \cs{Depth} and
+% \cs{Width}, which will evaluate to the appropriate dimensions of
+% the \meta{coffin}. For example, to create a pole running
+% horizontally through the coffin at one third of the distance from
% the base of the coffin to the top, the appropriate instruction would
% be
% \begin{verbatim}
@@ -230,7 +230,7 @@
% \end{verbatim}
% Note that poles which run \emph{horizontally} are described in terms
% of their \emph{vertical} location in the coffin. Also notice that
-% the total height of the coffin is described by the sum of
+% the total height of the coffin is described by the sum of
% \cs{Height} and \cs{Depth}: these are both measured from the
% horizontal baseline of the material in the coffin.
% \end{function}
@@ -243,10 +243,10 @@
% The \meta{pole} will be located at the \meta{offset} from the
% left-hand edge of the bounding box of the \meta{coffin}. The
% \meta{offset} should be given as a dimension expression; this may
-% include the terms \cs{TotalHeight}, \cs{Height}, \cs{Depth} and
-% \cs{Width}, which will evaluate to the appropriate dimensions of
+% include the terms \cs{TotalHeight}, \cs{Height}, \cs{Depth} and
+% \cs{Width}, which will evaluate to the appropriate dimensions of
% the \meta{coffin}. For example, to create a pole running vertically
-% through the coffin at one third of the distance from the left-hand
+% through the coffin at one third of the distance from the left-hand
% edge, the appropriate instruction would be
% \begin{verbatim}
% \SetVerticalPole \MyCoffin {width/3} {\Width/3}
@@ -260,7 +260,7 @@
% \cs{TotalHeight}
% \end{syntax}
% Within the \meta{offset} argument of \cs{SetHorizontalPole} and
-% \cs{SetVerticalPole}, \cs{TotalHeight} will give the distance from
+% \cs{SetVerticalPole}, \cs{TotalHeight} will give the distance from
% the base to the top of the bounding box of the relevant coffin.
% \end{function}
%
@@ -300,10 +300,10 @@
% \end{syntax}
% Rotates the \meta{coffin} by the given \meta{angle} about its reference
% point (given in degrees counter-clockwise) . This process will rotate both
-% the coffin content and poles. Multiple rotations will not result in
+% the coffin content and poles. Multiple rotations will not result in
% the bounding box of the coffin growing unnecessarily.
-%
-% The effect of rotation on a coffin is illustrated in
+%
+% The effect of rotation on a coffin is illustrated in
% Figure~\ref{fgr:rotation}. As is shown, the coffin handles will
% remain correctly positioned relative to the content of the coffin.
% The \enquote{top} of a rotated coffin may of course no longer be the
@@ -312,7 +312,7 @@
%
% \begin{figure}
% \hfil
-% \SetHorizontalCoffin\ExampleCoffin
+% \SetHorizontalCoffin\ExampleCoffin
% {^^A
% \color{black!10!white}\rule{0.5 in}{1 in}^^A
% \color{black!20!white}\rule{0.5 in}{1 in}^^A
@@ -339,8 +339,8 @@
% \end{syntax}
% Resized the \meta{coffin} to \meta{width} and \meta{total-height},
% both of which should be given as dimension expressions. These may
-% include the terms \cs{TotalHeight}, \cs{Height}, \cs{Depth} and
-% \cs{Width}, which will evaluate to the appropriate dimensions of
+% include the terms \cs{TotalHeight}, \cs{Height}, \cs{Depth} and
+% \cs{Width}, which will evaluate to the appropriate dimensions of
% the \meta{coffin}.
% \end{function}
%
@@ -348,8 +348,8 @@
% \begin{syntax}
% \cs{ScaleCoffin} \meta{coffin} \Arg{x-scale} \Arg{y-scale}
% \end{syntax}
-% Scales the \meta{coffin} by a factors \meta{x-scale} and
-% \meta{y-scale} in the horizontal and vertical directions,
+% Scales the \meta{coffin} by a factors \meta{x-scale} and
+% \meta{y-scale} in the horizontal and vertical directions,
% respectively. The two scale factors should be given as real numbers.
% \end{function}
%
@@ -359,7 +359,7 @@
%
% \begin{figure}
% \hfil
-% \SetHorizontalCoffin\ExampleCoffin
+% \SetHorizontalCoffin\ExampleCoffin
% {^^A
% \color{black!10!white}\rule{0.5 in}{1 in}^^A
% \color{black!20!white}\rule{0.5 in}{1 in}^^A
@@ -376,26 +376,26 @@
% \hfil
% \caption{Coffin resizing: left, resized to exactly $4$\,cm by
% $6$\,cm; right, scaled a factors of $2$ and $0.5$ in
-% $x$ and $y$, respectively (example coffin as in
+% $x$ and $y$, respectively (example coffin as in
% Figure~\ref{fgr:rotation}).}
-% \label{fgr:resizing}
+% \label{fgr:resizing}
% \end{figure}
%
% \section{Joining coffins}
%
% The key operation for coffins is joining coffins to each other. This
-% is always carried out such that the first coffin is the
+% is always carried out such that the first coffin is the
% \enquote{parent}, and is updated by the alignment. The second
% \enquote{child} coffin is not altered by the alignment process.
-%
+%
% \begin{function}{\JoinCoffins}
% \begin{syntax}
% \cs{JoinCoffins} *
-% ~~\meta{coffin1} [ \meta{coffin1-pole1} , \meta{coffin1-pole2} ]
+% ~~\meta{coffin1} [ \meta{coffin1-pole1} , \meta{coffin1-pole2} ]
% ~~\meta{coffin2} [ \meta{coffin2-pole1} , \meta{coffin2-pole2} ]
-% ~~( \meta{x-offset} , \meta{y-offset} )
+% ~~( \meta{x-offset} , \meta{y-offset} )
% \end{syntax}
-% Joining of two coffins is carried out by the \cs{JoinCoffins}
+% Joining of two coffins is carried out by the \cs{JoinCoffins}
% function, which takes two mandatory arguments: the \enquote{parent}
% \meta{coffin1} and the \enquote{child} \meta{coffin2}. All of the
% other arguments shown are optional.
@@ -403,15 +403,15 @@
%
% The standard \cs{JoinCoffins} functions joins \meta{coffin2} to
% \meta{coffin1} such that the bounding box of \meta{coffin1} after the
-% process will expand. The new bounding box will be the smallest
-% rectangle covering the bounding boxes of the two input coffins.
+% process will expand. The new bounding box will be the smallest
+% rectangle covering the bounding boxes of the two input coffins.
% When the starred variant of \cs{JoinCoffins} is used, the bounding
% box of \meta{coffin1} is not altered, \emph{i.e.}~\meta{coffin2} may
% protrude outside of the bounding box of the updated \meta{coffin1}.
% The difference between the two forms of alignment is best illustrated
% using a visual example. In Figure~\ref{fgr:alignment}, the two
% processes are contrasted. In both cases, the small red coffin has been
-% aligned with the large grey coffin. In the left-hand illustration,
+% aligned with the large grey coffin. In the left-hand illustration,
% the \cs{JoinCoffins} function was used, resulting in an expanded
% bounding box. In contrast, on the right \cs{AttachCoffin} was used,
% meaning that the bounding box does not include the area of the
@@ -419,9 +419,9 @@
%
% \begin{figure}
% \fboxsep 0 pt\relax
-% \SetHorizontalCoffin\ExampleCoffin
+% \SetHorizontalCoffin\ExampleCoffin
% {\color{black!20!white}\rule{1 in}{1 in}}
-% \SetHorizontalCoffin \SmallCoffin
+% \SetHorizontalCoffin \SmallCoffin
% {\color{red!20!white}\rule{0.1 in}{0.1 in}}
% \hfil
% \begin{minipage}{0.4\textwidth}
@@ -436,14 +436,14 @@
% \fbox{\TypesetCoffin\ExampleCoffin}%
% \end{minipage}
% \hfil
-% \caption{Contrast between \cs{JoinCoffins} (left) and
+% \caption{Contrast between \cs{JoinCoffins} (left) and
% \cs{JoinCoffins*} (right); the bounding box of the coffin is show
% in black.}
-% \label{fgr:alignment}
+% \label{fgr:alignment}
% \end{figure}
-%
+%
% The alignment is carried out by first calculating \meta{handle1}, the
-% point of intersection of \meta{coffin1-pole1} and
+% point of intersection of \meta{coffin1-pole1} and
% \meta{coffin1-pole2}, and \meta{handle2}, the point of intersection
% of \meta{coffin2-pole1} and \meta{coffin2-pole2}. If the two
% \meta{poles} are not specified, \cs{JoinCoffins} will use the
@@ -456,19 +456,19 @@
% |(0 pt, 0 pt)| is used.
%
% Notice that when \cs{JoinCoffins} is used the new bounding box is
-% the smallest rectangle containing the bounding boxes of the two input
+% the smallest rectangle containing the bounding boxes of the two input
% coffins. As a result, it will include additional white space unless
-% one coffin entirely overlaps the other (Figure~\ref{fgr:bounding},
+% one coffin entirely overlaps the other (Figure~\ref{fgr:bounding},
% left). Rotation of coffins will take account of the extent of the
% material after rotation when re-calculating the bounding box. This
-% means that no \emph{unnecessary} white space will be added on
+% means that no \emph{unnecessary} white space will be added on
% rotation (Figure~\ref{fgr:bounding}, right).
%
% \begin{figure}
% \fboxsep 0 pt\relax
-% \SetHorizontalCoffin\ExampleCoffin
+% \SetHorizontalCoffin\ExampleCoffin
% {\color{black!20!white}\rule{1 in}{1 in}}
-% \SetHorizontalCoffin\SmallCoffin
+% \SetHorizontalCoffin\SmallCoffin
% {\color{red!20!white}\rule{0.1 in}{0.1 in}}
% \JoinCoffins\ExampleCoffin[vc,r]\SmallCoffin[vc,l]
% \hfil
@@ -485,7 +485,7 @@
% \hfil
% \caption{The effect of rotation of a joined coffin: the black line
% shows the coffin bounding box.}
-% \label{fgr:bounding}
+% \label{fgr:bounding}
% \end{figure}
%
% As part of the joining procedure, the poles of the two input coffins
@@ -495,35 +495,35 @@
% \begin{enumerate}
% \item The \enquote{native} poles of the updated coffin, such as
% \texttt{l}, \texttt{r}, \texttt{hc}, \emph{etc}.
-% \item Poles derived from \meta{coffin1}, such as
+% \item Poles derived from \meta{coffin1}, such as
% \texttt{\meta{coffin1}-l}, \texttt{\meta{coffin1}-r},
% \texttt{\meta{coffin1}-hc}, \emph{etc.}
-% \item Poles derived from \meta{coffin2}, such as
+% \item Poles derived from \meta{coffin2}, such as
% \texttt{\meta{coffin2}-l}, \texttt{\meta{coffin2}-r},
% \texttt{\meta{coffin2}-hc}, \emph{etc.}
% \end{enumerate}
-%
+%
% Applying this ability allows a series of joining operations to
% take place, as illustrated in Figure~\ref{fgr:nested}. In this
% example, the scheme used for alignment was as follows:
% \begin{verbatim}
% \SetHorizontalCoffin\OutputCoffin{}
-% \SetHorizontalCoffin\RedCoffin
+% \SetHorizontalCoffin\RedCoffin
% {\color{red!20!white}\rule{0.2 in}{0.2 in}}
% \JoinCoffins\OutputCoffin[vc,hc]\RedCoffin[vc,hc]
-% \SetHorizontalCoffin\BlueCoffin
+% \SetHorizontalCoffin\BlueCoffin
% {\color{blue!20!white}\rule{0.2 in}{0.2 in}}
% \JoinCoffins\OutputCoffin[\RedCoffin-vc,\RedCoffin-hc]
% \BlueCoffin[b,l]
-% \SetHorizontalCoffin\GreenCoffin
+% \SetHorizontalCoffin\GreenCoffin
% {\color{green!20!white}\rule{0.2 in}{0.2 in}}
% \JoinCoffins\OutputCoffin[\BlueCoffin-vc,\BlueCoffin-hc]
% \GreenCoffin[b,l]
-% \SetHorizontalCoffin\YellowCoffin
+% \SetHorizontalCoffin\YellowCoffin
% {\color{yellow!20!white}\rule{0.2 in}{0.2 in}}
% \JoinCoffins\OutputCoffin[\GreenCoffin-vc,\GreenCoffin-hc]
% \YellowCoffin[b,l]
-% \SetHorizontalCoffin \OrangeCoffin
+% \SetHorizontalCoffin \OrangeCoffin
% {\color{orange!20!white}\rule{0.2 in}{0.2 in}}
% \JoinCoffins\OutputCoffin[\BlueCoffin-t,\BlueCoffin-l]
% \OrangeCoffin[b,r]
@@ -533,62 +533,62 @@
% output. Each join then takes place placing the new addition relative
% to the previous one. As each coffin joined has a unique name it is
% possible to align relative to each one of the component parts of the
-% assembly. This is illustrated by the addition of the final
+% assembly. This is illustrated by the addition of the final
% \cs{OrangeCoffin} based on the earlier placement of the
% \cs{BlueCoffin}.
%
% \begin{figure}
% \centering
% \SetHorizontalCoffin\OutputCoffin{}
-% \SetHorizontalCoffin\RedCoffin
+% \SetHorizontalCoffin\RedCoffin
% {\color{red!20!white}\rule{0.2 in}{0.2 in}}
% \JoinCoffins\OutputCoffin[vc,hc]\RedCoffin[vc,hc]
-% \SetHorizontalCoffin\BlueCoffin
+% \SetHorizontalCoffin\BlueCoffin
% {\color{blue!20!white}\rule{0.2 in}{0.2 in}}
% \JoinCoffins\OutputCoffin[\RedCoffin-vc,\RedCoffin-hc]
% \BlueCoffin[b,l]
-% \SetHorizontalCoffin\GreenCoffin
+% \SetHorizontalCoffin\GreenCoffin
% {\color{green!20!white}\rule{0.2 in}{0.2 in}}
% \JoinCoffins\OutputCoffin[\BlueCoffin-vc,\BlueCoffin-hc]
% \GreenCoffin[b,l]
-% \SetHorizontalCoffin\YellowCoffin
+% \SetHorizontalCoffin\YellowCoffin
% {\color{yellow!20!white}\rule{0.2 in}{0.2 in}}
% \JoinCoffins\OutputCoffin[\GreenCoffin-vc,\GreenCoffin-hc]
% \YellowCoffin[b,l]
-% \SetHorizontalCoffin \OrangeCoffin
+% \SetHorizontalCoffin \OrangeCoffin
% {\color{orange!20!white}\rule{0.2 in}{0.2 in}}
% \JoinCoffins\OutputCoffin[\BlueCoffin-t,\BlueCoffin-l]
% \OrangeCoffin[b,r]
% \TypesetCoffin\OutputCoffin
% \caption{Aligning coffins using poles from previous operations.}
-% \label{fgr:nested}
+% \label{fgr:nested}
% \end{figure}
%
% \section{Typesetting coffins}
%
% \begin{function}{\TypesetCoffin}
% \begin{syntax}
-% \cs{TypesetCoffin}
+% \cs{TypesetCoffin}
% ~~\meta{coffin} [ \meta{pole1} , \meta{pole2} ]
-% ~~( \meta{x-offset} , \meta{y-offset} )
+% ~~( \meta{x-offset} , \meta{y-offset} )
% \end{syntax}
% Typesetting is carried out by first calculating \meta{handle}, the
% point of intersection of \meta{pole1} and \meta{pole2}. This is an
% optional argument, and if not given then |(H,l)|, the \TeX{}
% reference point of the underlying box, is used. The coffin
-% is then typeset such that the relationship between the current
+% is then typeset such that the relationship between the current
% reference point in the document and the \meta{handle} is described
% by the \meta{x-offset} and \meta{y-offset}. This \meta{offset} is
% optional, and if not given |(0 pt, 0 pt)| is used. Typesetting a
% coffin is therefore analogous to carrying out an alignment where the
% \enquote{parent} coffin is the current insertion point.
% \end{function}
-%
+%
% \section{Measuring coffins}
%
% There are places in the design process where it is useful to be able to
% measure coffins outside of pole-setting procedures.
-%
+%
% \begin{function}{\CoffinDepth}
% \begin{syntax}
% \cs{CoffinDepth} \meta{coffin}
@@ -606,7 +606,7 @@
% in a form suitable for use in a \meta{dimension expression}, for example
% |\setlength{\mylength}{\CoffinHeight\ExampleCoffin}|.
% \end{function}
-%
+%
% \begin{function}{\CoffinTotalHeight}
% \begin{syntax}
% \cs{CoffinTotalHeight} \meta{coffin}
@@ -627,7 +627,7 @@
%
% \section{Diagnostic functions}
%
-% Diagnostic data for following the coffin-building process is
+% Diagnostic data for following the coffin-building process is
% available both graphically and at the terminal. This reflects the
% fact that coffins are visual constructs.
%
@@ -636,22 +636,22 @@
% \cs{DisplayCoffinHandles} \meta{coffin} \Arg{colour}
% \end{syntax}
% This function first calculates the intersections between all of
-% the \meta{poles} of the \meta{coffin} to give a set of
-% \meta{handles}. It then prints the \meta{coffin} at the current
-% location in the source, with the position of the \meta{handles}
-% marked on the coffin. The \meta{handles} will be labelled as part
-% of this process: the locations of the \meta{handles} and the labels
+% the \meta{poles} of the \meta{coffin} to give a set of
+% \meta{handles}. It then prints the \meta{coffin} at the current
+% location in the source, with the position of the \meta{handles}
+% marked on the coffin. The \meta{handles} will be labelled as part
+% of this process: the locations of the \meta{handles} and the labels
% are both printed in the \meta{colour} specified.
% \end{function}
%
% \begin{function}{\MarkCoffinHandle}
% \begin{syntax}
-% \cs{MarkCoffinHandle} \meta{coffin}
+% \cs{MarkCoffinHandle} \meta{coffin}
% ~~[ \meta{pole1} , \meta{pole2} ] \Arg{colour}
% \end{syntax}
-% This function first calculates the \meta{handle} for the
+% This function first calculates the \meta{handle} for the
% \meta{coffin} as defined by the intersection of \meta{pole1} and
-% \meta{pole2}. It then marks the position of the \meta{handle}
+% \meta{pole2}. It then marks the position of the \meta{handle}
% on the \meta{coffin}. The \meta{handle} will be labelled as part of
% this process: the location of the \meta{handle} and the label are
% both printed in the \meta{colour} specified. If no \meta{poles} are
@@ -663,9 +663,9 @@
% \cs{ShowCoffinStructure} \meta{coffin}
% \end{syntax}
% This function shows the structural information about the
-% \meta{coffin} in the terminal. The width, height and depth of the
-% typeset material are given, along with the location of all of the
-% poles of the coffin. For example, for the rotated coffin in
+% \meta{coffin} in the terminal. The width, height and depth of the
+% typeset material are given, along with the location of all of the
+% poles of the coffin. For example, for the rotated coffin in
% Figure~\ref{fgr:rotation}, the output of \cs{ShowCoffinStructure}
% is:
% \begin{verbatim}
@@ -673,7 +673,6 @@
% > ht = 72.26999pt
% > dp = 0.0pt
% > wd = 72.26999pt
-%
% Poles of coffin \ExampleCoffin:
% > l => {0pt}{0pt}{0pt}{1000pt}
% > B => {0pt}{0pt}{1000pt}{0pt}
@@ -684,11 +683,11 @@
% > vc => {0pt}{36.135pt}{1000pt}{0pt}
% > t => {0pt}{72.26999pt}{1000pt}{0pt}
% > b => {0pt}{0.0pt}{1000pt}{0pt}.
-% <argument> ...ffin_show_aux:w \l_coffin_tmp_tl }
+% <recently read> }
% \end{verbatim}
% Notice that the poles of a coffin are defined by four values:
-% the $x$ and $y$ co-ordinates of a point that the pole
-% passes through and the $x$- and $y$-components of a
+% the $x$ and $y$ co-ordinates of a point that the pole
+% passes through and the $x$- and $y$-components of a
% vector denoting the direction of the pole. It is the ratio between
% the later, rather than the absolute values, which determines the
% direction of the pole.
@@ -699,17 +698,17 @@
% \begin{implementation}
%
% \section{Implementation}
-%
+%
% \begin{macrocode}
%<*package>
% \end{macrocode}
-%
+%
% \begin{macrocode}
\ProvidesExplPackage
{\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
% \end{macrocode}
-%
-%
+%
+%
% \begin{variable}
% {
% \l_coffin_A_hpole_tl ,
@@ -722,7 +721,7 @@
% }
% Key--value definitions for the alignment system. With the exception
% of \texttt{grow-bounding-box}, all of these have to be given with a
-% value.
+% value.
% \begin{macrocode}
\keys_define:nn { coffin }
{
@@ -755,7 +754,7 @@
% \end{variable}
%
% A lot of this is more-or-less just passing data straight through.
-%
+%
% \begin{macro}{\NewCoffin}
% This is a very easy conversion.
% \begin{macrocode}
@@ -763,7 +762,7 @@
{ \coffin_new:N #1 }
% \end{macrocode}
% \end{macro}
-%
+%
% \begin{macro}{\BoxToCoffin}
% For allowing a user-level \LaTeXe{} box to be used as a coffin.
% \begin{macrocode}
@@ -855,12 +854,12 @@
% which need to be split. This is a simpler case of the code needed for
% \cs{JoinCoffins}.
% \begin{macrocode}
-\NewDocumentCommand \TypesetCoffin
+\NewDocumentCommand \TypesetCoffin
{
- m
- > { \SplitArgument { 1 } { , } } O { H , l }
+ m
+ > { \SplitArgument { 1 } { , } } O { H , l }
> { \SplitArgument { 1 } { , } } D ( ) { 0 pt , 0 pt }
- }
+ }
{ \coffin_typeset:Nnnnn #1 #2 #3 }
% \end{macrocode}
% \end{macro}
@@ -880,7 +879,7 @@
% \end{macro}
% \end{macro}
% \end{macro}
-%
+%
% \begin{macro}{\CoffinDepth, \CoffinHeight, \CoffinTotalHeigth, \CoffinWidth}
% Nothing too complex, except that the total height is set up as an
% expression so that it will act correctly if prefixed with a negative
@@ -911,10 +910,10 @@
% \end{macro}
%
% \begin{macro}{\MarkCoffinHandle}
-% Marking a handle requires a bit of work with the input, so that
+% Marking a handle requires a bit of work with the input, so that
% the design-level interface is \enquote{nice}.
% \begin{macrocode}
-\NewDocumentCommand \MarkCoffinHandle
+\NewDocumentCommand \MarkCoffinHandle
{ m > { \SplitArgument { 1 } { , } } O { H , l } m }
{
\cs_if_exist:cTF { l_coffin_poles_ \int_value:w #1 _prop }