summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/source
diff options
context:
space:
mode:
authorKarl Berry <karl@freefriends.org>2012-01-19 22:49:25 +0000
committerKarl Berry <karl@freefriends.org>2012-01-19 22:49:25 +0000
commit8764984f07460a77b8200111ddb513c8af6c23f3 (patch)
tree61782419c00b217fc7f14919b9e24bbae260274f /Master/texmf-dist/source
parent17d590a6bcf898806aeec2d7aa57d641a440ca10 (diff)
l3experimental 3209 (19jan12)
git-svn-id: svn://tug.org/texlive/trunk@25158 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/source')
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx10
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx1361
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx298
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx612
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-test.tex380
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example-keyval.tex281
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example.tex185
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx63
8 files changed, 2210 insertions, 980 deletions
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx
index eed657ad231..46bd4c80cf7 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx
@@ -1,6 +1,6 @@
% \iffalse meta-comment
%
-%% File: l3flag.dtx Copyright (C) 2011 The LaTeX3 Project
+%% File: l3flag.dtx Copyright (C) 2011-2012 The LaTeX3 Project
%%
%% It may be distributed and/or modified under the conditions of the
%% LaTeX Project Public License (LPPL), either version 1.3c of this
@@ -70,7 +70,7 @@
% \begin{documentation}
%
% Flags are the only data structure on which \TeX{} can perform
-% assignments in expansion only contexts. This module is meant
+% assignments in expansion-only contexts. This module is meant
% mostly for kernel use: in almost all cases, booleans should
% be preferred to flags, because they are faster.
%
@@ -155,16 +155,14 @@
% \begin{macrocode}
\cs_new_protected:Npn \flag_new:N #1
{
+ \cs_new_eq:NN #1 \c_undefined:D
\tl_if_in:NnTF \g_flag_list_tl {#1}
{
\msg_kernel_error:nnx
{ flag } { already-defined }
{ \token_to_str:N #1 }
}
- {
- \tl_gput_right:Nn \g_flag_list_tl {#1}
- \cs_new_eq:NN #1 \c_undefined:D
- }
+ { \tl_gput_right:Nn \g_flag_list_tl {#1} }
}
\cs_generate_variant:Nn \flag_new:N { c }
% \end{macrocode}
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx
index 9c276b6922d..d921b0b7e56 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx
@@ -1,6 +1,6 @@
% \iffalse meta-comment
%
-%% File: l3regex.dtx Copyright (C) 2011 The LaTeX3 Project
+%% File: l3regex.dtx Copyright (C) 2011-2012 The LaTeX3 Project
%%
%% It may be distributed and/or modified under the conditions of the
%% LaTeX Project Public License (LPPL), either version 1.3c of this
@@ -35,7 +35,7 @@
%
%<*driver|package>
\RequirePackage{expl3}
-\GetIdInfo$Id: l3regex.dtx 3073 2011-12-21 01:53:06Z bruno $
+\GetIdInfo$Id: l3regex.dtx 3200 2012-01-16 03:50:54Z bruno $
{L3 Experimental Regular Expressions}
%</driver|package>
%<*driver>
@@ -104,16 +104,16 @@
% it can be compiled once, and stored in a token list
% variable using \cs{regex_const:Nn}. For example,
% \begin{verbatim}
-% \regex_const:Nn \l_foo_regex_tl { \c{begin} \cB (\CB*?) \cE }
+% \regex_const:Nn \c_foo_regex_tl { \c{begin} \cB. (\c[^BE].*) \cE. }
% \end{verbatim}
-% stores in \cs{l_foo_regex_tl} a regular expression which matches
-% the starting marker for an environment: \cs{begin}, followed
-% by a begin-group token (|\cB|), then tokens which are not
-% begin-group characters (|\CB|), as few as possible (|*?|),
-% followed by an end-group token (|\cE|). As explained in the
-% next section, the parentheses \enquote{capture} the result of
-% |\CB*?|, giving us access to the name of the environment
-% when doing replacements.
+% stores in \cs{c_foo_regex_tl} a regular expression which matches the
+% starting marker for an environment: \cs{begin}, followed by a
+% begin-group token (|\cB.|), then any number of tokens which are
+% neither begin-group nor end-group character tokens (|\c[^BE].*|),
+% ending with an end-group token (|\cE.|). As explained in the next
+% section, the parentheses \enquote{capture} the result of |\c[^BE].*|,
+% giving us access to the name of the environment when doing
+% replacements.
%
% \subsection{Syntax of regular expressions}
%
@@ -191,7 +191,7 @@
% Matches any token other than the specified characters.
% \item[{[x-y]}] Range (can be used with escaped characters).
% \end{l3regex-syntax}
-% For instance, |[a-oq-z\cC]| matches any lowercase latin letter
+% For instance, |[a-oq-z\cC.]| matches any lowercase latin letter
% except |p|, as well as control sequences (see below for a description
% of |\c|).
%
@@ -240,29 +240,47 @@
% group number.
% \end{l3regex-syntax}
%
-% Each catcode is represented by a single uppercase letter:
-% |C| for control sequences;
-% |B| for begin-group tokens;
-% |E| for end-group tokens;
-% |M| for math shift;
-% |T| for alignment tab tokens;
-% |P| for macro parameter tokens;
-% |U| for superscript tokens (up);
-% |D| for subscript tokens (down);
-% |S| for spaces;
-% |L| for letters;
-% |O| for others; and
-% |A| for active characters.
+% The |\c| escape sequence allows to test the category code of tokens,
+% and match control sequences. Each character category is represented
+% by a single uppercase letter:
+% \begin{itemize}
+% \item |C| for control sequences;
+% \item |B| for begin-group tokens;
+% \item |E| for end-group tokens;
+% \item |M| for math shift;
+% \item |T| for alignment tab tokens;
+% \item |P| for macro parameter tokens;
+% \item |U| for superscript tokens (up);
+% \item |D| for subscript tokens (down);
+% \item |S| for spaces;
+% \item |L| for letters;
+% \item |O| for others; and
+% \item |A| for active characters.
+% \end{itemize}
+% The |\c| escape sequence is used as follows.
% \begin{l3regex-syntax}
-% \item[\\cX] Any character token with catcode |X|,
-% where |X| can be any of |CBEMTPUDSLOA|.
-% \item[\\CX] Any token with catcode different from |X|,
-% where |X| can be any of |CBEMTPUDSLOA|.
-% \item[\\c\Arg{regex}] A control sequence whose csname matches
-% the \meta{regex}, anchored at the beginning and end,
-% so that |\c{begin}| matches exactly \cs{begin}, and nothing
-% else.
+% \item[\\c\Arg{regex}] A control sequence whose csname matches the
+% \meta{regex}, anchored at the beginning and end, so that |\c{begin}|
+% matches exactly \cs{begin}, and nothing else.
+% \item[\\cX\meta{character or class}] Matches a token with category
+% code |X| (any of |CBEMTPUDSLOA|) if it is also matched by the
+% \meta{character or class}. For instance, |\cL[A-Z]| matches
+% uppercase letters of category code letter, |\cC.| matches any
+% control sequence, and |\cO\d| matches digits of category other.
+% \item[{\\c[XYZ]\meta{character or class}}] Matches a token with
+% category |X|, |Y|, or |Z| (each being any of |CBEMTPUDSLOA|),
+% if it is also matched by the \meta{character or class}. For instance,
+% |\c[LSO].| matches tokens of category letter, space, or other.
+% \item[{\\c[\char`\^XYZ]\meta{character or class}}] Matches a token with
+% category different from |X|, |Y|, or |Z| (each being any of
+% |CBEMTPUDSLOA|), if it is also matched by the \meta{character or
+% class}. For instance, |\c[LSO].| matches tokens of category
+% letter, space, or other.
% \end{l3regex-syntax}
+% The category code tests can be used inside classes; for instance,
+% |[\cO\d \c[LO][A-F]]| matches what \TeX{} considers as hexadecimal
+% digits, namely digits with category other, or uppercase letters from
+% |A| to |F| with category either letter or other.
%
% Options can be set with |(?|\meta{option}|)| and
% unset with |(?-|\meta{option}|)|. Options are local
@@ -526,6 +544,8 @@
% \begin{itemize}
% \item The \texttt{\{\}} quantifiers are only partially implemented.
% \item Clean up the use of messages.
+% \item Triple check magic numbers.
+% \item Test \cs{par}.
% \item Add tests for every regex and replacement feature.
% \item Make sure the documentation is correct, and the code comments
% are as well.
@@ -533,11 +553,12 @@
%
% Code improvements to come.
% \begin{itemize}
+% \item \cs{regex_show:N} to show how a given regular expression is
+% interpreted.
% \item Test for the maximum register \cs{c_max_register_int}.
% \item Use \tn{dimen} registers rather than \cs{l_regex_nesting_tl}
% to build \cs{regex_nesting:n}.
% \item Reduce the number of epsilon-transitions in alternatives.
-% \item Merge the \texttt{build} and \texttt{class} functions where possible.
% \item Improve nesting of |\c| in the replacement text.
% \item Move the \enquote{reconstruction} part of \pkg{l3regex}
% to \pkg{l3tl-analysis}.
@@ -577,22 +598,24 @@
% the matching, because the regex code uses registers in an unsafe way;
% \item Conditional subpatterns (other than with a look-ahead
% or look-behind condition): this is non-regular, isn't it?
+% \item Named subpatterns: \TeX{} programmers have lived so far without
+% any need for named macro parameters.
% \end{itemize}
%
% The following features of PCRE or perl will definitely not be implemented.
% \begin{itemize}
% \item |\cx|, similar to \TeX{}'s own |\^^x|;
% \item Comments: \TeX{} already has its own system for comments.
-% \item Named subpatterns: \TeX{} programmers have lived so far without
-% any need for named macro parameters.
% \item |\Q...\E| escaping: this would require to read the argument
% verbatim, which is not in the scope of this module.
% \item Atomic grouping, possessive quantifiers: those tools, mostly
% meant to fix catastrophic backtracking, are unnecessary in a
% non-backtracking algorithm, and difficult to implement.
-% \item Subroutine calls: this syntactic sugar is difficult to
-% include in a non-backtracking algorithm, in particular because
-% the corresponding group should be treated as atomic.
+% \item Subroutine calls: this syntactic sugar is difficult to include
+% in a non-backtracking algorithm, in particular because the
+% corresponding group should be treated as atomic. Also, we cannot
+% afford to run user code within the regular expression matching,
+% because of our \enquote{misuse} of registers.
% \item Recursion: this is a non-regular feature.
% \item Back-references: non-regular feature, this requires backtracking,
% which is prohibitively slow.
@@ -665,6 +688,15 @@
%
% \subsubsection{Variables used while building}
%
+% \begin{variable}{\l_regex_build_mode_int}
+% While building, ten modes are recognized, labelled $-63$, $-23$,
+% $-6$, $-2$, $0$, $2$, $3$, $6$, $23$, $63$. See
+% section~\ref{sec:regex-modes}.
+% \begin{macrocode}
+\int_new:N \l_regex_build_mode_int
+% \end{macrocode}
+% \end{variable}
+%
% \begin{variable}{\l_regex_max_state_int}
% \begin{variable}{\l_regex_left_state_int, \l_regex_right_state_int}
% The end-point states of the last group (which any quantifier
@@ -723,16 +755,6 @@
% \end{variable}
% \end{variable}
%
-% \begin{variable}{\l_regex_build_or_class_tl}
-% The actions of various escape sequences and special characters
-% are different within a class and outside a class. Rather than
-% using a boolean, we store either \texttt{build} or \texttt{class}
-% in this token list.
-% \begin{macrocode}
-\tl_new:N \l_regex_build_or_class_tl
-% \end{macrocode}
-% \end{variable}
-%
% \begin{variable}{\l_regex_one_or_group_tl}
% When looking for quantifiers, this variable holds either
% \enquote{one} or \enquote{group} depending on whether the
@@ -743,11 +765,52 @@
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\l_regex_in_csname_bool}
-% This boolean is true when building the regular expression
-% for a given csname.
+% \begin{variable}{\l_regex_catcodes_int, \l_regex_catcodes_default_int}
+% \begin{variable}
+% {
+% \c_regex_catcode_C_int, \c_regex_catcode_B_int, \c_regex_catcode_E_int,
+% \c_regex_catcode_M_int, \c_regex_catcode_T_int, \c_regex_catcode_P_int,
+% \c_regex_catcode_U_int, \c_regex_catcode_D_int, \c_regex_catcode_S_int,
+% \c_regex_catcode_L_int, \c_regex_catcode_O_int, \c_regex_catcode_A_int
+% }
+% \begin{variable}{\c_regex_catcodes_all_int}
+% We wish to allow constructions such as |\c[^BE](..\cL[a-z]..)|,
+% matching two tokens which are neither a begin-group nor an end-group
+% token, followed by a token of category letter and character code in
+% |[a-z]|, followed by two more tokens which are neither begin-group
+% nor end-group tokens. For this to work, we need to keep track of
+% lists of allowed category codes: \cs{l_regex_catcodes_int} and
+% \cs{l_regex_catcodes_default_int} are bitmaps, sums of $4^c$, for
+% all allowed catcodes $c$. The latter is local to each capturing
+% group, and we reset \cs{l_regex_catcodes_int} to that value after
+% each character or class, changing it only when encountering a |\c|
+% escape.
+% \begin{macrocode}
+\int_new:N \l_regex_catcodes_int
+\int_new:N \l_regex_catcodes_default_int
+\int_const:Nn \c_regex_catcode_C_int { "1 }
+\int_const:Nn \c_regex_catcode_B_int { "4 }
+\int_const:Nn \c_regex_catcode_E_int { "10 }
+\int_const:Nn \c_regex_catcode_M_int { "40 }
+\int_const:Nn \c_regex_catcode_T_int { "100 }
+\int_const:Nn \c_regex_catcode_P_int { "1000 }
+\int_const:Nn \c_regex_catcode_U_int { "4000 }
+\int_const:Nn \c_regex_catcode_D_int { "10000 }
+\int_const:Nn \c_regex_catcode_S_int { "100000 }
+\int_const:Nn \c_regex_catcode_L_int { "400000 }
+\int_const:Nn \c_regex_catcode_O_int { "1000000 }
+\int_const:Nn \c_regex_catcode_A_int { "4000000 }
+\int_const:Nn \c_regex_catcodes_all_int { "5515155 }
+% \end{macrocode}
+% \end{variable}
+% \end{variable}
+% \end{variable}
+%
+% \begin{variable}{\l_regex_catcodes_bool}
+% Controls whether the bitmap of category codes built should be
+% inverted or not.
% \begin{macrocode}
-\bool_new:N \l_regex_in_csname_bool
+\bool_new:N \l_regex_catcodes_bool
% \end{macrocode}
% \end{variable}
%
@@ -761,22 +824,22 @@
%
% \subsubsection{Character classes}
%
-% \begin{macro}{\regex_build_tmp_class:n}
% \begin{variable}{\l_regex_class_bool,\l_regex_class_tl}
+% \begin{variable}{\l_regex_class_saved_bool,\l_regex_class_saved_tl}
% \cs{l_regex_class_bool} is false for negative character classes.
-% \cs{l_regex_class_tl} holds the tests which should be performed
-% to decide whether the \cs{l_regex_current_char_int} matches that
-% character class. Once the class is read completely, the full
-% instructions are stored in \cs{regex_build_tmp_class:n}, whose
-% argument is the target state if the test succeeds. It is also used
-% for all other single-character situations (\emph{e.g.}, |\d|, or |.|).
-% \begin{macrocode}
-\cs_new_eq:NN \regex_build_tmp_class:n \use_none:n
+% \cs{l_regex_class_tl} holds the tests which should be performed to
+% decide whether the \cs{l_regex_current_char_int} matches that
+% character class. In nested class, which can only occur as
+% |[\c[...]]|, the two variables from the outer class must be saved
+% while processing the inner class.
+% \begin{macrocode}
\bool_new:N \l_regex_class_bool
\tl_new:N \l_regex_class_tl
+\bool_new:N \l_regex_class_saved_bool
+\tl_new:N \l_regex_class_saved_tl
% \end{macrocode}
% \end{variable}
-% \end{macro}
+% \end{variable}
%
% \begin{variable}{\c_regex_d_tl,\c_regex_D_tl}
% \begin{variable}{\c_regex_h_tl,\c_regex_H_tl}
@@ -1202,12 +1265,12 @@
% This is done \enquote{by hand}, in sequences whose items should
% be run immediately.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_seq_pop_use:N #1
+\cs_new_protected:Npn \regex_seq_pop_use:N #1
{
\seq_pop:NN #1 \l_regex_tmpa_tl
\l_regex_tmpa_tl
}
-\cs_new_protected_nopar:Npn \regex_seq_get_use:N #1
+\cs_new_protected:Npn \regex_seq_get_use:N #1
{
\seq_get:NN #1 \l_regex_tmpa_tl
\l_regex_tmpa_tl
@@ -1216,45 +1279,10 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}[int, EXP]{\regex_seq_item:Nn}
-% \begin{macro}[aux, EXP]{\regex_seq_item_aux:wNn}
-% \begin{macrocode}
-\cs_new:Npn \regex_seq_item:Nn #1#2
- {
- \exp_after:wN \regex_seq_item_aux:wNn
- \int_use:N \int_eval:w #2 \exp_after:wN ;
- #1 { ? \prg_map_break: } { }
- \prg_break_point:n { }
- }
-\cs_new:Npn \regex_seq_item_aux:wNn #1; #2#3
- {
- \use_none:n #2
- \if_num:w #1 = \c_zero
- \exp_after:wN \prg_map_break:n
- \else:
- \exp_after:wN \use_none:n
- \fi:
- { \exp_not:n {#3} }
- \exp_after:wN \regex_seq_item_aux:wNn
- \int_use:N \int_eval:w #1 - \c_one ;
- }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-%
% \subsubsection{Testing characters}
%
-% \begin{macro}{\regex_item_dot:T}
-% The dot meta-character matches any character,
-% except the end marker.
-% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_item_dot:T
- { \int_compare:nNnF \l_regex_current_char_int < \c_minus_one }
-% \end{macrocode}
-% \end{macro}
-%
% \begin{macro}[int]{\regex_break_point:TF}
-% \begin{macro}[int]{\regex_break_true:w,\regex_break_false:w}
+% \begin{macro}[int]{\regex_break_true:w}
% When testing whether a character of the query token list matches
% a given character class in the regular expression, we often
% have to test it against several ranges of characters, checking
@@ -1268,27 +1296,38 @@
% Otherwise, \cs{regex_break_point:TF} leaves the \meta{false code}
% in the input stream.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_break_true:w
+\cs_new_protected:Npn \regex_break_true:w
#1 \regex_break_point:TF #2 #3 {#2}
-\cs_new_protected_nopar:Npn \regex_break_false:w
- #1 \regex_break_point:TF #2 #3 {#3}
-\cs_new_protected_nopar:Npn \regex_break_point:TF #1 #2 { #2 }
+\cs_new_protected:Npn \regex_break_point:TF #1 #2 { #2 }
% \end{macrocode}
% \end{macro}
% \end{macro}
%
+% \begin{macro}{\regex_item_dot:}
+% The dot meta-character matches any character,
+% except the end marker.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \regex_item_dot:
+ {
+ \if_num:w \l_regex_current_char_int > - \c_two
+ \exp_after:wN \regex_break_true:w
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
% \begin{macro}[int]{\regex_item_caseful_equal:n}
% \begin{macro}[int]{\regex_item_caseful_range:nn}
% \begin{macro}[int]{\regex_item_caseful_geq:n}
% Simple comparisons triggering \cs{regex_break_true:w} when true.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_item_caseful_equal:n #1
+\cs_new_protected:Npn \regex_item_caseful_equal:n #1
{
\if_num:w #1 = \l_regex_current_char_int
\exp_after:wN \regex_break_true:w
\fi:
}
-\cs_new_protected_nopar:Npn \regex_item_caseful_range:nn #1 #2
+\cs_new_protected:Npn \regex_item_caseful_range:nn #1 #2
{
\reverse_if:N \if_num:w #1 > \l_regex_current_char_int
\reverse_if:N \if_num:w #2 < \l_regex_current_char_int
@@ -1296,7 +1335,7 @@
\fi:
\fi:
}
-\cs_new_protected_nopar:Npn \regex_item_caseful_geq:n #1
+\cs_new_protected:Npn \regex_item_caseful_geq:n #1
{
\reverse_if:N \if_num:w #1 > \l_regex_current_char_int
\exp_after:wN \regex_break_true:w
@@ -1314,7 +1353,7 @@
% \cs{l_regex_current_char_int} and on
% \cs{l_regex_case_changed_char_int}.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_item_caseless_equal:n #1
+\cs_new_protected:Npn \regex_item_caseless_equal:n #1
{
\if_num:w #1 = \l_regex_current_char_int
\exp_after:wN \regex_break_true:w
@@ -1323,7 +1362,7 @@
\exp_after:wN \regex_break_true:w
\fi:
}
-\cs_new_protected_nopar:Npn \regex_item_caseless_range:nn #1 #2
+\cs_new_protected:Npn \regex_item_caseless_range:nn #1 #2
{
\reverse_if:N \if_num:w #1 > \l_regex_current_char_int
\reverse_if:N \if_num:w #2 < \l_regex_current_char_int
@@ -1336,7 +1375,7 @@
\fi:
\fi:
}
-\cs_new_protected_nopar:Npn \regex_item_caseless_geq:n #1
+\cs_new_protected:Npn \regex_item_caseless_geq:n #1
{
\reverse_if:N \if_num:w #1 > \l_regex_current_char_int
\exp_after:wN \regex_break_true:w
@@ -1387,26 +1426,30 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[int]{\regex_item_catcode:n}
-% \begin{macro}[int]{\regex_item_catcode_neq:n}
-% Checking the category code is quite straightforward.
+% \begin{macro}{\regex_item_catcode:nT}
+% The argument is a sum of powers of $4$ with exponents given by the
+% allowed category codes (between $0$ and $13$). Dividing by a given
+% power of $4$ gives an odd result if and only if that category code
+% is allowed. If the catcode does not match, then skip the character
+% code tests which follow.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_item_catcode:n #1
- {
- \if_num:w #1 = \l_regex_current_catcode_int
- \exp_after:wN \regex_break_true:w
- \fi:
- }
-\cs_new_protected_nopar:Npn \regex_item_catcode_neq:n #1
+\cs_new_protected:Npn \regex_item_catcode:nT #1
{
- \if_num:w #1 = \l_regex_current_catcode_int
+ \if_int_odd:w \int_eval:w #1 / "
+ \if_case:w \l_regex_current_catcode_int
+ 1 \or: 4 \or: 10 \or: 40
+ \or: 100 \or: \or: 1000 \or: 4000
+ \or: 10000 \or: \or: 100000 \or: 400000
+ \or: 1000000 \or: 4000000 \else: 1*\c_zero
+ \fi:
+ \int_eval_end:
+ \exp_after:wN \use:n
\else:
- \exp_after:wN \regex_break_true:w
+ \exp_after:wN \use_none:n
\fi:
}
% \end{macrocode}
% \end{macro}
-% \end{macro}
%
% \begin{macro}[int]{\regex_item_cs:n}
% Match a control sequence (the argument is a pre-compiled regex).
@@ -1455,7 +1498,7 @@
{ \if_false: } } \fi:
\regex_get_digits_aux:NN
}
-\cs_new_nopar:Npn \regex_get_digits_aux:NN #1#2
+\cs_new:Npn \regex_get_digits_aux:NN #1#2
{
\if_meaning:w \regex_build_raw:N #1
\if_charcode:w \c_space_token \exp_not:N #2
@@ -1471,7 +1514,7 @@
\fi:
\regex_get_digits_aux:NN
}
-\cs_new_nopar:Npn \regex_get_digits_end:w #1 \fi: #2 \regex_get_digits_aux:NN
+\cs_new:Npn \regex_get_digits_end:w #1 \fi: #2 \regex_get_digits_aux:NN
{
\fi: #2
\if_false: { { \fi: } }
@@ -1568,6 +1611,66 @@
%
% \subsection{Building}
%
+% \subsubsection{Build mode}
+% \label{sec:regex-modes}
+%
+% When building the NFA corresponding to a given regex, we can be in
+% ten distinct modes, which we label by some magic numbers:
+% \begin{itemize}
+% \item[-6] |[\c{...}]| control sequence in a class,
+% \item[-2] |\c{...}| control sequence,
+% \item[0] |...| outer,
+% \item[2] |\c...| catcode test,
+% \item[6] |[\c...]| catcode test in a class,
+% \item[-63] |[\c{[...]}]| class inside mode $-6$,
+% \item[-23] |\c{[...]}| class inside mode $-2$,
+% \item[3] |[...]| class inside mode $-3$,
+% \item[23] |\c[...]| class inside mode $2$,
+% \item[63] |[\c[...]]| class inside mode $6$.
+% \end{itemize}
+% This list is exhaustive, because |\c| escape sequences cannot be
+% nested, and character classes cannot be nested directly. The choice
+% of numbers is such as to optimize the most useful tests, and make
+% transitions from one mode to another as simple as possible.
+% \begin{itemize}
+% \item Even modes mean that we are not directly in a character class.
+% In this case, a left bracket appends $3$ to the mode. In a character
+% class, a right bracket changes the mode as $m\to (m-15)/13$,
+% truncated.
+% \item Grouping, assertion, and anchors are allowed in non-positive
+% even modes ($0$, $-2$, $-6$), and do not change the mode. Otherwise,
+% they trigger an error.
+% \item A left bracket is special in even modes, appending $3$ to the
+% mode; in those modes, quantifiers and the dot are recognized, and
+% the right bracket is normal. In odd modes (within classes), the left
+% bracket is normal, but the right bracket ends the class, changing
+% the mode from $m$ to $(m-15)/13$, truncated; also, ranges are
+% recognized.
+% \item In non-negative modes, left and right braces are normal. In
+% negative modes, however, left braces trigger a warning; right braces
+% end the control sequence, going from $-2$ to $0$ or $-6$ to $3$,
+% with error recovery for odd modes.
+% \item Properties (such as the |\d| character class) can appear in any
+% mode.
+% \end{itemize}
+%
+% \begin{macro}[int, EXP]{\regex_build_if_in_class:TF}
+% Test whether we are currently in a character class (at the
+% inner-most level of nesting). There, many escape sequences are not
+% recognized, and special characters are normal. Also, for every raw
+% character, we must look ahead for a possible raw dash.
+% \begin{macrocode}
+\cs_new_nopar:Npn \regex_build_if_in_class:TF
+ {
+ \if_int_odd:w \l_regex_build_mode_int
+ \exp_after:wN \use_i:nn
+ \else:
+ \exp_after:wN \use_ii:nn
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
% \subsubsection{Helpers for building an NFA}
%
% \begin{macro}[int]{\regex_build_new_state:}
@@ -1586,18 +1689,18 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_transition_aux:NN}
-% \begin{macro}[aux]{\regex_build_transitions_aux:NNNN}
+% \begin{macro}[aux]{\regex_build_transition:NN}
+% \begin{macro}[aux]{\regex_build_transitions:NNNN}
% These functions create a new state, and put one or two transitions
% starting from the old current state.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_transition_aux:NN #1#2
+\cs_new_protected:Npn \regex_build_transition:NN #1#2
{
\regex_build_new_state:
\regex_toks_put_right:Nx \l_regex_left_state_int
{ #1 { \int_eval:n { #2 - \l_regex_left_state_int } } }
}
-\cs_new_protected_nopar:Npn \regex_build_transitions_aux:NNNN #1#2#3#4
+\cs_new_protected:Npn \regex_build_transitions:NNNN #1#2#3#4
{
\regex_build_new_state:
\regex_toks_put_right:Nx \l_regex_left_state_int
@@ -1612,12 +1715,11 @@
%
% \subsubsection{From regex to NFA: framework}
%
-% In order for the construction \verb"ab|cd" to work, we enclose
-% the whole pattern within parentheses (in the code below,
-% \cs{regex_build_open_aux:} and \cs{regex_build_close_aux:}).
-% These have the added benefit
-% to form a capturing group: hence we get the data of the whole match
-% for free.
+%^^A In order for the construction \verb"ab|cd" to work, we enclose the
+%^^A whole pattern within parentheses (in the code below,
+%^^A \cs{regex_build_open_aux:} and \cs{regex_build_close_aux:}). These
+%^^A have the added benefit to form a capturing group: hence we get the
+%^^A data of the whole match for free.
%
% \begin{macro}[int]{\regex_build:n}
% First, reset a few variables. Then use the generic framework defined
@@ -1633,11 +1735,11 @@
% \begin{macrocode}
\cs_new_protected:Npn \regex_build:n #1
{
- \bool_set_false:N \l_regex_in_csname_bool
+ \int_set_eq:NN \l_regex_build_mode_int \c_zero
\regex_build:w
\regex_build_new_state:
\regex_toks_put_right:Nx \l_regex_left_state_int
- { \regex_action_start_wildcard:nn {0} {1} }
+ { \regex_action_start_wildcard: }
\regex_build_open_aux:
\str_escape_use:NNNn
\regex_build_i_unescaped:N
@@ -1645,6 +1747,7 @@
\regex_build_i_raw:N
{ #1 }
\prg_do_nothing: \prg_do_nothing:
+ \prg_do_nothing: \prg_do_nothing:
\regex_build_end:
}
% \end{macrocode}
@@ -1663,55 +1766,60 @@
% special meanings, while everything else should be treated as a raw
% character.
% \begin{macrocode}
-\cs_new_nopar:Npn \regex_build_i_unescaped:N #1
+\cs_new:Npn \regex_build_i_unescaped:N #1
{
\regex_aux_char_if_special:NTF #1
- { \exp_not:N \regex_build_control:N #1 }
+ { \exp_not:N \regex_build_special:N #1 }
{ \exp_not:N \regex_build_raw:N #1 }
}
-\cs_new_nopar:Npn \regex_build_i_escaped:N #1
+\cs_new:Npn \regex_build_i_escaped:N #1
{
\regex_aux_char_if_alphanumeric:NTF #1
- { \exp_not:N \regex_build_control:N #1 }
+ { \exp_not:N \regex_build_escaped:N #1 }
{ \exp_not:N \regex_build_raw:N #1 }
}
-\cs_new_nopar:Npn \regex_build_i_raw:N #1
+\cs_new:Npn \regex_build_i_raw:N #1
{ \exp_not:N \regex_build_raw:N #1 }
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_control:N}
-% If the control character has a particular meaning in regexes,
-% the corresponding function is used. Otherwise, it is interpreted
-% as a raw character. The \cs{regex_build/class_default_raw:N}
-% functions are defined later.
+% \begin{macro}[aux]{\regex_build_special:N}
+% \begin{macro}[aux]{\regex_build_escaped:N}
+% If the control character has a particular meaning in regexes, the
+% corresponding function is used. Otherwise, it is interpreted as a
+% raw character. We distinguish special characters from escaped
+% alphanumeric characters because in character classes, unknown
+% escaped alphanumeric characters raise an error, while special
+% charcters are silently converted to raw characters.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_control:N #1
+\cs_new_protected:Npn \regex_build_special:N #1
{
- \cs_if_exist_use:cF { regex_ \l_regex_build_or_class_tl _#1: }
- { \use:c { regex_ \l_regex_build_or_class_tl _default_raw:N } #1 }
+ \cs_if_exist_use:cF { regex_build_#1: }
+ { \regex_build_raw:N #1 }
+ }
+\cs_new_protected:Npn \regex_build_escaped:N #1
+ {
+ \cs_if_exist_use:cF { regex_build_/#1: }
+ { \regex_build_raw:N #1 }
}
% \end{macrocode}
% \end{macro}
-%
-%^^A todo: update
-% This function is similar to \cs{regex_build_control:N}. If the control
-% character has a meaning in character classes, call the corresponding
-% function, otherwise, treat it as a raw character, with the
-% \cs{regex_class_default_raw:N} function, defined later.
+% \end{macro}
%
% \begin{macro}[int]{\regex_build:w}
% Hopefully, we didn't forget to initialize anything here.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \regex_build:w
{
- \cs_set_eq:NN \regex_build_raw:N \regex_build_default_raw:N
- \tl_set:Nn \l_regex_build_or_class_tl { build }
- \int_set_eq:NN \l_regex_capturing_group_int \c_zero
+ \int_set_eq:NN \l_regex_catcodes_default_int \c_regex_catcodes_all_int
+ \int_set_eq:NN \l_regex_capturing_group_int \c_zero
\int_zero:N \l_regex_max_state_int
\regex_build_new_state:
+ \tl_clear:N \l_regex_class_tl
+ \bool_set_true:N \l_regex_class_bool
+ \int_set_eq:NN \l_regex_catcodes_int \l_regex_catcodes_default_int
}
% \end{macrocode}
% \end{macro}
@@ -1740,179 +1848,162 @@
% \end{macrocode}
% \end{macro}
%
-% \subsubsection{Anchoring and simple assertions}
-%
-% \begin{macro}[int]{\regex_build_A:}
-% \begin{macro}[int]+\regex_build_^:+
-% \begin{macro}[int]{\regex_build_G:}
-% \begin{macro}[aux]{\regex_build_anchor_start:N}
-% Anchoring at the start corresponds to checking that the current
-% character is the first in the token list. Anchoring to the beginning
-% of the match attempt uses \cs{l_regex_start_step_int} instead of
-% \cs{c_zero}.
+% \begin{macro}[int]{\regex_build_one:n, \regex_build_one:x}
+% In a class, add the argument to the current class. Outside a class,
+% this argument is the whole \enquote{class}, and we look for
+% quantifiers.
% \begin{macrocode}
-\cs_new_protected_nopar:cpn { regex_build_^: }
- { \regex_build_anchor_start:N \l_regex_min_step_int }
-\cs_new_protected_nopar:Npn \regex_build_A:
- { \regex_build_anchor_start:N \l_regex_min_step_int }
-\cs_new_protected_nopar:Npn \regex_build_G:
- { \regex_build_anchor_start:N \l_regex_start_step_int }
-\cs_new_protected_nopar:Npn \regex_build_anchor_start:N #1
+\cs_new_protected:Npn \regex_build_one:n #1
+ { \regex_build_one:x { \exp_not:n {#1} } }
+\cs_new_protected:Npn \regex_build_one:x #1
{
- \regex_build_new_state:
- \regex_toks_put_right:Nx \l_regex_left_state_int
+ \tl_put_right:Nx \l_regex_class_tl
{
- \exp_not:N \int_compare:nNnT #1 = \l_regex_current_step_int
- {
- \regex_action_free:n
- {
- \int_eval:n
- { \l_regex_right_state_int - \l_regex_left_state_int }
- }
- }
+ \if_num:w \l_regex_catcodes_int = \c_regex_catcodes_all_int
+ \exp_after:wN \use:n
+ \else:
+ \regex_item_catcode:nT { \int_use:N \l_regex_catcodes_int }
+ \fi:
+ {#1}
}
+ \if_num:w \l_regex_build_mode_int = \c_two
+ \l_regex_build_mode_int = \c_zero
+ \else:
+ \if_num:w \l_regex_build_mode_int = \c_six
+ \l_regex_build_mode_int = \c_three
+ \fi:
+ \fi:
+ \if_int_odd:w \l_regex_build_mode_int \else:
+ \exp_after:wN \regex_build_one_quantifier:
+ \fi:
}
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_Z:}
-% \begin{macro}[aux]{\regex_build_z:}
-% \begin{macro}[aux]+\regex_build_$:+
-% This matches the end of the token list, marked by a character code of $-2$.
+% \begin{macro}{\regex_build_tmp_class:n}
+% ^^A todo: rename, not tmp, and used for class & single.
+% The argument is the target state if the test succeeds.
% \begin{macrocode}
-\cs_new_protected_nopar:cpn { regex_build_$: } % $
+\cs_new:Npn \regex_build_tmp_class:n #1
{
- \regex_build_new_state:
- \regex_toks_put_right:Nx \l_regex_left_state_int
- {
- \exp_not:N \int_compare:nNnT
- \l_regex_current_char_int < \c_minus_one
- {
- \regex_action_free:n
- {
- \int_eval:n
- { \l_regex_right_state_int - \l_regex_left_state_int }
- }
- }
- }
+ \exp_not:o \l_regex_class_tl
+ \bool_if:NTF \l_regex_class_bool
+ { \regex_break_point:TF { \regex_action_cost:n {#1} } { } }
+ { \regex_break_point:TF { } { \regex_action_cost:n {#1} } }
}
-\cs_new_eq:Nc \regex_build_Z: { regex_build_$: } %$
-\cs_new_eq:Nc \regex_build_z: { regex_build_$: } %$
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}[int]{\regex_build_b:}
-% \begin{macro}[int]{\regex_build_B:}
-% \begin{macro}[aux]{\regex_if_word_boundary:TF}
-% Contrarily to |^| and |$|, which could be implemented without
-% really knowing what precedes in the token list, this requires
-% more information, namely, the knowledge of the last character
-% code. Case sensitivity does not change word boundaries.
+% \begin{macro}[int]{\regex_class_put:N}
+% \begin{macro}[int]{\regex_class_put:NN}
+% This function is used to rapidly put a character in a character
+% class being built, avoiding some tests implied by \cs{regex_build_one:x}.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_b:
+\cs_new_protected:Npn \regex_class_put:N #1
{
- \regex_build_new_state:
- \regex_toks_put_right:Nx \l_regex_left_state_int
- {
- \exp_not:N \regex_if_word_boundary:TF
- {
- \regex_action_free:n
- {
- \int_eval:n
- { \l_regex_right_state_int - \l_regex_left_state_int }
- }
- }
- { }
- }
+ \tl_put_right:Nx \l_regex_class_tl
+ { \regex_item_equal:n { \int_value:w `#1 ~ } }
}
-\cs_new_protected_nopar:Npn \regex_build_B:
- {
- \regex_build_new_state:
- \regex_toks_put_right:Nx \l_regex_left_state_int
- {
- \exp_not:N \regex_if_word_boundary:TF
- { }
- {
- \regex_action_free:n
- {
- \int_eval:n
- { \l_regex_right_state_int - \l_regex_left_state_int }
- }
- }
- }
- }
-\cs_new_protected_nopar:Npn \regex_if_word_boundary:TF
+\cs_new_protected:Npn \regex_class_put:NN #1#2
{
- \group_begin:
- \int_set_eq:NN \l_regex_current_char_int \l_regex_last_char_int
- \c_regex_w_tl
- \regex_break_point:TF
- { \group_end: \c_regex_W_tl \regex_item_equal:n { -2 } }
- { \group_end: \c_regex_w_tl }
- \regex_break_point:TF
+ \tl_put_right:Nx \l_regex_class_tl
+ { \regex_item_range:nn { \int_value:w `#1 ~ } { \int_value:w `#2 ~ } }
}
% \end{macrocode}
% \end{macro}
% \end{macro}
-% \end{macro}
-%
-% \subsubsection{Normal character, and simple character classes}
%
-% \begin{macro}[int]{\regex_build_one:n, \regex_build_one:x}
+% \begin{macro}[int]{\regex_build_raw_alphanum_is_error:N}
+% Within character classes, some special character and escape
+% sequences do not have any meaning. Special characters are
+% interpreted as their \enquote{raw} counterpart, but escape sequences
+% without a meaning, \emph{e.g.}, |\A|, are most likely an error:
+% this should be |A|.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_build_raw_alphanum_is_error:N #1
+ {
+ \regex_aux_char_if_alphanumeric:NTF #1
+ { \msg_kernel_error:nnx { regex } { class-bad-escape } { #1 } }
+ { \regex_build_raw:N #1 }
+ }
+% \end{macrocode}
+% \end{macro}
%
+% \begin{macro}[int]{\regex_build_raw:N}
+% If we are in a character class and the next character is an
+% unescaped dash, this denotes a range. Otherwise, the current
+% character |#1| matches itself.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_one:n #1
- { \regex_build_one:x { \exp_not:n {#1} } }
-\cs_new_protected:Npn \regex_build_one:x #1
+\cs_new_protected:Npn \regex_build_raw:N #1#2#3
{
- \cs_set_nopar:Npx \regex_build_tmp_class:n ##1
+ \regex_build_if_in_class:TF
+ { \str_if_eq:nnTF {#2#3} { \regex_build_special:N - } }
+ { \use_ii:nn }
+ { \regex_class_range:Nw #1 }
{
- \exp_not:N \exp_not:n {#1}
- \regex_break_point:TF { \regex_action_cost:n {##1} } { }
+ \regex_build_one:x { \regex_item_equal:n { \int_value:w `#1 ~ } }
+ #2 #3
}
- \regex_build_one_quantifier:
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_default_raw:N}
-% A normal alphanumeric or an escaped non-alphanumeric
-% (actually, any unknown combination) will match itself
-% and the thread will fail otherwise. We prepare
-% \cs{regex_build_tmp_class:n} with the relevant test and
-% commands. The state shift to be inserted in those
-% commands will come as |##1|: we don't know
-% yet what this will be before checking for quantifiers.
+% \begin{macro}[aux]{\regex_class_range:Nw}
+% We have just read a raw character followed by a dash; this should be
+% followed by an end-point for the range. If the following character
+% is an escaped alphanumeric, or if it is an unescaped right bracket,
+% then we have an error, so we put the initial character and the dash
+% back, as raw characters. Otherwise, build the range, checking that
+% it is in the right order, and optimizing for equal end-points.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_default_raw:N #1
- { \regex_build_one:x { \regex_item_equal:n { \int_value:w `#1 ~ } } }
+\cs_new_protected:Npn \regex_class_range:Nw #1#2#3
+ {
+ \token_if_eq_meaning:NNTF #2 \regex_build_escaped:N % [
+ { \use_i:nn } { \str_if_eq:nnTF { #2#3 } { \regex_build_special:N ] } }
+ {
+ \msg_kernel_warning:nnx { regex } { end-range-missing } % [
+ { #1 - \if_meaning:w #3 ] \else: \c_backslash_str \fi: #3 }
+ \regex_class_put:N #1
+ \regex_class_put:N -
+ #2#3
+ }
+ {
+ \if_num:w `#1 > `#3 \exp_stop_f:
+ \msg_kernel_error:nnxx { regex } { backwards-range } {#1} {#3}
+ \else:
+ \if_num:w `#1 = `#3 \exp_stop_f:
+ \regex_class_put:N #3
+ \else:
+ \regex_class_put:NN #1#3
+ \fi:
+ \fi:
+ }
+ }
% \end{macrocode}
% \end{macro}
%
+% \subsubsection{Character properties}
+%
% \begin{macro}[aux]{\regex_build_.:}
-% Similar to \cs{regex_build_default_raw:N} but accepts any character,
-% and refuses $-1$, which marks the end of the token list.
+% In a class, the dot has no special meaning. Outside, insert
+% \cs{regex_item_dot:}, which matches any character or control
+% sequence, and refuses $-2$, which marks the end of the token list.
% \begin{macrocode}
\cs_new_protected_nopar:cpn { regex_build_.: }
{
- \cs_set:Npn \regex_build_tmp_class:n ##1
- { \regex_item_dot:T { \regex_action_cost:n {##1} } }
- \regex_build_one_quantifier:
+ \regex_build_if_in_class:TF
+ { \regex_build_raw:N . }
+ { \regex_build_one:x \regex_item_dot: }
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_d:,\regex_build_D:}
-% \begin{macro}[aux]{\regex_build_h:,\regex_build_H:}
-% \begin{macro}[aux]{\regex_build_s:,\regex_build_S:}
-% \begin{macro}[aux]{\regex_build_v:,\regex_build_V:}
-% \begin{macro}[aux]{\regex_build_w:,\regex_build_W:}
-% \begin{macro}[aux]{\regex_build_N:}
+% \begin{macro}[aux]{\regex_build_/d:,\regex_build_/D:}
+% \begin{macro}[aux]{\regex_build_/h:,\regex_build_/H:}
+% \begin{macro}[aux]{\regex_build_/s:,\regex_build_/S:}
+% \begin{macro}[aux]{\regex_build_/v:,\regex_build_/V:}
+% \begin{macro}[aux]{\regex_build_/w:,\regex_build_/W:}
+% \begin{macro}[aux]{\regex_build_/N:}
% The constants \cs{c_regex_d_tl}, \emph{etc.} hold
% a list of tests which match the corresponding character
% class, and jump to the \cs{regex_break_point:TF} marker.
@@ -1920,7 +2011,7 @@
% \begin{macrocode}
\tl_map_inline:nn { dDhHsSvVwWN }
{
- \cs_new_protected_nopar:cpx { regex_build_#1: }
+ \cs_new_protected_nopar:cpx { regex_build_/#1: }
{ \regex_build_one:n \exp_not:c { c_regex_#1_tl } }
}
% \end{macrocode}
@@ -1931,263 +2022,338 @@
% \end{macro}
% \end{macro}
%
-% \subsubsection{Character classes}
+% \subsubsection{Anchoring and simple assertions}
%
-% \begin{macro}[aux]{\regex_class_one:n, \regex_class_one:x}
-% This is the function used to add material
-% to the class.
+% \begin{macro}[aux]{\regex_build_simple_assertion:Nn}
+% Assertions are not allowed within character classes: the raw
+% character |#1| is inserted instead. Otherwise, use the test |#2|;
+% if the assertion is successful, move from the \texttt{left} state to
+% the \texttt{right} state.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_class_one:n
- { \tl_put_right:Nn \l_regex_class_tl }
-\cs_new_protected:Npn \regex_class_one:x
- { \tl_put_right:Nx \l_regex_class_tl }
+\cs_new_protected:Npn \regex_build_simple_assertion:Nn #1#2
+ {
+ \regex_build_if_in_class:TF
+ { \regex_build_raw_alphanum_is_error:N #1 }
+ {
+ \regex_build_new_state:
+ \regex_toks_put_right:Nx \l_regex_left_state_int
+ {
+ \exp_not:n {#2}
+ {
+ \regex_action_free:n
+ {
+ \int_eval:n
+ { \l_regex_right_state_int - \l_regex_left_state_int }
+ }
+ }
+ }
+ %^^A \regex_assertion_quantifier:
+ }
+ }
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_[:}
-% This starts a class. The code for the class is collected
-% in \cs{l_regex_class_tl}. The first character is special.
+% \begin{macro}[aux]+\regex_build_^:+
+% \begin{macro}[aux]{\regex_build_/A:}
+% \begin{macro}[aux]{\regex_build_/G:}
+% \begin{macro}[aux]+\regex_build_$:+
+% \begin{macro}[aux]{\regex_build_/Z:}
+% \begin{macro}[aux]{\regex_build_/z:}
+% Anchoring at the start corresponds to checking that the current
+% character is the first in the token list. Anchoring to the beginning
+% of the match attempt uses \cs{l_regex_start_step_int} instead of
+% \cs{c_zero}. End anchors match the end of the token list, marked by
+% a character code of $-2$.
% \begin{macrocode}
-\cs_new_protected_nopar:cpn { regex_build_[: }
+\cs_new_protected_nopar:cpn { regex_build_^: }
{
- \tl_clear:N \l_regex_class_tl
- \cs_set_eq:NN \regex_build_raw:N \regex_class_default_raw:N
- \tl_set:Nn \l_regex_build_or_class_tl { class }
- \regex_class_first:NN
+ \regex_build_simple_assertion:Nn ^
+ { \int_compare:nNnT \l_regex_min_step_int = \l_regex_current_step_int }
+ }
+\cs_new_protected_nopar:cpn { regex_build_/A: }
+ {
+ \regex_build_simple_assertion:Nn A
+ { \int_compare:nNnT \l_regex_min_step_int = \l_regex_current_step_int }
+ }
+\cs_new_protected_nopar:cpn { regex_build_/G: }
+ {
+ \regex_build_simple_assertion:Nn G
+ { \int_compare:nNnT \l_regex_start_step_int = \l_regex_current_step_int }
+ }
+\cs_new_protected_nopar:cpn { regex_build_$: } % $
+ {
+ \regex_build_simple_assertion:Nn $ % $
+ { \int_compare:nNnT \l_regex_current_char_int < \c_minus_one }
+ }
+\cs_new_protected_nopar:cpn { regex_build_/Z: }
+ {
+ \regex_build_simple_assertion:Nn Z
+ { \int_compare:nNnT \l_regex_current_char_int < \c_minus_one }
+ }
+\cs_new_protected_nopar:cpn { regex_build_/z: }
+ {
+ \regex_build_simple_assertion:Nn z
+ { \int_compare:nNnT \l_regex_current_char_int < \c_minus_one }
}
% \end{macrocode}
% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
%
-% \begin{macro}[aux]{\regex_class_]:}
-% If \texttt{]} appears as the first item of a class, then
-% it doesn't end the class. Otherwise, it's the end,
-% act just as for a single character, but with a more
-% complicated test. And restore \cs{regex_build_raw:N}.
+% \begin{macro}[int]{\regex_build_/b:}
+% \begin{macro}[int]{\regex_build_/B:}
+% \begin{macro}[aux]{\regex_if_word_boundary:TF}
+% Contrarily to |^| and |$|, which could be implemented without
+% really knowing what precedes in the token list, this requires
+% more information, namely, the knowledge of the last character
+% code. Case sensitivity does not change word boundaries.
% \begin{macrocode}
-\cs_new_protected_nopar:cpn { regex_class_]: }
+\cs_new_protected_nopar:cpn { regex_build_/b: }
{
- \tl_if_empty:NTF \l_regex_class_tl %[
- { \regex_class_default_raw:N ] }
- {
- \cs_set_eq:NN \regex_build_raw:N \regex_build_default_raw:N
- \tl_set:Nn \l_regex_build_or_class_tl { build }
- \cs_set:Npn \regex_build_tmp_class:n ##1
- {
- \exp_not:o \l_regex_class_tl
- \bool_if:NTF \l_regex_class_bool
- { \regex_break_point:TF { \regex_action_cost:n {##1} } { } }
- { \regex_break_point:TF { } { \regex_action_cost:n {##1} } }
- }
- \regex_build_one_quantifier:
- }
+ \regex_build_simple_assertion:Nn b
+ { \regex_if_word_boundary:TF \use:n \use_none:n }
+ }
+\cs_new_protected_nopar:cpn { regex_build_/B: }
+ {
+ \regex_build_simple_assertion:Nn B
+ { \regex_if_word_boundary:TF { } }
+ }
+\cs_new_protected_nopar:Npn \regex_if_word_boundary:TF
+ {
+ \group_begin:
+ \int_set_eq:NN \l_regex_current_char_int \l_regex_last_char_int
+ \c_regex_w_tl
+ \regex_break_point:TF
+ { \group_end: \c_regex_W_tl \regex_item_equal:n { -2 } }
+ { \group_end: \c_regex_w_tl }
+ \regex_break_point:TF
}
% \end{macrocode}
% \end{macro}
+% \end{macro}
+% \end{macro}
%
-% \begin{macro}[aux]{\regex_class_first:NN}
-% If the first non-space character is |^|, then the class is inverted.
-% We keep track of this in \cs{l_regex_class_bool}.
+% \subsubsection{Entering and exiting character classes}
+%
+% \begin{macro}[aux]{\regex_build_[:}
+% In a class, left brackets mean nothing. Outside a class, this starts
+% a class, whose first characters may have a special meaning.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_class_first:NN #1#2
+\cs_new_protected_nopar:cpn { regex_build_[: }
{
- \str_if_eq:nnTF {#1#2} { \regex_build_control:N ^ }
- { \bool_set_false:N \l_regex_class_bool }
- {
- \bool_set_true:N \l_regex_class_bool
- #1 #2
- }
+ \regex_build_if_in_class:TF
+ { \regex_build_raw:N [ }
+ { \regex_class_first:NNNN }
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_class_default_raw:N}
-% \begin{macro}[aux]{\regex_class_single:N}
-% Most characters are treated here. We look ahead for an unescaped dash.
-% If there is none, then the character matches itself.
+% \begin{macro}[aux]{\regex_build_]:}
+% Outside a class, right brackets have no meaning. In a class, change
+% the mode ($m\to (m-15)/13$, truncated) to reflect the fact that we
+% are leaving the class. If we are still in a class after leaving one,
+% then this is the case |[...\cL[...]...]|, and we insert the relevant
+% closing material in \cs{l_regex_class_tl}. Otherwise look for
+% quantifiers.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_class_default_raw:N #1#2#3
+\cs_new_protected:cpn { regex_build_]: }
{
- \str_if_eq:nnTF {#2#3} { \regex_build_control:N - }
- { \regex_class_range:Nw #1 }
+ \regex_build_if_in_class:TF
{
- \regex_class_single:N #1
- #2 #3
+ \if_num:w \l_regex_build_mode_int > \c_sixteen
+ \tl_set:Nx \l_regex_class_tl
+ {
+ \exp_not:o \l_regex_class_saved_tl
+ \if_num:w \l_regex_catcodes_int < \c_regex_catcodes_all_int
+ \regex_item_catcode:nT { \int_use:N \l_regex_catcodes_int }
+ \else:
+ \exp_after:wN \use:n
+ \fi:
+ {
+ \exp_not:o \l_regex_class_tl
+ \bool_if:NF \l_regex_class_bool
+ { \regex_break_point:TF { } { \regex_break_true:w } }
+ }
+ }
+ \bool_set_eq:NN \l_regex_class_bool \l_regex_class_saved_bool
+ \fi:
+ \tex_advance:D \l_regex_build_mode_int - \c_fifteen
+ \tex_divide:D \l_regex_build_mode_int \c_thirteen
+ \if_int_odd:w \l_regex_build_mode_int \else:
+ \exp_after:wN \regex_build_one_quantifier:
+ \fi:
}
+ { \regex_build_raw:N ] }
}
-\cs_new_protected_nopar:Npn \regex_class_single:N #1
- { \regex_class_one:x { \regex_item_equal:n { \int_value:w `#1 } } }
% \end{macrocode}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}[aux]{\regex_class_range:Nw}
-% \begin{macro}[aux]{\regex_class_range_put:NN}
-% If the character is followed by a dash, we look for
-% the end-point of the range. For \enquote{raw} characters,
-% that's simply |#3|. Most \enquote{control} characters also
-% have no meaning, and can serve as an end-point, but those
-% with a meaning interrupt the range.
-% In the case of a true range, check whether the end-points
-% are in the right order, and optimize in the case of equal
-% end-points.
-% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_class_range:Nw #1#2#3
- {
- \token_if_eq_meaning:NNTF #2 \regex_build_control:N
+%^^A todo: ignore spaces everywhere!!
+%
+% \begin{macro}[aux]{\regex_class_first:NNNN}
+% This starts a class. Change the mode by appending $3$ to it, and
+% reset the variables \cs{l_regex_class_tl} and
+% \cs{l_regex_bool_tl}. In the special case of mode $63$
+% (|[\c[...]]|), we open a group, to avoid overriding the setting of
+% \cs{l_regex_class_bool} and \cs{l_regex_class_tl}; the group ends at
+% the matching right bracket. If the first character is |^|, then the
+% class is inverted. We keep track of this in
+% \cs{l_regex_class_bool}. If the next character is a right bracket,
+% then it should be changed to a raw one (dirty hack here; the |F|
+% argument of \cs{str_if_eq:nnTF} is the trailing |#3|).
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_class_first:NNNN #1#2#3#4
+ {
+ \l_regex_build_mode_int = \int_value:w \l_regex_build_mode_int 3 ~ %
+ \if_num:w \l_regex_build_mode_int > \c_sixteen
+ \tl_set_eq:NN \l_regex_class_saved_tl \l_regex_class_tl
+ \bool_set_eq:NN \l_regex_class_saved_bool \l_regex_class_bool
+ \fi:
+ \token_if_eq_meaning:NNTF #1 \regex_build_special:N
{
- \cs_if_exist:cTF { regex_class_#3: }
+ \token_if_eq_charcode:NNTF #2 ^
{
- \regex_class_single:N #1
- \regex_class_single:N -
- #2#3
+ \bool_set_false:N \l_regex_class_bool % [
+ \str_if_eq:nnTF {#3#4} { \regex_build_special:N ] }
+ { \regex_build_raw:N }
+ }
+ { % [
+ \token_if_eq_charcode:NNTF #2 ]
+ { \regex_build_raw:N #2 }
+ { #1 #2 }
}
- { \regex_class_range_put:NN #1#3 }
}
- { \regex_class_range_put:NN #1#3 }
- }
-\cs_new_protected_nopar:Npn \regex_class_range_put:NN #1#2
- {
- \if_num:w `#1 > `#2 \exp_stop_f:
- \msg_kernel_error:nnxx { regex } { backwards-range } {#1} {#2}
- \else:
- \regex_class_one:x
- {
- \if_num:w `#1 = `#2 \exp_stop_f:
- \regex_item_equal:n
- \else:
- \regex_item_range:nn { \int_value:w `#1 }
- \fi:
- { \int_value:w `#2 }
- }
- \fi:
+ { #1 #2 }
+ #3 #4
}
% \end{macrocode}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}[aux]{\regex_class_d:,\regex_class_D:}
-% \begin{macro}[aux]{\regex_class_h:,\regex_class_H:}
-% \begin{macro}[aux]{\regex_class_s:,\regex_class_S:}
-% \begin{macro}[aux]{\regex_class_v:,\regex_class_V:}
-% \begin{macro}[aux]{\regex_class_w:,\regex_class_W:}
-% Similar to \cs{regex_class_single:N}, adding the appropriate
-% ranges of characters to the class. The token lists are not
-% expanded because it is more memory efficient, with a tiny
-% overhead on execution.
+% \subsubsection{Catcodes and csnames}
+%
+% \begin{macro}[aux]{\regex_build_/c:}
+% The |\c| escape sequence is only allowed in modes $0$ and $3$,
+% \emph{i.e.}, not within any other |\c| escape sequence.
% \begin{macrocode}
-\tl_map_inline:nn { dDhHsSvVwWN }
+\cs_new_protected:cpn { regex_build_/c: }
{
- \cs_new_protected_nopar:cpx { regex_class_#1: }
- { \regex_class_one:n \exp_not:c { c_regex_#1_tl } }
+ \if_case:w \l_regex_build_mode_int
+ \exp_after:wN \regex_build_c_aux:wNN
+ \or: \or: \or: \exp_after:wN \regex_build_c_aux:wNN
+ \fi:
+ \msg_kernel_error:nn { regex } { c-bad-mode }
+ \s_stop
}
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-%
-% \subsubsection{Catcodes and csnames}
-%
-% \begin{macro}[aux]{\regex_build_c:}
-% \begin{macro}[aux]{\regex_build_C:}
-% \begin{macro}[aux]{\regex_class_c:}
-% \begin{macro}[aux]{\regex_class_C:}
-% \begin{macro}[aux]{\regex_aux_c:nnNN}
%
+% \begin{macro}[aux]{\regex_build_c_aux:wNN}
+% The |\c| escape sequence can be followed by a capital letter
+% representing a character category, by a left bracket which starts a
+% list of categories, or by a brace group holding a regular expression
+% for a control sequence name. Otherwise, raise an error.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_c:
- { \regex_aux_c:NNNN \regex_item_catcode:n \use_i:nn }
-\cs_new_protected_nopar:Npn \regex_build_C:
- { \regex_aux_c:NNNN \regex_item_catcode_neq:n \use_ii:nn }
-\cs_new_eq:NN \regex_class_c: \regex_build_c:
-\cs_new_eq:NN \regex_class_C: \regex_build_C:
-\cs_new_protected:Npn \regex_aux_c:NNNN #1#2#3#4
+\cs_new_protected:Npn \regex_build_c_aux:wNN #1 \s_stop #2#3
{
- \token_if_eq_meaning:NNTF #3 \regex_build_raw:N
+ \token_if_eq_meaning:NNTF #2 \regex_build_raw:N
{
- \regex_aux_c_test:NTF #4
+ \cs_if_exist:cTF { c_regex_catcode_#3_int }
{
- \use:c { regex_ \l_regex_build_or_class_tl _one:x }
- { #1 { \int_use:N \l_regex_current_catcode_int } }
+ \int_set_eq:Nc \l_regex_catcodes_int { c_regex_catcode_#3_int }
+ \l_regex_build_mode_int
+ = \if_case:w \l_regex_build_mode_int \c_two \else: \c_six \fi:
}
+ }
+ { \cs_if_exist_use:cF { regex_build_c_#3: } }
{
- \msg_kernel_error:nnx { regex } { build-c } { #4 }
- #3 #4
+ \msg_kernel_error:nnxx { regex } { c-command }
+ { regular~expression } { #3 }
+ #2 #3
}
- }
- {
- \if_charcode:w \c_lbrace_str \exp_not:N #4
- \exp_after:wN #2
- \else:
- \exp_after:wN \use_ii:nn
- \fi:
- {
- \group_begin:
- \bool_set_true:N \l_regex_in_csname_bool
- \regex_build:w
- }
- {
- \msg_kernel_error:nnx { regex } { build-c } { #4 }
- #3 #4
- }
- }
}
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
+%
+% \begin{macro}+\regex_build_c_{:+ ^^A }
+% The case of a left brace is easy, based on what we have done so far:
+% in a group, build the regular expression, after changing the mode to
+% forbid nesting |\c|.
+% \begin{macrocode}
+\cs_new_protected:cpn { regex_build_c_ \c_lbrace_str : }
+ {
+ \group_begin:
+ \l_regex_build_mode_int
+ = - \if_case:w \l_regex_build_mode_int \c_two \else: \c_six \fi:
+ \regex_build:w
+ }
+% \end{macrocode}
% \end{macro}
%
-% The |\c| and |\C| escape sequences are used both in the
-% regex and the replacement texts. They can be followed either
-% by a capital letter representing a category code, or by a left
-% brace (in the case of |\c| only).
+% \begin{macro}[aux]{\regex_build_c_[:} ^^A]
+% \begin{macro}[aux]{\regex_build_c_lbrack_loop:NN}
+% \begin{macro}[aux]{\regex_build_c_lbrack_end:}
+% \begin{macro}[aux]{\regex_build_add:N}
%
-% \begin{macro}{\regex_aux_c_test:NTF}
-% If the first argument is a known catcode identifier
-% (capital letters listed below), then
-% \cs{l_regex_current_catcode_int} is set to the relevant
-% catcode value (between $0$ for cs and $13$ for active chars),
-% and the \texttt{true} branch is taken.
-% Otherwise, the \texttt{false} branch is taken,
-% and the current catcode is set to $-1$.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_aux_c_test:NTF #1
+\cs_new_protected:cpn { regex_build_c_[: } #1#2
{
- \int_set:Nn \l_regex_current_catcode_int
+ \int_zero:N \l_regex_catcodes_int
+ \str_if_eq:nnTF { #1 #2 } { \regex_build_special:N ^ }
{
- \if_case:w \int_eval:w `#1 - `A \int_eval_end:
- \c_thirteen % Active
- \or: \c_one % Begin-group
- \or: \c_zero % Control sequence
- \or: \c_eight % Down (subscript)
- \or: \c_two % End-group
- \else:
- \if_case:w \int_eval:w `#1 - `L \int_eval_end:
- \c_eleven % Letter
- \or: \c_three % Math toggle
- \or: \c_minus_one
- \or: \c_twelve % Other
- \or: \c_six % Parameter
- \or: \c_minus_one
- \or: \c_minus_one
- \or: \c_ten % Space
- \or: \c_four % Tab (alignment)
- \or: \c_seven % Up (superscript)
- \else: \c_minus_one
- \fi:
- \fi:
+ \bool_set_false:N \l_regex_catcodes_bool
+ \regex_build_c_lbrack_loop:NN
}
- \if_num:w \l_regex_current_catcode_int = \c_minus_one
- \exp_after:wN \use_ii:nn
+ {
+ \bool_set_true:N \l_regex_catcodes_bool
+ \regex_build_c_lbrack_loop:NN
+ #1#2
+ }
+ }
+\cs_new_protected:Npn \regex_build_c_lbrack_loop:NN #1#2
+ {
+ \token_if_eq_meaning:NNTF #1 \regex_build_raw:N
+ {
+ \cs_if_exist:cTF { c_regex_catcode_#2_int }
+ {
+ \exp_args:Nc \regex_build_c_lbrack_add:N
+ { c_regex_catcode_#2_int }
+ \regex_build_c_lbrack_loop:NN
+ }
+ }
+ { % [
+ \token_if_eq_charcode:NNTF #2 ]
+ { \regex_build_c_lbrack_end: }
+ }
+ {
+ \msg_kernel_error:nnxx { regex } { c-command }
+ { regular-expression } { #2 }
+ \regex_build_c_lbrack_end:
+ }
+ }
+\cs_new_protected_nopar:Npn \regex_build_c_lbrack_end:
+ {
+ \l_regex_build_mode_int
+ = \if_case:w \l_regex_build_mode_int \c_two \else: \c_six \fi:
+ \if_meaning:w \c_false_bool \l_regex_catcodes_bool
+ \int_set:Nn \l_regex_catcodes_int
+ { \c_regex_catcodes_all_int - \l_regex_catcodes_int }
+ \fi:
+ }
+\cs_new_protected:Npn \regex_build_c_lbrack_add:N #1
+ {
+ \if_int_odd:w \int_eval:w \l_regex_catcodes_int / #1 \int_eval_end:
\else:
- \exp_after:wN \use_i:nn
+ \tex_advance:D \l_regex_catcodes_int #1
\fi:
}
% \end{macrocode}
% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
%
% ^^A{
% \begin{macro}+\regex_build}:+
@@ -2196,18 +2362,18 @@
% replace the brace with an escaped brace.
% \begin{macrocode}
\cs_new_protected:cpn { regex_build_ \c_rbrace_str : }
- {
- \bool_if:NTF \l_regex_in_csname_bool
+ { %^^A incorrect recovery for \c{...(...[...} with missing ) or ].
+ \int_compare:nNnTF \l_regex_build_mode_int < \c_zero
{
- \regex_build_Z:
+ \use:c { regex_build_/Z: }
\regex_toks_put_right:Nx \l_regex_right_state_int
{ \regex_action_success: }
\int_incr:N \l_regex_capturing_group_int
\use:x
{
\group_end:
- \exp_not:n { \use:c { regex_ \l_regex_build_or_class_tl _one:n } }
- { \regex_item_cs:n { \regex_set_aux:N ? } }
+ \regex_build_one:n
+ { \regex_item_cs:n { \regex_set_aux:N ? } }
}
}
{ \exp_after:wN \regex_build_raw:N \c_rbrace_str }
@@ -2223,18 +2389,18 @@
% When all characters for the quantifier are found, the corresponding
% function is called.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_quantifier:w #1#2
+\cs_new_protected:Npn \regex_build_quantifier:w #1#2
{
- \token_if_eq_meaning:NNTF #1 \regex_build_control:N
+ \token_if_eq_meaning:NNTF #1 \regex_build_special:N
{
\cs_if_exist_use:cF { regex_build_quantifier_#2:w }
{
- \regex_build_quantifier_end:n { }
+ \regex_build_quantifier_end:nn { } { }
#1 #2
}
}
{
- \regex_build_quantifier_end:n { }
+ \regex_build_quantifier_end:nn { } { }
#1 #2
}
}
@@ -2266,12 +2432,12 @@
% arguments that the final function needs in the case of braced
% constructions, and is empty otherwise.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_quantifier_aux:nnNN #1#2#3#4
+\cs_new_protected:Npn \regex_build_quantifier_aux:nnNN #1#2#3#4
{
- \str_if_eq:nnTF { #3 #4 } { \regex_build_control:N ? }
- { \regex_build_quantifier_end:n { #2 #4 } #1 }
+ \str_if_eq:nnTF { #3 #4 } { \regex_build_special:N ? }
+ { \regex_build_quantifier_end:nn { #2 #4 } {#1} }
{
- \regex_build_quantifier_end:n { #2 } #1
+ \regex_build_quantifier_end:nn { #2 } {#1}
#3 #4
}
}
@@ -2287,44 +2453,44 @@
% \begin{macrocode}
\cs_new_protected_nopar:cpn { regex_build_quantifier_ \c_lbrace_str :w }
{ \regex_get_digits:nw { \regex_build_quantifier_lbrace:n } }
-\cs_new_protected_nopar:Npn \regex_build_quantifier_lbrace:n #1
+\cs_new_protected:Npn \regex_build_quantifier_lbrace:n #1
{
\tl_if_empty:nTF {#1}
{
- \regex_build_quantifier_end:n { }
+ \regex_build_quantifier_end:nn { } { }
\exp_after:wN \regex_build_raw:N \c_lbrace_str
}
{ \regex_build_quantifier_lbrace:nw {#1} }
}
-\cs_new_protected_nopar:Npx \regex_build_quantifier_lbrace:nw #1#2#3
+\cs_new_protected:Npx \regex_build_quantifier_lbrace:nw #1#2#3
{
\exp_not:N \prg_case_str:nnn { #2 #3 }
{
- { \exp_not:N \regex_build_control:N , }
+ { \exp_not:N \regex_build_special:N , }
{
\exp_not:N \regex_get_digits:nw
{ \exp_not:N \regex_build_quantifier_lbrace:nnw {#1} }
}
- { \exp_not:N \regex_build_control:N \c_rbrace_str }
- { \exp_not:N \regex_build_quantifier_end:n {n} {#1} }
+ { \exp_not:N \regex_build_special:N \c_rbrace_str }
+ { \exp_not:N \regex_build_quantifier_end:nn {n} { {#1} } }
}
{
- \exp_not:N \regex_build_quantifier_end:n { }
+ \exp_not:N \regex_build_quantifier_end:nn { } { }
\exp_not:N \regex_build_raw:N \c_lbrace_str #1#2
}
}
-\cs_new_protected_nopar:Npn \regex_build_quantifier_lbrace:nnw #1#2#3#4
+\cs_new_protected:Npn \regex_build_quantifier_lbrace:nnw #1#2#3#4
{
\str_if_eq:xxTF
{ \exp_not:n {#3#4} }
- { \exp_not:N \regex_build_control:N \c_rbrace_str }
+ { \exp_not:N \regex_build_special:N \c_rbrace_str }
{
\tl_if_empty:nTF {#2}
{ \regex_build_quantifier_aux:nnNN { {#1} } { n* } }
{ \regex_build_quantifier_aux:nnNN { {#1} {#2} } { nn } }
} %^^A todo: catch the case m>n
{
- \regex_build_quantifier_end:n { }
+ \regex_build_quantifier_end:nn { } { }
\use:x
{
\exp_args:No \tl_map_function:nN
@@ -2340,12 +2506,17 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_quantifier_end:n}
+% \begin{macro}[aux]{\regex_build_quantifier_end:nn}
% When all quantifiers are found, we will call the relevant
% \cs{regex_build_one/group_\meta{quantifiers}:} function.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_quantifier_end:n #1
- { \use:c { regex_build_ \l_regex_one_or_group_tl _ #1 : } }
+\cs_new_protected:Npn \regex_build_quantifier_end:nn #1#2
+ {
+ \use:c { regex_build_ \l_regex_one_or_group_tl _ #1 : } #2
+ \tl_clear:N \l_regex_class_tl
+ \bool_set_true:N \l_regex_class_bool
+ \int_set_eq:NN \l_regex_catcodes_int \l_regex_catcodes_default_int
+ }
% \end{macrocode}
% \end{macro}
%
@@ -2372,7 +2543,7 @@
% \begin{macrocode}
\cs_new_protected_nopar:Npn \regex_build_one_:
{
- \regex_build_transition_aux:NN
+ \regex_build_transition:NN
\regex_build_tmp_class:n \l_regex_right_state_int
}
% \end{macrocode}
@@ -2387,13 +2558,13 @@
% \begin{macrocode}
\cs_new_protected_nopar:cpn { regex_build_one_?: }
{
- \regex_build_transitions_aux:NNNN
+ \regex_build_transitions:NNNN
\regex_build_tmp_class:n \l_regex_right_state_int
\regex_action_free:n \l_regex_right_state_int
}
\cs_new_protected_nopar:cpn { regex_build_one_??: }
{
- \regex_build_transitions_aux:NNNN
+ \regex_build_transitions:NNNN
\regex_action_free:n \l_regex_right_state_int
\regex_build_tmp_class:n \l_regex_right_state_int
}
@@ -2408,13 +2579,13 @@
% \begin{macrocode}
\cs_new_protected_nopar:cpn { regex_build_one_*: }
{
- \regex_build_transitions_aux:NNNN
+ \regex_build_transitions:NNNN
\regex_build_tmp_class:n \l_regex_left_state_int
\regex_action_free:n \l_regex_right_state_int
}
\cs_new_protected_nopar:cpn { regex_build_one_*?: }
{
- \regex_build_transitions_aux:NNNN
+ \regex_build_transitions:NNNN
\regex_action_free:n \l_regex_right_state_int
\regex_build_tmp_class:n \l_regex_left_state_int
}
@@ -2433,7 +2604,7 @@
{
\regex_build_one_:
\int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int
- \regex_build_transitions_aux:NNNN
+ \regex_build_transitions:NNNN
\regex_action_free:n \l_regex_tmpa_int
\regex_action_free:n \l_regex_right_state_int
}
@@ -2441,7 +2612,7 @@
{
\regex_build_one_:
\int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int
- \regex_build_transitions_aux:NNNN
+ \regex_build_transitions:NNNN
\regex_action_free:n \l_regex_right_state_int
\regex_action_free:n \l_regex_tmpa_int
}
@@ -2456,7 +2627,7 @@
% are identical, since the number of repetitions is fixed.
% Simply repeat |#1| times the effect of \cs{regex_build_one_:}.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_one_n: #1
+\cs_new_protected:Npn \regex_build_one_n: #1
{ \prg_replicate:nn {#1} { \regex_build_one_: } }
\cs_new_eq:cN { regex_build_one_n?: } \regex_build_one_n:
% \end{macrocode}
@@ -2468,12 +2639,12 @@
% This function is called in case the syntax is
% \texttt{\{\meta{int},\}}.
% \begin{macrocode}
-\cs_new_protected_nopar:cpx { regex_build_one_n*: } #1
+\cs_new_protected:cpx { regex_build_one_n*: } #1
{
\exp_not:N \prg_replicate:nn {#1} { \exp_not:N \regex_build_one_: }
\exp_not:c { regex_build_one_*: }
}
-\cs_new_protected_nopar:cpx { regex_build_one_n*?: } #1
+\cs_new_protected:cpx { regex_build_one_n*?: } #1
{
\exp_not:N \prg_replicate:nn {#1} { \exp_not:N \regex_build_one_: }
\exp_not:c { regex_build_one_*?: }
@@ -2488,7 +2659,7 @@
% This function is called when the syntax is
% \texttt{\{\meta{int},\meta{int}\}}.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_one_nn_aux:Nnn #1#2#3
+\cs_new_protected:Npn \regex_build_one_nn_aux:Nnn #1#2#3
{
\prg_replicate:nn {#2} { \regex_build_one_: }
\prg_replicate:nn {#3-#2} {#1}
@@ -2524,17 +2695,24 @@
% vertex to a new vertex.
% \end{itemize}
% \begin{macrocode}
-\cs_new_protected_nopar:cpn { regex_build_(: } #1#2
+\cs_new_protected:cpn { regex_build_(: } #1#2
{
- \str_if_eq:nnTF { #1 #2 } { \regex_build_control:N ? }
- { \regex_build_special_group:NN }
+ \regex_build_if_in_class:TF
{
- \int_incr:N \l_regex_capturing_group_int
- \regex_seq_push_int:NN
- \l_regex_capturing_group_seq \l_regex_capturing_group_int
- \regex_build_open_aux:
+ \regex_build_raw:N (
#1 #2
}
+ {
+ \str_if_eq:nnTF { #1 #2 } { \regex_build_special:N ? }
+ { \regex_build_special_group:NN }
+ {
+ \int_incr:N \l_regex_capturing_group_int
+ \regex_seq_push_int:NN
+ \l_regex_capturing_group_seq \l_regex_capturing_group_int
+ \regex_build_open_aux:
+ #1 #2
+ }
+ }
}
\cs_new_protected_nopar:Npn \regex_build_open_aux:
{
@@ -2549,16 +2727,22 @@
}
\cs_new_protected_nopar:cpn { regex_build_|: }
{
- \regex_build_end_alternation:
- \regex_build_begin_alternation:
+ \regex_build_if_in_class:TF { \regex_build_raw:N | }
+ {
+ \regex_build_end_alternation:
+ \regex_build_begin_alternation:
+ }
}
\cs_new_protected_nopar:cpn { regex_build_): }
{
- \seq_if_empty:NTF \l_regex_capturing_group_seq
- { \msg_kernel_error:nn { regex } { extra-rparen } }
+ \regex_build_if_in_class:TF { \regex_build_raw:N ) }
{
- \regex_build_close_aux:
- \regex_build_group_quantifier:
+ \seq_if_empty:NTF \l_regex_capturing_group_seq
+ { \msg_kernel_error:nn { regex } { extra-rparen } }
+ {
+ \regex_build_close_aux:
+ \regex_build_group_quantifier:
+ }
}
}
\cs_new_protected_nopar:Npn \regex_build_close_aux:
@@ -2610,13 +2794,13 @@
% is known, then use that. Otherwise, treat the question mark
% as if it had been escaped.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_special_group:NN #1#2
+\cs_new_protected:Npn \regex_build_special_group:NN #1#2
{
\cs_if_exist_use:cF { regex_build_special_group_\token_to_str:N #2 : }
{
\msg_kernel_error:nnx { regex } { unsupported }
{ (? \token_to_str:N #2 } %)
- \regex_build_control:N ( % )
+ \regex_build_special:N ( % )
\regex_build_raw:N ?
#1 #2
}
@@ -2699,7 +2883,7 @@
{
\regex_build_options:NNN -
}
-\cs_new_protected_nopar:Npn \regex_build_options:NNN #1#2#3
+\cs_new_protected:Npn \regex_build_options:NNN #1#2#3
{
\token_if_eq_meaning:NNTF \regex_build_raw:N #2
{
@@ -2752,7 +2936,7 @@
% Once the quantifier is found by \cs{regex_build_quantifier:w},
% we insert the code for tracking submatches.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_group_submatches:NN #1#2
+\cs_new_protected:Npn \regex_build_group_submatches:NN #1#2
{
\seq_pop:NN \l_regex_capturing_group_seq \l_regex_tmpa_tl
\int_compare:nNnF { \l_regex_tmpa_tl } < \c_zero
@@ -2774,7 +2958,7 @@
{
\regex_build_group_submatches:NN
\l_regex_left_state_int \l_regex_right_state_int
- \regex_build_transition_aux:NN
+ \regex_build_transition:NN
\regex_action_free:n \l_regex_right_state_int
}
% \end{macrocode}
@@ -2785,7 +2969,7 @@
% This is done by shifting the current contents of the \cs{tex_toks:D}
% \cs{l_regex_tmpa_int} to a new state.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_group_shift:N #1
+\cs_new_protected:Npn \regex_build_group_shift:N #1
{
\int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int
\regex_build_new_state:
@@ -2827,11 +3011,11 @@
% operators, and higher priority (\texttt{put_left}) for
% lazy operators.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_group_qs_aux:NN #1#2
+\cs_new_protected:Npn \regex_build_group_qs_aux:NN #1#2
{
\regex_build_group_shift:N \regex_action_free:n
\int_set_eq:NN \l_regex_right_state_int \l_regex_left_state_int
- \regex_build_transition_aux:NN \regex_action_free:n #1
+ \regex_build_transition:NN \regex_action_free:n #1
#2 \l_regex_tmpa_int
{
\regex_action_free:n
@@ -2876,7 +3060,7 @@
\regex_build_group_submatches:NN
\l_regex_left_state_int \l_regex_right_state_int
\int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int
- \regex_build_transitions_aux:NNNN
+ \regex_build_transitions:NNNN
\regex_action_free:n \l_regex_tmpa_int
\regex_action_free:n \l_regex_right_state_int
}
@@ -2885,7 +3069,7 @@
\regex_build_group_submatches:NN
\l_regex_left_state_int \l_regex_right_state_int
\int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int
- \regex_build_transitions_aux:NNNN
+ \regex_build_transitions:NNNN
\regex_action_free:n \l_regex_right_state_int
\regex_action_free:n \l_regex_tmpa_int
}
@@ -2905,7 +3089,7 @@
% \cs{l_regex_max_state_int} is set appropriately to the largest
% allocated \tn{toks} register.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_group_n_aux:n #1
+\cs_new_protected:Npn \regex_build_group_n_aux:n #1
{
\regex_toks_put_right:Nx \l_regex_right_state_int
{
@@ -2939,10 +3123,10 @@
% are identical, since the number of repetitions is fixed.
% We only record the submatch information at the last repetition.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_group_n: #1
+\cs_new_protected:Npn \regex_build_group_n: #1
{ % ^^A todo: catch case #1 <= 0.
\regex_build_group_n_aux:n {#1}
- \regex_build_transition_aux:NN
+ \regex_build_transition:NN
\regex_action_free:n \l_regex_right_state_int
\regex_build_group_submatches:NN
\l_regex_tmpa_int \l_regex_left_state_int
@@ -2962,19 +3146,19 @@
% case no iteration was made, here, we know that the submatch
% information is overwritten in any case.
% \begin{macrocode}
-\cs_new_protected_nopar:cpn { regex_build_group_n*: } #1
+\cs_new_protected:cpn { regex_build_group_n*: } #1
{ % ^^A todo: catch case #1 <= 0.
\regex_build_group_n_aux:n {#1}
- \regex_build_transitions_aux:NNNN
+ \regex_build_transitions:NNNN
\regex_action_free:n \l_regex_tmpa_int
\regex_action_free:n \l_regex_right_state_int
\regex_build_group_submatches:NN
\l_regex_tmpa_int \l_regex_left_state_int
}
-\cs_new_protected_nopar:cpn { regex_build_group_n*?: } #1
+\cs_new_protected:cpn { regex_build_group_n*?: } #1
{ % ^^A todo: catch case #1 <= 0.
\regex_build_group_n_aux:n {#1}
- \regex_build_transitions_aux:NNNN
+ \regex_build_transitions:NNNN
\regex_action_free:n \l_regex_right_state_int
\regex_action_free:n \l_regex_tmpa_int
\regex_build_group_submatches:NN
@@ -2989,12 +3173,12 @@
% These functions are called when the syntax is either
% \texttt{\{\meta{int},\}} or \texttt{\{\meta{int},\meta{int}\}}.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_group_nn: #1#2
+\cs_new_protected:Npn \regex_build_group_nn: #1#2
{ % ^^A Not Implemented Yet!
\msg_expandable_error:n { Quantifier~{m,n}~not~implemented~yet }
\use:c { regex_build_group_n*: } {#1}
}
-\cs_new_protected_nopar:cpn { regex_build_group_nn?: } #1#2
+\cs_new_protected:cpn { regex_build_group_nn?: } #1#2
{ % ^^A Not Implemented Yet!
\msg_expandable_error:n { Quantifier~{m,n}~not~implemented~yet }
\use:c { regex_build_group_n*?: } {#1}
@@ -3115,7 +3299,7 @@
{ \regex_state_use_aux:n { } }
\cs_new_protected_nopar:Npn \regex_state_use:
{ \regex_state_use_aux:n { \exp_after:wN \use_none_delimit_by_s_stop:w } }
-\cs_new_protected_nopar:Npn \regex_state_use_aux:n #1
+\cs_new_protected:Npn \regex_state_use_aux:n #1
{
\if_num:w \tex_dimen:D \l_regex_current_state_int
< \l_regex_unique_id_int
@@ -3227,7 +3411,7 @@
\int_set_eq:NN \l_regex_min_step_int \l_regex_current_step_int
\exp_after:wN \regex_query_set_loop:ww
\l_regex_tmpa_tl
- \q_mark { ? \prg_map_break: } ,
+ \s_tl { ? \prg_map_break: } \s_tl
\prg_break_point:n { }
\int_set_eq:NN \l_regex_max_step_int \l_regex_current_step_int
\regex_query_set:nnn { } { -1 } { -2 }
@@ -3235,7 +3419,7 @@
\regex_match_initial_setup:
\regex_match_once:
}
-\cs_new:Npn \regex_query_set_loop:ww #1 \q_mark #2#3 ,
+\cs_new:Npn \regex_query_set_loop:ww #1 \s_tl #2#3 \s_tl
{
\use_none:n #2
\regex_query_set:nnn {#1} {"#2} {#3}
@@ -3369,7 +3553,7 @@
\fi:
\regex_match_loop:
}
-\cs_new_nopar:Npn \regex_match_one_index:w #1;
+\cs_new:Npn \regex_match_one_index:w #1;
{
\if_num:w #1 > \l_regex_max_index_int
\exp_after:wN \prg_map_break:
@@ -3379,7 +3563,7 @@
\exp_after:wN \regex_match_one_index:w
\int_use:N \int_eval:w #1 + \c_one ;
}
-\cs_new_protected_nopar:Npn \regex_match_one_index_aux:n #1
+\cs_new_protected:Npn \regex_match_one_index_aux:n #1
{
\int_set:Nn \l_regex_current_state_int {#1}
\prop_clear:N \l_regex_current_submatches_prop
@@ -3420,7 +3604,7 @@
%
% \subsubsection{Actions when matching}
%
-% \begin{macro}[aux]{\regex_action_start_wildcard:nn}
+% \begin{macro}[aux]{\regex_action_start_wildcard:}
%: the first state has a free transition to the second
% state, where the regular expression really begins, and a costly
% transition to itself, to try again at the next character. ^^A ??
@@ -3433,12 +3617,12 @@
% this character. For that, we use the boolean
% \cs{l_regex_fresh_thread_bool}.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_action_start_wildcard:nn #1#2
+\cs_new_protected_nopar:Npn \regex_action_start_wildcard:
{
\bool_set_true:N \l_regex_fresh_thread_bool
- \regex_action_free:n {#2}
+ \regex_action_free:n {1}
\bool_set_false:N \l_regex_fresh_thread_bool
- \regex_action_cost:n {#1}
+ \regex_action_cost:n {0}
}
% \end{macrocode}
% \end{macro}
@@ -3447,7 +3631,7 @@
% A transition which consumes the current character and moves
% to state |#1|.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_action_cost:n #1
+\cs_new_protected:Npn \regex_action_cost:n #1
{
\exp_args:Nf \regex_store_state:n
{ \int_eval:n { \l_regex_current_state_int + #1 } }
@@ -3488,7 +3672,7 @@
% Then restore the old value of \cs{l_regex_current_state_int}
% and of the current submatches.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_action_free:n #1
+\cs_new_protected:Npn \regex_action_free:n #1
{
\cs_set_nopar:Npx \regex_tmp:w
{
@@ -3508,7 +3692,7 @@
% Update the current submatches with the information
% from the current step.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_action_submatch:n #1
+\cs_new_protected:Npn \regex_action_submatch:n #1
{
\prop_put:Nno \l_regex_current_submatches_prop {#1}
{ \int_use:N \l_regex_current_step_int }
@@ -3552,7 +3736,7 @@
\regex_replacement_raw:N
{#1}
\prg_do_nothing: \prg_do_nothing:
- \cs_set_nopar:Npx \regex_nesting:n ##1
+ \cs_set:Npx \regex_nesting:n ##1
{
+ \int_use:N \l_regex_nesting_int
\l_regex_nesting_tl
@@ -3560,7 +3744,7 @@
}
\use:x
{
- \exp_not:n { \cs_set_nopar:Npn \regex_replacement_tl:n ##1 }
+ \exp_not:n { \cs_set:Npn \regex_replacement_tl:n ##1 }
{ \str_aux_toks_range:nn \c_zero \l_regex_replacement_int }
}
% ^^A rename! Careful with \cP\#.
@@ -3633,7 +3817,8 @@
}
\cs_new_protected:Npn \regex_replacement_error:NNN #1#2#3
{
- \msg_kernel_error:nn { regex } { replacement-#1 }
+ \msg_kernel_error:nnxx { regex } { #1-command }
+ { replacement~text } {#3}
#2 #3
}
\cs_new_protected:Npn \regex_replacement_g:w #1#2
@@ -3670,7 +3855,7 @@
{ \regex_replacement_error:NNN g }
#1 #2
}
-\cs_new_protected_nopar:Npn \regex_replacement_c:w #1#2
+\cs_new_protected:Npn \regex_replacement_c:w #1#2
{
\token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N
{
@@ -3685,7 +3870,9 @@
\regex_replacement_put:n
{ \exp_not:n { \exp_after:wN \regex_replacement_exp_not:N \cs:w } }
}
-\cs_new_nopar:Npn \regex_replacement_exp_not:N #1 { \exp_not:n {#1} }
+\cs_new_eq:cc { regex_replacement_c_C:w }
+ { regex_replacement_c_ \c_lbrace_str :w }
+\cs_new:Npn \regex_replacement_exp_not:N #1 { \exp_not:n {#1} }
\group_begin:
\char_set_catcode_math_superscript:N \^^@
@@ -3696,7 +3883,7 @@
\cs_new_protected_nopar:Npn \regex_replacement_c_T:w
{ \regex_replacement_char:nNN { ^^@ } }
- \cs_new_protected_nopar:Npn \regex_replacement_c_S:w #1#2
+ \cs_new_protected:Npn \regex_replacement_c_S:w #1#2
{
\int_compare:nNnTF { `#2 } = \c_zero
{ \regex_replacement_error:NNN c #1#2 }
@@ -3708,7 +3895,7 @@
\char_set_catcode_parameter:N \^^@
\cs_new_protected_nopar:Npn \regex_replacement_c_P:w
- { \regex_replacement_char:nNN { ^^@^^@ } }
+ { \regex_replacement_char:nNN { ^^@^^@^^@^^@^^@^^@^^@^^@ } }
\char_set_catcode_other:N \^^@
\cs_new_protected_nopar:Npn \regex_replacement_c_O:w
@@ -3746,7 +3933,7 @@
\group_end:
-\cs_new_protected_nopar:Npn \regex_replacement_char:nNN #1#2#3
+\cs_new_protected:Npn \regex_replacement_char:nNN #1#2#3
{
\char_set_lccode:nn \c_zero { `#3 }
\tl_to_lowercase:n
@@ -3843,7 +4030,7 @@
\regex_set_aux:n
\s_stop
}
-\cs_new_nopar:Npn \regex_set_aux:n #1
+\cs_new:Npn \regex_set_aux:n #1
{ \tex_toks:D #1 { \tex_the:D \tex_toks:D #1 } }
\cs_new:Npn \regex_nfa:Nw #1
{
@@ -3870,13 +4057,13 @@
% token list variables. We do some more processing to start each
% line with \cs{tex_toks:D} instead of a single very long line.
% \begin{macrocode}
-\cs_new_nopar:Npn \regex_to_str:N #1
+\cs_new:Npn \regex_to_str:N #1
{
\exp_after:wN \regex_to_str_aux:Nw #1
\tex_toks:D \q_stop \prg_map_break: \tex_toks:D
\prg_break_point:n { }
}
-\cs_new_nopar:Npn \regex_to_str_aux:Nw #1#2\tex_toks:D
+\cs_new:Npn \regex_to_str_aux:Nw #1#2\tex_toks:D
{
\use_none_delimit_by_q_stop:w #2 \q_stop
\tl_to_str:n {#1#2} \iow_newline:
@@ -4027,12 +4214,12 @@
}
\fi:
}
-\cs_new_protected_nopar:Npn \regex_extract_aux_b:wn #1 < #2
+\cs_new_protected:Npn \regex_extract_aux_b:wn #1 < #2
{
\tex_skip:D #1 = #2 sp
plus \etex_gluestretch:D \tex_skip:D #1 \scan_stop:
}
-\cs_new_protected_nopar:Npn \regex_extract_aux_e:wn #1 > #2
+\cs_new_protected:Npn \regex_extract_aux_e:wn #1 > #2
{
\tex_skip:D #1
= 1 \tex_skip:D #1 plus #2 sp \scan_stop:
@@ -4066,7 +4253,7 @@
\exp_args:NNNo \group_end:
\tl_set:Nn #1 \l_regex_tmpa_tl
}
-\cs_new_nopar:Npn \regex_extract_seq_aux:n #1
+\cs_new:Npn \regex_extract_seq_aux:n #1
{
\seq_item:n
{
@@ -4074,7 +4261,7 @@
\int_value:w \regex_submatch_nesting_aux:n {#1} ; #1;
}
}
-\cs_new_nopar:Npn \regex_extract_seq_aux:ww #1; #2;
+\cs_new:Npn \regex_extract_seq_aux:ww #1; #2;
{ %^^A todo: use flags
\if_num:w #1 < \c_zero
\flag_raise:N \l_regex_group_end_flag
@@ -4293,7 +4480,7 @@
\group_end:
\fi:
}
-\cs_new_protected_nopar:Npn \regex_replace_once_aux:w #1;
+\cs_new_protected:Npn \regex_replace_once_aux:w #1;
{
}
% \end{macrocode}
@@ -4354,7 +4541,7 @@
\exp_args:NNNo \group_end:
\tl_set:Nn #4 \l_regex_tmpa_tl
}
-\cs_new_nopar:Npn \regex_replace_all_aux:n #1
+\cs_new:Npn \regex_replace_all_aux:n #1
{
\regex_query_substr:nn
{ \etex_glueshrink:D \tex_skip:D #1 } { \tex_skip:D #1 }
@@ -4467,12 +4654,12 @@
% \end{macrocode}
%
% \begin{macrocode}
-\msg_kernel_new:nnnn { regex } { build-c }
- { Misused~\iow_char:N\\c~or\iow_char:N\\C~command~in~regular~expression. }
+\msg_kernel_new:nnnn { regex } { c-command }
+ { Misused~\iow_char:N\\c~or\iow_char:N\\C~command~in~a~#1. }
{
- The~\iow_char:N\\C~escape~sequence~can~be~followed~by~
- one~of~the~letters~ABCDELMOPSTU.~The~\iow_char:N\\c~
- escape~sequence~can~also~be~followed~by~a~brace~group.
+ In~a~#1,~the~\iow_char:N\\C~escape~sequence~
+ can~be~followed~by~one~of~the~letters~ABCDELMOPSTU~
+ or~a~brace~group,~not~by~'#2'.
}
\msg_kernel_new:nnnn { regex } { unknown-option }
{ Unknown~option~`#1'~for~regular~expressions. }
@@ -4486,19 +4673,13 @@
The~character~or~escape~sequence~`#1'~is~not~defined~
as~an~option~within~regular~expressions.
}
-\msg_kernel_new:nnnn { regex } { replacement-g }
+\msg_kernel_new:nnnn { regex } { g-command }
{ Missing~brace~for~the~\iow_char:N\\g~construction~in~a~replacement~text. }
{
In~the~replacement~text~for~a~regular~expression~search,~
submatches~are~represented~either~as~\iow_char:N \\g{dd..d},~
or~\\d,~where~`d'~are~single~digits.~Here,~a~brace~is~missing.
}
-\msg_kernel_new:nnnn { regex } { replacement-c }
- { Something~is~wrong~with~the~replacement~text. }
- {
- The~\iow_char:N\\c~escape~sequence~can~only~be~followed~by~
- ABCDELMN~or~by~a~brace.
- }
% \end{macrocode}
%
% \begin{macrocode}
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx
index a33d4909a1a..cddd7349adc 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx
@@ -1,6 +1,6 @@
% \iffalse meta-comment
%
-%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project
+%% File: l3str.dtx Copyright (C) 2011-2012 The LaTeX3 Project
%%
%% It may be distributed and/or modified under the conditions of the
%% LaTeX Project Public License (LPPL), either version 1.3c of this
@@ -35,7 +35,7 @@
%
%<*driver|package>
\RequirePackage{expl3}
-\GetIdInfo$Id: l3str.dtx 3088 2011-12-23 02:00:14Z bruno $
+\GetIdInfo$Id: l3str.dtx 3159 2012-01-09 00:51:58Z bruno $
{L3 Experimental Strings}
%</driver|package>
%<*driver>
@@ -67,6 +67,7 @@
%
% \maketitle
%
+% \newcommand{\hexnum}[1]{\text{\texttt{\char`\"}#1}}
% \begin{documentation}
%
% \LaTeX3 provides a set of functions to manipulate token lists
@@ -106,7 +107,7 @@
% \c_percent_str
% }
% Constant strings, containing a single character, with category code $12$.
-% Other characters can be accessed as \cs{iow_char:N} |\|\meta{character}.
+% Any character can be accessed as \cs{iow_char:N} |\|\meta{character}.
% \end{variable}
%
% \begin{function}[EXP]{\tl_to_str:N, \tl_to_str:n}
@@ -116,6 +117,16 @@
% \end{syntax}
% Converts the \meta{token list} to a \meta{string}, leaving the resulting
% tokens in the input stream.
+% \begin{texnote}
+% The string representation of a token list may depend
+% on the category codes in effect when it is measured,
+% and the value of the \tn{escapechar}: for instance
+% |\tl_to_str:n {\a}| may produce $1$, $2$ or $3$
+% characters depending on the escape character, and
+% the category code of \texttt{a}. This impacts almost
+% all functions in the module, which use \cs{tl_to_str:n}
+% internally.
+% \end{texnote}
% \end{function}
%
% \begin{function}{\str_const:Nn, \str_const:Nx, \str_const:cn, \str_const:cx}
@@ -191,16 +202,16 @@
% (an arbitrary number of hexadecimal digits are read):
% this is mostly useful for Unicode-aware engines.
% \item |\a|, |\e|, |\f|, |\n|, |\r|, |\t| stand for specific characters:
-% \begin{center}
-% \begin{tabular}{cccc}
-% |\a| & |\^^G| & alarm & hex \texttt{07} \\
-% |\e| & |\^^[| & escape & hex \texttt{1B} \\
-% |\f| & |\^^L| & form feed & hex \texttt{0C} \\
-% |\n| & |\^^J| & new line & hex \texttt{0A} \\
-% |\r| & |\^^M| & carriage return & hex \texttt{0D} \\
-% |\t| & |\^^I| & horizontal tab & hex \texttt{09} \\
-% \end{tabular}
-% \end{center}
+% \begin{center}
+% \begin{tabular}{cccc}
+% |\a| & |\^^G| & alarm & hex \texttt{07} \\
+% |\e| & |\^^[| & escape & hex \texttt{1B} \\
+% |\f| & |\^^L| & form feed & hex \texttt{0C} \\
+% |\n| & |\^^J| & new line & hex \texttt{0A} \\
+% |\r| & |\^^M| & carriage return & hex \texttt{0D} \\
+% |\t| & |\^^I| & horizontal tab & hex \texttt{09} \\
+% \end{tabular}
+% \end{center}
% \end{itemize}
% For instance,
% \begin{quote}
@@ -226,13 +237,6 @@
% all characters including spaces are counted.
% The \cs{str_length_ignore_spaces:n} leaves the number of non-space
% characters in the input stream.
-% \begin{texnote}
-% The \cs{str_length:n} of a given token list may depend
-% on the category codes in effect when it is measured,
-% and the value of the \tn{escapechar}: for instance
-% |\str_length:n {\a}| may return $1$, $2$ or $3$ depending
-% on the escape character, and the category code of \texttt{a}.
-% \end{texnote}
% \end{function}
%
% \begin{function}[EXP]{\str_head:N, \str_head:n, \str_head_ignore_spaces:n}
@@ -262,8 +266,11 @@
% with category codes $12$ and $10$ (for spaces).
% The functions differ in the case where the first character
% is a space: \cs{str_tail:N} and \cs{str_tail:n} will trim
-% only that space, while \cs{str_tail_ignore_spaces:n} trims
+% only that space, while \cs{str_tail_ignore_spaces:n} removes
% the first non-space character and any space before it.
+% If the \meta{token list} is empty (or blank in the case of the
+% \texttt{_ignore_spaces} variant), then nothing is left on the
+% input stream.
% \end{function}
%
% \begin{function}[EXP]{\str_item:Nn, \str_item:nn, \str_item_ignore_spaces:nn}
@@ -432,14 +439,14 @@
% by converting each character in the string \texttt{Hello!}
% to the \textsc{utf-16} encoding, and expressing each byte
% as a pair of hexadecimal digits. Note the presence of a
-% (big-endian) byte order mark \texttt{FEFF}, which can be
+% (big-endian) byte order mark \hexnum{FEFF}, which can be
% avoided by specifying the encoding \texttt{utf16be/hex}.
%
% Parts of the \meta{string} which cannot be converted to bytes
% using the \meta{escaping 1} are silently ignored (including non-byte
% characters which could be present in the string).
% Sequences of bytes which are not valid in the \meta{encoding 1}
-% are converted to the replacement character |"FFFD|, and raise
+% are converted to the replacement character \hexnum{FFFD}, and raise
% an error.
% Characters which cannot be encoded in the \meta{encoding 2}
% are silently ignored.
@@ -478,7 +485,7 @@
% ~~\Arg{true code} \Arg{false code}
% \end{syntax}
% If the \meta{token} is a hexdecimal digit (upper case or lower case),
-% its upper-case version is left in the input strem, \emph{followed}
+% its upper-case version is left in the input stream, \emph{followed}
% by the \meta{true code}. Otherwise, the \meta{false code} is left
% in the input stream.
% \begin{texnote}
@@ -552,6 +559,7 @@
% \begin{macrocode}
\ProvidesExplPackage
{\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
+\RequirePackage{l3tl-analysis}
% \end{macrocode}
%
% Those string-related functions are defined in \pkg{l3kernel}.
@@ -669,7 +677,7 @@
%
% \begin{variable}{\c_str_replacement_char_int}
% When converting, invalid bytes are replaced by the Unicode
-% replacement character \textsc{u-fffd}.
+% replacement character \hexnum{FFFD}.
% \begin{macrocode}
\int_const:Nn \c_str_replacement_char_int { "FFFD }
% \end{macrocode}
@@ -1431,11 +1439,11 @@
{
\exp_after:wN \str_aux_gmap_internal_result_loop:Nww
\exp_after:wN #1
- \g_str_result_tl \q_mark \q_stop \prg_map_break: ,
+ \g_str_result_tl \s_tl \q_stop \prg_map_break: \s_tl
\prg_break_point:n { }
}
}
-\cs_new_nopar:Npn \str_aux_gmap_internal_result_loop:Nww #1#2\q_mark #3,
+\cs_new_nopar:Npn \str_aux_gmap_internal_result_loop:Nww #1 #2 \s_tl #3 \s_tl
{
\use_none_delimit_by_q_stop:w #3 \q_stop
#1 {#3}
@@ -1982,19 +1990,20 @@
\use_none_delimit_by_q_stop:w #1 \q_stop
\tex_dimen:D "#1 = \l_str_tmpa_int sp \scan_stop:
\tex_skip:D \l_str_tmpa_int = "#1 sp \scan_stop:
- \tex_toks:D \l_str_tmpa_int \exp_after:wN { \int_value:w "FFFD }
+ \tex_toks:D \l_str_tmpa_int \exp_after:wN
+ { \int_use:N \c_str_replacement_char_int }
\tex_advance:D \l_str_tmpa_int \c_one
\str_convert_from_eight_bit_load_missing:n
}
\cs_new:Npn \str_convert_from_eight_bit_aux:N #1
{
- #1 \exp_not:N \q_mark
+ #1 \s_tl
\if_num:w \tex_dimen:D `#1 < \l_str_tmpa_int
\if_num:w \tex_skip:D \tex_dimen:D `#1 = `#1 \exp_stop_f:
\tex_the:D \tex_toks:D \tex_dimen:D
\fi:
\fi:
- \int_value:w `#1 ,
+ \int_value:w `#1 \s_tl
}
% \end{macrocode}
% \end{macro}
@@ -2031,7 +2040,10 @@
}
\cs_new:Npn \str_convert_to_eight_bit_aux:n #1
{
- \if_num:w #1 < "8000 \exp_stop_f:
+ \if_num:w #1 > \c_max_register_int
+ \msg_expandable_kernel_error:nnn
+ { str } { eight-bit-to-byte-overflow } {#1}
+ \else:
\if_num:w \tex_dimen:D #1 < \l_str_tmpa_int
\if_num:w \tex_skip:D \tex_dimen:D #1 = #1 \exp_stop_f:
\tex_the:D \tex_toks:D \tex_dimen:D #1 \exp_stop_f:
@@ -2039,9 +2051,6 @@
\fi:
\fi:
\str_convert_to_eight_bit_aux_ii:n {#1}
- \else:
- \msg_expandable_kernel_error:nnn
- { str } { eight-bit-to-byte-overflow } {#1}
\fi:
}
\cs_new:Npn \str_convert_to_eight_bit_aux_ii:n #1
@@ -2145,7 +2154,7 @@
\cs_new_protected_nopar:Npn \str_convert_from_native:
{ \str_aux_gmap_result:N \str_convert_from_native_aux:N }
\cs_new:Npn \str_convert_from_native_aux:N #1
- { #1 \exp_not:N \q_mark \int_value:w `#1 , }
+ { #1 \s_tl \int_value:w `#1 \s_tl }
% \end{macrocode}
% \end{macro}
% \end{macro}
@@ -2169,10 +2178,10 @@
\str_aux_convert_to_native:n
{
\exp_after:wN \str_convert_to_native_aux:w
- \l_str_tmpa_tl \q_mark { \q_stop \prg_map_break: } ,
+ \l_str_tmpa_tl \s_tl { \q_stop \prg_map_break: } \s_tl
}
}
-\cs_new_protected_nopar:Npn \str_convert_to_native_aux:w #1\q_mark #2,
+\cs_new_protected_nopar:Npn \str_convert_to_native_aux:w #1 \s_tl #2 \s_tl
{
\use_none_delimit_by_q_stop:w #2 \q_stop
\str_aux_convert_to_native_step:n {#2}
@@ -2294,6 +2303,8 @@
{ Overlong~UTF-8~byte~sequence~for~code~point~#1. }
\msg_kernel_new:nnn { str } { utf8-premature-end }
{ Incomplete~last~UTF-8~character. }
+\msg_kernel_new:nnn { str } { utf8-overflow }
+ { Code~point~#1~too~large~(UTF-8). }
\msg_kernel_new:nnn { str } { utf32-overflow }
{ Code~point~too~large~(UTF-32~`#1'). }
\msg_kernel_new:nnn { str } { utf32-truncated }
@@ -2584,7 +2595,7 @@
% \begin{variable}{\c_str_convert_output_name_not_str}
% For each byte, test whether it should
% be output as is, or be \enquote{hash-encoded}.
-% Roughly, bytes outside the range from |"2A| to |"7E|
+% Roughly, bytes outside the range $[\hexnum{2A},\hexnum{7E}]$
% are hash-encoded. We keep two lists of exceptions:
% characters in \cs{c_str_convert_output_name_not_str}
% are not hash-encoded, and characters in the
@@ -2767,22 +2778,21 @@
% \begin{macro}[aux, rEXP]
% {
% \str_aux_from_utf_viii:N,
-% \str_aux_from_utf_viii:wwNN,
-% \str_aux_from_utf_viii:wNNww
+% \str_aux_from_utf_viii:wwN,
+% \str_aux_from_utf_viii:wNnnwN
% }
% \begin{macro}[aux, rEXP]
% {
-% \str_aux_from_utf_viii_error:w,
-% \str_aux_from_utf_viii_error:NwNN,
+% \str_aux_from_utf_viii_overflow:w ,
% \str_aux_from_utf_viii_error:
% }
-% In the \textsc{utf-8} encoding, bytes in the range $[0,127]$ stand
-% for themselves, bytes in the range $[128,191]$ are continuation bytes,
+% In the \textsc{utf-8} encoding, bytes in the range
+% $[0,\hexnum{7F}]$ stand for themselves, bytes in the range
+% $[\hexnum{80},\hexnum{BF}]$ are continuation bytes,
% and larger bytes must be followed by a number of continuation bytes.
% Documentation to come.
% ^^A todo: refuse "D800 -- "DFFF?
% ^^A todo: document
-% ^^A todo: what error recovery for code points > "10FFFF?
% \begin{macrocode}
\cs_new_protected_nopar:cpn { str_convert_from_utf8: }
{
@@ -2797,7 +2807,7 @@
{
#1
\if_num:w `#1 < "C0 \exp_stop_f:
- \exp_not:N \q_mark
+ \s_tl
\if_num:w `#1 < "80 \exp_stop_f:
\int_value:w `#1
\else:
@@ -2807,71 +2817,73 @@
\fi:
\else:
\if_num:w `#1 < "F5 \exp_stop_f:
- \exp_after:wN \str_aux_from_utf_viii:wwNN
- \int_use:N \int_eval:w `#1 - "C0
+ \exp_after:wN \str_aux_from_utf_viii:wwN
+ \int_value:w \int_eval:n { `#1 - "C0 }
\else:
- \exp_not:N \q_mark
+ \s_tl
\msg_expandable_kernel_error:nnn
{ str } { utf8-invalid-byte } {#1}
\int_use:N \c_str_replacement_char_int
\fi:
\fi:
- ,
- \use_none_delimit_by_q_stop:w
- "80 , "800 , "10000 , "10FFFF ,
- \q_stop
+ \s_tl
+ \use_none_delimit_by_q_stop:w {"80} {"800} {"10000} {"110000} \q_stop
\str_aux_from_utf_viii:N
}
-\cs_new_nopar:Npn \str_aux_from_utf_viii:wwNN #1 , #2 \q_stop #3#4
+\cs_new_nopar:Npn \str_aux_from_utf_viii:wwN
+ #1 \s_tl #2 \str_aux_from_utf_viii:N #3
{
- \use_none:n #4
- \exp_after:wN \str_aux_from_utf_viii:wNNww
- \int_use:N \int_eval:w #1 * "40 + `#4 - "80 ,
- #4
- #2
- \q_stop #3
+ \use_none:n #3
+ \if_num:w \int_eval:w ( `#3 + "20 ) / "40 = \c_three
+ #3
+ \exp_after:wN \str_aux_from_utf_viii:wNnnwN
+ \int_value:w \int_eval:n { #1 * "40 + `#3 - "80 }
+ \else:
+ \s_tl
+ \msg_expandable_kernel_error:nnn
+ { str } { utf8-missing-conti } {#3}
+ \int_use:N \c_str_replacement_char_int
+ \fi:
+ \s_tl
+ #2
+ \str_aux_from_utf_viii:N #3
}
-\cs_new_nopar:Npn \str_aux_from_utf_viii:wNNww #1, #2 #3#4, #5,
+\cs_new_nopar:Npn \str_aux_from_utf_viii:wNnnwN
+ #1 \s_tl #2#3#4 #5 \str_aux_from_utf_viii:N #6
{
- \if_num:w \int_eval:w ( `#2 + \c_thirty_two ) / "40 = \c_three
- \if_num:w #1 < #4 \exp_stop_f:
- \str_aux_from_utf_viii_error:w #1 ,
+ \if_num:w #1 < #4
+ \s_tl
+ \if_num:w #1 < #3 \exp_stop_f:
+ \msg_expandable_kernel_error:nnn
+ { str } { utf8-overlong } {#1}
+ \int_use:N \c_str_replacement_char_int
\else:
- #2
- \if_num:w #1 < #5 \exp_stop_f:
- \exp_not:N \q_mark #1
- \else:
- \exp_after:wN \str_aux_from_utf_viii:wwNN
- \int_use:N \int_eval:w #1 - #5
- \fi:
+ #1
\fi:
\else:
- \exp_after:wN \str_aux_from_utf_viii_error:NwN
- \exp_after:wN #2
+ \if_meaning:w \q_stop #5
+ \str_aux_from_utf_viii_overflow:w #1
+ \fi:
+ \exp_after:wN \str_aux_from_utf_viii:wwN
+ \int_value:w \int_eval:n { #1 - #4 }
\fi:
- , #3#5,
- }
-\cs_new_nopar:Npn \str_aux_from_utf_viii_error:w #1 ,
- {
- \exp_not:N \q_mark
- \msg_expandable_kernel_error:nnn
- { str } { utf8-overlong } {#1}
- \c_str_replacement_char_int
+ \s_tl
+ #2 {#4} #5
+ \str_aux_from_utf_viii:N
}
-\cs_new_nopar:Npn \str_aux_from_utf_viii_error:NwN #1 #2 \q_stop #3
+\cs_new_nopar:Npn \str_aux_from_utf_viii_overflow:w #1 \fi: #2 \fi:
{
- \exp_not:N \q_mark
+ \fi: \fi:
\msg_expandable_kernel_error:nnn
- { str } { utf8-missing-conti } {#1}
- \c_str_replacement_char_int ,
- #3#1
+ { str } { utf8-overflow } {#1}
+ \int_use:N \c_str_replacement_char_int
}
\cs_new_nopar:Npn \str_aux_from_utf_viii_error:
{
- \exp_not:N \q_mark
+ \s_tl
\msg_expandable_kernel_error:nn
{ str } { utf8-premature-end }
- \c_str_replacement_char_int ,
+ \int_use:N \c_str_replacement_char_int \s_tl
\prg_map_break:
}
% \end{macrocode}
@@ -3018,19 +3030,32 @@
% \end{macro}
%
% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi:NN}
-% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi:wNNN}
-% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_error:wNNw}
+% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi:NNwNNN}
% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_end:Nw}
-% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_end:wNNww}
+% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_error:NNw}
+% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_error:NNN}
% When \cs{str_aux_from_utf_xvi:NN} is called, \cs{str_tmp:w}
% is such that |\str_tmp:w #1#2| expands to |`#1| if the string
% is big-endian, and |`#2| if the string is little-endian.
-% If the most significant byte is between |"D8| and |"DB|
-% inclusive, then the pair of bytes is a high surrogate,
-% and we fetch the following pair, which must be a low surrogate,
-% \emph{i.e.}, between |"DC| and |"DF| inclusive.
-% Unpaired surrogates are errors, handled with
-% \cs{str_aux_from_utf_xvi_error:wNNw}.
+% If the code unit (a pair of characters) does not represent
+% a surrogate, the \cs{if_case:w} construction expands to nothing,
+% and we leave in the input stream \meta{characters} \cs{s_tl}
+% \meta{char code} \cs{s_tl}, and move on to the next code unit.
+% When the end is found, we check for the case of an odd-length
+% string (an error) and end the mapping.
+%
+% Surrogates are characterized by a most significant byte in the
+% ranges $[\hexnum{D8},\hexnum{DB}]$ and $[\hexnum{DC},\hexnum{DF}]$,
+% detected by dividing the character code by $4$ (\eTeX{} rounds to
+% nearest, ties away from zero). Here, a trail surrogate is invalid,
+% replaced by the replacement character. Lead surrogates are treated by
+% \cs{str_aux_from_utf_xvi:NNwNNN}, which grabs the next two
+% characters, checks for a premature end, and checks whether
+% the code unit is a trailing surrogate. If it is, leave
+% \meta{characters} \cs{s_tl} \meta{char code} \cs{s_tl} in the
+% input stream and move on to the next code unit. Otherwise, raise
+% an error, and continue parsing, replacing only the first code unit
+% (the leading surrogate) by the replacement character.
% \begin{macrocode}
\cs_new:Npn \str_aux_from_utf_xvi:NN #1#2
{
@@ -3039,60 +3064,63 @@
\fi:
\if_case:w
\int_eval:w ( \str_tmp:w #1#2 - "D6 ) / \c_four \int_eval_end:
- \or: #1#2 \exp_after:wN \str_aux_from_utf_xvi:wNNN
- \or: \exp_after:wN \str_aux_from_utf_xvi_error:wNNw
- \else: #1#2 \exp_not:N \q_mark \int_use:N \int_eval:w
+ \or: \exp_after:wN \str_aux_from_utf_xvi:NNwNNN
+ \or: \exp_after:wN \str_aux_from_utf_xvi_error:NNw
\fi:
- "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 ,
+ #1#2 \s_tl
+ \int_eval:n { "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 } \s_tl
\str_aux_from_utf_xvi:NN
}
- \cs_new:Npn \str_aux_from_utf_xvi:wNNN #1, #2#3#4
+ \cs_new:Npn \str_aux_from_utf_xvi:NNwNNN #1#2 \s_tl #3 \s_tl #4#5#6
{
- \if_meaning:w \scan_stop: #4
- \str_aux_from_utf_xvi_end:wNNww #1 , #3
+ \if_meaning:w \scan_stop: #6
+ \str_aux_from_utf_xvi_error:NNN #1#2#5
\fi:
- \if_num:w \int_eval:w ( \str_tmp:w #3#4 - "DA ) / \c_four = \c_one
- #3 #4
- \int_use:N \int_eval:w
- ( #1 - "D800 ) * "400
- + \str_tmp:w #3#4 * "100 + \str_tmp:w #4#3 - "DC00 ,
+ \if_num:w \int_eval:w ( \str_tmp:w #5#6 - "DA ) / \c_four = \c_one
+ #1 #2 #5 #6 \s_tl
+ \int_eval:n
+ {
+ ( "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 - "D800 ) * "400
+ + "100 * \str_tmp:w #4#5 + \str_tmp:w #5#4 - "DC00
+ }
+ \s_tl
\exp_after:wN \use_i:nnn
\else:
- \str_aux_from_utf_xvi_error:wNNw #1 ,
+ \str_aux_from_utf_xvi_error:NNw #1#2 \s_tl \s_tl
\fi:
- #2#3#4
- }
- \cs_new:Npn \str_aux_from_utf_xvi_error:wNNw #1 \str_tmp:w #2#3 #4,
- {
- \exp_not:N \q_mark
- \msg_expandable_kernel_error:nnn
- { str } { utf16-surrogate } { #2#3 }
- \c_str_replacement_char_int ,
+ #4#5#6
}
\cs_new:Npn \str_aux_from_utf_xvi_end:Nw #1 \fi:
{
\fi:
\if_meaning:w \scan_stop: #1
\else:
- #1 \exp_not:N \q_mark
+ #1 \s_tl
\msg_expandable_kernel_error:nnn
{ str } { utf16-odd } {#1}
- \int_use:N \c_str_replacement_char_int ,
+ \int_use:N \c_str_replacement_char_int \s_tl
\fi:
\prg_map_break:
}
- \cs_new:Npn \str_aux_from_utf_xvi_end:wNNww #1 `#2#3 #4, #5 \fi:
+ \cs_new:Npn \str_aux_from_utf_xvi_error:NNw #1#2 \s_tl #3 \s_tl
{
- \fi:
- \exp_not:N \q_mark
- \if_meaning:w \scan_stop: #5
+ #1 #2 \s_tl
+ \msg_expandable_kernel_error:nnn
+ { str } { utf16-surrogate } { #1#2 }
+ \int_use:N \c_str_replacement_char_int \s_tl
+ }
+ \cs_new:Npn \str_aux_from_utf_xvi_error:NNN #1#2#3
+ {
+ \if_meaning:w \scan_stop: #3
+ #1#2 \s_tl
\msg_expandable_kernel_error:nnn
- { str } { utf16-surrogate } { #2#3 }
+ { str } { utf16-surrogate } { #1#2 }
\else:
+ #1#2#3 \s_tl
\msg_expandable_kernel_error:nnn
- { str } { utf16-odd } { #2#3#5 }
+ { str } { utf16-odd } { #1#2#3 }
\fi:
- \int_use:N \c_str_replacement_char_int ,
+ \int_use:N \c_str_replacement_char_int \s_tl
\prg_map_break:
}
% \end{macrocode}
@@ -3180,7 +3208,7 @@
% \begin{macro}[aux, rEXP]
% {
% \str_aux_from_utf_xxxii:NNNN,
-% \str_aux_from_utf_xxxii_end:ww
+% \str_aux_from_utf_xxxii_end:w
% }
% See the conversion functions from \textsc{utf-16} encodings.
% The \cs{str_aux_from_utf_xxxii:No} auxiliary defines
@@ -3226,7 +3254,7 @@
\if_meaning:w \scan_stop: #4
\exp_after:wN \str_aux_from_utf_xxxii_end:w
\fi:
- #1#2#3#4 \exp_not:N \q_mark
+ #1#2#3#4 \s_tl
\if_num:w \int_eval:w
\str_tmp:w #1#4 * \c_two_hundred_fifty_six
+ \str_tmp:w #2#3
@@ -3235,12 +3263,10 @@
{ str } { utf32-overflow } { #1#2#3#4 }
\int_use:N \c_str_replacement_char_int
\else:
- \int_use:N \int_eval:w
- \str_tmp:w #2#3*"10000
- + \str_tmp:w #3#2 * \c_two_hundred_fifty_six
- + \str_tmp:w #4#1
+ \int_eval:n
+ { \str_tmp:w #2#3*"10000 + \str_tmp:w #3#2*"100 + \str_tmp:w #4#1 }
\fi:
- ,
+ \s_tl
\str_aux_from_utf_xxxii:NNNN
}
\cs_new:Npn \str_aux_from_utf_xxxii_end:w #1 \scan_stop:
@@ -3249,8 +3275,8 @@
{
\msg_expandable_kernel_error:nnn
{ str } { utf32-truncated } { #1 }
- #1 \exp_not:N \q_mark
- \int_use:N \c_str_replacement_char_int ,
+ #1 \s_tl
+ \int_use:N \c_str_replacement_char_int \s_tl
}
\prg_map_break:
}
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx
index c3310545ac0..ca9ac5399b0 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx
@@ -1,6 +1,6 @@
% \iffalse meta-comment
%
-%% File: l3tl-analysis.dtx Copyright (C) 2011 The LaTeX3 Project
+%% File: l3tl-analysis.dtx Copyright (C) 2011-2012 The LaTeX3 Project
%%
%% It may be distributed and/or modified under the conditions of the
%% LaTeX Project Public License (LPPL), either version 1.3c of this
@@ -88,22 +88,83 @@
%
% \subsection{Internal}
%
+% \begin{variable}{\s_tl}
+% The format used to store token lists internally uses
+% the scan mark \cs{s_tl} as a delimiter.
+% \end{variable}
+%
+% The task of the \pkg{l3tl-analysis} module is to convert
+% token lists to an internal format which allows us to extract
+% all the relevant information about individual tokens
+% (category code, character code), as well as reconstruct
+% the token list quickly. This internal format is used in
+% \pkg{l3regex} where we need to support arbitrary tokens,
+% and it is used in conversion functions in \pkg{l3str},
+% where we wish to support clusters of characters instead
+% of single characters.
+%
+% The internal format thus takes the form of a succession
+% of items of the form
+% \begin{quote}
+% \meta{tokens} \cs{s_tl} \meta{catcode} \meta{char code} \cs{s_tl}
+% \end{quote}
+% Here, the \meta{tokens} \texttt{o}- and \texttt{x}-expand
+% to the original token in the token list (even special
+% category codes) or to the cluster of tokens corresponding
+% to one Unicode character in the given encoding.
+% The \meta{catcode} is given as a single hexadecimal digit,
+% $0$ for control sequences. The \meta{char code} is given
+% as a decimal number, $-1$ for control sequences.
+%
+% Using delimited arguments lets us build the \meta{tokens}
+% progressively when doing an encoding conversion in \pkg{l3str}.
+% On the other hand, the delimiter \cs{s_tl} may not appear
+% unbraced in \meta{tokens}. This is not a problem because we
+% are careful to wrap control sequences in braces (as an argument
+% to \cs{exp_not:n}) when converting from a general token list
+% to the internal format.
+%
% \begin{function}{\tl_set_analysis:Nn}
% \begin{syntax}
-% \cs{tl_analysis:Nn} \meta{tl var} \Arg{token list}
+% \cs{tl_set_analysis:Nn} \meta{tl var} \Arg{token list}
% \end{syntax}
-% This function analysiss each token in the \meta{token list},
-% converting each \meta{token} to
-% \begin{quote}
-% \Arg{tokens} \cs{q_mark} \meta{catcode} \meta{char code} |,|
-% \end{quote}
-% where \meta{tokens} expands in one step to the \meta{token},
-% and \meta{catcode} and \meta{char code} are the category code
-% and character code of the token if it is a character token,
-% and otherwise are $0$ and $-1$ respectively.
-% The result of the conversion is then stored in \meta{tl var}.
+% This function analyzes each \meta{token} in the \meta{token list},
+% converting each \meta{token} to one item in the internal format.
+% The result of the analysis is then stored in \meta{tl var}.
+% The following transformation is performed:
+% \begin{itemize}
+% \item a control sequence |\cs| becomes
+% |\exp_not:n { \cs } \s_tl 0 -1 \s_tl|;
+% \item a begin-group character |{| becomes
+% |\exp_after:wN { \if_false: } \fi: \s_tl 1| \meta{char code} \cs{s_tl};
+% \item an end-group character |}| becomes
+% |\if_false: { \fi: } \s_tl 2| \meta{char code} \cs{s_tl};
+% \item a character with any other category code becomes
+% |\exp_not:n| \Arg{character} \cs{s_tl} \meta{hex catcode}
+% \meta{char code} \cs{s_tl}.
+% \end{itemize}
+% We can note that in every case the part before \cs{s_tl}
+% both \texttt{o}-expands and \texttt{x}-expands to the token
+% which is represented.
% \end{function}
%
+% \begin{function}{\tl_set_analysis_from_str:Nn}
+% \begin{syntax}
+% \cs{tl_set_analysis:Nn} \meta{tl var} \Arg{token list}
+% \end{syntax}
+% This function analyzes each \meta{token} in the \meta{token list},
+% after conversion to a string.
+% The result of the analysis is then stored in \meta{tl var}.
+% \begin{itemize}
+% \item Spaces become \verb*| |\cs{s_tl} |A32| \cs{s_tl}.
+% \item Other characters become
+% \meta{character} \cs{s_tl} |C| \meta{char code} \cs{s_tl}.
+% \end{itemize}
+% \end{function}
+%
+% ^^A todo: ask LuaTeX list for an \ifx\undefined <active char>
+% ^^A which does not add the <active char> in memory.
+%
% \end{documentation}
%
% \begin{implementation}
@@ -122,6 +183,33 @@
%
% \subsection{Variables and helper functions}
%
+% \begin{variable}{\s_tl}
+% The scan mark \cs{s_tl} is used as a delimiter in the internal
+% format. This is more practical than using a quark, because we
+% would then need to control expansion much more carefully: compare
+% \cs{int_value:w} |`#1| \cs{s_tl} with
+% \cs{int_value:w} |`#1| \cs{exp_stop_f:} \cs{exp_not:N} \cs{q_mark}
+% to extract a character code followed by the delimiter in an
+% \texttt{x}-expansion.
+% \begin{macrocode}
+\scan_new:N \s_tl
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_tl_analysis_token}
+% \begin{variable}{\l_tl_analysis_char_token}
+% The tokens in the token list are probed with the \TeX{}
+% primitive \tn{futurelet}. We use \cs{l_tl_analysis_token}
+% in that construction. In some cases, we convert the
+% following token to a string before probing it: then the
+% token variable used is \cs{l_tl_analysis_char_token}.
+% \begin{macrocode}
+\cs_new_eq:NN \l_tl_analysis_token ?
+\cs_new_eq:NN \l_tl_analysis_char_token ?
+% \end{macrocode}
+% \end{variable}
+% \end{variable}
+%
% \begin{variable}{\l_tl_analysis_normal_int}
% The number of normal (\texttt{N}-type argument) tokens
% since the last special token.
@@ -148,14 +236,6 @@
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\l_tl_analysis_char_int}
-% Holds the character code of the token currently considered,
-% in cases where it is a character.
-% \begin{macrocode}
-\int_new:N \l_tl_analysis_char_int
-% \end{macrocode}
-% \end{variable}
-%
% \begin{variable}{\l_tl_analysis_type_int}
% When encountering special characters, we record their \enquote{type}
% in this integer.
@@ -168,36 +248,40 @@
% The result of the conversion is stored in this token list,
% with a succession of items of the form
% \begin{quote}
-% \meta{tokens} \cs{q_mark} \meta{catcode} \meta{char code} |,|
+% \meta{tokens} \cs{s_tl} \meta{catcode} \meta{char code} \cs{s_tl}
% \end{quote}
% \begin{macrocode}
\tl_new:N \g_tl_analysis_result_tl
% \end{macrocode}
% \end{variable}
%
-% \begin{macro}[int]{\tl_analysis_extract_charcode:}
-% \begin{macro}[aux]{\tl_analysis_extract_charcode_aux:w}
-% Extracting the character code from the meaning of \cs{l_tl_analysis_token}.
-% This has no error checking, and should only be assumed to work
-% for begin-group and end-group tokens.
+% \begin{macro}[int, EXP]{\tl_analysis_extract_charcode:}
+% \begin{macro}[aux, EXP]{\tl_analysis_extract_charcode_aux:w}
+% Extracting the character code from the meaning of
+% \cs{l_tl_analysis_token}. This has no error checking,
+% and should only be assumed to work for begin-group
+% and end-group character tokens. It produces a number
+% in the form |`|\meta{char}.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \tl_analysis_extract_charcode:
+\cs_new_nopar:Npn \tl_analysis_extract_charcode:
{
\exp_after:wN \tl_analysis_extract_charcode_aux:w
\token_to_meaning:N \l_tl_analysis_token
}
-\cs_new_protected_nopar:Npn \tl_analysis_extract_charcode_aux:w #1 ~ #2 ~ { ` }
+\cs_new:Npn \tl_analysis_extract_charcode_aux:w #1 ~ #2 ~ { ` }
% \end{macrocode}
% \end{macro}
% \end{macro}
%
-% \begin{macro}[int]{\tl_analysis_cs_space_count:NN}
-% \begin{macro}[aux]{\tl_analysis_cs_space_count:w}
-% \begin{macro}[aux]{\tl_analysis_cs_space_count_end:w}
+% \begin{macro}[int, EXP]{\tl_analysis_cs_space_count:NN}
+% \begin{macro}[aux, EXP]{\tl_analysis_cs_space_count:w}
+% \begin{macro}[aux, EXP]{\tl_analysis_cs_space_count_end:w}
% Counts the number of spaces in the string representation of its
% second argument, as well as the number of characters following
% the last space in that representation, and feeds the two numbers
% as semicolon-delimited arguments to the first argument.
+% When this function is used, the escape character is printable
+% and non-space.
% \begin{macrocode}
\cs_new:Npn \tl_analysis_cs_space_count:NN #1 #2
{
@@ -224,10 +308,10 @@
%
% Our goal is to produce a token list of the form roughly
% \begin{quote}
-% \Arg{token 1} \cs{q_mark} \meta{catcode 1} \meta{char code 1} |,| \\
-% \Arg{token 2} \cs{q_mark} \meta{catcode 2} \meta{char code 2} |,| \\
+% \meta{token 1} \cs{s_tl} \meta{catcode 1} \meta{char code 1} \cs{s_tl} \\
+% \meta{token 2} \cs{s_tl} \meta{catcode 2} \meta{char code 2} \cs{s_tl} \\
% \ldots{}
-% \Arg{token N} \cs{q_mark} \meta{catcode N} \meta{char code N} |,|
+% \meta{token N} \cs{s_tl} \meta{catcode N} \meta{char code N} \cs{s_tl}
% \end{quote}
% Most but not all tokens can be grabbed as an undelimited
% (\texttt{N}-type) argument by \TeX{}.
@@ -239,12 +323,12 @@
% registers, and removed from the input stream by some means.
%
% To ease the difficult first pass, we first do some setup
-% with \cs{tl_analysis_i_setup:n}.
+% with \cs{tl_analysis_setup:n}.
% Active characters set equal to non-active characters
% cause trouble, so we disable all active characters by
-% setting them equal to \texttt{undefined}.
+% setting them equal to \texttt{undefined} locally.
% We also set there the escape character to be printable
-% (backslash, but this later changes to a forward slash):
+% (backslash, but this later oscillates between solidus and backslash):
% this makes it possible to distinguish characters from
% control sequences.
%
@@ -259,10 +343,6 @@
% \item an end-group token (category code $2$),
% either space (character code $32$), or non-space;
% \item a space token (category code $10$, character code $32$);
-%^^A \item a character with category code $3$, $4$, $6$, $7$, $8$, $11$,
-%^^A or $12$, or with category code $10$ but character code $\neq 32$;
-%^^A \item a non-character (primitive, macro, register, font,
-%^^A undefined, \emph{etc.}).
% \item anything else (then the token is always an \texttt{N}-type argument).
% \end{itemize}
% The token itself can \enquote{look like} one of the following
@@ -270,19 +350,18 @@
% \item a non-active character, in which case its meaning
% is automatically that associated to its character code
% and category code, we call it \enquote{true} character;
-% \item an active character;
+% \item an active character (we eliminate those in the setup step);
% \item a control sequence.
% \end{itemize}
% The only tokens which are not valid \texttt{N}-type arguments are
% true begin-group characters, true end-group characters, and true spaces.
% We will detect those characters by scanning ahead with \tn{futurelet},
-% then distinguishing true characters from tokens set equal to them
-% using the \tn{string} representation.
+% then distinguishing true characters from control sequences set equal
+% to them using the \tn{string} representation.
%
% The second pass is a simple exercise in expandable loops.
%
% \begin{macro}[int]{\tl_analysis:n}
-% \begin{macro}[int]{\tl_set_analysis:Nn}
% Everything is done within a group, and all definitions will be local.
% We use \cs{group_align_safe_begin/end:} to avoid problems in case
% \cs{tl_analysis:n} is used within an alignment and its argument
@@ -298,6 +377,13 @@
\group_align_safe_end:
\group_end:
}
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[int]{\tl_set_analysis:Nn}
+% This function is used in \pkg{l3regex}. Simply copy the result
+% of \cs{tl_analysis:n} into the given token list.
+% \begin{macrocode}
\cs_new_protected:Npn \tl_set_analysis:Nn #1#2
{
\tl_analysis:n {#2}
@@ -305,6 +391,41 @@
}
% \end{macrocode}
% \end{macro}
+%
+% \begin{macro}[int]{\tl_set_analysis_from_str:Nn}
+% \begin{macro}[aux, EXP]{\tl_analysis_from_str_loop:N}
+% This function, used in \pkg{l3str} could be emulated by passing the
+% token list through \cs{tl_to_str:n} and using
+% \cs{tl_set_analysis:Nn}, but this would be terribly inefficient.
+% Instead, convert to a string with spaces of category code other,
+% and for each character output one item of the analysis:
+% \meta{character} \cs{s_tl} \meta{catcode} \meta{char code} \cs{s_tl}
+% where the \meta{catcode} is either |A| (for spaces) or |C|
+% (for other characters).
+% \begin{macrocode}
+\cs_new_protected:Npn \tl_set_analysis_from_str:Nn #1#2
+ {
+ \str_gset_other:Nn \g_tl_analysis_result_tl {#2}
+ \tl_set:Nx #1
+ {
+ \exp_after:wN \tl_analysis_from_str_loop:N
+ \g_tl_analysis_result_tl \prg_map_break:
+ \prg_break_point:n { }
+ }
+ }
+\group_begin:
+ \char_set_catcode_other:N A
+ \char_set_catcode_other:N C
+ \cs_new_protected:Npn \tl_analysis_from_str_loop:N #1
+ {
+ #1 \s_tl
+ \if_charcode:w #1 ~ A \else: C \fi:
+ \int_value:w `#1 \s_tl
+ \tl_analysis_from_str_loop:N
+ }
+\group_end:
+% \end{macrocode}
+% \end{macro}
% \end{macro}
%
% \subsection{Setup}
@@ -316,20 +437,22 @@
% \texttt{undefined}. Since Unicode contains too many characters
% to loop over all of them, we instead loop over the input token
% list as a string: any active character in the token list
-% must appear in its string representation.
-% ^^A todo: ask LuaTeX list for an \ifx\undefined <active char>
-% ^^A which does not add the <active char> in memory.
+% must appear in its string representation. The string is shortened
+% a little by making the escape character unprintable. The active
+% space must be disabled separately (the loop skips over it otherwise),
+% and we end the loop by feeding an odd non-\texttt{N}-type
+% argument to the looping macro.
% \begin{macrocode}
\cs_new_protected:Npn \tl_analysis_setup:n #1
{
- \int_set:Nn \tex_escapechar:D { 92 }
+ \int_set_eq:NN \tex_escapechar:D \c_minus_one
\exp_after:wN \tl_analysis_disable_loop:N
\tl_to_str:n {#1} { ~ } { ? ~ \prg_map_break: }
\prg_break_point:n { }
}
\group_begin:
\char_set_catcode_active:N \^^@
- \cs_new_protected_nopar:Npn \tl_analysis_disable_loop:N #1
+ \cs_new_protected:Npn \tl_analysis_disable_loop:N #1
{
\tex_lccode:D \c_zero `#1 ~
\tl_to_lowercase:n { \tex_let:D ^^@ } \c_undefined:D
@@ -349,22 +472,22 @@
%
% After the setup step, we have $11$ types of tokens:
% \begin{itemize}
-% \item[1] a true non-space begin-group character;
-% \item[2] a true space begin-group character;
-% \item[3] a true non-space end-group character;
-% \item[4] a true space end-group character;
-% \item[5] a true space blank space character;
-% \item[6] an undefined active character;
-% \item[7] any other true character;
-% \item[8] a control sequence equal to a begin-group token (category code $1$);
-% \item[9] a control sequence equal to an end-group token (category code $2$);
-% \item[10] a control sequence equal to a space token
+% \item[1.] a true non-space begin-group character;
+% \item[2.] a true space begin-group character;
+% \item[3.] a true non-space end-group character;
+% \item[4.] a true space end-group character;
+% \item[5.] a true space blank space character;
+% \item[6.] an undefined active character;
+% \item[7.] any other true character;
+% \item[8.] a control sequence equal to a begin-group token (category code $1$);
+% \item[9.] a control sequence equal to an end-group token (category code $2$);
+% \item[10.] a control sequence equal to a space token
% (character code $32$, category code $10$);
-% \item[11] any other control sequence.
+% \item[11.] any other control sequence.
% \end{itemize}
% Our first tool is \tn{futurelet}. This cannot distinguish
-% cases $1$ and $2$ from $8$, nor cases $3$ and $4$ from $9$,
-% nor case $5$ from case $10$. Those cases will be distinguished
+% case $8$ from $1$ or $2$, nor case $9$ from $3$ or $4$,
+% nor case $10$ from case $5$. Those cases will be distinguished
% by applying the \tn{string} primitive to the following token,
% after possibly changing the escape character to ensure that
% a control sequence's string representation cannot be mistaken
@@ -376,7 +499,7 @@
% \cs{str_tail:n} \Arg{token} is non-empty, because the
% escape character is printable.
%
-% \begin{macro}{\tl_analysis_i:n}
+% \begin{macro}[int]{\tl_analysis_i:n}
% We read tokens one by one using \tn{futurelet}.
% While performing the loop, we keep track of the number of
% true begin-group characters minus the number of
@@ -385,6 +508,7 @@
% \begin{macrocode}
\cs_new_protected:Npn \tl_analysis_i:n #1
{
+ \int_set:Nn \tex_escapechar:D { 92 }
\int_zero:N \l_tl_analysis_normal_int
\int_zero:N \l_tl_analysis_index_int
\int_zero:N \l_tl_analysis_nesting_int
@@ -403,17 +527,21 @@
% \end{macro}
%
% \begin{macro}[int]{\tl_analysis_i_type:w}
-% At this point, \cs{l_tl_analysis_token} holds the meaning of the following
-% token. We store in \cs{l_tl_analysis_type_int} the meaning of the token ahead:
+% At this point, \cs{l_tl_analysis_token} holds the meaning
+% of the following token. We store in \cs{l_tl_analysis_type_int}
+% the meaning of the token ahead:
% \begin{itemize}
-% \item[0] space token;
-% \item[1] begin-group token;
-% \item[-1] end-group token;
-% \item[2] other.
+% \item 0 space token;
+% \item 1 begin-group token;
+% \item -1 end-group token;
+% \item 2 other.
% \end{itemize}
-% The values $0$, $1$, $-1$ correspond to how much a true such character
-% changes the nesting level ($2$ is used only here, and is irrelevant later).
-% Then call the auxiliary for each case.
+% The values $0$, $1$, $-1$ correspond to how much a true such
+% character changes the nesting level ($2$ is used only here,
+% and is irrelevant later). Then call the auxiliary for each case.
+% Note that nesting conditionals here is safe because we only skip
+% over \cs{l_tl_analysis_token} if it matches with one of the
+% character tokens (hence is not a primitive conditional).
% \begin{macrocode}
\cs_new_protected_nopar:Npn \tl_analysis_i_type:w
{
@@ -441,65 +569,38 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[int]{\tl_analysis_i_safe:N}
-% \begin{macro}[aux]{\tl_analysis_i_cs:ww}
-% The upcoming token is safe: we can simply grab it in a second pass.
-% Since other branches of the code must pass their tokens through
-% \tn{string}, we do it here as well. It's been optimized in an evil way,
-% sorry.
-%^^A todo: document
-% \begin{macrocode}
-\cs_new_protected:Npn \tl_analysis_i_safe:N #1
- {
- \if_charcode:w
- \scan_stop:
- \exp_after:wN \use_none:n \token_to_str:N #1 \prg_do_nothing:
- \scan_stop:
- \int_incr:N \l_tl_analysis_normal_int
- \else:
- \tl_analysis_cs_space_count:NN \tl_analysis_i_cs:ww #1
- \fi:
- \tl_analysis_i_loop:w
- }
-\cs_new_protected:Npn \tl_analysis_i_cs:ww #1; #2;
- {
- \if_num:w #1 > \c_zero
- \tex_skip:D \l_tl_analysis_index_int
- = \int_eval:w \l_tl_analysis_normal_int + \c_one sp \scan_stop:
- \tex_advance:D \l_tl_analysis_index_int #1 \exp_stop_f:
- \l_tl_analysis_normal_int #2 \exp_stop_f:
- \else:
- \tex_advance:D \l_tl_analysis_normal_int #2 \exp_stop_f:
- \fi:
- }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-%
-% \begin{macro}[aux]{\tl_analysis_i_space:w}
+% \begin{macro}[int]{\tl_analysis_i_space:w}
% \begin{macro}[aux]{\tl_analysis_i_space_test:w}
% In this branch, the following token's meaning is a blank space.
% Apply \tn{string} to that token: if it is a control sequence
% the result starts with the escape character; otherwise it is
% a true blank space, whose string representation is also a blank space.
% We test for that in \cs{tl_analysis_i_space_test:w},
-% after grabbing the first character of the string representation
-% as \cs{l_tl_analysis_token}.
+% after grabbing as \cs{l_tl_analysis_char_token} the first character
+% of the string representation.
% Also, since \cs{tl_analysis_i_store:} expects the special token to be
% stored in the relevant \tn{toks} register, we do that. The extra
% \cs{exp_not:n} is unnecessary of course, but it makes the treatment
% of all tokens more homogeneous.
+% If we discover that the next token was actually a control sequence
+% instead of a true space, then we step the counter of normal tokens.
+% We now have in front of us the whole string representation of
+% the control sequence, including potential spaces; those will appear
+% to be true spaces later in this pass. Hence, all other branches of
+% the code in this first pass need to consider the string representation,
+% so that the second pass does not need to test the meaning of tokens,
+% only strings.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \tl_analysis_i_space:w
{
\tex_afterassignment:D \tl_analysis_i_space_test:w
\exp_after:wN \cs_set_eq:NN
- \exp_after:wN \l_tl_analysis_token
+ \exp_after:wN \l_tl_analysis_char_token
\token_to_str:N
}
\cs_new_protected_nopar:Npn \tl_analysis_i_space_test:w
{
- \if_meaning:w \l_tl_analysis_token \c_space_token
+ \if_meaning:w \l_tl_analysis_char_token \c_space_token
\tex_toks:D \l_tl_analysis_index_int { \exp_not:n { ~ } }
\tl_analysis_i_store:
\else:
@@ -511,17 +612,20 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}[int]{\tl_analysis_i_bgroup:w}
-% \begin{macro}[int]{\tl_analysis_i_egroup:w}
+% \begin{macro}[int]{\tl_analysis_i_bgroup:w, \tl_analysis_i_egroup:w}
% \begin{macro}[aux]{\tl_analysis_i_group:nw}
% \begin{macro}[aux]{\tl_analysis_i_group_test:w}
% The token might be either a true character token with
% catcode $1$ or $2$, or it could be a control sequence.
% The only tricky case is if the character code happens
% to be equal to the escape character: then we change
-% the escape character, so that the string representation
-% of the true character and of a control sequence set equal
-% to it start differently.
+% the escape character from backslash to solidus or back,
+% so that the string representation of the true character
+% and of a control sequence set equal to it start differently.
+% Then probe what the first character of that string
+% representation is: this is the place where we need
+% \cs{l_tl_analysis_char_token} to be a separate control
+% sequence from \cs{l_tl_analysis_token}, to compare them.
% \begin{macrocode}
\group_begin:
\char_set_catcode_group_begin:N \^^@
@@ -533,22 +637,21 @@
\cs_new_protected_nopar:Npn \tl_analysis_i_egroup:w
{ \tl_analysis_i_group:nw { \if_false: ^^B \fi: ^^@ } }
\group_end:
-\cs_new_protected_nopar:Npn \tl_analysis_i_group:nw #1
+\cs_new_protected:Npn \tl_analysis_i_group:nw #1
{
- \l_tl_analysis_char_int = \tl_analysis_extract_charcode: \scan_stop:
- \tex_lccode:D \c_zero \l_tl_analysis_char_int
+ \tex_lccode:D \c_zero = \tl_analysis_extract_charcode: \scan_stop:
\tl_to_lowercase:n { \tex_toks:D \l_tl_analysis_index_int {#1} }
- \if_num:w \l_tl_analysis_char_int = \tex_escapechar:D
+ \if_num:w \tex_lccode:D \c_zero = \tex_escapechar:D
\int_set:Nn \tex_escapechar:D { 139 - \tex_escapechar:D }
\fi:
\tex_afterassignment:D \tl_analysis_i_group_test:w
\exp_after:wN \cs_set_eq:NN
- \exp_after:wN \l_tl_analysis_token
+ \exp_after:wN \l_tl_analysis_char_token
\token_to_str:N
}
\cs_new_protected_nopar:Npn \tl_analysis_i_group_test:w
{
- \if_num:w \tl_analysis_extract_charcode: = \l_tl_analysis_char_int
+ \if_charcode:w \l_tl_analysis_token \l_tl_analysis_char_token
\tl_analysis_i_store:
\else:
\int_incr:N \l_tl_analysis_normal_int
@@ -559,37 +662,41 @@
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
%
% \begin{macro}[int]{\tl_analysis_i_store:}
-% This function is called each time we meet a special token,
-% and the value of \cs{l_tl_analysis_type_int} indicates which case
+% This function is called each time we meet a special token;
+% at this point, the \tn{toks} register \cs{l_tl_analysis_index_int}
+% holds a token list which expands to the given special token.
+% Also, the value of \cs{l_tl_analysis_type_int} indicates which case
% we are in:
% \begin{itemize}
-% \item[-1] end-group character;
-% \item[0] space character;
-% \item[1] begin-group character.
+% \item -1 end-group character;
+% \item 0 space character;
+% \item 1 begin-group character.
% \end{itemize}
% We need to distinguish further the case of a space character
% (code $32$) from other character codes, because those will
-% behave differently in the second pass. Namely, we change the
-% cases above to
+% behave differently in the second pass. Namely, after testing
+% the \tn{lccode} of $0$ (which holds the present character code)
+% we change the cases above to
% \begin{itemize}
-% \item[-2] space end-group character;
-% \item[-1] non-space end-group character;
-% \item[0] space blank space character;
-% \item[1] non-space begin-group character;
-% \item[2] space begin-group character.
+% \item -2 space end-group character;
+% \item -1 non-space end-group character;
+% \item 0 space blank space character;
+% \item 1 non-space begin-group character;
+% \item 2 space begin-group character.
% \end{itemize}
% This has the property that non-space characters correspond to odd
% values of \cs{l_tl_analysis_type_int}.
-% Also, the \tn{toks} register number \cs{l_tl_analysis_index_int}
-% holds a token list which expands to the given special token.
+% The number of normal tokens, and the type of special token,
+% are packed into a \tn{skip} register.
+% Finally, we check whether we reached the last closing brace, in which
+% case we stop by disabling the looping function (locally).
% \begin{macrocode}
\cs_new_protected_nopar:Npn \tl_analysis_i_store:
{
\tex_advance:D \l_tl_analysis_nesting_int \l_tl_analysis_type_int
- \if_num:w \l_tl_analysis_char_int = \c_thirty_two
+ \if_num:w \tex_lccode:D \c_zero = \c_thirty_two
\tex_multiply:D \l_tl_analysis_type_int \c_two
\fi:
\tex_skip:D \l_tl_analysis_index_int
@@ -603,12 +710,61 @@
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}[int]{\tl_analysis_i_safe:N}
+% \begin{macro}[aux]{\tl_analysis_i_cs:ww}
+% This should be the simplest case: since the upcoming token is safe,
+% we can simply grab it in a second pass. However, other branches of
+% the code must pass their tokens through \tn{string}, hence we do it
+% here as well, with some optimizations. If the token is a single
+% character (including space), the \cs{if_charcode:w} test yields
+% true, and we simply count one \enquote{normal} token. On the other
+% hand, if the token is a control sequence, we should replace it by
+% its string representation for compatibility with other code
+% branches. Instead of slowly looping through the characters with
+% the main code, we use the knowledge of how the second pass works:
+% if the control sequence name contains no space, count that token
+% as a number of normal tokens equal to its string length. If the
+% control sequence contains spaces, they should be registered as
+% special characters by increasing \cs{l_tl_analysis_index_int}
+% (no need to carefully count character between each space), and
+% all characters after the last space should be counted in the
+% following sequence of \enquote{normal} tokens.
+% \begin{macrocode}
+\cs_new_protected:Npn \tl_analysis_i_safe:N #1
+ {
+ \if_charcode:w
+ \scan_stop:
+ \exp_after:wN \use_none:n \token_to_str:N #1 \prg_do_nothing:
+ \scan_stop:
+ \int_incr:N \l_tl_analysis_normal_int
+ \else:
+ \tl_analysis_cs_space_count:NN \tl_analysis_i_cs:ww #1
+ \fi:
+ \tl_analysis_i_loop:w
+ }
+\cs_new_protected:Npn \tl_analysis_i_cs:ww #1; #2;
+ {
+ \if_num:w #1 > \c_zero
+ \tex_skip:D \l_tl_analysis_index_int
+ = \int_eval:w \l_tl_analysis_normal_int + \c_one sp \scan_stop:
+ \tex_advance:D \l_tl_analysis_index_int #1 \exp_stop_f:
+ \l_tl_analysis_normal_int #2 \exp_stop_f:
+ \else:
+ \tex_advance:D \l_tl_analysis_normal_int #2 \exp_stop_f:
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
% \subsection{Second pass}
%
% The second pass is an exercise in expandable loops.
+% All the necessary information is stored in \tn{skip}
+% and \tn{toks} registers.
%
% \begin{macro}[int]{\tl_analysis_ii:n}
-% \begin{macro}[aux]{\tl_analysis_ii_loop:w}
+% \begin{macro}[int, EXP]{\tl_analysis_ii_loop:w}
% Start the loop with the index $0$. No need for an end-marker:
% the loop will stop by itself when the last index is read.
% We will repeatedly oscillate between reading long stretches
@@ -631,16 +787,18 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}[aux]{\tl_analysis_ii_normals:ww}
-% \begin{macro}[aux]{\tl_analysis_ii_normal:wwN}
+% \begin{macro}[int, EXP]{\tl_analysis_ii_normals:ww}
+% \begin{macro}[aux, EXP]{\tl_analysis_ii_normal:wwN}
% The first argument is the number of normal tokens which remain
% to be read, and the second argument is the index in the array
% produced in the first step.
% A character's string representation is always one character long,
% while a control sequence is always longer (we have set the escape
% character to a printable value). In both cases, we leave
-% \cs{exp_not:n} \Arg{token} \cs{q_mark} in the input stream
-% (after \texttt{x}-expansion).
+% \cs{exp_not:n} \Arg{token} \cs{s_tl} in the input stream
+% (after \texttt{x}-expansion). Here, \cs{exp_not:n} is used
+% rather than \cs{exp_not:N} because |#3| could be \cs{s_tl},
+% hence must be hidden behind braces in the result.
% \begin{macrocode}
\cs_new:Npn \tl_analysis_ii_normals:ww #1;
{
@@ -651,7 +809,7 @@
}
\cs_new:Npn \tl_analysis_ii_normal:wwN #1; #2; #3
{
- \exp_not:n { \exp_not:n { #3 } } \exp_not:N \q_mark
+ \exp_not:n { \exp_not:n { #3 } } \s_tl
\if_charcode:w
\scan_stop:
\exp_after:wN \use_none:n \token_to_str:N #3 \prg_do_nothing:
@@ -666,41 +824,53 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}[aux]{\tl_analysis_ii_char:Nww}
+% \begin{macro}[int, EXP]{\tl_analysis_ii_char:Nww}
% If the normal token we grab is a character, leave
-% \meta{catcode} \meta{charcode} followed by a comma
+% \meta{catcode} \meta{charcode} followed by \cs{s_tl}
% in the input stream, and call \cs{tl_analysis_ii_normals:ww}
% with its first argument decremented.
% \begin{macrocode}
-\cs_new:Npn \tl_analysis_ii_char:Nww #1
- {
- \if_meaning:w #1 \c_undefined:D D \else:
- \if_catcode:w #1 \c_catcode_other_token C \else:
- \if_catcode:w #1 \c_catcode_letter_token B \else:
- \if_catcode:w #1 \c_math_toggle_token 3 \else:
- \if_catcode:w #1 \c_alignment_token 4 \else:
- \if_catcode:w #1 \c_math_superscript_token 7 \else:
- \if_catcode:w #1 \c_math_subscript_token 8 \else:
- 6
- \fi: \fi: \fi: \fi: \fi: \fi: \fi:
- \int_value:w `#1 ,
- \exp_after:wN \tl_analysis_ii_normals:ww
- \int_use:N \int_eval:w \c_minus_one +
- }
+\group_begin:
+ \char_set_catcode_other:N A
+ \char_set_catcode_other:N B
+ \char_set_catcode_other:N C
+ \char_set_uccode:nn { `? } { `D }
+ \tl_to_uppercase:n
+ {
+ \cs_new:Npn \tl_analysis_ii_char:Nww #1
+ {
+ \if_meaning:w #1 \c_undefined:D ? \else:
+ \if_catcode:w #1 \c_catcode_other_token C \else:
+ \if_catcode:w #1 \c_catcode_letter_token B \else:
+ \if_catcode:w #1 \c_math_toggle_token 3 \else:
+ \if_catcode:w #1 \c_alignment_token 4 \else:
+ \if_catcode:w #1 \c_math_superscript_token 7 \else:
+ \if_catcode:w #1 \c_math_subscript_token 8 \else:
+ \if_catcode:w #1 \c_space_token A \else:
+ 6
+ \fi: \fi: \fi: \fi: \fi: \fi: \fi: \fi:
+ \int_value:w `#1 \s_tl
+ \exp_after:wN \tl_analysis_ii_normals:ww
+ \int_use:N \int_eval:w \c_minus_one +
+ }
+ }
+\group_end:
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\tl_analysis_ii_cs:Nww}
+% \begin{macro}[int, EXP]{\tl_analysis_ii_cs:Nww}
+% \begin{macro}[aux, EXP]{\tl_analysis_ii_cs_test:ww}
% If the token we grab is a control sequence, leave
-% |0-1,| (as category code and character code) in the input stream,
+% |0 -1| (as category code and character code) in the input stream,
+% followed by \cs{s_tl},
% and call \cs{tl_analysis_ii_normals:ww} with updated arguments.
% \begin{macrocode}
\cs_new:Npn \tl_analysis_ii_cs:Nww #1
{
- 0 -1 ,
+ 0 -1 \s_tl
\tl_analysis_cs_space_count:NN \tl_analysis_ii_cs_test:ww #1
}
-\cs_new:Npn \tl_analysis_ii_cs_test:ww #1 ; #2 ; #3; #4;
+\cs_new:Npn \tl_analysis_ii_cs_test:ww #1 ; #2 ; #3 ; #4 ;
{
\exp_after:wN \tl_analysis_ii_normals:ww
\int_use:N \int_eval:w
@@ -711,16 +881,17 @@
\fi:
- #2
\exp_after:wN ;
- \int_use:N \int_eval:w #4 + #1;
+ \int_use:N \int_eval:w #4 + #1 ;
}
% \end{macrocode}
% \end{macro}
+% \end{macro}
%
-% \begin{macro}[aux]{\tl_analysis_ii_special:w}
-% \begin{macro}[aux]{\tl_analysis_ii_special_char:wN}
-% \begin{macro}[aux]{\tl_analysis_ii_special_space:w}
+% \begin{macro}[int, EXP]{\tl_analysis_ii_special:w}
+% \begin{macro}[aux, EXP]{\tl_analysis_ii_special_char:wN}
+% \begin{macro}[aux, EXP]{\tl_analysis_ii_special_space:w}
% Here, |#1| is the current index in the array built in the first pass.
-% Check whether we reached the end (we shouldn't insert the trailing
+% Check now whether we reached the end (we shouldn't keep the trailing
% end-group character that marked the end of the token list in the
% first pass).
% Unpack the \tn{toks} register: when \texttt{x}-expanding again,
@@ -728,35 +899,39 @@
% Then leave the category code in the input stream, followed by
% the character code, and call \cs{tl_analysis_ii_loop:w} with the next index.
% \begin{macrocode}
-\cs_new:Npn \tl_analysis_ii_special:w \fi: \tl_analysis_ii_normal:wwN 0; #1;
- {
- \fi:
- \if_num:w #1 = \l_tl_analysis_index_int
- \exp_after:wN \prg_map_break:
- \fi:
- \tex_the:D \tex_toks:D #1 \exp_stop_f: \exp_not:N \q_mark
- \if_case:w \etex_gluestretch:D \tex_skip:D #1 \exp_stop_f:
- A
- \or: 1
- \or: 1
- \else: 2
- \fi:
- \if_int_odd:w \etex_gluestretch:D \tex_skip:D #1 \exp_stop_f:
- \exp_after:wN \tl_analysis_ii_special_char:wN \int_use:N
- \else:
- \exp_after:wN \tl_analysis_ii_special_space:w \int_use:N
- \fi:
- \int_eval:w \c_one + #1 \exp_after:wN ;
- \token_to_str:N
- }
+\group_begin:
+ \char_set_catcode_other:N A
+ \cs_new:Npn \tl_analysis_ii_special:w
+ \fi: \tl_analysis_ii_normal:wwN 0 ; #1 ;
+ {
+ \fi:
+ \if_num:w #1 = \l_tl_analysis_index_int
+ \exp_after:wN \prg_map_break:
+ \fi:
+ \tex_the:D \tex_toks:D #1 \s_tl
+ \if_case:w \etex_gluestretch:D \tex_skip:D #1 \exp_stop_f:
+ A
+ \or: 1
+ \or: 1
+ \else: 2
+ \fi:
+ \if_int_odd:w \etex_gluestretch:D \tex_skip:D #1 \exp_stop_f:
+ \exp_after:wN \tl_analysis_ii_special_char:wN \int_use:N
+ \else:
+ \exp_after:wN \tl_analysis_ii_special_space:w \int_use:N
+ \fi:
+ \int_eval:w \c_one + #1 \exp_after:wN ;
+ \token_to_str:N
+ }
+\group_end:
\cs_new:Npn \tl_analysis_ii_special_char:wN #1 ; #2
{
- \int_value:w `#2 ,
+ \int_value:w `#2 \s_tl
\tl_analysis_ii_loop:w #1 ;
}
\cs_new:Npn \tl_analysis_ii_special_space:w #1 ; ~
{
- 32 ,
+ 32 \s_tl
\tl_analysis_ii_loop:w #1 ;
}
% \end{macrocode}
@@ -777,7 +952,7 @@
{ tl-analysis }
{
\exp_after:wN \tl_show_analysis_loop:wNw \g_tl_analysis_result_tl
- \q_mark { ? \prg_map_break: } ,
+ \s_tl { ? \prg_map_break: } \s_tl
\prg_break_point:n { }
}
}
@@ -790,16 +965,16 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}[aux, EXP]{\tl_show_analysis_loop:wNw}
-% Here, |#1| \texttt{o}-expands to the token;
-% |#2| is the category code (one hexadecimal digit),
+% \begin{macro}[aux, rEXP]{\tl_show_analysis_loop:wNw}
+% Here, |#1| \texttt{o}- and \texttt{x}-expands to the token;
+% |#2| is the category code (one uppercase hexadecimal digit),
% $0$ for control sequences;
% |#3| is the character code, which we ignore.
% In the cases of control sequences and active characters,
% the meaning may overflow one line, and we want to truncate
% it. Those cases are thus separated out.
% \begin{macrocode}
-\cs_new:Npn \tl_show_analysis_loop:wNw #1 \q_mark #2 #3,
+\cs_new:Npn \tl_show_analysis_loop:wNw #1 \s_tl #2 #3 \s_tl
{
\use_none:n #2
\iow_newline: > \c_space_tl \c_space_tl
@@ -820,9 +995,9 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\tl_show_analysis_normal:n}
+% \begin{macro}[aux, rEXP]{\tl_show_analysis_normal:n}
% Non-active characters are a simple matter of printing
-% the character, and its meaning. One can check that
+% the character, and its meaning. Our test suite checks that
% begin-group and end-group characters do not mess up
% \TeX{}'s alignment status.
% \begin{macrocode}
@@ -834,10 +1009,10 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\tl_show_analysis_cs:n}
-% \begin{macro}[aux]{\tl_show_analysis_active:n}
-% \begin{macro}[aux]{\tl_show_analysis_long:nn}
-% \begin{macro}[aux]{\tl_show_analysis_long_aux:nnn}
+% \begin{macro}[aux, rEXP]{\tl_show_analysis_cs:n}
+% \begin{macro}[aux, rEXP]{\tl_show_analysis_active:n}
+% \begin{macro}[aux, rEXP]{\tl_show_analysis_long:nn}
+% \begin{macro}[aux, rEXP]{\tl_show_analysis_long_aux:nnn}
% Control sequences and active characters are printed in the same way,
% making sure not to go beyond the \cs{l_iow_line_length_int}. In case
% of an overflow, we replace the last characters by
@@ -853,17 +1028,20 @@
{ \token_to_str:N #1 }
{ \token_to_meaning:N #1 }
}
-\cs_new_nopar:Npn \tl_show_analysis_long_aux:nnn #1#2#3
+\cs_new:Npn \tl_show_analysis_long_aux:nnn #1#2#3
{
\int_compare:nNnTF
- { \str_length:n { >>> #1 ~ ( #3 #2 ) } }
- > { \l_iow_line_length_int }
+ { \str_length:n { #1 ~ ( #3 #2 ) } }
+ > { \l_iow_line_length_int - \c_three }
{
- \str_substr:nnn { >>> #1 ~ ( #3 #2 ) } \c_three
- { \l_iow_line_length_int - \str_length:N \c_tl_show_analysis_etc_str }
+ \str_substr:nnn { #1 ~ ( #3 #2 ) } \c_zero
+ {
+ \l_iow_line_length_int - \c_three
+ - \str_length:N \c_tl_show_analysis_etc_str
+ }
\c_tl_show_analysis_etc_str
}
- {#1~(#3#2)}
+ { #1 ~ ( #3 #2 ) }
}
% \end{macrocode}
% \end{macro}
@@ -871,21 +1049,23 @@
% \end{macro}
% \end{macro}
%
+% \subsection{Messages}
+%
% \begin{variable}{\c_tl_show_analysis_etc_str}
% When a control sequence (or active character)
% and its meaning are too long to fit in one line
% of the terminal, the end is replaced by this token list.
% \begin{macrocode}
-\str_const:Nx \c_tl_show_analysis_etc_str % (
- { \iow_char:N \\ETC. ) }
+\tl_const:Nx \c_tl_show_analysis_etc_str % (
+ { \token_to_str:N \ETC.) }
% \end{macrocode}
% \end{variable}
%
% \begin{macrocode}
\msg_kernel_new:nnn { tl-analysis } { show }
{
- Token~list~
- \str_if_eq:nnF {#1} { \l_tl_tmpa_tl } { \token_to_str:N #1~}
+ The~token~list~
+ \str_if_eq:nnF {#1} { \l_tl_tmpa_tl } { \token_to_str:N #1 ~ }
\tl_if_empty:NTF #1
{ is~empty }
{ contains~the~tokens: }
diff --git a/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-test.tex b/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-test.tex
new file mode 100644
index 00000000000..1ede1770c08
--- /dev/null
+++ b/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-test.tex
@@ -0,0 +1,380 @@
+
+\documentclass{article}
+
+\usepackage[T1]{fontenc}
+\usepackage{times,multicol}
+\usepackage{xcoffins}
+
+\usepackage{color}
+\newcommand\cbox[2][.8]{{\setlength\fboxsep{0pt}\colorbox[gray]{#1}{#2}}}
+
+\newcommand\hrulebox [2]{\setbox#1\hbox to#2{{\scriptsize\itshape
+ \color{blue}\hrulefill #2\hrulefill}}}
+\newcommand\vrulebox [2]{\setbox#1\vbox to#2{{\hsize 1pt\centering\scriptsize
+ \itshape\color{blue}\leaders\vrule\vfill
+ \hbox to0pt{\hss #2\hss}\leaders\vrule\vfill\par}}}
+
+
+\addtolength\textwidth{10pt}
+
+\showboxdepth 9999
+\showboxbreadth 9999
+\tracingonline 1
+
+
+\scrollmode
+
+\newbox\zzz
+
+
+\begin{document}
+
+
+\title{Test file for coffins}
+\author{FMi}
+\maketitle
+
+First we add a few test coffins:
+\begin{verbatim}
+ \newcoffin \aaa
+ \newcoffin \bbb
+ \newcoffin \ccc
+ \newcoffin \ddd
+\end{verbatim}
+ \newcoffin \aaa
+ \newcoffin \bbb
+ \newcoffin \ccc
+ \newcoffin \ddd
+and some boxes not set up as ordinary boxes (without extra poles):
+\begin{verbatim}
+ \newbox \xxx
+ \newbox \yyy
+\end{verbatim}
+ \newbox \xxx
+ \newbox \yyy
+
+
+If a coffin receives data a set of ``natural'' default poles are automatically added.
+\begin{verbatim}
+\sbox\aaa{\fbox{\begin{tabular}[b]{l}123\\4\\5\end{tabular}}}
+\showcoffindata \aaa
+\end{verbatim}
+\sbox\aaa{\fbox{\begin{tabular}[b]{l}123\\4\\5\end{tabular}}}
+\showcoffindata \aaa
+
+
+In contrast, boxes not declared as coffins have no poles defined. However, if
+used as coffins at least the natural default poles can be used without
+further problems, i.e.,they are changed to coffins automatically on use.
+\begin{verbatim}
+\sbox\xxx{\fbox{Some very looooonggg Caption Text}}
+\showpoles \xxx
+\end{verbatim}
+\sbox\xxx{\fbox{Some very looooonggg Caption Text}}
+\showcoffindata \xxx
+
+
+Using \verb|\setvcoffin| instead of \verb|\sbox| gives us the codes with
+\texttt{T} and \texttt{B}. Similar to \verb|\parbox| this command requires to
+specify a target width of the box:
+\begin{verbatim}
+\setvcoffin \aaa {1.5cm} {\centering
+ \fbox{\begin{tabular}[b]{l}123\\4\\5\end{tabular}}}
+\showcoffindata \aaa
+\end{verbatim}
+\setvcoffin \aaa {1.5cm} {\centering
+ \fbox{\begin{tabular}[b]{l}123\\4\\5\end{tabular}}}
+\showcoffindata \aaa
+
+There are a dozen natural poles per box (some with multiple names); you can
+move each of the poles to someplace else (either to an absolute position in
+``box space'' or relative to its previous position). The coordinates can
+refer to box dimensions (using \verb|\height|, \verb|\depth|,
+ and \verb|width|). First we move some pole and add one
+additional one:
+\begin{verbatim}
+\setcoffinpole \aaa [h]{t}(\height -3pt)
+\setcoffinpole \aaa [v]{l}(0pt)
+\setcoffinpole \aaa [h]{mybottom}(\depth +2pt)
+\showcoffindata \aaa
+\end{verbatim}
+\setcoffinpole \aaa [h]{t}(\height -3pt)
+\setcoffinpole \aaa [v]{l}(0pt)
+\setcoffinpole \aaa [h]{mybottom}(\depth +2pt)
+\showcoffindata \aaa
+
+
+More interesting in many cases is the relative move of poles. For this the
+poles better exist or one gets an error and \texttt{0pt,0pt} is used:
+\begin{verbatim}
+\adjustcoffinpole \aaa [h]{H}(1pt)
+\adjustcoffinpole \aaa [v]{l}(1pt)
+\adjustcoffinpole \aaa [h]{undefined}(1pt)
+\showcoffindata \aaa
+\end{verbatim}
+\adjustcoffinpole \aaa [h]{H}(1pt)
+\adjustcoffinpole \aaa [v]{l}(1pt)
+\adjustcoffinpole \aaa [h]{undefined}(1pt)
+\showcoffindata \aaa
+
+
+
+\sbox \aaa {\fontsize{14.4}{5.5pc minus .5pc}\sffamily C\,H\,A\,P\,T\,E\,R
+ \fontsize{36}{40pt}\selectfont 2}
+
+\setvcoffin \bbb {11cm}
+ {\raggedleft\normalfont\fontsize{36}{38pt}\bfseries
+ The Structure of a \LaTeX{} Document}
+
+\setvcoffin \ccc {13cm}
+ {Some sample blind text to produce paragraph data after a heading.
+ Some sample blind text to produce paragraph data after a heading.
+ Some sample blind text to produce paragraph data after a heading.
+ Some sample blind text to produce paragraph data after a heading.
+ \endgraf
+ Some more sample blind text to produce paragraph data after a heading.
+ Some more sample blind text to produce paragraph data after a heading.
+}
+
+
+Suppose you have the following blocks of text:
+
+\bigskip
+
+\noindent\cbox{\usebox\aaa}
+
+\begin{multicols}{2}
+\ttfamily\tiny
+\ExplSyntaxOn
+ \noindent \coffin_print_pole_values:Nn \aaa \\
+\ExplSyntaxOff
+\end{multicols}
+
+\medskip
+
+\noindent\cbox{\usebox\bbb}
+
+\medskip
+
+\noindent\cbox{\usebox\ccc}
+
+\bigskip
+
+produced by:
+\begin{verbatim}
+\sbox \aaa {\fontsize{14.4}{5.5pc minus .5pc}\sffamily
+ C\,H\,A\,P\,T\,E\,R
+ \fontsize{36}{40pt}\selectfont 2}
+\setvcoffin \bbb {11cm}
+ {\raggedleft\normalfont\fontsize{36}{38pt}\bfseries
+ The Structure of a \LaTeX{} Document}
+\setvcoffin \ccc {13cm}
+ {Some sample blind text to produce paragraph
+ data after a heading. ... }
+\end{verbatim}
+
+How do you turn this into a heading of TLC2?
+
+
+\newbox\RBi
+\newbox\RBii
+\newbox\RBiii
+
+\hrulebox\RBi{62pt}
+\vrulebox\RBii{90pt}
+\vrulebox\RBiii{60pt}
+
+\aligncoffins \bbb [T,r] \aaa [H,r](-62pt,60pt)
+\aligncoffins \ccc[T,r] \bbb[B,r](62pt,90pt)
+
+ \aligncoffins * \ccc[\bbb-T,\bbb-hc] \RBiii[b,hc](28pt,0pt)
+ \aligncoffins * \ccc[\bbb-b,\bbb-r] \RBi[H,r](0pt,-5pt)
+ \aligncoffins * \ccc[\bbb-B,\bbb-hc] \RBii[t,r](14pt,0pt)
+
+\newpage
+
+How do you best define/describe the following design?
+
+\medskip
+
+\noindent\cbox{\usebox\ccc}
+
+
+
+
+
+
+\newpage \pagestyle{empty}
+
+\newcoffin \eee
+\newcoffin \fff
+\newcoffin \ggg
+
+\sbox \aaa {\small\scshape les vases communicants}
+\sbox \bbb {\scshape comunicating}
+\sbox \ccc {\fontsize{70pt}{60pt} \bfseries Ve\S els}
+\sbox \ddd {\scshape andr\'e breton}
+\setvcoffin \eee {4.7cm}{\noindent Translated by Mary Ann Caws \&\break
+ Geoffrey T.\,Harris, with notes \&\break
+ introduction by Mary Ann Caws\parfillskip=0pt\relax
+ }
+\sbox \fff {University of Nebraska Press: Lincoln \& London}
+
+\setvcoffin \ggg {100mm}{\noindent\fbox{\parbox{97mm}{\leavevmode\vspace*{228mm}}}}
+
+
+\aligncoffins \ccc [H,r] \ddd [H,r](0pt,-12pt)
+\aligncoffins \ccc [t,l] \bbb (0pt,6pt)
+\aligncoffins \ccc [H,r] \aaa [H,r](0pt,138pt)
+\aligncoffins \eee [T,l] \ccc (0pt,22pc)
+\aligncoffins \fff \eee [B,l](0pt,4pc)
+\aligncoffins \ggg [b,l] \fff [B,l](54pt,11pc)
+
+
+\vspace*{-2cm}
+\noindent\cbox{\usebox\ggg}
+
+
+\newpage
+
+\section{Rotation}
+
+\sbox \aaa {\tabcolsep0pt\begin{tabular}[c]{|c|}\hline a\\b\\cccccccccccccc\\d
+ \\e\\e\\e\\e\\\hline\end{tabular}}
+\sbox \bbb {\fbox{A sample Text}}
+
+% just for the sake of it:
+\adjustcoffinpole \aaa {T}(24pt)
+\displaycoffinpoles \aaa {black}
+x\cbox{\usebox \aaa }x
+%
+\rotatecoffin \aaa {45}
+\displaycoffinhandle \aaa {b}{hc} {blue}
+\displaycoffinhandle \aaa {b}{l} {blue}
+\displaycoffinhandle \aaa {t}{r} {blue}
+\displaycoffinhandle \aaa {vc}{r} {blue}
+\displaycoffinhandle \aaa {vc}{l} {blue}
+\quad
+x\cbox{\usebox \aaa }x
+\rotatecoffin \aaa {45}
+\displaycoffinhandle \aaa {b}{r} {red}
+\displaycoffinhandle \aaa {vc}{l} {red}
+\quad
+x\cbox{\usebox \aaa }x
+
+\vspace{1cm}
+
+\rotatecoffin \aaa {45}
+\displaycoffinhandle \aaa {b}{l} {yellow}
+%
+x\cbox{\usebox \aaa }x
+%
+
+\vspace{1cm}
+
+x\cbox{\usebox \bbb }x
+%
+\rotatecoffin \bbb {45}
+\displaycoffinhandle \bbb {B}{hc} {yellow}
+\quad
+x\cbox{\usebox \bbb }x
+
+
+\section{Rotation + alignment}
+
+\aligncoffins \aaa [b,l] \bbb[B,hc](30pt,0pt)
+x\cbox{\usebox \aaa }x
+
+
+After we have aligned a roated box with some other box we need to decide about
+the bounding box of the new box. Right now this become the enclosing box and
+we do not maintain information about the inner boxes. So when we rotate that
+new box there seems to be unnecessary space in the enclosing bounding box,
+even though that is correct if you think of the aligned box being unrotated.
+
+There is at least the possibility to refer to the handles of the inner boxes
+as one can see by two of the green handles
+
+\medskip
+
+\rotatecoffin \aaa {45}
+\displaycoffinhandle \aaa {vc}{hc} {green}
+\displaycoffinhandle \aaa {H}{l} {green}
+\displaycoffinhandle \aaa {\aaa-T}{\aaa-r} {green}
+\displaycoffinhandle \aaa {\bbb-H}{\bbb-r} {green}
+%\quad
+x\cbox{\usebox \aaa }x
+
+We could do better, if we want to, by actually checking for max and
+min of all inner bounding box corners and construct the result BB box from
+that---but is it worth it?
+
+Perhaps it is! After all, the current implementation shows different results
+depending on when you align boxes and when you rotate, e.g., aligning first
+gives totally different bounding box results.
+
+\newpage
+
+Aligning first and then 135 + 45 rotation gives this:
+
+\sbox \aaa {\tabcolsep0pt\begin{tabular}[c]{|c|}\hline a\\b\\cccccccccccccc\\d
+ \\e\\e\\e\\e\\\hline\end{tabular}}
+\sbox \bbb {\fbox{A sample Text}}
+\rotatecoffin \bbb {90}
+
+\aligncoffins \aaa [b,l] \bbb[B,hc](-30pt,0pt)
+x\cbox{\usebox \aaa }x
+\qquad
+\rotatecoffin \aaa {135}
+x\cbox{\usebox \aaa }x
+\qquad
+\rotatecoffin \aaa {45}
+x\cbox{\usebox \aaa }x
+
+
+
+
+\newpage
+
+\section{Rotation by small amounts}
+
+\sbox \aaa {\tabcolsep0pt\begin{tabular}[c]{|c|}\hline a\\b\\ccccccccc\\d
+ \\e\\e\\e\\\hline\end{tabular}}
+
+\subsection{30 + 30 +30 }
+\rotatecoffin \aaa {30}
+x\cbox{\usebox \aaa }x
+\rotatecoffin \aaa {30}
+x\cbox{\usebox \aaa }x
+\rotatecoffin \aaa {30}
+x\cbox{\usebox \aaa }x
+
+\subsection{6 * 10 + 45 + 45 + 30}
+
+\rotatecoffin \aaa {10}
+x\cbox{\usebox \aaa }x
+\rotatecoffin \aaa {10}
+x\cbox{\usebox \aaa }x
+\rotatecoffin \aaa {10}
+x\cbox{\usebox \aaa }x
+
+\vspace{1cm}
+\rotatecoffin \aaa {10}
+x\cbox{\usebox \aaa }x
+\rotatecoffin \aaa {10}
+x\cbox{\usebox \aaa }x
+\rotatecoffin \aaa {10}
+x\cbox{\usebox \aaa }x
+
+\vspace{1cm}
+\rotatecoffin \aaa {45}
+x\cbox{\usebox \aaa }x
+\rotatecoffin \aaa {45}
+x\cbox{\usebox \aaa }x
+\rotatecoffin \aaa {30}
+x\cbox{\usebox \aaa }x
+
+
+\end{document}
+
+
diff --git a/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example-keyval.tex b/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example-keyval.tex
new file mode 100644
index 00000000000..e83c0b490b6
--- /dev/null
+++ b/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example-keyval.tex
@@ -0,0 +1,281 @@
+
+\documentclass{article}
+
+\usepackage[a4paper,margin=5pt]{geometry}
+
+\usepackage[T1]{fontenc}
+\usepackage{times,multicol,graphicx}
+\usepackage{xcoffins}
+
+\usepackage{times,color}
+\newcommand\cbox[2][.8]{{\setlength\fboxsep{0pt}\colorbox[gray]{#1}{#2}}}
+
+\ExplSyntaxOn
+
+\ExplSyntaxOff
+
+
+
+%\scrollmode
+
+\pagestyle{empty}
+
+\begin{document}
+
+ \NewCoffin \result
+ \NewCoffin \aaa
+ \NewCoffin \bbb
+ \NewCoffin \ccc
+ \NewCoffin \ddd
+ \NewCoffin \eee
+ \NewCoffin \fff
+ \NewCoffin \rulei
+ \NewCoffin \ruleii
+ \NewCoffin \ruleiii
+
+\SetHorizontalCoffin \result {}
+\SetHorizontalCoffin \aaa {\fontsize{52}{50}\sffamily\bfseries mitteilungen}
+\SetHorizontalCoffin \bbb {\fontsize{52}{50}\sffamily\bfseries typographische}
+\SetHorizontalCoffin \ccc {\fontsize{12}{10}\sffamily
+ \quad zeitschrift des bildungsverbandes der
+ deutschen buchdrucker leipzig
+ \textbullet{} oktoberheft 1925}
+\SetHorizontalCoffin \ddd {\fontsize{28}{20}\sffamily sonderheft}
+\SetVerticalCoffin \eee {180pt}
+ {\raggedleft\fontsize{31}{36}\sffamily\bfseries
+ elementare\\
+ typographie}
+\SetVerticalCoffin \fff {140pt}
+ {\raggedright \fontsize{13}{14}\sffamily\bfseries
+ natan altman \\
+ otto baumberger \\
+ herbert mayer \\
+ max burchartz \\
+ el lissitzky \\
+ ladislaus moholy-nagy \\
+ moln\'ar f.~farkas \\
+ johannes molzahn \\
+ kurt schwitters \\
+ mart stam \\
+ ivan tschichold}
+
+\RotateCoffin \bbb {90}
+\RotateCoffin \ccc {270}
+
+\SetHorizontalCoffin \rulei {\color{red}\rule{6.5in}{1pc}}
+\SetHorizontalCoffin \ruleii {\color{red}\rule{1pc}{23.5cm}}
+\SetHorizontalCoffin \ruleiii{\color{black}\rule{10pt}{152pt}}
+
+
+\JoinCoffins \result \aaa
+\JoinCoffins [
+ coffin1-hpole = \aaa-t ,
+ coffin1-vpole = \aaa-r ,
+ coffin2-hpole = b ,
+ coffin2-vpole = r ,
+ voffset = 2 mm ,
+] \result \rulei
+\JoinCoffins [
+ coffin1-hpole = \aaa-b ,
+ coffin1-vpole = \aaa-l ,
+ coffin2-hpole = B ,
+ coffin2-vpole = r ,
+ hoffset = 2 pt ,
+] \result \bbb
+\JoinCoffins [
+ coffin1-hpole = \bbb-t ,
+ coffin1-vpole = \bbb-r ,
+ coffin2-hpole = t ,
+ coffin2-vpole = r ,
+ hoffset = -2 mm ,
+] \result \ruleii
+\JoinCoffins [
+ coffin1-hpole = \aaa-B ,
+ coffin1-vpole = \aaa-r ,
+ coffin2-hpole = B ,
+ coffin2-vpole = l ,
+ hoffset = 66 pt ,
+ voffset = 14 pc ,
+] \result \ccc
+\JoinCoffins [
+ coffin1-hpole = \bbb-l ,
+ coffin1-vpole = \ccc-B ,
+ coffin2-hpole = t ,
+ coffin2-vpole = r ,
+ hoffset = -2 mm ,
+] \result \fff
+\JoinCoffins [
+ coffin1-hpole = \fff-b ,
+ coffin1-vpole = \fff-r ,
+ coffin2-hpole = b ,
+ coffin2-vpole = l ,
+ hoffset = 2 mm ,
+] \result \ruleiii
+\JoinCoffins [
+ coffin1-hpole = \ccc-r ,
+ coffin1-vpole = \fff-l ,
+ coffin2-hpole = B ,
+ coffin2-vpole = r ,
+] \result \eee
+\JoinCoffins [
+ coffin1-hpole = \eee-T ,
+ coffin1-vpole = \eee-r ,
+ coffin2-hpole = B ,
+ coffin2-vpole = r ,
+ voffset = 4 pc ,
+] \result \ddd
+
+
+\vspace*{3cm}
+\begin{center}
+ {\Large Title page of ``elementare typographie'' by Ivan Tschichold\par}
+
+\large
+\vspace*{1cm}
+
+ 1. first the scanned original from 1925
+
+\vspace*{6mm}
+
+ 2. then the recreated \TeX{} version from 2010 using coffins---not
+ attempting\\ to match the fonts and size but the structure
+
+\vspace*{6mm}
+
+ 3. and finally the source code used.
+
+ This document uses the new
+ implementation by Joseph in \texttt{l3coffins-new}.
+
+\end{center}
+
+\newpage
+
+
+\begin{center}
+ \null
+ \fbox{\includegraphics[scale=.95]{elementare-typographie-title.jpg}}
+\end{center}
+
+\newpage
+
+\TypesetCoffin \result
+
+\newpage
+
+\vspace*{3cm}
+\begin{center}
+ {\Large Code used: \par}
+\vspace*{1cm}
+
+
+\begin{minipage}{14cm}
+\begin{verbatim}
+\SetHorizontalCoffin\result{}
+\SetHorizontalCoffin \aaa {\fontsize{52}{50}\sffamily\bfseries mitteilungen}
+\SetHorizontalCoffin \bbb {\fontsize{52}{50}\sffamily\bfseries typographische}
+\SetHorizontalCoffin \ccc {\fontsize{12}{10}\sffamily
+ \quad zeitschrift des bildungsverbandes der
+ deutschen buchdrucker leipzig
+ \textbullet{} oktoberheft 1925}
+\SetHorizontalCoffin \ddd {\fontsize{28}{20}\sffamily sonderheft}
+\SetVerticalCoffin \eee {180pt}
+ {\raggedleft\fontsize{31}{36}\sffamily\bfseries
+ elementare\\
+ typographie}
+\SetVerticalCoffin \fff {140pt}
+ {\raggedright \fontsize{13}{14}\sffamily\bfseries
+ natan altman \\
+ otto baumberger \\
+ herbert mayer \\
+ max burchartz \\
+ el lissitzky \\
+ ladislaus moholy-nagy \\
+ moln\'ar f.~farkas \\
+ jahannes molzahn \\
+ kurt schwitters \\
+ mart stam \\
+ ivan tschichold}
+
+\RotateCoffin \bbb {90}
+\RotateCoffin \ccc {270}
+
+\SetHorizontalCoffin \rulei {\color{red}\rule{6.5in}{1pc}}
+\SetHorizontalCoffin \ruleii {\color{red}\rule{1pc}{23.5cm}}
+\SetHorizontalCoffin \ruleiii{\color{black}\rule{10pt}{152pt}}
+
+\JoinCoffins \result \aaa
+\JoinCoffins [
+ coffin1-hpole = \aaa-t ,
+ coffin1-vpole = \aaa-r ,
+ coffin2-hpole = b ,
+ coffin2-vpole = r ,
+ voffset = 2 mm ,
+] \result \rulei
+\JoinCoffins [
+ coffin1-hpole = \aaa-b ,
+ coffin1-vpole = \aaa-l ,
+ coffin2-hpole = B ,
+ coffin2-vpole = r ,
+ hoffset = 2 pt ,
+] \result \bbb
+\JoinCoffins [
+ coffin1-hpole = \bbb-t ,
+ coffin1-vpole = \bbb-r ,
+ coffin2-hpole = t ,
+ coffin2-vpole = r ,
+ hoffset = -2 mm ,
+] \result \ruleii
+\JoinCoffins [
+ coffin1-hpole = \aaa-B ,
+ coffin1-vpole = \aaa-r ,
+ coffin2-hpole = B ,
+ coffin2-vpole = l ,
+ hoffset = 66 pt ,
+ voffset = 14 pc ,
+] \result \ccc
+\JoinCoffins [
+ coffin1-hpole = \bbb-l ,
+ coffin1-vpole = \ccc-B ,
+ coffin2-hpole = t ,
+ coffin2-vpole = r ,
+ hoffset = -2 mm ,
+] \result \fff
+\JoinCoffins [
+ coffin1-hpole = \fff-b ,
+ coffin1-vpole = \fff-r ,
+ coffin2-hpole = b ,
+ coffin2-vpole = l ,
+ hoffset = 2 mm ,
+] \result \ruleiii
+\JoinCoffins [
+ coffin1-hpole = \ccc-r ,
+ coffin1-vpole = \fff-l ,
+ coffin2-hpole = B ,
+ coffin2-vpole = r ,
+] \result \eee
+\JoinCoffins [
+ coffin1-hpole = \eee-T ,
+ coffin1-vpole = \eee-r ,
+ coffin2-hpole = B ,
+ coffin2-vpole = r ,
+ voffset = 4 pc ,
+] \result \ddd
+\SetHorizontalCoffin\result{}
+\SetHorizontalCoffin \aaa {\fontsize{52}{50}\sffamily\bfseries
+ mitteilungen}
+\SetHorizontalCoffin \bbb {\fontsize{52}{50}\rotatebox{90}{\sffamily\bfseries
+ typographische}}
+
+\TypesetCoffin \result
+\end{verbatim}
+
+This is not necessarily the final syntax but for now it does its job. For
+example, flexible support for adding ornaments (lines, \ldots) is still
+missing, so above the rules got added as predefined individual coffins.
+
+\end{minipage}
+\end{center}
+
+\end{document}
+
diff --git a/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example.tex b/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example.tex
new file mode 100644
index 00000000000..343c9d7e29f
--- /dev/null
+++ b/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example.tex
@@ -0,0 +1,185 @@
+
+\documentclass{article}
+
+\usepackage[a4paper,margin=5pt]{geometry}
+
+\usepackage[T1]{fontenc}
+\usepackage{times,multicol,graphicx}
+\usepackage{xcoffins}
+
+\usepackage{times,color}
+\newcommand\cbox[2][.8]{{\setlength\fboxsep{0pt}\colorbox[gray]{#1}{#2}}}
+
+\ExplSyntaxOn
+
+\ExplSyntaxOff
+
+
+
+%\scrollmode
+
+\pagestyle{empty}
+
+\begin{document}
+
+ \NewCoffin \result
+ \NewCoffin \aaa
+ \NewCoffin \bbb
+ \NewCoffin \ccc
+ \NewCoffin \ddd
+ \NewCoffin \eee
+ \NewCoffin \fff
+ \NewCoffin \rulei
+ \NewCoffin \ruleii
+ \NewCoffin \ruleiii
+
+\SetHorizontalCoffin \result {}
+\SetHorizontalCoffin \aaa {\fontsize{52}{50}\sffamily\bfseries mitteilungen}
+\SetHorizontalCoffin \bbb {\fontsize{52}{50}\sffamily\bfseries typographische}
+\SetHorizontalCoffin \ccc {\fontsize{12}{10}\sffamily
+ \quad zeitschrift des bildungsverbandes der
+ deutschen buchdrucker leipzig
+ \textbullet{} oktoberheft 1925}
+\SetHorizontalCoffin \ddd {\fontsize{28}{20}\sffamily sonderheft}
+\SetVerticalCoffin \eee {180pt}
+ {\raggedleft\fontsize{31}{36}\sffamily\bfseries
+ elementare\\
+ typographie}
+\SetVerticalCoffin \fff {140pt}
+ {\raggedright \fontsize{13}{14}\sffamily\bfseries
+ natan altman \\
+ otto baumberger \\
+ herbert mayer \\
+ max burchartz \\
+ el lissitzky \\
+ ladislaus moholy-nagy \\
+ moln\'ar f.~farkas \\
+ johannes molzahn \\
+ kurt schwitters \\
+ mart stam \\
+ ivan tschichold}
+
+\RotateCoffin \bbb {90}
+\RotateCoffin \ccc {270}
+
+\SetHorizontalCoffin \rulei {\color{red}\rule{6.5in}{1pc}}
+\SetHorizontalCoffin \ruleii {\color{red}\rule{1pc}{23.5cm}}
+\SetHorizontalCoffin \ruleiii{\color{black}\rule{10pt}{152pt}}
+
+
+\JoinCoffins \result \aaa
+\JoinCoffins \result[\aaa-t,\aaa-r] \rulei [b,r](0pt,2mm)
+\JoinCoffins \result[\aaa-b,\aaa-l] \bbb [B,r](2pt,0pt)
+\JoinCoffins \result[\bbb-t,\bbb-r] \ruleii [t,r](-2mm,0pt)
+\JoinCoffins \result[\aaa-B,\aaa-r] \ccc [B,l](66pt,14pc)
+\JoinCoffins \result[\bbb-l,\ccc-B] \fff [t,r](-2mm,0pt)
+\JoinCoffins \result[\fff-b,\fff-r] \ruleiii [b,l](2mm,0pt)
+\JoinCoffins \result[\ccc-r,\fff-l] \eee [B,r]
+\JoinCoffins \result[\eee-T,\eee-r] \ddd [B,r](0pt,4pc)
+
+
+\vspace*{3cm}
+\begin{center}
+ {\Large Title page of ``elementare typographie'' by Ivan Tschichold\par}
+
+\large
+\vspace*{1cm}
+
+ 1. first the scanned original from 1925
+
+\vspace*{6mm}
+
+ 2. then the recreated \TeX{} version from 2010 using coffins---not
+ attempting\\ to match the fonts and size but the structure
+
+\vspace*{6mm}
+
+ 3. and finally the source code used.
+
+ This document uses the new
+ implementation by Joseph in \texttt{l3coffins-new}.
+
+\end{center}
+
+\newpage
+
+
+\begin{center}
+ \null
+ \fbox{\includegraphics[scale=.95]{elementare-typographie-title.jpg}}
+\end{center}
+
+\newpage
+
+\TypesetCoffin \result
+
+\newpage
+
+\vspace*{3cm}
+\begin{center}
+ {\Large Code used: \par}
+\vspace*{1cm}
+
+
+\begin{minipage}{14cm}
+\begin{verbatim}
+\SetHorizontalCoffin\result{}
+\SetHorizontalCoffin \aaa {\fontsize{52}{50}\sffamily\bfseries mitteilungen}
+\SetHorizontalCoffin \bbb {\fontsize{52}{50}\sffamily\bfseries typographische}
+\SetHorizontalCoffin \ccc {\fontsize{12}{10}\sffamily
+ \quad zeitschrift des bildungsverbandes der
+ deutschen buchdrucker leipzig
+ \textbullet{} oktoberheft 1925}
+\SetHorizontalCoffin \ddd {\fontsize{28}{20}\sffamily sonderheft}
+\SetVerticalCoffin \eee {180pt}
+ {\raggedleft\fontsize{31}{36}\sffamily\bfseries
+ elementare\\
+ typographie}
+\SetVerticalCoffin \fff {140pt}
+ {\raggedright \fontsize{13}{14}\sffamily\bfseries
+ natan altman \\
+ otto baumberger \\
+ herbert mayer \\
+ max burchartz \\
+ el lissitzky \\
+ ladislaus moholy-nagy \\
+ moln\'ar f.~farkas \\
+ jahannes molzahn \\
+ kurt schwitters \\
+ mart stam \\
+ ivan tschichold}
+
+\RotateCoffin \bbb {90}
+\RotateCoffin \ccc {270}
+
+\SetHorizontalCoffin \rulei {\color{red}\rule{6.5in}{1pc}}
+\SetHorizontalCoffin \ruleii {\color{red}\rule{1pc}{23.5cm}}
+\SetHorizontalCoffin \ruleiii{\color{black}\rule{10pt}{152pt}}
+
+\JoinCoffins \result \aaa
+\JoinCoffins \result[\aaa-t,\aaa-r] \rulei [b,r](0pt,2mm)
+\JoinCoffins \result[\aaa-b,\aaa-l] \bbb [B,r](2pt,0pt)
+\JoinCoffins \result[\bbb-t,\bbb-r] \ruleii [t,r](-2mm,0pt)
+\JoinCoffins \result[\aaa-B,\aaa-r] \ccc [B,l](66pt,14pc)
+\JoinCoffins \result[\bbb-l,\ccc-B] \fff [t,r](-2mm,0pt)
+\JoinCoffins \result[\fff-b,\fff-r] \ruleiii [b,l](2mm,0pt)
+\JoinCoffins \result[\ccc-r,\fff-l] \eee [B,r]
+\JoinCoffins \result[\eee-T,\eee-r] \ddd [B,r](0pt,4pc)
+\SetHorizontalCoffin\result{}
+\SetHorizontalCoffin \aaa {\fontsize{52}{50}\sffamily\bfseries
+ mitteilungen}
+\SetHorizontalCoffin \bbb {\fontsize{52}{50}\rotatebox{90}{\sffamily\bfseries
+ typographische}}
+
+\TypesetCoffin \result
+\end{verbatim}
+
+This is not necessarily the final syntax but for now it does its job. For
+example, flexible support for adding ornaments (lines, \ldots) is still
+missing, so above the rules got added as predefined individual coffins.
+
+\end{minipage}
+\end{center}
+
+\end{document}
+
diff --git a/Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx b/Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx
index 0d3fdafcdf8..dc6279914af 100644
--- a/Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx
@@ -37,7 +37,7 @@
%
%<*driver|package>
\RequirePackage{l3names}
-\GetIdInfo$Id: l3galley.dtx 3071 2011-12-20 20:14:00Z bruno $
+\GetIdInfo$Id: l3galley.dtx 3205 2012-01-18 22:09:05Z joseph $
{L3 Experimental galley code}
%</driver|package>
%<*driver>
@@ -372,7 +372,7 @@
%
% \begin{variable}{\l_galley_final_hyphen_demerits_int}
% Extra demerit charge if the second last line is hyphenated.
-% \end{variable}
+% \end{variable}
%
% \begin{variable}{\l_galley_linebreak_badness_int}
% Boundary that if exceeded will cause \TeX{} to report an
@@ -388,12 +388,12 @@
% this penalty higher \TeX{} will try harder to produce compact
% paragraphs.
% \end{variable}
-%
+%
% \begin{variable}{\l_galley_linebreak_pretolerance_int}
% Maximum tolerance allowed for individual lines to break the
% paragraph without attempting hyphenation.
% \end{variable}
-%
+%
% \begin{variable}{\l_galley_linebreak_tolerance_int}
% Maximum tolerance allowed for individual lines when breaking a
% paragraph while attempting hyphenation (if this limit can't be
@@ -416,7 +416,7 @@
% Boundary that if exceeded will cause \TeX{} to report an
% underfull vertical box.
% \end{variable}
-%
+%
% \begin{variable}{\l_galley_parbreak_fuzz_dim}
% Boundary below which overfull vertical boxes are not reported.
% \end{variable}
@@ -432,7 +432,7 @@
% \begin{variable}{\l_galley_post_display_penalty_int}
% Penalty for breaking between immediately after display math material.
% \end{variable}
-%
+%
% \begin{function}
% {
% \galley_set_club_penalties:n,
@@ -468,7 +468,7 @@
% In all cases, these penalties apply in addition to the general interline
% penalty or to any \enquote{special} line penalties.
% \end{function}
-%
+%
% \begin{function}{\galley_set_interline_penalty:n}
% \begin{syntax}
% \cs{galley_set_interline_penalty:n} \Arg{penalty}
@@ -476,7 +476,7 @@
% Sets the standard interline penalty applied between lines of a paragraph.
% This value is added to any (display) club or widow penalty in force.
% \end{function}
-%
+%
% \begin{function}
% {
% \galley_set_interline_penalties:n,
@@ -490,7 +490,7 @@
% The \meta{penalties} apply to the first, second, third, \emph{etc.}~line
% of the paragraph.
% \end{function}
-%
+%
% \begin{function}
% {
% \galley_save_club_penalties:N,
@@ -502,10 +502,10 @@
% \begin{syntax}
% \cs{galley_save_club_penalties:N} \Arg{comma list}
% \end{syntax}
-% These functions save the current value of the appropriate to the
+% These functions save the current value of the appropriate to the
% comma list specified, within the current \TeX{} group.
% \end{function}
-%
+%
% \begin{function}[EXP]{\galley_interline_penalty:}
% \begin{syntax}
% \cs{galley_interline_penalty:}
@@ -524,7 +524,7 @@
% \begin{variable}{\g_galley_par_end_hook_tl}
% Token list inserted at the end of every paragraph in horizontal mode.
% \end{variable}
-%
+%
% \begin{variable}{\g_galley_par_after_hook_tl}
% Token list inserted after each paragraph. This is used for resetting
% galley parameters, and is therefore cleared after use.
@@ -539,7 +539,7 @@
% Token list for whatsits to be inserted at the very end of the last
% paragraph started.
% \end{variable}
-%
+%
% \section{Additional effects}
%
% \begin{function}{\galley_end_par:n}
@@ -663,7 +663,7 @@
% The standard mode to leave vertical mode, starting a paragraph.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \galley_leave_vmode:
- { \tex_unhbox:D \c_empty_box }
+ { \hbox_unpack:N \c_empty_box }
% \end{macrocode}
% \end{macro}
%
@@ -815,7 +815,7 @@
\tl_new:N \l_galley_par_end_hook_tl
% \end{macrocode}
% \end{variable}
-%
+%
% \begin{variable}
% {\g_galley_par_after_hook_tl, \l_galley_par_after_hook_tl}
% This one is used by the galley: it happens \enquote{after} the current
@@ -1082,7 +1082,7 @@
{ \tex_par:D }
% \end{macrocode}
% In horizontal mode, the paragraph shape is set \enquote{just in time}
-% before inserting \cs{tex_par:D}. The \cs{tex_par:D} is inside a
+% before inserting \cs{tex_par:D}. The \cs{tex_par:D} is inside a
% group to preserve some dynamic settings (for example
% \cs{etex_interlinepenalties}). Once the
% paragraph has been typeset, the number of lines is \emph{added} to the
@@ -1112,7 +1112,7 @@
% \end{macro}
% \end{macro}
% \end{macro}
-%
+%
% \begin{macro}{\galley_end_par:n}
% Inserts tokens such that they are appended to the end of the last
% paragraph, using the paragraph-omitting system.
@@ -1830,7 +1830,7 @@
\cs_new_eq:NN \l_galley_pre_display_penalty_int \tex_predisplaypenalty:D
% \end{macrocode}
% \end{variable}
-%
+%
% \begin{macro}{\l_galley_club_penalties_clist, \l_galley_line_penalties_clist}
% These are used to keep a track of information which cannot be
% extracted out of the primitives due to the overlapping nature of
@@ -1840,7 +1840,7 @@
\clist_new:N \l_galley_line_penalties_clist
% \end{macrocode}
% \end{macro}
-%
+%
% \begin{macro}
% {
% \galley_set_display_widow_penalties:n,
@@ -1878,7 +1878,7 @@
% \end{macrocode}
% \end{macro}
% \end{macro}
-%
+%
% \begin{macro}
% {
% \galley_set_club_penalties:n,
@@ -1907,7 +1907,7 @@
\cs_generate_variant:Nn \galley_set_interline_penalties:n { V , v }
% \end{macrocode}
% \end{macro}
-%
+%
% \begin{macro}
% {
% \galley_set_display_club_penalties:n,
@@ -1928,7 +1928,7 @@
\cs_generate_variant:Nn \galley_set_display_club_penalties:n { V , v }
% \end{macrocode}
% \end{macro}
-%
+%
% \begin{macro}{\galley_set_interline_penalty:n}
% \begin{macro}[aux]{\galley_set_interline_penalty_aux:nn}
% \begin{macro}[aux]
@@ -1957,17 +1957,17 @@
\etex_interlinepenalties:D ##1
- \etex_interlinepenalties:D \etex_interlinepenalties:D \c_zero
+ #1
- \int_eval_end:
+ \int_eval_end:
}
\exp_args:Nf \galley_set_interline_penalty_aux:nn
{ \clist_length:N \l_galley_line_penalties_clist } {#1}
}
}
\cs_new_protected_nopar:Npn \galley_set_interline_penalty_aux:nn #1#2
- {
+ {
\etex_interlinepenalties:D
\etex_interlinepenalties:D \c_zero
- \prg_stepwise_function:nnnN \c_one \c_one {#1}
+ \prg_stepwise_function:nnnN \c_one \c_one {#1}
\galley_set_interline_penalty_aux_i:n
\prg_stepwise_function:nnnN { #1 + \c_one } \c_one
{ \etex_interlinepenalties:D \c_zero - \c_one }
@@ -1981,7 +1981,7 @@
% \end{macro}
% \end{macro}
% \end{macro}
-%
+%
% \begin{macro}[int]{\galley_calc_interline_penalties:}
% \begin{macro}[aux]{\galley_calc_interline_penalties_aux:nn}
% \begin{macro}[aux]
@@ -2012,7 +2012,7 @@
{
\clist_length:N \l_galley_line_penalties_clist
+ \c_one
- }
+ }
{ \etex_clubpenalties:D \c_zero }
}
}
@@ -2049,7 +2049,7 @@
#1
{
\int_eval:w
- \etex_interlinepenalties:D \etex_interlinepenalties:D \c_zero
+ \etex_interlinepenalties:D \etex_interlinepenalties:D \c_zero
+ 0 \clist_item:Nn \l_galley_club_penalties_clist
{ #1 - \c_one }
- \etex_clubpenalties:D #1 ~
@@ -2059,7 +2059,7 @@
% \end{macro}
% \end{macro}
% \end{macro}
-%
+%
% \begin{macro}
% {
% \galley_save_club_penalties:N,
@@ -2130,10 +2130,9 @@
% \end{macro}
% \end{macro}
% \end{macro}
-%
%
% \subsection{\LaTeXe{} functions}
-%
+%
% \begin{macrocode}
%<*package>
% \end{macrocode}
@@ -2174,7 +2173,7 @@
% Purely for testing, some internal \LaTeXe{} functions are altered to work
% with the mechanism here. This material is not comprehensive: additions are
% made as-needed for test purposes.
-%
+%
% \begin{macro}{\@@par}
% The primitive is moved as otherwise the clever skipping code will fail.
% \begin{macrocode}