From 8764984f07460a77b8200111ddb513c8af6c23f3 Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Thu, 19 Jan 2012 22:49:25 +0000 Subject: l3experimental 3209 (19jan12) git-svn-id: svn://tug.org/texlive/trunk@25158 c570f23f-e606-0410-a88d-b1316a301751 --- .../source/latex/l3experimental/l3str/l3flag.dtx | 10 +- .../source/latex/l3experimental/l3str/l3regex.dtx | 1361 +++++++++++--------- .../source/latex/l3experimental/l3str/l3str.dtx | 298 +++-- .../latex/l3experimental/l3str/l3tl-analysis.dtx | 612 +++++---- .../l3experimental/xcoffins/xcoffins-test.tex | 380 ++++++ .../xcoffins-tschichold-example-keyval.tex | 281 ++++ .../xcoffins/xcoffins-tschichold-example.tex | 185 +++ .../latex/l3experimental/xgalley/l3galley.dtx | 63 +- 8 files changed, 2210 insertions(+), 980 deletions(-) create mode 100644 Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-test.tex create mode 100644 Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example-keyval.tex create mode 100644 Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example.tex (limited to 'Master/texmf-dist/source') diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx index eed657ad231..46bd4c80cf7 100644 --- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx @@ -1,6 +1,6 @@ % \iffalse meta-comment % -%% File: l3flag.dtx Copyright (C) 2011 The LaTeX3 Project +%% File: l3flag.dtx Copyright (C) 2011-2012 The LaTeX3 Project %% %% It may be distributed and/or modified under the conditions of the %% LaTeX Project Public License (LPPL), either version 1.3c of this @@ -70,7 +70,7 @@ % \begin{documentation} % % Flags are the only data structure on which \TeX{} can perform -% assignments in expansion only contexts. This module is meant +% assignments in expansion-only contexts. This module is meant % mostly for kernel use: in almost all cases, booleans should % be preferred to flags, because they are faster. % @@ -155,16 +155,14 @@ % \begin{macrocode} \cs_new_protected:Npn \flag_new:N #1 { + \cs_new_eq:NN #1 \c_undefined:D \tl_if_in:NnTF \g_flag_list_tl {#1} { \msg_kernel_error:nnx { flag } { already-defined } { \token_to_str:N #1 } } - { - \tl_gput_right:Nn \g_flag_list_tl {#1} - \cs_new_eq:NN #1 \c_undefined:D - } + { \tl_gput_right:Nn \g_flag_list_tl {#1} } } \cs_generate_variant:Nn \flag_new:N { c } % \end{macrocode} diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx index 9c276b6922d..d921b0b7e56 100644 --- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx @@ -1,6 +1,6 @@ % \iffalse meta-comment % -%% File: l3regex.dtx Copyright (C) 2011 The LaTeX3 Project +%% File: l3regex.dtx Copyright (C) 2011-2012 The LaTeX3 Project %% %% It may be distributed and/or modified under the conditions of the %% LaTeX Project Public License (LPPL), either version 1.3c of this @@ -35,7 +35,7 @@ % %<*driver|package> \RequirePackage{expl3} -\GetIdInfo$Id: l3regex.dtx 3073 2011-12-21 01:53:06Z bruno $ +\GetIdInfo$Id: l3regex.dtx 3200 2012-01-16 03:50:54Z bruno $ {L3 Experimental Regular Expressions} % %<*driver> @@ -104,16 +104,16 @@ % it can be compiled once, and stored in a token list % variable using \cs{regex_const:Nn}. For example, % \begin{verbatim} -% \regex_const:Nn \l_foo_regex_tl { \c{begin} \cB (\CB*?) \cE } +% \regex_const:Nn \c_foo_regex_tl { \c{begin} \cB. (\c[^BE].*) \cE. } % \end{verbatim} -% stores in \cs{l_foo_regex_tl} a regular expression which matches -% the starting marker for an environment: \cs{begin}, followed -% by a begin-group token (|\cB|), then tokens which are not -% begin-group characters (|\CB|), as few as possible (|*?|), -% followed by an end-group token (|\cE|). As explained in the -% next section, the parentheses \enquote{capture} the result of -% |\CB*?|, giving us access to the name of the environment -% when doing replacements. +% stores in \cs{c_foo_regex_tl} a regular expression which matches the +% starting marker for an environment: \cs{begin}, followed by a +% begin-group token (|\cB.|), then any number of tokens which are +% neither begin-group nor end-group character tokens (|\c[^BE].*|), +% ending with an end-group token (|\cE.|). As explained in the next +% section, the parentheses \enquote{capture} the result of |\c[^BE].*|, +% giving us access to the name of the environment when doing +% replacements. % % \subsection{Syntax of regular expressions} % @@ -191,7 +191,7 @@ % Matches any token other than the specified characters. % \item[{[x-y]}] Range (can be used with escaped characters). % \end{l3regex-syntax} -% For instance, |[a-oq-z\cC]| matches any lowercase latin letter +% For instance, |[a-oq-z\cC.]| matches any lowercase latin letter % except |p|, as well as control sequences (see below for a description % of |\c|). % @@ -240,29 +240,47 @@ % group number. % \end{l3regex-syntax} % -% Each catcode is represented by a single uppercase letter: -% |C| for control sequences; -% |B| for begin-group tokens; -% |E| for end-group tokens; -% |M| for math shift; -% |T| for alignment tab tokens; -% |P| for macro parameter tokens; -% |U| for superscript tokens (up); -% |D| for subscript tokens (down); -% |S| for spaces; -% |L| for letters; -% |O| for others; and -% |A| for active characters. +% The |\c| escape sequence allows to test the category code of tokens, +% and match control sequences. Each character category is represented +% by a single uppercase letter: +% \begin{itemize} +% \item |C| for control sequences; +% \item |B| for begin-group tokens; +% \item |E| for end-group tokens; +% \item |M| for math shift; +% \item |T| for alignment tab tokens; +% \item |P| for macro parameter tokens; +% \item |U| for superscript tokens (up); +% \item |D| for subscript tokens (down); +% \item |S| for spaces; +% \item |L| for letters; +% \item |O| for others; and +% \item |A| for active characters. +% \end{itemize} +% The |\c| escape sequence is used as follows. % \begin{l3regex-syntax} -% \item[\\cX] Any character token with catcode |X|, -% where |X| can be any of |CBEMTPUDSLOA|. -% \item[\\CX] Any token with catcode different from |X|, -% where |X| can be any of |CBEMTPUDSLOA|. -% \item[\\c\Arg{regex}] A control sequence whose csname matches -% the \meta{regex}, anchored at the beginning and end, -% so that |\c{begin}| matches exactly \cs{begin}, and nothing -% else. +% \item[\\c\Arg{regex}] A control sequence whose csname matches the +% \meta{regex}, anchored at the beginning and end, so that |\c{begin}| +% matches exactly \cs{begin}, and nothing else. +% \item[\\cX\meta{character or class}] Matches a token with category +% code |X| (any of |CBEMTPUDSLOA|) if it is also matched by the +% \meta{character or class}. For instance, |\cL[A-Z]| matches +% uppercase letters of category code letter, |\cC.| matches any +% control sequence, and |\cO\d| matches digits of category other. +% \item[{\\c[XYZ]\meta{character or class}}] Matches a token with +% category |X|, |Y|, or |Z| (each being any of |CBEMTPUDSLOA|), +% if it is also matched by the \meta{character or class}. For instance, +% |\c[LSO].| matches tokens of category letter, space, or other. +% \item[{\\c[\char`\^XYZ]\meta{character or class}}] Matches a token with +% category different from |X|, |Y|, or |Z| (each being any of +% |CBEMTPUDSLOA|), if it is also matched by the \meta{character or +% class}. For instance, |\c[LSO].| matches tokens of category +% letter, space, or other. % \end{l3regex-syntax} +% The category code tests can be used inside classes; for instance, +% |[\cO\d \c[LO][A-F]]| matches what \TeX{} considers as hexadecimal +% digits, namely digits with category other, or uppercase letters from +% |A| to |F| with category either letter or other. % % Options can be set with |(?|\meta{option}|)| and % unset with |(?-|\meta{option}|)|. Options are local @@ -526,6 +544,8 @@ % \begin{itemize} % \item The \texttt{\{\}} quantifiers are only partially implemented. % \item Clean up the use of messages. +% \item Triple check magic numbers. +% \item Test \cs{par}. % \item Add tests for every regex and replacement feature. % \item Make sure the documentation is correct, and the code comments % are as well. @@ -533,11 +553,12 @@ % % Code improvements to come. % \begin{itemize} +% \item \cs{regex_show:N} to show how a given regular expression is +% interpreted. % \item Test for the maximum register \cs{c_max_register_int}. % \item Use \tn{dimen} registers rather than \cs{l_regex_nesting_tl} % to build \cs{regex_nesting:n}. % \item Reduce the number of epsilon-transitions in alternatives. -% \item Merge the \texttt{build} and \texttt{class} functions where possible. % \item Improve nesting of |\c| in the replacement text. % \item Move the \enquote{reconstruction} part of \pkg{l3regex} % to \pkg{l3tl-analysis}. @@ -577,22 +598,24 @@ % the matching, because the regex code uses registers in an unsafe way; % \item Conditional subpatterns (other than with a look-ahead % or look-behind condition): this is non-regular, isn't it? +% \item Named subpatterns: \TeX{} programmers have lived so far without +% any need for named macro parameters. % \end{itemize} % % The following features of PCRE or perl will definitely not be implemented. % \begin{itemize} % \item |\cx|, similar to \TeX{}'s own |\^^x|; % \item Comments: \TeX{} already has its own system for comments. -% \item Named subpatterns: \TeX{} programmers have lived so far without -% any need for named macro parameters. % \item |\Q...\E| escaping: this would require to read the argument % verbatim, which is not in the scope of this module. % \item Atomic grouping, possessive quantifiers: those tools, mostly % meant to fix catastrophic backtracking, are unnecessary in a % non-backtracking algorithm, and difficult to implement. -% \item Subroutine calls: this syntactic sugar is difficult to -% include in a non-backtracking algorithm, in particular because -% the corresponding group should be treated as atomic. +% \item Subroutine calls: this syntactic sugar is difficult to include +% in a non-backtracking algorithm, in particular because the +% corresponding group should be treated as atomic. Also, we cannot +% afford to run user code within the regular expression matching, +% because of our \enquote{misuse} of registers. % \item Recursion: this is a non-regular feature. % \item Back-references: non-regular feature, this requires backtracking, % which is prohibitively slow. @@ -665,6 +688,15 @@ % % \subsubsection{Variables used while building} % +% \begin{variable}{\l_regex_build_mode_int} +% While building, ten modes are recognized, labelled $-63$, $-23$, +% $-6$, $-2$, $0$, $2$, $3$, $6$, $23$, $63$. See +% section~\ref{sec:regex-modes}. +% \begin{macrocode} +\int_new:N \l_regex_build_mode_int +% \end{macrocode} +% \end{variable} +% % \begin{variable}{\l_regex_max_state_int} % \begin{variable}{\l_regex_left_state_int, \l_regex_right_state_int} % The end-point states of the last group (which any quantifier @@ -723,16 +755,6 @@ % \end{variable} % \end{variable} % -% \begin{variable}{\l_regex_build_or_class_tl} -% The actions of various escape sequences and special characters -% are different within a class and outside a class. Rather than -% using a boolean, we store either \texttt{build} or \texttt{class} -% in this token list. -% \begin{macrocode} -\tl_new:N \l_regex_build_or_class_tl -% \end{macrocode} -% \end{variable} -% % \begin{variable}{\l_regex_one_or_group_tl} % When looking for quantifiers, this variable holds either % \enquote{one} or \enquote{group} depending on whether the @@ -743,11 +765,52 @@ % \end{macrocode} % \end{variable} % -% \begin{variable}{\l_regex_in_csname_bool} -% This boolean is true when building the regular expression -% for a given csname. +% \begin{variable}{\l_regex_catcodes_int, \l_regex_catcodes_default_int} +% \begin{variable} +% { +% \c_regex_catcode_C_int, \c_regex_catcode_B_int, \c_regex_catcode_E_int, +% \c_regex_catcode_M_int, \c_regex_catcode_T_int, \c_regex_catcode_P_int, +% \c_regex_catcode_U_int, \c_regex_catcode_D_int, \c_regex_catcode_S_int, +% \c_regex_catcode_L_int, \c_regex_catcode_O_int, \c_regex_catcode_A_int +% } +% \begin{variable}{\c_regex_catcodes_all_int} +% We wish to allow constructions such as |\c[^BE](..\cL[a-z]..)|, +% matching two tokens which are neither a begin-group nor an end-group +% token, followed by a token of category letter and character code in +% |[a-z]|, followed by two more tokens which are neither begin-group +% nor end-group tokens. For this to work, we need to keep track of +% lists of allowed category codes: \cs{l_regex_catcodes_int} and +% \cs{l_regex_catcodes_default_int} are bitmaps, sums of $4^c$, for +% all allowed catcodes $c$. The latter is local to each capturing +% group, and we reset \cs{l_regex_catcodes_int} to that value after +% each character or class, changing it only when encountering a |\c| +% escape. +% \begin{macrocode} +\int_new:N \l_regex_catcodes_int +\int_new:N \l_regex_catcodes_default_int +\int_const:Nn \c_regex_catcode_C_int { "1 } +\int_const:Nn \c_regex_catcode_B_int { "4 } +\int_const:Nn \c_regex_catcode_E_int { "10 } +\int_const:Nn \c_regex_catcode_M_int { "40 } +\int_const:Nn \c_regex_catcode_T_int { "100 } +\int_const:Nn \c_regex_catcode_P_int { "1000 } +\int_const:Nn \c_regex_catcode_U_int { "4000 } +\int_const:Nn \c_regex_catcode_D_int { "10000 } +\int_const:Nn \c_regex_catcode_S_int { "100000 } +\int_const:Nn \c_regex_catcode_L_int { "400000 } +\int_const:Nn \c_regex_catcode_O_int { "1000000 } +\int_const:Nn \c_regex_catcode_A_int { "4000000 } +\int_const:Nn \c_regex_catcodes_all_int { "5515155 } +% \end{macrocode} +% \end{variable} +% \end{variable} +% \end{variable} +% +% \begin{variable}{\l_regex_catcodes_bool} +% Controls whether the bitmap of category codes built should be +% inverted or not. % \begin{macrocode} -\bool_new:N \l_regex_in_csname_bool +\bool_new:N \l_regex_catcodes_bool % \end{macrocode} % \end{variable} % @@ -761,22 +824,22 @@ % % \subsubsection{Character classes} % -% \begin{macro}{\regex_build_tmp_class:n} % \begin{variable}{\l_regex_class_bool,\l_regex_class_tl} +% \begin{variable}{\l_regex_class_saved_bool,\l_regex_class_saved_tl} % \cs{l_regex_class_bool} is false for negative character classes. -% \cs{l_regex_class_tl} holds the tests which should be performed -% to decide whether the \cs{l_regex_current_char_int} matches that -% character class. Once the class is read completely, the full -% instructions are stored in \cs{regex_build_tmp_class:n}, whose -% argument is the target state if the test succeeds. It is also used -% for all other single-character situations (\emph{e.g.}, |\d|, or |.|). -% \begin{macrocode} -\cs_new_eq:NN \regex_build_tmp_class:n \use_none:n +% \cs{l_regex_class_tl} holds the tests which should be performed to +% decide whether the \cs{l_regex_current_char_int} matches that +% character class. In nested class, which can only occur as +% |[\c[...]]|, the two variables from the outer class must be saved +% while processing the inner class. +% \begin{macrocode} \bool_new:N \l_regex_class_bool \tl_new:N \l_regex_class_tl +\bool_new:N \l_regex_class_saved_bool +\tl_new:N \l_regex_class_saved_tl % \end{macrocode} % \end{variable} -% \end{macro} +% \end{variable} % % \begin{variable}{\c_regex_d_tl,\c_regex_D_tl} % \begin{variable}{\c_regex_h_tl,\c_regex_H_tl} @@ -1202,12 +1265,12 @@ % This is done \enquote{by hand}, in sequences whose items should % be run immediately. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_seq_pop_use:N #1 +\cs_new_protected:Npn \regex_seq_pop_use:N #1 { \seq_pop:NN #1 \l_regex_tmpa_tl \l_regex_tmpa_tl } -\cs_new_protected_nopar:Npn \regex_seq_get_use:N #1 +\cs_new_protected:Npn \regex_seq_get_use:N #1 { \seq_get:NN #1 \l_regex_tmpa_tl \l_regex_tmpa_tl @@ -1216,45 +1279,10 @@ % \end{macro} % \end{macro} % -% \begin{macro}[int, EXP]{\regex_seq_item:Nn} -% \begin{macro}[aux, EXP]{\regex_seq_item_aux:wNn} -% \begin{macrocode} -\cs_new:Npn \regex_seq_item:Nn #1#2 - { - \exp_after:wN \regex_seq_item_aux:wNn - \int_use:N \int_eval:w #2 \exp_after:wN ; - #1 { ? \prg_map_break: } { } - \prg_break_point:n { } - } -\cs_new:Npn \regex_seq_item_aux:wNn #1; #2#3 - { - \use_none:n #2 - \if_num:w #1 = \c_zero - \exp_after:wN \prg_map_break:n - \else: - \exp_after:wN \use_none:n - \fi: - { \exp_not:n {#3} } - \exp_after:wN \regex_seq_item_aux:wNn - \int_use:N \int_eval:w #1 - \c_one ; - } -% \end{macrocode} -% \end{macro} -% \end{macro} -% % \subsubsection{Testing characters} % -% \begin{macro}{\regex_item_dot:T} -% The dot meta-character matches any character, -% except the end marker. -% \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_item_dot:T - { \int_compare:nNnF \l_regex_current_char_int < \c_minus_one } -% \end{macrocode} -% \end{macro} -% % \begin{macro}[int]{\regex_break_point:TF} -% \begin{macro}[int]{\regex_break_true:w,\regex_break_false:w} +% \begin{macro}[int]{\regex_break_true:w} % When testing whether a character of the query token list matches % a given character class in the regular expression, we often % have to test it against several ranges of characters, checking @@ -1268,27 +1296,38 @@ % Otherwise, \cs{regex_break_point:TF} leaves the \meta{false code} % in the input stream. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_break_true:w +\cs_new_protected:Npn \regex_break_true:w #1 \regex_break_point:TF #2 #3 {#2} -\cs_new_protected_nopar:Npn \regex_break_false:w - #1 \regex_break_point:TF #2 #3 {#3} -\cs_new_protected_nopar:Npn \regex_break_point:TF #1 #2 { #2 } +\cs_new_protected:Npn \regex_break_point:TF #1 #2 { #2 } % \end{macrocode} % \end{macro} % \end{macro} % +% \begin{macro}{\regex_item_dot:} +% The dot meta-character matches any character, +% except the end marker. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \regex_item_dot: + { + \if_num:w \l_regex_current_char_int > - \c_two + \exp_after:wN \regex_break_true:w + \fi: + } +% \end{macrocode} +% \end{macro} +% % \begin{macro}[int]{\regex_item_caseful_equal:n} % \begin{macro}[int]{\regex_item_caseful_range:nn} % \begin{macro}[int]{\regex_item_caseful_geq:n} % Simple comparisons triggering \cs{regex_break_true:w} when true. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_item_caseful_equal:n #1 +\cs_new_protected:Npn \regex_item_caseful_equal:n #1 { \if_num:w #1 = \l_regex_current_char_int \exp_after:wN \regex_break_true:w \fi: } -\cs_new_protected_nopar:Npn \regex_item_caseful_range:nn #1 #2 +\cs_new_protected:Npn \regex_item_caseful_range:nn #1 #2 { \reverse_if:N \if_num:w #1 > \l_regex_current_char_int \reverse_if:N \if_num:w #2 < \l_regex_current_char_int @@ -1296,7 +1335,7 @@ \fi: \fi: } -\cs_new_protected_nopar:Npn \regex_item_caseful_geq:n #1 +\cs_new_protected:Npn \regex_item_caseful_geq:n #1 { \reverse_if:N \if_num:w #1 > \l_regex_current_char_int \exp_after:wN \regex_break_true:w @@ -1314,7 +1353,7 @@ % \cs{l_regex_current_char_int} and on % \cs{l_regex_case_changed_char_int}. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_item_caseless_equal:n #1 +\cs_new_protected:Npn \regex_item_caseless_equal:n #1 { \if_num:w #1 = \l_regex_current_char_int \exp_after:wN \regex_break_true:w @@ -1323,7 +1362,7 @@ \exp_after:wN \regex_break_true:w \fi: } -\cs_new_protected_nopar:Npn \regex_item_caseless_range:nn #1 #2 +\cs_new_protected:Npn \regex_item_caseless_range:nn #1 #2 { \reverse_if:N \if_num:w #1 > \l_regex_current_char_int \reverse_if:N \if_num:w #2 < \l_regex_current_char_int @@ -1336,7 +1375,7 @@ \fi: \fi: } -\cs_new_protected_nopar:Npn \regex_item_caseless_geq:n #1 +\cs_new_protected:Npn \regex_item_caseless_geq:n #1 { \reverse_if:N \if_num:w #1 > \l_regex_current_char_int \exp_after:wN \regex_break_true:w @@ -1387,26 +1426,30 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}[int]{\regex_item_catcode:n} -% \begin{macro}[int]{\regex_item_catcode_neq:n} -% Checking the category code is quite straightforward. +% \begin{macro}{\regex_item_catcode:nT} +% The argument is a sum of powers of $4$ with exponents given by the +% allowed category codes (between $0$ and $13$). Dividing by a given +% power of $4$ gives an odd result if and only if that category code +% is allowed. If the catcode does not match, then skip the character +% code tests which follow. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_item_catcode:n #1 - { - \if_num:w #1 = \l_regex_current_catcode_int - \exp_after:wN \regex_break_true:w - \fi: - } -\cs_new_protected_nopar:Npn \regex_item_catcode_neq:n #1 +\cs_new_protected:Npn \regex_item_catcode:nT #1 { - \if_num:w #1 = \l_regex_current_catcode_int + \if_int_odd:w \int_eval:w #1 / " + \if_case:w \l_regex_current_catcode_int + 1 \or: 4 \or: 10 \or: 40 + \or: 100 \or: \or: 1000 \or: 4000 + \or: 10000 \or: \or: 100000 \or: 400000 + \or: 1000000 \or: 4000000 \else: 1*\c_zero + \fi: + \int_eval_end: + \exp_after:wN \use:n \else: - \exp_after:wN \regex_break_true:w + \exp_after:wN \use_none:n \fi: } % \end{macrocode} % \end{macro} -% \end{macro} % % \begin{macro}[int]{\regex_item_cs:n} % Match a control sequence (the argument is a pre-compiled regex). @@ -1455,7 +1498,7 @@ { \if_false: } } \fi: \regex_get_digits_aux:NN } -\cs_new_nopar:Npn \regex_get_digits_aux:NN #1#2 +\cs_new:Npn \regex_get_digits_aux:NN #1#2 { \if_meaning:w \regex_build_raw:N #1 \if_charcode:w \c_space_token \exp_not:N #2 @@ -1471,7 +1514,7 @@ \fi: \regex_get_digits_aux:NN } -\cs_new_nopar:Npn \regex_get_digits_end:w #1 \fi: #2 \regex_get_digits_aux:NN +\cs_new:Npn \regex_get_digits_end:w #1 \fi: #2 \regex_get_digits_aux:NN { \fi: #2 \if_false: { { \fi: } } @@ -1568,6 +1611,66 @@ % % \subsection{Building} % +% \subsubsection{Build mode} +% \label{sec:regex-modes} +% +% When building the NFA corresponding to a given regex, we can be in +% ten distinct modes, which we label by some magic numbers: +% \begin{itemize} +% \item[-6] |[\c{...}]| control sequence in a class, +% \item[-2] |\c{...}| control sequence, +% \item[0] |...| outer, +% \item[2] |\c...| catcode test, +% \item[6] |[\c...]| catcode test in a class, +% \item[-63] |[\c{[...]}]| class inside mode $-6$, +% \item[-23] |\c{[...]}| class inside mode $-2$, +% \item[3] |[...]| class inside mode $-3$, +% \item[23] |\c[...]| class inside mode $2$, +% \item[63] |[\c[...]]| class inside mode $6$. +% \end{itemize} +% This list is exhaustive, because |\c| escape sequences cannot be +% nested, and character classes cannot be nested directly. The choice +% of numbers is such as to optimize the most useful tests, and make +% transitions from one mode to another as simple as possible. +% \begin{itemize} +% \item Even modes mean that we are not directly in a character class. +% In this case, a left bracket appends $3$ to the mode. In a character +% class, a right bracket changes the mode as $m\to (m-15)/13$, +% truncated. +% \item Grouping, assertion, and anchors are allowed in non-positive +% even modes ($0$, $-2$, $-6$), and do not change the mode. Otherwise, +% they trigger an error. +% \item A left bracket is special in even modes, appending $3$ to the +% mode; in those modes, quantifiers and the dot are recognized, and +% the right bracket is normal. In odd modes (within classes), the left +% bracket is normal, but the right bracket ends the class, changing +% the mode from $m$ to $(m-15)/13$, truncated; also, ranges are +% recognized. +% \item In non-negative modes, left and right braces are normal. In +% negative modes, however, left braces trigger a warning; right braces +% end the control sequence, going from $-2$ to $0$ or $-6$ to $3$, +% with error recovery for odd modes. +% \item Properties (such as the |\d| character class) can appear in any +% mode. +% \end{itemize} +% +% \begin{macro}[int, EXP]{\regex_build_if_in_class:TF} +% Test whether we are currently in a character class (at the +% inner-most level of nesting). There, many escape sequences are not +% recognized, and special characters are normal. Also, for every raw +% character, we must look ahead for a possible raw dash. +% \begin{macrocode} +\cs_new_nopar:Npn \regex_build_if_in_class:TF + { + \if_int_odd:w \l_regex_build_mode_int + \exp_after:wN \use_i:nn + \else: + \exp_after:wN \use_ii:nn + \fi: + } +% \end{macrocode} +% \end{macro} +% % \subsubsection{Helpers for building an NFA} % % \begin{macro}[int]{\regex_build_new_state:} @@ -1586,18 +1689,18 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_transition_aux:NN} -% \begin{macro}[aux]{\regex_build_transitions_aux:NNNN} +% \begin{macro}[aux]{\regex_build_transition:NN} +% \begin{macro}[aux]{\regex_build_transitions:NNNN} % These functions create a new state, and put one or two transitions % starting from the old current state. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_transition_aux:NN #1#2 +\cs_new_protected:Npn \regex_build_transition:NN #1#2 { \regex_build_new_state: \regex_toks_put_right:Nx \l_regex_left_state_int { #1 { \int_eval:n { #2 - \l_regex_left_state_int } } } } -\cs_new_protected_nopar:Npn \regex_build_transitions_aux:NNNN #1#2#3#4 +\cs_new_protected:Npn \regex_build_transitions:NNNN #1#2#3#4 { \regex_build_new_state: \regex_toks_put_right:Nx \l_regex_left_state_int @@ -1612,12 +1715,11 @@ % % \subsubsection{From regex to NFA: framework} % -% In order for the construction \verb"ab|cd" to work, we enclose -% the whole pattern within parentheses (in the code below, -% \cs{regex_build_open_aux:} and \cs{regex_build_close_aux:}). -% These have the added benefit -% to form a capturing group: hence we get the data of the whole match -% for free. +%^^A In order for the construction \verb"ab|cd" to work, we enclose the +%^^A whole pattern within parentheses (in the code below, +%^^A \cs{regex_build_open_aux:} and \cs{regex_build_close_aux:}). These +%^^A have the added benefit to form a capturing group: hence we get the +%^^A data of the whole match for free. % % \begin{macro}[int]{\regex_build:n} % First, reset a few variables. Then use the generic framework defined @@ -1633,11 +1735,11 @@ % \begin{macrocode} \cs_new_protected:Npn \regex_build:n #1 { - \bool_set_false:N \l_regex_in_csname_bool + \int_set_eq:NN \l_regex_build_mode_int \c_zero \regex_build:w \regex_build_new_state: \regex_toks_put_right:Nx \l_regex_left_state_int - { \regex_action_start_wildcard:nn {0} {1} } + { \regex_action_start_wildcard: } \regex_build_open_aux: \str_escape_use:NNNn \regex_build_i_unescaped:N @@ -1645,6 +1747,7 @@ \regex_build_i_raw:N { #1 } \prg_do_nothing: \prg_do_nothing: + \prg_do_nothing: \prg_do_nothing: \regex_build_end: } % \end{macrocode} @@ -1663,55 +1766,60 @@ % special meanings, while everything else should be treated as a raw % character. % \begin{macrocode} -\cs_new_nopar:Npn \regex_build_i_unescaped:N #1 +\cs_new:Npn \regex_build_i_unescaped:N #1 { \regex_aux_char_if_special:NTF #1 - { \exp_not:N \regex_build_control:N #1 } + { \exp_not:N \regex_build_special:N #1 } { \exp_not:N \regex_build_raw:N #1 } } -\cs_new_nopar:Npn \regex_build_i_escaped:N #1 +\cs_new:Npn \regex_build_i_escaped:N #1 { \regex_aux_char_if_alphanumeric:NTF #1 - { \exp_not:N \regex_build_control:N #1 } + { \exp_not:N \regex_build_escaped:N #1 } { \exp_not:N \regex_build_raw:N #1 } } -\cs_new_nopar:Npn \regex_build_i_raw:N #1 +\cs_new:Npn \regex_build_i_raw:N #1 { \exp_not:N \regex_build_raw:N #1 } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % -% \begin{macro}[aux]{\regex_build_control:N} -% If the control character has a particular meaning in regexes, -% the corresponding function is used. Otherwise, it is interpreted -% as a raw character. The \cs{regex_build/class_default_raw:N} -% functions are defined later. +% \begin{macro}[aux]{\regex_build_special:N} +% \begin{macro}[aux]{\regex_build_escaped:N} +% If the control character has a particular meaning in regexes, the +% corresponding function is used. Otherwise, it is interpreted as a +% raw character. We distinguish special characters from escaped +% alphanumeric characters because in character classes, unknown +% escaped alphanumeric characters raise an error, while special +% charcters are silently converted to raw characters. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_control:N #1 +\cs_new_protected:Npn \regex_build_special:N #1 { - \cs_if_exist_use:cF { regex_ \l_regex_build_or_class_tl _#1: } - { \use:c { regex_ \l_regex_build_or_class_tl _default_raw:N } #1 } + \cs_if_exist_use:cF { regex_build_#1: } + { \regex_build_raw:N #1 } + } +\cs_new_protected:Npn \regex_build_escaped:N #1 + { + \cs_if_exist_use:cF { regex_build_/#1: } + { \regex_build_raw:N #1 } } % \end{macrocode} % \end{macro} -% -%^^A todo: update -% This function is similar to \cs{regex_build_control:N}. If the control -% character has a meaning in character classes, call the corresponding -% function, otherwise, treat it as a raw character, with the -% \cs{regex_class_default_raw:N} function, defined later. +% \end{macro} % % \begin{macro}[int]{\regex_build:w} % Hopefully, we didn't forget to initialize anything here. % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_build:w { - \cs_set_eq:NN \regex_build_raw:N \regex_build_default_raw:N - \tl_set:Nn \l_regex_build_or_class_tl { build } - \int_set_eq:NN \l_regex_capturing_group_int \c_zero + \int_set_eq:NN \l_regex_catcodes_default_int \c_regex_catcodes_all_int + \int_set_eq:NN \l_regex_capturing_group_int \c_zero \int_zero:N \l_regex_max_state_int \regex_build_new_state: + \tl_clear:N \l_regex_class_tl + \bool_set_true:N \l_regex_class_bool + \int_set_eq:NN \l_regex_catcodes_int \l_regex_catcodes_default_int } % \end{macrocode} % \end{macro} @@ -1740,179 +1848,162 @@ % \end{macrocode} % \end{macro} % -% \subsubsection{Anchoring and simple assertions} -% -% \begin{macro}[int]{\regex_build_A:} -% \begin{macro}[int]+\regex_build_^:+ -% \begin{macro}[int]{\regex_build_G:} -% \begin{macro}[aux]{\regex_build_anchor_start:N} -% Anchoring at the start corresponds to checking that the current -% character is the first in the token list. Anchoring to the beginning -% of the match attempt uses \cs{l_regex_start_step_int} instead of -% \cs{c_zero}. +% \begin{macro}[int]{\regex_build_one:n, \regex_build_one:x} +% In a class, add the argument to the current class. Outside a class, +% this argument is the whole \enquote{class}, and we look for +% quantifiers. % \begin{macrocode} -\cs_new_protected_nopar:cpn { regex_build_^: } - { \regex_build_anchor_start:N \l_regex_min_step_int } -\cs_new_protected_nopar:Npn \regex_build_A: - { \regex_build_anchor_start:N \l_regex_min_step_int } -\cs_new_protected_nopar:Npn \regex_build_G: - { \regex_build_anchor_start:N \l_regex_start_step_int } -\cs_new_protected_nopar:Npn \regex_build_anchor_start:N #1 +\cs_new_protected:Npn \regex_build_one:n #1 + { \regex_build_one:x { \exp_not:n {#1} } } +\cs_new_protected:Npn \regex_build_one:x #1 { - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int + \tl_put_right:Nx \l_regex_class_tl { - \exp_not:N \int_compare:nNnT #1 = \l_regex_current_step_int - { - \regex_action_free:n - { - \int_eval:n - { \l_regex_right_state_int - \l_regex_left_state_int } - } - } + \if_num:w \l_regex_catcodes_int = \c_regex_catcodes_all_int + \exp_after:wN \use:n + \else: + \regex_item_catcode:nT { \int_use:N \l_regex_catcodes_int } + \fi: + {#1} } + \if_num:w \l_regex_build_mode_int = \c_two + \l_regex_build_mode_int = \c_zero + \else: + \if_num:w \l_regex_build_mode_int = \c_six + \l_regex_build_mode_int = \c_three + \fi: + \fi: + \if_int_odd:w \l_regex_build_mode_int \else: + \exp_after:wN \regex_build_one_quantifier: + \fi: } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}[aux]{\regex_build_Z:} -% \begin{macro}[aux]{\regex_build_z:} -% \begin{macro}[aux]+\regex_build_$:+ -% This matches the end of the token list, marked by a character code of $-2$. +% \begin{macro}{\regex_build_tmp_class:n} +% ^^A todo: rename, not tmp, and used for class & single. +% The argument is the target state if the test succeeds. % \begin{macrocode} -\cs_new_protected_nopar:cpn { regex_build_$: } % $ +\cs_new:Npn \regex_build_tmp_class:n #1 { - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int - { - \exp_not:N \int_compare:nNnT - \l_regex_current_char_int < \c_minus_one - { - \regex_action_free:n - { - \int_eval:n - { \l_regex_right_state_int - \l_regex_left_state_int } - } - } - } + \exp_not:o \l_regex_class_tl + \bool_if:NTF \l_regex_class_bool + { \regex_break_point:TF { \regex_action_cost:n {#1} } { } } + { \regex_break_point:TF { } { \regex_action_cost:n {#1} } } } -\cs_new_eq:Nc \regex_build_Z: { regex_build_$: } %$ -\cs_new_eq:Nc \regex_build_z: { regex_build_$: } %$ % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}[int]{\regex_build_b:} -% \begin{macro}[int]{\regex_build_B:} -% \begin{macro}[aux]{\regex_if_word_boundary:TF} -% Contrarily to |^| and |$|, which could be implemented without -% really knowing what precedes in the token list, this requires -% more information, namely, the knowledge of the last character -% code. Case sensitivity does not change word boundaries. +% \begin{macro}[int]{\regex_class_put:N} +% \begin{macro}[int]{\regex_class_put:NN} +% This function is used to rapidly put a character in a character +% class being built, avoiding some tests implied by \cs{regex_build_one:x}. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_b: +\cs_new_protected:Npn \regex_class_put:N #1 { - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int - { - \exp_not:N \regex_if_word_boundary:TF - { - \regex_action_free:n - { - \int_eval:n - { \l_regex_right_state_int - \l_regex_left_state_int } - } - } - { } - } + \tl_put_right:Nx \l_regex_class_tl + { \regex_item_equal:n { \int_value:w `#1 ~ } } } -\cs_new_protected_nopar:Npn \regex_build_B: - { - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int - { - \exp_not:N \regex_if_word_boundary:TF - { } - { - \regex_action_free:n - { - \int_eval:n - { \l_regex_right_state_int - \l_regex_left_state_int } - } - } - } - } -\cs_new_protected_nopar:Npn \regex_if_word_boundary:TF +\cs_new_protected:Npn \regex_class_put:NN #1#2 { - \group_begin: - \int_set_eq:NN \l_regex_current_char_int \l_regex_last_char_int - \c_regex_w_tl - \regex_break_point:TF - { \group_end: \c_regex_W_tl \regex_item_equal:n { -2 } } - { \group_end: \c_regex_w_tl } - \regex_break_point:TF + \tl_put_right:Nx \l_regex_class_tl + { \regex_item_range:nn { \int_value:w `#1 ~ } { \int_value:w `#2 ~ } } } % \end{macrocode} % \end{macro} % \end{macro} -% \end{macro} -% -% \subsubsection{Normal character, and simple character classes} % -% \begin{macro}[int]{\regex_build_one:n, \regex_build_one:x} +% \begin{macro}[int]{\regex_build_raw_alphanum_is_error:N} +% Within character classes, some special character and escape +% sequences do not have any meaning. Special characters are +% interpreted as their \enquote{raw} counterpart, but escape sequences +% without a meaning, \emph{e.g.}, |\A|, are most likely an error: +% this should be |A|. +% \begin{macrocode} +\cs_new_protected:Npn \regex_build_raw_alphanum_is_error:N #1 + { + \regex_aux_char_if_alphanumeric:NTF #1 + { \msg_kernel_error:nnx { regex } { class-bad-escape } { #1 } } + { \regex_build_raw:N #1 } + } +% \end{macrocode} +% \end{macro} % +% \begin{macro}[int]{\regex_build_raw:N} +% If we are in a character class and the next character is an +% unescaped dash, this denotes a range. Otherwise, the current +% character |#1| matches itself. % \begin{macrocode} -\cs_new_protected:Npn \regex_build_one:n #1 - { \regex_build_one:x { \exp_not:n {#1} } } -\cs_new_protected:Npn \regex_build_one:x #1 +\cs_new_protected:Npn \regex_build_raw:N #1#2#3 { - \cs_set_nopar:Npx \regex_build_tmp_class:n ##1 + \regex_build_if_in_class:TF + { \str_if_eq:nnTF {#2#3} { \regex_build_special:N - } } + { \use_ii:nn } + { \regex_class_range:Nw #1 } { - \exp_not:N \exp_not:n {#1} - \regex_break_point:TF { \regex_action_cost:n {##1} } { } + \regex_build_one:x { \regex_item_equal:n { \int_value:w `#1 ~ } } + #2 #3 } - \regex_build_one_quantifier: } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_default_raw:N} -% A normal alphanumeric or an escaped non-alphanumeric -% (actually, any unknown combination) will match itself -% and the thread will fail otherwise. We prepare -% \cs{regex_build_tmp_class:n} with the relevant test and -% commands. The state shift to be inserted in those -% commands will come as |##1|: we don't know -% yet what this will be before checking for quantifiers. +% \begin{macro}[aux]{\regex_class_range:Nw} +% We have just read a raw character followed by a dash; this should be +% followed by an end-point for the range. If the following character +% is an escaped alphanumeric, or if it is an unescaped right bracket, +% then we have an error, so we put the initial character and the dash +% back, as raw characters. Otherwise, build the range, checking that +% it is in the right order, and optimizing for equal end-points. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_default_raw:N #1 - { \regex_build_one:x { \regex_item_equal:n { \int_value:w `#1 ~ } } } +\cs_new_protected:Npn \regex_class_range:Nw #1#2#3 + { + \token_if_eq_meaning:NNTF #2 \regex_build_escaped:N % [ + { \use_i:nn } { \str_if_eq:nnTF { #2#3 } { \regex_build_special:N ] } } + { + \msg_kernel_warning:nnx { regex } { end-range-missing } % [ + { #1 - \if_meaning:w #3 ] \else: \c_backslash_str \fi: #3 } + \regex_class_put:N #1 + \regex_class_put:N - + #2#3 + } + { + \if_num:w `#1 > `#3 \exp_stop_f: + \msg_kernel_error:nnxx { regex } { backwards-range } {#1} {#3} + \else: + \if_num:w `#1 = `#3 \exp_stop_f: + \regex_class_put:N #3 + \else: + \regex_class_put:NN #1#3 + \fi: + \fi: + } + } % \end{macrocode} % \end{macro} % +% \subsubsection{Character properties} +% % \begin{macro}[aux]{\regex_build_.:} -% Similar to \cs{regex_build_default_raw:N} but accepts any character, -% and refuses $-1$, which marks the end of the token list. +% In a class, the dot has no special meaning. Outside, insert +% \cs{regex_item_dot:}, which matches any character or control +% sequence, and refuses $-2$, which marks the end of the token list. % \begin{macrocode} \cs_new_protected_nopar:cpn { regex_build_.: } { - \cs_set:Npn \regex_build_tmp_class:n ##1 - { \regex_item_dot:T { \regex_action_cost:n {##1} } } - \regex_build_one_quantifier: + \regex_build_if_in_class:TF + { \regex_build_raw:N . } + { \regex_build_one:x \regex_item_dot: } } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_d:,\regex_build_D:} -% \begin{macro}[aux]{\regex_build_h:,\regex_build_H:} -% \begin{macro}[aux]{\regex_build_s:,\regex_build_S:} -% \begin{macro}[aux]{\regex_build_v:,\regex_build_V:} -% \begin{macro}[aux]{\regex_build_w:,\regex_build_W:} -% \begin{macro}[aux]{\regex_build_N:} +% \begin{macro}[aux]{\regex_build_/d:,\regex_build_/D:} +% \begin{macro}[aux]{\regex_build_/h:,\regex_build_/H:} +% \begin{macro}[aux]{\regex_build_/s:,\regex_build_/S:} +% \begin{macro}[aux]{\regex_build_/v:,\regex_build_/V:} +% \begin{macro}[aux]{\regex_build_/w:,\regex_build_/W:} +% \begin{macro}[aux]{\regex_build_/N:} % The constants \cs{c_regex_d_tl}, \emph{etc.} hold % a list of tests which match the corresponding character % class, and jump to the \cs{regex_break_point:TF} marker. @@ -1920,7 +2011,7 @@ % \begin{macrocode} \tl_map_inline:nn { dDhHsSvVwWN } { - \cs_new_protected_nopar:cpx { regex_build_#1: } + \cs_new_protected_nopar:cpx { regex_build_/#1: } { \regex_build_one:n \exp_not:c { c_regex_#1_tl } } } % \end{macrocode} @@ -1931,263 +2022,338 @@ % \end{macro} % \end{macro} % -% \subsubsection{Character classes} +% \subsubsection{Anchoring and simple assertions} % -% \begin{macro}[aux]{\regex_class_one:n, \regex_class_one:x} -% This is the function used to add material -% to the class. +% \begin{macro}[aux]{\regex_build_simple_assertion:Nn} +% Assertions are not allowed within character classes: the raw +% character |#1| is inserted instead. Otherwise, use the test |#2|; +% if the assertion is successful, move from the \texttt{left} state to +% the \texttt{right} state. % \begin{macrocode} -\cs_new_protected:Npn \regex_class_one:n - { \tl_put_right:Nn \l_regex_class_tl } -\cs_new_protected:Npn \regex_class_one:x - { \tl_put_right:Nx \l_regex_class_tl } +\cs_new_protected:Npn \regex_build_simple_assertion:Nn #1#2 + { + \regex_build_if_in_class:TF + { \regex_build_raw_alphanum_is_error:N #1 } + { + \regex_build_new_state: + \regex_toks_put_right:Nx \l_regex_left_state_int + { + \exp_not:n {#2} + { + \regex_action_free:n + { + \int_eval:n + { \l_regex_right_state_int - \l_regex_left_state_int } + } + } + } + %^^A \regex_assertion_quantifier: + } + } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_build_[:} -% This starts a class. The code for the class is collected -% in \cs{l_regex_class_tl}. The first character is special. +% \begin{macro}[aux]+\regex_build_^:+ +% \begin{macro}[aux]{\regex_build_/A:} +% \begin{macro}[aux]{\regex_build_/G:} +% \begin{macro}[aux]+\regex_build_$:+ +% \begin{macro}[aux]{\regex_build_/Z:} +% \begin{macro}[aux]{\regex_build_/z:} +% Anchoring at the start corresponds to checking that the current +% character is the first in the token list. Anchoring to the beginning +% of the match attempt uses \cs{l_regex_start_step_int} instead of +% \cs{c_zero}. End anchors match the end of the token list, marked by +% a character code of $-2$. % \begin{macrocode} -\cs_new_protected_nopar:cpn { regex_build_[: } +\cs_new_protected_nopar:cpn { regex_build_^: } { - \tl_clear:N \l_regex_class_tl - \cs_set_eq:NN \regex_build_raw:N \regex_class_default_raw:N - \tl_set:Nn \l_regex_build_or_class_tl { class } - \regex_class_first:NN + \regex_build_simple_assertion:Nn ^ + { \int_compare:nNnT \l_regex_min_step_int = \l_regex_current_step_int } + } +\cs_new_protected_nopar:cpn { regex_build_/A: } + { + \regex_build_simple_assertion:Nn A + { \int_compare:nNnT \l_regex_min_step_int = \l_regex_current_step_int } + } +\cs_new_protected_nopar:cpn { regex_build_/G: } + { + \regex_build_simple_assertion:Nn G + { \int_compare:nNnT \l_regex_start_step_int = \l_regex_current_step_int } + } +\cs_new_protected_nopar:cpn { regex_build_$: } % $ + { + \regex_build_simple_assertion:Nn $ % $ + { \int_compare:nNnT \l_regex_current_char_int < \c_minus_one } + } +\cs_new_protected_nopar:cpn { regex_build_/Z: } + { + \regex_build_simple_assertion:Nn Z + { \int_compare:nNnT \l_regex_current_char_int < \c_minus_one } + } +\cs_new_protected_nopar:cpn { regex_build_/z: } + { + \regex_build_simple_assertion:Nn z + { \int_compare:nNnT \l_regex_current_char_int < \c_minus_one } } % \end{macrocode} % \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} % -% \begin{macro}[aux]{\regex_class_]:} -% If \texttt{]} appears as the first item of a class, then -% it doesn't end the class. Otherwise, it's the end, -% act just as for a single character, but with a more -% complicated test. And restore \cs{regex_build_raw:N}. +% \begin{macro}[int]{\regex_build_/b:} +% \begin{macro}[int]{\regex_build_/B:} +% \begin{macro}[aux]{\regex_if_word_boundary:TF} +% Contrarily to |^| and |$|, which could be implemented without +% really knowing what precedes in the token list, this requires +% more information, namely, the knowledge of the last character +% code. Case sensitivity does not change word boundaries. % \begin{macrocode} -\cs_new_protected_nopar:cpn { regex_class_]: } +\cs_new_protected_nopar:cpn { regex_build_/b: } { - \tl_if_empty:NTF \l_regex_class_tl %[ - { \regex_class_default_raw:N ] } - { - \cs_set_eq:NN \regex_build_raw:N \regex_build_default_raw:N - \tl_set:Nn \l_regex_build_or_class_tl { build } - \cs_set:Npn \regex_build_tmp_class:n ##1 - { - \exp_not:o \l_regex_class_tl - \bool_if:NTF \l_regex_class_bool - { \regex_break_point:TF { \regex_action_cost:n {##1} } { } } - { \regex_break_point:TF { } { \regex_action_cost:n {##1} } } - } - \regex_build_one_quantifier: - } + \regex_build_simple_assertion:Nn b + { \regex_if_word_boundary:TF \use:n \use_none:n } + } +\cs_new_protected_nopar:cpn { regex_build_/B: } + { + \regex_build_simple_assertion:Nn B + { \regex_if_word_boundary:TF { } } + } +\cs_new_protected_nopar:Npn \regex_if_word_boundary:TF + { + \group_begin: + \int_set_eq:NN \l_regex_current_char_int \l_regex_last_char_int + \c_regex_w_tl + \regex_break_point:TF + { \group_end: \c_regex_W_tl \regex_item_equal:n { -2 } } + { \group_end: \c_regex_w_tl } + \regex_break_point:TF } % \end{macrocode} % \end{macro} +% \end{macro} +% \end{macro} % -% \begin{macro}[aux]{\regex_class_first:NN} -% If the first non-space character is |^|, then the class is inverted. -% We keep track of this in \cs{l_regex_class_bool}. +% \subsubsection{Entering and exiting character classes} +% +% \begin{macro}[aux]{\regex_build_[:} +% In a class, left brackets mean nothing. Outside a class, this starts +% a class, whose first characters may have a special meaning. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_class_first:NN #1#2 +\cs_new_protected_nopar:cpn { regex_build_[: } { - \str_if_eq:nnTF {#1#2} { \regex_build_control:N ^ } - { \bool_set_false:N \l_regex_class_bool } - { - \bool_set_true:N \l_regex_class_bool - #1 #2 - } + \regex_build_if_in_class:TF + { \regex_build_raw:N [ } + { \regex_class_first:NNNN } } % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\regex_class_default_raw:N} -% \begin{macro}[aux]{\regex_class_single:N} -% Most characters are treated here. We look ahead for an unescaped dash. -% If there is none, then the character matches itself. +% \begin{macro}[aux]{\regex_build_]:} +% Outside a class, right brackets have no meaning. In a class, change +% the mode ($m\to (m-15)/13$, truncated) to reflect the fact that we +% are leaving the class. If we are still in a class after leaving one, +% then this is the case |[...\cL[...]...]|, and we insert the relevant +% closing material in \cs{l_regex_class_tl}. Otherwise look for +% quantifiers. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_class_default_raw:N #1#2#3 +\cs_new_protected:cpn { regex_build_]: } { - \str_if_eq:nnTF {#2#3} { \regex_build_control:N - } - { \regex_class_range:Nw #1 } + \regex_build_if_in_class:TF { - \regex_class_single:N #1 - #2 #3 + \if_num:w \l_regex_build_mode_int > \c_sixteen + \tl_set:Nx \l_regex_class_tl + { + \exp_not:o \l_regex_class_saved_tl + \if_num:w \l_regex_catcodes_int < \c_regex_catcodes_all_int + \regex_item_catcode:nT { \int_use:N \l_regex_catcodes_int } + \else: + \exp_after:wN \use:n + \fi: + { + \exp_not:o \l_regex_class_tl + \bool_if:NF \l_regex_class_bool + { \regex_break_point:TF { } { \regex_break_true:w } } + } + } + \bool_set_eq:NN \l_regex_class_bool \l_regex_class_saved_bool + \fi: + \tex_advance:D \l_regex_build_mode_int - \c_fifteen + \tex_divide:D \l_regex_build_mode_int \c_thirteen + \if_int_odd:w \l_regex_build_mode_int \else: + \exp_after:wN \regex_build_one_quantifier: + \fi: } + { \regex_build_raw:N ] } } -\cs_new_protected_nopar:Npn \regex_class_single:N #1 - { \regex_class_one:x { \regex_item_equal:n { \int_value:w `#1 } } } % \end{macrocode} % \end{macro} -% \end{macro} % -% \begin{macro}[aux]{\regex_class_range:Nw} -% \begin{macro}[aux]{\regex_class_range_put:NN} -% If the character is followed by a dash, we look for -% the end-point of the range. For \enquote{raw} characters, -% that's simply |#3|. Most \enquote{control} characters also -% have no meaning, and can serve as an end-point, but those -% with a meaning interrupt the range. -% In the case of a true range, check whether the end-points -% are in the right order, and optimize in the case of equal -% end-points. -% \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_class_range:Nw #1#2#3 - { - \token_if_eq_meaning:NNTF #2 \regex_build_control:N +%^^A todo: ignore spaces everywhere!! +% +% \begin{macro}[aux]{\regex_class_first:NNNN} +% This starts a class. Change the mode by appending $3$ to it, and +% reset the variables \cs{l_regex_class_tl} and +% \cs{l_regex_bool_tl}. In the special case of mode $63$ +% (|[\c[...]]|), we open a group, to avoid overriding the setting of +% \cs{l_regex_class_bool} and \cs{l_regex_class_tl}; the group ends at +% the matching right bracket. If the first character is |^|, then the +% class is inverted. We keep track of this in +% \cs{l_regex_class_bool}. If the next character is a right bracket, +% then it should be changed to a raw one (dirty hack here; the |F| +% argument of \cs{str_if_eq:nnTF} is the trailing |#3|). +% \begin{macrocode} +\cs_new_protected:Npn \regex_class_first:NNNN #1#2#3#4 + { + \l_regex_build_mode_int = \int_value:w \l_regex_build_mode_int 3 ~ % + \if_num:w \l_regex_build_mode_int > \c_sixteen + \tl_set_eq:NN \l_regex_class_saved_tl \l_regex_class_tl + \bool_set_eq:NN \l_regex_class_saved_bool \l_regex_class_bool + \fi: + \token_if_eq_meaning:NNTF #1 \regex_build_special:N { - \cs_if_exist:cTF { regex_class_#3: } + \token_if_eq_charcode:NNTF #2 ^ { - \regex_class_single:N #1 - \regex_class_single:N - - #2#3 + \bool_set_false:N \l_regex_class_bool % [ + \str_if_eq:nnTF {#3#4} { \regex_build_special:N ] } + { \regex_build_raw:N } + } + { % [ + \token_if_eq_charcode:NNTF #2 ] + { \regex_build_raw:N #2 } + { #1 #2 } } - { \regex_class_range_put:NN #1#3 } } - { \regex_class_range_put:NN #1#3 } - } -\cs_new_protected_nopar:Npn \regex_class_range_put:NN #1#2 - { - \if_num:w `#1 > `#2 \exp_stop_f: - \msg_kernel_error:nnxx { regex } { backwards-range } {#1} {#2} - \else: - \regex_class_one:x - { - \if_num:w `#1 = `#2 \exp_stop_f: - \regex_item_equal:n - \else: - \regex_item_range:nn { \int_value:w `#1 } - \fi: - { \int_value:w `#2 } - } - \fi: + { #1 #2 } + #3 #4 } % \end{macrocode} % \end{macro} -% \end{macro} % -% \begin{macro}[aux]{\regex_class_d:,\regex_class_D:} -% \begin{macro}[aux]{\regex_class_h:,\regex_class_H:} -% \begin{macro}[aux]{\regex_class_s:,\regex_class_S:} -% \begin{macro}[aux]{\regex_class_v:,\regex_class_V:} -% \begin{macro}[aux]{\regex_class_w:,\regex_class_W:} -% Similar to \cs{regex_class_single:N}, adding the appropriate -% ranges of characters to the class. The token lists are not -% expanded because it is more memory efficient, with a tiny -% overhead on execution. +% \subsubsection{Catcodes and csnames} +% +% \begin{macro}[aux]{\regex_build_/c:} +% The |\c| escape sequence is only allowed in modes $0$ and $3$, +% \emph{i.e.}, not within any other |\c| escape sequence. % \begin{macrocode} -\tl_map_inline:nn { dDhHsSvVwWN } +\cs_new_protected:cpn { regex_build_/c: } { - \cs_new_protected_nopar:cpx { regex_class_#1: } - { \regex_class_one:n \exp_not:c { c_regex_#1_tl } } + \if_case:w \l_regex_build_mode_int + \exp_after:wN \regex_build_c_aux:wNN + \or: \or: \or: \exp_after:wN \regex_build_c_aux:wNN + \fi: + \msg_kernel_error:nn { regex } { c-bad-mode } + \s_stop } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% -% \subsubsection{Catcodes and csnames} -% -% \begin{macro}[aux]{\regex_build_c:} -% \begin{macro}[aux]{\regex_build_C:} -% \begin{macro}[aux]{\regex_class_c:} -% \begin{macro}[aux]{\regex_class_C:} -% \begin{macro}[aux]{\regex_aux_c:nnNN} % +% \begin{macro}[aux]{\regex_build_c_aux:wNN} +% The |\c| escape sequence can be followed by a capital letter +% representing a character category, by a left bracket which starts a +% list of categories, or by a brace group holding a regular expression +% for a control sequence name. Otherwise, raise an error. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_c: - { \regex_aux_c:NNNN \regex_item_catcode:n \use_i:nn } -\cs_new_protected_nopar:Npn \regex_build_C: - { \regex_aux_c:NNNN \regex_item_catcode_neq:n \use_ii:nn } -\cs_new_eq:NN \regex_class_c: \regex_build_c: -\cs_new_eq:NN \regex_class_C: \regex_build_C: -\cs_new_protected:Npn \regex_aux_c:NNNN #1#2#3#4 +\cs_new_protected:Npn \regex_build_c_aux:wNN #1 \s_stop #2#3 { - \token_if_eq_meaning:NNTF #3 \regex_build_raw:N + \token_if_eq_meaning:NNTF #2 \regex_build_raw:N { - \regex_aux_c_test:NTF #4 + \cs_if_exist:cTF { c_regex_catcode_#3_int } { - \use:c { regex_ \l_regex_build_or_class_tl _one:x } - { #1 { \int_use:N \l_regex_current_catcode_int } } + \int_set_eq:Nc \l_regex_catcodes_int { c_regex_catcode_#3_int } + \l_regex_build_mode_int + = \if_case:w \l_regex_build_mode_int \c_two \else: \c_six \fi: } + } + { \cs_if_exist_use:cF { regex_build_c_#3: } } { - \msg_kernel_error:nnx { regex } { build-c } { #4 } - #3 #4 + \msg_kernel_error:nnxx { regex } { c-command } + { regular~expression } { #3 } + #2 #3 } - } - { - \if_charcode:w \c_lbrace_str \exp_not:N #4 - \exp_after:wN #2 - \else: - \exp_after:wN \use_ii:nn - \fi: - { - \group_begin: - \bool_set_true:N \l_regex_in_csname_bool - \regex_build:w - } - { - \msg_kernel_error:nnx { regex } { build-c } { #4 } - #3 #4 - } - } } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} +% +% \begin{macro}+\regex_build_c_{:+ ^^A } +% The case of a left brace is easy, based on what we have done so far: +% in a group, build the regular expression, after changing the mode to +% forbid nesting |\c|. +% \begin{macrocode} +\cs_new_protected:cpn { regex_build_c_ \c_lbrace_str : } + { + \group_begin: + \l_regex_build_mode_int + = - \if_case:w \l_regex_build_mode_int \c_two \else: \c_six \fi: + \regex_build:w + } +% \end{macrocode} % \end{macro} % -% The |\c| and |\C| escape sequences are used both in the -% regex and the replacement texts. They can be followed either -% by a capital letter representing a category code, or by a left -% brace (in the case of |\c| only). +% \begin{macro}[aux]{\regex_build_c_[:} ^^A] +% \begin{macro}[aux]{\regex_build_c_lbrack_loop:NN} +% \begin{macro}[aux]{\regex_build_c_lbrack_end:} +% \begin{macro}[aux]{\regex_build_add:N} % -% \begin{macro}{\regex_aux_c_test:NTF} -% If the first argument is a known catcode identifier -% (capital letters listed below), then -% \cs{l_regex_current_catcode_int} is set to the relevant -% catcode value (between $0$ for cs and $13$ for active chars), -% and the \texttt{true} branch is taken. -% Otherwise, the \texttt{false} branch is taken, -% and the current catcode is set to $-1$. % \begin{macrocode} -\cs_new_protected:Npn \regex_aux_c_test:NTF #1 +\cs_new_protected:cpn { regex_build_c_[: } #1#2 { - \int_set:Nn \l_regex_current_catcode_int + \int_zero:N \l_regex_catcodes_int + \str_if_eq:nnTF { #1 #2 } { \regex_build_special:N ^ } { - \if_case:w \int_eval:w `#1 - `A \int_eval_end: - \c_thirteen % Active - \or: \c_one % Begin-group - \or: \c_zero % Control sequence - \or: \c_eight % Down (subscript) - \or: \c_two % End-group - \else: - \if_case:w \int_eval:w `#1 - `L \int_eval_end: - \c_eleven % Letter - \or: \c_three % Math toggle - \or: \c_minus_one - \or: \c_twelve % Other - \or: \c_six % Parameter - \or: \c_minus_one - \or: \c_minus_one - \or: \c_ten % Space - \or: \c_four % Tab (alignment) - \or: \c_seven % Up (superscript) - \else: \c_minus_one - \fi: - \fi: + \bool_set_false:N \l_regex_catcodes_bool + \regex_build_c_lbrack_loop:NN } - \if_num:w \l_regex_current_catcode_int = \c_minus_one - \exp_after:wN \use_ii:nn + { + \bool_set_true:N \l_regex_catcodes_bool + \regex_build_c_lbrack_loop:NN + #1#2 + } + } +\cs_new_protected:Npn \regex_build_c_lbrack_loop:NN #1#2 + { + \token_if_eq_meaning:NNTF #1 \regex_build_raw:N + { + \cs_if_exist:cTF { c_regex_catcode_#2_int } + { + \exp_args:Nc \regex_build_c_lbrack_add:N + { c_regex_catcode_#2_int } + \regex_build_c_lbrack_loop:NN + } + } + { % [ + \token_if_eq_charcode:NNTF #2 ] + { \regex_build_c_lbrack_end: } + } + { + \msg_kernel_error:nnxx { regex } { c-command } + { regular-expression } { #2 } + \regex_build_c_lbrack_end: + } + } +\cs_new_protected_nopar:Npn \regex_build_c_lbrack_end: + { + \l_regex_build_mode_int + = \if_case:w \l_regex_build_mode_int \c_two \else: \c_six \fi: + \if_meaning:w \c_false_bool \l_regex_catcodes_bool + \int_set:Nn \l_regex_catcodes_int + { \c_regex_catcodes_all_int - \l_regex_catcodes_int } + \fi: + } +\cs_new_protected:Npn \regex_build_c_lbrack_add:N #1 + { + \if_int_odd:w \int_eval:w \l_regex_catcodes_int / #1 \int_eval_end: \else: - \exp_after:wN \use_i:nn + \tex_advance:D \l_regex_catcodes_int #1 \fi: } % \end{macrocode} % \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} % % ^^A{ % \begin{macro}+\regex_build}:+ @@ -2196,18 +2362,18 @@ % replace the brace with an escaped brace. % \begin{macrocode} \cs_new_protected:cpn { regex_build_ \c_rbrace_str : } - { - \bool_if:NTF \l_regex_in_csname_bool + { %^^A incorrect recovery for \c{...(...[...} with missing ) or ]. + \int_compare:nNnTF \l_regex_build_mode_int < \c_zero { - \regex_build_Z: + \use:c { regex_build_/Z: } \regex_toks_put_right:Nx \l_regex_right_state_int { \regex_action_success: } \int_incr:N \l_regex_capturing_group_int \use:x { \group_end: - \exp_not:n { \use:c { regex_ \l_regex_build_or_class_tl _one:n } } - { \regex_item_cs:n { \regex_set_aux:N ? } } + \regex_build_one:n + { \regex_item_cs:n { \regex_set_aux:N ? } } } } { \exp_after:wN \regex_build_raw:N \c_rbrace_str } @@ -2223,18 +2389,18 @@ % When all characters for the quantifier are found, the corresponding % function is called. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_quantifier:w #1#2 +\cs_new_protected:Npn \regex_build_quantifier:w #1#2 { - \token_if_eq_meaning:NNTF #1 \regex_build_control:N + \token_if_eq_meaning:NNTF #1 \regex_build_special:N { \cs_if_exist_use:cF { regex_build_quantifier_#2:w } { - \regex_build_quantifier_end:n { } + \regex_build_quantifier_end:nn { } { } #1 #2 } } { - \regex_build_quantifier_end:n { } + \regex_build_quantifier_end:nn { } { } #1 #2 } } @@ -2266,12 +2432,12 @@ % arguments that the final function needs in the case of braced % constructions, and is empty otherwise. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_quantifier_aux:nnNN #1#2#3#4 +\cs_new_protected:Npn \regex_build_quantifier_aux:nnNN #1#2#3#4 { - \str_if_eq:nnTF { #3 #4 } { \regex_build_control:N ? } - { \regex_build_quantifier_end:n { #2 #4 } #1 } + \str_if_eq:nnTF { #3 #4 } { \regex_build_special:N ? } + { \regex_build_quantifier_end:nn { #2 #4 } {#1} } { - \regex_build_quantifier_end:n { #2 } #1 + \regex_build_quantifier_end:nn { #2 } {#1} #3 #4 } } @@ -2287,44 +2453,44 @@ % \begin{macrocode} \cs_new_protected_nopar:cpn { regex_build_quantifier_ \c_lbrace_str :w } { \regex_get_digits:nw { \regex_build_quantifier_lbrace:n } } -\cs_new_protected_nopar:Npn \regex_build_quantifier_lbrace:n #1 +\cs_new_protected:Npn \regex_build_quantifier_lbrace:n #1 { \tl_if_empty:nTF {#1} { - \regex_build_quantifier_end:n { } + \regex_build_quantifier_end:nn { } { } \exp_after:wN \regex_build_raw:N \c_lbrace_str } { \regex_build_quantifier_lbrace:nw {#1} } } -\cs_new_protected_nopar:Npx \regex_build_quantifier_lbrace:nw #1#2#3 +\cs_new_protected:Npx \regex_build_quantifier_lbrace:nw #1#2#3 { \exp_not:N \prg_case_str:nnn { #2 #3 } { - { \exp_not:N \regex_build_control:N , } + { \exp_not:N \regex_build_special:N , } { \exp_not:N \regex_get_digits:nw { \exp_not:N \regex_build_quantifier_lbrace:nnw {#1} } } - { \exp_not:N \regex_build_control:N \c_rbrace_str } - { \exp_not:N \regex_build_quantifier_end:n {n} {#1} } + { \exp_not:N \regex_build_special:N \c_rbrace_str } + { \exp_not:N \regex_build_quantifier_end:nn {n} { {#1} } } } { - \exp_not:N \regex_build_quantifier_end:n { } + \exp_not:N \regex_build_quantifier_end:nn { } { } \exp_not:N \regex_build_raw:N \c_lbrace_str #1#2 } } -\cs_new_protected_nopar:Npn \regex_build_quantifier_lbrace:nnw #1#2#3#4 +\cs_new_protected:Npn \regex_build_quantifier_lbrace:nnw #1#2#3#4 { \str_if_eq:xxTF { \exp_not:n {#3#4} } - { \exp_not:N \regex_build_control:N \c_rbrace_str } + { \exp_not:N \regex_build_special:N \c_rbrace_str } { \tl_if_empty:nTF {#2} { \regex_build_quantifier_aux:nnNN { {#1} } { n* } } { \regex_build_quantifier_aux:nnNN { {#1} {#2} } { nn } } } %^^A todo: catch the case m>n { - \regex_build_quantifier_end:n { } + \regex_build_quantifier_end:nn { } { } \use:x { \exp_args:No \tl_map_function:nN @@ -2340,12 +2506,17 @@ % \end{macro} % \end{macro} % -% \begin{macro}[aux]{\regex_build_quantifier_end:n} +% \begin{macro}[aux]{\regex_build_quantifier_end:nn} % When all quantifiers are found, we will call the relevant % \cs{regex_build_one/group_\meta{quantifiers}:} function. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_quantifier_end:n #1 - { \use:c { regex_build_ \l_regex_one_or_group_tl _ #1 : } } +\cs_new_protected:Npn \regex_build_quantifier_end:nn #1#2 + { + \use:c { regex_build_ \l_regex_one_or_group_tl _ #1 : } #2 + \tl_clear:N \l_regex_class_tl + \bool_set_true:N \l_regex_class_bool + \int_set_eq:NN \l_regex_catcodes_int \l_regex_catcodes_default_int + } % \end{macrocode} % \end{macro} % @@ -2372,7 +2543,7 @@ % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_build_one_: { - \regex_build_transition_aux:NN + \regex_build_transition:NN \regex_build_tmp_class:n \l_regex_right_state_int } % \end{macrocode} @@ -2387,13 +2558,13 @@ % \begin{macrocode} \cs_new_protected_nopar:cpn { regex_build_one_?: } { - \regex_build_transitions_aux:NNNN + \regex_build_transitions:NNNN \regex_build_tmp_class:n \l_regex_right_state_int \regex_action_free:n \l_regex_right_state_int } \cs_new_protected_nopar:cpn { regex_build_one_??: } { - \regex_build_transitions_aux:NNNN + \regex_build_transitions:NNNN \regex_action_free:n \l_regex_right_state_int \regex_build_tmp_class:n \l_regex_right_state_int } @@ -2408,13 +2579,13 @@ % \begin{macrocode} \cs_new_protected_nopar:cpn { regex_build_one_*: } { - \regex_build_transitions_aux:NNNN + \regex_build_transitions:NNNN \regex_build_tmp_class:n \l_regex_left_state_int \regex_action_free:n \l_regex_right_state_int } \cs_new_protected_nopar:cpn { regex_build_one_*?: } { - \regex_build_transitions_aux:NNNN + \regex_build_transitions:NNNN \regex_action_free:n \l_regex_right_state_int \regex_build_tmp_class:n \l_regex_left_state_int } @@ -2433,7 +2604,7 @@ { \regex_build_one_: \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int - \regex_build_transitions_aux:NNNN + \regex_build_transitions:NNNN \regex_action_free:n \l_regex_tmpa_int \regex_action_free:n \l_regex_right_state_int } @@ -2441,7 +2612,7 @@ { \regex_build_one_: \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int - \regex_build_transitions_aux:NNNN + \regex_build_transitions:NNNN \regex_action_free:n \l_regex_right_state_int \regex_action_free:n \l_regex_tmpa_int } @@ -2456,7 +2627,7 @@ % are identical, since the number of repetitions is fixed. % Simply repeat |#1| times the effect of \cs{regex_build_one_:}. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_one_n: #1 +\cs_new_protected:Npn \regex_build_one_n: #1 { \prg_replicate:nn {#1} { \regex_build_one_: } } \cs_new_eq:cN { regex_build_one_n?: } \regex_build_one_n: % \end{macrocode} @@ -2468,12 +2639,12 @@ % This function is called in case the syntax is % \texttt{\{\meta{int},\}}. % \begin{macrocode} -\cs_new_protected_nopar:cpx { regex_build_one_n*: } #1 +\cs_new_protected:cpx { regex_build_one_n*: } #1 { \exp_not:N \prg_replicate:nn {#1} { \exp_not:N \regex_build_one_: } \exp_not:c { regex_build_one_*: } } -\cs_new_protected_nopar:cpx { regex_build_one_n*?: } #1 +\cs_new_protected:cpx { regex_build_one_n*?: } #1 { \exp_not:N \prg_replicate:nn {#1} { \exp_not:N \regex_build_one_: } \exp_not:c { regex_build_one_*?: } @@ -2488,7 +2659,7 @@ % This function is called when the syntax is % \texttt{\{\meta{int},\meta{int}\}}. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_one_nn_aux:Nnn #1#2#3 +\cs_new_protected:Npn \regex_build_one_nn_aux:Nnn #1#2#3 { \prg_replicate:nn {#2} { \regex_build_one_: } \prg_replicate:nn {#3-#2} {#1} @@ -2524,17 +2695,24 @@ % vertex to a new vertex. % \end{itemize} % \begin{macrocode} -\cs_new_protected_nopar:cpn { regex_build_(: } #1#2 +\cs_new_protected:cpn { regex_build_(: } #1#2 { - \str_if_eq:nnTF { #1 #2 } { \regex_build_control:N ? } - { \regex_build_special_group:NN } + \regex_build_if_in_class:TF { - \int_incr:N \l_regex_capturing_group_int - \regex_seq_push_int:NN - \l_regex_capturing_group_seq \l_regex_capturing_group_int - \regex_build_open_aux: + \regex_build_raw:N ( #1 #2 } + { + \str_if_eq:nnTF { #1 #2 } { \regex_build_special:N ? } + { \regex_build_special_group:NN } + { + \int_incr:N \l_regex_capturing_group_int + \regex_seq_push_int:NN + \l_regex_capturing_group_seq \l_regex_capturing_group_int + \regex_build_open_aux: + #1 #2 + } + } } \cs_new_protected_nopar:Npn \regex_build_open_aux: { @@ -2549,16 +2727,22 @@ } \cs_new_protected_nopar:cpn { regex_build_|: } { - \regex_build_end_alternation: - \regex_build_begin_alternation: + \regex_build_if_in_class:TF { \regex_build_raw:N | } + { + \regex_build_end_alternation: + \regex_build_begin_alternation: + } } \cs_new_protected_nopar:cpn { regex_build_): } { - \seq_if_empty:NTF \l_regex_capturing_group_seq - { \msg_kernel_error:nn { regex } { extra-rparen } } + \regex_build_if_in_class:TF { \regex_build_raw:N ) } { - \regex_build_close_aux: - \regex_build_group_quantifier: + \seq_if_empty:NTF \l_regex_capturing_group_seq + { \msg_kernel_error:nn { regex } { extra-rparen } } + { + \regex_build_close_aux: + \regex_build_group_quantifier: + } } } \cs_new_protected_nopar:Npn \regex_build_close_aux: @@ -2610,13 +2794,13 @@ % is known, then use that. Otherwise, treat the question mark % as if it had been escaped. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_special_group:NN #1#2 +\cs_new_protected:Npn \regex_build_special_group:NN #1#2 { \cs_if_exist_use:cF { regex_build_special_group_\token_to_str:N #2 : } { \msg_kernel_error:nnx { regex } { unsupported } { (? \token_to_str:N #2 } %) - \regex_build_control:N ( % ) + \regex_build_special:N ( % ) \regex_build_raw:N ? #1 #2 } @@ -2699,7 +2883,7 @@ { \regex_build_options:NNN - } -\cs_new_protected_nopar:Npn \regex_build_options:NNN #1#2#3 +\cs_new_protected:Npn \regex_build_options:NNN #1#2#3 { \token_if_eq_meaning:NNTF \regex_build_raw:N #2 { @@ -2752,7 +2936,7 @@ % Once the quantifier is found by \cs{regex_build_quantifier:w}, % we insert the code for tracking submatches. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_group_submatches:NN #1#2 +\cs_new_protected:Npn \regex_build_group_submatches:NN #1#2 { \seq_pop:NN \l_regex_capturing_group_seq \l_regex_tmpa_tl \int_compare:nNnF { \l_regex_tmpa_tl } < \c_zero @@ -2774,7 +2958,7 @@ { \regex_build_group_submatches:NN \l_regex_left_state_int \l_regex_right_state_int - \regex_build_transition_aux:NN + \regex_build_transition:NN \regex_action_free:n \l_regex_right_state_int } % \end{macrocode} @@ -2785,7 +2969,7 @@ % This is done by shifting the current contents of the \cs{tex_toks:D} % \cs{l_regex_tmpa_int} to a new state. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_group_shift:N #1 +\cs_new_protected:Npn \regex_build_group_shift:N #1 { \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int \regex_build_new_state: @@ -2827,11 +3011,11 @@ % operators, and higher priority (\texttt{put_left}) for % lazy operators. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_group_qs_aux:NN #1#2 +\cs_new_protected:Npn \regex_build_group_qs_aux:NN #1#2 { \regex_build_group_shift:N \regex_action_free:n \int_set_eq:NN \l_regex_right_state_int \l_regex_left_state_int - \regex_build_transition_aux:NN \regex_action_free:n #1 + \regex_build_transition:NN \regex_action_free:n #1 #2 \l_regex_tmpa_int { \regex_action_free:n @@ -2876,7 +3060,7 @@ \regex_build_group_submatches:NN \l_regex_left_state_int \l_regex_right_state_int \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int - \regex_build_transitions_aux:NNNN + \regex_build_transitions:NNNN \regex_action_free:n \l_regex_tmpa_int \regex_action_free:n \l_regex_right_state_int } @@ -2885,7 +3069,7 @@ \regex_build_group_submatches:NN \l_regex_left_state_int \l_regex_right_state_int \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int - \regex_build_transitions_aux:NNNN + \regex_build_transitions:NNNN \regex_action_free:n \l_regex_right_state_int \regex_action_free:n \l_regex_tmpa_int } @@ -2905,7 +3089,7 @@ % \cs{l_regex_max_state_int} is set appropriately to the largest % allocated \tn{toks} register. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_group_n_aux:n #1 +\cs_new_protected:Npn \regex_build_group_n_aux:n #1 { \regex_toks_put_right:Nx \l_regex_right_state_int { @@ -2939,10 +3123,10 @@ % are identical, since the number of repetitions is fixed. % We only record the submatch information at the last repetition. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_group_n: #1 +\cs_new_protected:Npn \regex_build_group_n: #1 { % ^^A todo: catch case #1 <= 0. \regex_build_group_n_aux:n {#1} - \regex_build_transition_aux:NN + \regex_build_transition:NN \regex_action_free:n \l_regex_right_state_int \regex_build_group_submatches:NN \l_regex_tmpa_int \l_regex_left_state_int @@ -2962,19 +3146,19 @@ % case no iteration was made, here, we know that the submatch % information is overwritten in any case. % \begin{macrocode} -\cs_new_protected_nopar:cpn { regex_build_group_n*: } #1 +\cs_new_protected:cpn { regex_build_group_n*: } #1 { % ^^A todo: catch case #1 <= 0. \regex_build_group_n_aux:n {#1} - \regex_build_transitions_aux:NNNN + \regex_build_transitions:NNNN \regex_action_free:n \l_regex_tmpa_int \regex_action_free:n \l_regex_right_state_int \regex_build_group_submatches:NN \l_regex_tmpa_int \l_regex_left_state_int } -\cs_new_protected_nopar:cpn { regex_build_group_n*?: } #1 +\cs_new_protected:cpn { regex_build_group_n*?: } #1 { % ^^A todo: catch case #1 <= 0. \regex_build_group_n_aux:n {#1} - \regex_build_transitions_aux:NNNN + \regex_build_transitions:NNNN \regex_action_free:n \l_regex_right_state_int \regex_action_free:n \l_regex_tmpa_int \regex_build_group_submatches:NN @@ -2989,12 +3173,12 @@ % These functions are called when the syntax is either % \texttt{\{\meta{int},\}} or \texttt{\{\meta{int},\meta{int}\}}. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_build_group_nn: #1#2 +\cs_new_protected:Npn \regex_build_group_nn: #1#2 { % ^^A Not Implemented Yet! \msg_expandable_error:n { Quantifier~{m,n}~not~implemented~yet } \use:c { regex_build_group_n*: } {#1} } -\cs_new_protected_nopar:cpn { regex_build_group_nn?: } #1#2 +\cs_new_protected:cpn { regex_build_group_nn?: } #1#2 { % ^^A Not Implemented Yet! \msg_expandable_error:n { Quantifier~{m,n}~not~implemented~yet } \use:c { regex_build_group_n*?: } {#1} @@ -3115,7 +3299,7 @@ { \regex_state_use_aux:n { } } \cs_new_protected_nopar:Npn \regex_state_use: { \regex_state_use_aux:n { \exp_after:wN \use_none_delimit_by_s_stop:w } } -\cs_new_protected_nopar:Npn \regex_state_use_aux:n #1 +\cs_new_protected:Npn \regex_state_use_aux:n #1 { \if_num:w \tex_dimen:D \l_regex_current_state_int < \l_regex_unique_id_int @@ -3227,7 +3411,7 @@ \int_set_eq:NN \l_regex_min_step_int \l_regex_current_step_int \exp_after:wN \regex_query_set_loop:ww \l_regex_tmpa_tl - \q_mark { ? \prg_map_break: } , + \s_tl { ? \prg_map_break: } \s_tl \prg_break_point:n { } \int_set_eq:NN \l_regex_max_step_int \l_regex_current_step_int \regex_query_set:nnn { } { -1 } { -2 } @@ -3235,7 +3419,7 @@ \regex_match_initial_setup: \regex_match_once: } -\cs_new:Npn \regex_query_set_loop:ww #1 \q_mark #2#3 , +\cs_new:Npn \regex_query_set_loop:ww #1 \s_tl #2#3 \s_tl { \use_none:n #2 \regex_query_set:nnn {#1} {"#2} {#3} @@ -3369,7 +3553,7 @@ \fi: \regex_match_loop: } -\cs_new_nopar:Npn \regex_match_one_index:w #1; +\cs_new:Npn \regex_match_one_index:w #1; { \if_num:w #1 > \l_regex_max_index_int \exp_after:wN \prg_map_break: @@ -3379,7 +3563,7 @@ \exp_after:wN \regex_match_one_index:w \int_use:N \int_eval:w #1 + \c_one ; } -\cs_new_protected_nopar:Npn \regex_match_one_index_aux:n #1 +\cs_new_protected:Npn \regex_match_one_index_aux:n #1 { \int_set:Nn \l_regex_current_state_int {#1} \prop_clear:N \l_regex_current_submatches_prop @@ -3420,7 +3604,7 @@ % % \subsubsection{Actions when matching} % -% \begin{macro}[aux]{\regex_action_start_wildcard:nn} +% \begin{macro}[aux]{\regex_action_start_wildcard:} %: the first state has a free transition to the second % state, where the regular expression really begins, and a costly % transition to itself, to try again at the next character. ^^A ?? @@ -3433,12 +3617,12 @@ % this character. For that, we use the boolean % \cs{l_regex_fresh_thread_bool}. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_action_start_wildcard:nn #1#2 +\cs_new_protected_nopar:Npn \regex_action_start_wildcard: { \bool_set_true:N \l_regex_fresh_thread_bool - \regex_action_free:n {#2} + \regex_action_free:n {1} \bool_set_false:N \l_regex_fresh_thread_bool - \regex_action_cost:n {#1} + \regex_action_cost:n {0} } % \end{macrocode} % \end{macro} @@ -3447,7 +3631,7 @@ % A transition which consumes the current character and moves % to state |#1|. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_action_cost:n #1 +\cs_new_protected:Npn \regex_action_cost:n #1 { \exp_args:Nf \regex_store_state:n { \int_eval:n { \l_regex_current_state_int + #1 } } @@ -3488,7 +3672,7 @@ % Then restore the old value of \cs{l_regex_current_state_int} % and of the current submatches. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_action_free:n #1 +\cs_new_protected:Npn \regex_action_free:n #1 { \cs_set_nopar:Npx \regex_tmp:w { @@ -3508,7 +3692,7 @@ % Update the current submatches with the information % from the current step. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_action_submatch:n #1 +\cs_new_protected:Npn \regex_action_submatch:n #1 { \prop_put:Nno \l_regex_current_submatches_prop {#1} { \int_use:N \l_regex_current_step_int } @@ -3552,7 +3736,7 @@ \regex_replacement_raw:N {#1} \prg_do_nothing: \prg_do_nothing: - \cs_set_nopar:Npx \regex_nesting:n ##1 + \cs_set:Npx \regex_nesting:n ##1 { + \int_use:N \l_regex_nesting_int \l_regex_nesting_tl @@ -3560,7 +3744,7 @@ } \use:x { - \exp_not:n { \cs_set_nopar:Npn \regex_replacement_tl:n ##1 } + \exp_not:n { \cs_set:Npn \regex_replacement_tl:n ##1 } { \str_aux_toks_range:nn \c_zero \l_regex_replacement_int } } % ^^A rename! Careful with \cP\#. @@ -3633,7 +3817,8 @@ } \cs_new_protected:Npn \regex_replacement_error:NNN #1#2#3 { - \msg_kernel_error:nn { regex } { replacement-#1 } + \msg_kernel_error:nnxx { regex } { #1-command } + { replacement~text } {#3} #2 #3 } \cs_new_protected:Npn \regex_replacement_g:w #1#2 @@ -3670,7 +3855,7 @@ { \regex_replacement_error:NNN g } #1 #2 } -\cs_new_protected_nopar:Npn \regex_replacement_c:w #1#2 +\cs_new_protected:Npn \regex_replacement_c:w #1#2 { \token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N { @@ -3685,7 +3870,9 @@ \regex_replacement_put:n { \exp_not:n { \exp_after:wN \regex_replacement_exp_not:N \cs:w } } } -\cs_new_nopar:Npn \regex_replacement_exp_not:N #1 { \exp_not:n {#1} } +\cs_new_eq:cc { regex_replacement_c_C:w } + { regex_replacement_c_ \c_lbrace_str :w } +\cs_new:Npn \regex_replacement_exp_not:N #1 { \exp_not:n {#1} } \group_begin: \char_set_catcode_math_superscript:N \^^@ @@ -3696,7 +3883,7 @@ \cs_new_protected_nopar:Npn \regex_replacement_c_T:w { \regex_replacement_char:nNN { ^^@ } } - \cs_new_protected_nopar:Npn \regex_replacement_c_S:w #1#2 + \cs_new_protected:Npn \regex_replacement_c_S:w #1#2 { \int_compare:nNnTF { `#2 } = \c_zero { \regex_replacement_error:NNN c #1#2 } @@ -3708,7 +3895,7 @@ \char_set_catcode_parameter:N \^^@ \cs_new_protected_nopar:Npn \regex_replacement_c_P:w - { \regex_replacement_char:nNN { ^^@^^@ } } + { \regex_replacement_char:nNN { ^^@^^@^^@^^@^^@^^@^^@^^@ } } \char_set_catcode_other:N \^^@ \cs_new_protected_nopar:Npn \regex_replacement_c_O:w @@ -3746,7 +3933,7 @@ \group_end: -\cs_new_protected_nopar:Npn \regex_replacement_char:nNN #1#2#3 +\cs_new_protected:Npn \regex_replacement_char:nNN #1#2#3 { \char_set_lccode:nn \c_zero { `#3 } \tl_to_lowercase:n @@ -3843,7 +4030,7 @@ \regex_set_aux:n \s_stop } -\cs_new_nopar:Npn \regex_set_aux:n #1 +\cs_new:Npn \regex_set_aux:n #1 { \tex_toks:D #1 { \tex_the:D \tex_toks:D #1 } } \cs_new:Npn \regex_nfa:Nw #1 { @@ -3870,13 +4057,13 @@ % token list variables. We do some more processing to start each % line with \cs{tex_toks:D} instead of a single very long line. % \begin{macrocode} -\cs_new_nopar:Npn \regex_to_str:N #1 +\cs_new:Npn \regex_to_str:N #1 { \exp_after:wN \regex_to_str_aux:Nw #1 \tex_toks:D \q_stop \prg_map_break: \tex_toks:D \prg_break_point:n { } } -\cs_new_nopar:Npn \regex_to_str_aux:Nw #1#2\tex_toks:D +\cs_new:Npn \regex_to_str_aux:Nw #1#2\tex_toks:D { \use_none_delimit_by_q_stop:w #2 \q_stop \tl_to_str:n {#1#2} \iow_newline: @@ -4027,12 +4214,12 @@ } \fi: } -\cs_new_protected_nopar:Npn \regex_extract_aux_b:wn #1 < #2 +\cs_new_protected:Npn \regex_extract_aux_b:wn #1 < #2 { \tex_skip:D #1 = #2 sp plus \etex_gluestretch:D \tex_skip:D #1 \scan_stop: } -\cs_new_protected_nopar:Npn \regex_extract_aux_e:wn #1 > #2 +\cs_new_protected:Npn \regex_extract_aux_e:wn #1 > #2 { \tex_skip:D #1 = 1 \tex_skip:D #1 plus #2 sp \scan_stop: @@ -4066,7 +4253,7 @@ \exp_args:NNNo \group_end: \tl_set:Nn #1 \l_regex_tmpa_tl } -\cs_new_nopar:Npn \regex_extract_seq_aux:n #1 +\cs_new:Npn \regex_extract_seq_aux:n #1 { \seq_item:n { @@ -4074,7 +4261,7 @@ \int_value:w \regex_submatch_nesting_aux:n {#1} ; #1; } } -\cs_new_nopar:Npn \regex_extract_seq_aux:ww #1; #2; +\cs_new:Npn \regex_extract_seq_aux:ww #1; #2; { %^^A todo: use flags \if_num:w #1 < \c_zero \flag_raise:N \l_regex_group_end_flag @@ -4293,7 +4480,7 @@ \group_end: \fi: } -\cs_new_protected_nopar:Npn \regex_replace_once_aux:w #1; +\cs_new_protected:Npn \regex_replace_once_aux:w #1; { } % \end{macrocode} @@ -4354,7 +4541,7 @@ \exp_args:NNNo \group_end: \tl_set:Nn #4 \l_regex_tmpa_tl } -\cs_new_nopar:Npn \regex_replace_all_aux:n #1 +\cs_new:Npn \regex_replace_all_aux:n #1 { \regex_query_substr:nn { \etex_glueshrink:D \tex_skip:D #1 } { \tex_skip:D #1 } @@ -4467,12 +4654,12 @@ % \end{macrocode} % % \begin{macrocode} -\msg_kernel_new:nnnn { regex } { build-c } - { Misused~\iow_char:N\\c~or\iow_char:N\\C~command~in~regular~expression. } +\msg_kernel_new:nnnn { regex } { c-command } + { Misused~\iow_char:N\\c~or\iow_char:N\\C~command~in~a~#1. } { - The~\iow_char:N\\C~escape~sequence~can~be~followed~by~ - one~of~the~letters~ABCDELMOPSTU.~The~\iow_char:N\\c~ - escape~sequence~can~also~be~followed~by~a~brace~group. + In~a~#1,~the~\iow_char:N\\C~escape~sequence~ + can~be~followed~by~one~of~the~letters~ABCDELMOPSTU~ + or~a~brace~group,~not~by~'#2'. } \msg_kernel_new:nnnn { regex } { unknown-option } { Unknown~option~`#1'~for~regular~expressions. } @@ -4486,19 +4673,13 @@ The~character~or~escape~sequence~`#1'~is~not~defined~ as~an~option~within~regular~expressions. } -\msg_kernel_new:nnnn { regex } { replacement-g } +\msg_kernel_new:nnnn { regex } { g-command } { Missing~brace~for~the~\iow_char:N\\g~construction~in~a~replacement~text. } { In~the~replacement~text~for~a~regular~expression~search,~ submatches~are~represented~either~as~\iow_char:N \\g{dd..d},~ or~\\d,~where~`d'~are~single~digits.~Here,~a~brace~is~missing. } -\msg_kernel_new:nnnn { regex } { replacement-c } - { Something~is~wrong~with~the~replacement~text. } - { - The~\iow_char:N\\c~escape~sequence~can~only~be~followed~by~ - ABCDELMN~or~by~a~brace. - } % \end{macrocode} % % \begin{macrocode} diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx index a33d4909a1a..cddd7349adc 100644 --- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx @@ -1,6 +1,6 @@ % \iffalse meta-comment % -%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% File: l3str.dtx Copyright (C) 2011-2012 The LaTeX3 Project %% %% It may be distributed and/or modified under the conditions of the %% LaTeX Project Public License (LPPL), either version 1.3c of this @@ -35,7 +35,7 @@ % %<*driver|package> \RequirePackage{expl3} -\GetIdInfo$Id: l3str.dtx 3088 2011-12-23 02:00:14Z bruno $ +\GetIdInfo$Id: l3str.dtx 3159 2012-01-09 00:51:58Z bruno $ {L3 Experimental Strings} % %<*driver> @@ -67,6 +67,7 @@ % % \maketitle % +% \newcommand{\hexnum}[1]{\text{\texttt{\char`\"}#1}} % \begin{documentation} % % \LaTeX3 provides a set of functions to manipulate token lists @@ -106,7 +107,7 @@ % \c_percent_str % } % Constant strings, containing a single character, with category code $12$. -% Other characters can be accessed as \cs{iow_char:N} |\|\meta{character}. +% Any character can be accessed as \cs{iow_char:N} |\|\meta{character}. % \end{variable} % % \begin{function}[EXP]{\tl_to_str:N, \tl_to_str:n} @@ -116,6 +117,16 @@ % \end{syntax} % Converts the \meta{token list} to a \meta{string}, leaving the resulting % tokens in the input stream. +% \begin{texnote} +% The string representation of a token list may depend +% on the category codes in effect when it is measured, +% and the value of the \tn{escapechar}: for instance +% |\tl_to_str:n {\a}| may produce $1$, $2$ or $3$ +% characters depending on the escape character, and +% the category code of \texttt{a}. This impacts almost +% all functions in the module, which use \cs{tl_to_str:n} +% internally. +% \end{texnote} % \end{function} % % \begin{function}{\str_const:Nn, \str_const:Nx, \str_const:cn, \str_const:cx} @@ -191,16 +202,16 @@ % (an arbitrary number of hexadecimal digits are read): % this is mostly useful for Unicode-aware engines. % \item |\a|, |\e|, |\f|, |\n|, |\r|, |\t| stand for specific characters: -% \begin{center} -% \begin{tabular}{cccc} -% |\a| & |\^^G| & alarm & hex \texttt{07} \\ -% |\e| & |\^^[| & escape & hex \texttt{1B} \\ -% |\f| & |\^^L| & form feed & hex \texttt{0C} \\ -% |\n| & |\^^J| & new line & hex \texttt{0A} \\ -% |\r| & |\^^M| & carriage return & hex \texttt{0D} \\ -% |\t| & |\^^I| & horizontal tab & hex \texttt{09} \\ -% \end{tabular} -% \end{center} +% \begin{center} +% \begin{tabular}{cccc} +% |\a| & |\^^G| & alarm & hex \texttt{07} \\ +% |\e| & |\^^[| & escape & hex \texttt{1B} \\ +% |\f| & |\^^L| & form feed & hex \texttt{0C} \\ +% |\n| & |\^^J| & new line & hex \texttt{0A} \\ +% |\r| & |\^^M| & carriage return & hex \texttt{0D} \\ +% |\t| & |\^^I| & horizontal tab & hex \texttt{09} \\ +% \end{tabular} +% \end{center} % \end{itemize} % For instance, % \begin{quote} @@ -226,13 +237,6 @@ % all characters including spaces are counted. % The \cs{str_length_ignore_spaces:n} leaves the number of non-space % characters in the input stream. -% \begin{texnote} -% The \cs{str_length:n} of a given token list may depend -% on the category codes in effect when it is measured, -% and the value of the \tn{escapechar}: for instance -% |\str_length:n {\a}| may return $1$, $2$ or $3$ depending -% on the escape character, and the category code of \texttt{a}. -% \end{texnote} % \end{function} % % \begin{function}[EXP]{\str_head:N, \str_head:n, \str_head_ignore_spaces:n} @@ -262,8 +266,11 @@ % with category codes $12$ and $10$ (for spaces). % The functions differ in the case where the first character % is a space: \cs{str_tail:N} and \cs{str_tail:n} will trim -% only that space, while \cs{str_tail_ignore_spaces:n} trims +% only that space, while \cs{str_tail_ignore_spaces:n} removes % the first non-space character and any space before it. +% If the \meta{token list} is empty (or blank in the case of the +% \texttt{_ignore_spaces} variant), then nothing is left on the +% input stream. % \end{function} % % \begin{function}[EXP]{\str_item:Nn, \str_item:nn, \str_item_ignore_spaces:nn} @@ -432,14 +439,14 @@ % by converting each character in the string \texttt{Hello!} % to the \textsc{utf-16} encoding, and expressing each byte % as a pair of hexadecimal digits. Note the presence of a -% (big-endian) byte order mark \texttt{FEFF}, which can be +% (big-endian) byte order mark \hexnum{FEFF}, which can be % avoided by specifying the encoding \texttt{utf16be/hex}. % % Parts of the \meta{string} which cannot be converted to bytes % using the \meta{escaping 1} are silently ignored (including non-byte % characters which could be present in the string). % Sequences of bytes which are not valid in the \meta{encoding 1} -% are converted to the replacement character |"FFFD|, and raise +% are converted to the replacement character \hexnum{FFFD}, and raise % an error. % Characters which cannot be encoded in the \meta{encoding 2} % are silently ignored. @@ -478,7 +485,7 @@ % ~~\Arg{true code} \Arg{false code} % \end{syntax} % If the \meta{token} is a hexdecimal digit (upper case or lower case), -% its upper-case version is left in the input strem, \emph{followed} +% its upper-case version is left in the input stream, \emph{followed} % by the \meta{true code}. Otherwise, the \meta{false code} is left % in the input stream. % \begin{texnote} @@ -552,6 +559,7 @@ % \begin{macrocode} \ProvidesExplPackage {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription} +\RequirePackage{l3tl-analysis} % \end{macrocode} % % Those string-related functions are defined in \pkg{l3kernel}. @@ -669,7 +677,7 @@ % % \begin{variable}{\c_str_replacement_char_int} % When converting, invalid bytes are replaced by the Unicode -% replacement character \textsc{u-fffd}. +% replacement character \hexnum{FFFD}. % \begin{macrocode} \int_const:Nn \c_str_replacement_char_int { "FFFD } % \end{macrocode} @@ -1431,11 +1439,11 @@ { \exp_after:wN \str_aux_gmap_internal_result_loop:Nww \exp_after:wN #1 - \g_str_result_tl \q_mark \q_stop \prg_map_break: , + \g_str_result_tl \s_tl \q_stop \prg_map_break: \s_tl \prg_break_point:n { } } } -\cs_new_nopar:Npn \str_aux_gmap_internal_result_loop:Nww #1#2\q_mark #3, +\cs_new_nopar:Npn \str_aux_gmap_internal_result_loop:Nww #1 #2 \s_tl #3 \s_tl { \use_none_delimit_by_q_stop:w #3 \q_stop #1 {#3} @@ -1982,19 +1990,20 @@ \use_none_delimit_by_q_stop:w #1 \q_stop \tex_dimen:D "#1 = \l_str_tmpa_int sp \scan_stop: \tex_skip:D \l_str_tmpa_int = "#1 sp \scan_stop: - \tex_toks:D \l_str_tmpa_int \exp_after:wN { \int_value:w "FFFD } + \tex_toks:D \l_str_tmpa_int \exp_after:wN + { \int_use:N \c_str_replacement_char_int } \tex_advance:D \l_str_tmpa_int \c_one \str_convert_from_eight_bit_load_missing:n } \cs_new:Npn \str_convert_from_eight_bit_aux:N #1 { - #1 \exp_not:N \q_mark + #1 \s_tl \if_num:w \tex_dimen:D `#1 < \l_str_tmpa_int \if_num:w \tex_skip:D \tex_dimen:D `#1 = `#1 \exp_stop_f: \tex_the:D \tex_toks:D \tex_dimen:D \fi: \fi: - \int_value:w `#1 , + \int_value:w `#1 \s_tl } % \end{macrocode} % \end{macro} @@ -2031,7 +2040,10 @@ } \cs_new:Npn \str_convert_to_eight_bit_aux:n #1 { - \if_num:w #1 < "8000 \exp_stop_f: + \if_num:w #1 > \c_max_register_int + \msg_expandable_kernel_error:nnn + { str } { eight-bit-to-byte-overflow } {#1} + \else: \if_num:w \tex_dimen:D #1 < \l_str_tmpa_int \if_num:w \tex_skip:D \tex_dimen:D #1 = #1 \exp_stop_f: \tex_the:D \tex_toks:D \tex_dimen:D #1 \exp_stop_f: @@ -2039,9 +2051,6 @@ \fi: \fi: \str_convert_to_eight_bit_aux_ii:n {#1} - \else: - \msg_expandable_kernel_error:nnn - { str } { eight-bit-to-byte-overflow } {#1} \fi: } \cs_new:Npn \str_convert_to_eight_bit_aux_ii:n #1 @@ -2145,7 +2154,7 @@ \cs_new_protected_nopar:Npn \str_convert_from_native: { \str_aux_gmap_result:N \str_convert_from_native_aux:N } \cs_new:Npn \str_convert_from_native_aux:N #1 - { #1 \exp_not:N \q_mark \int_value:w `#1 , } + { #1 \s_tl \int_value:w `#1 \s_tl } % \end{macrocode} % \end{macro} % \end{macro} @@ -2169,10 +2178,10 @@ \str_aux_convert_to_native:n { \exp_after:wN \str_convert_to_native_aux:w - \l_str_tmpa_tl \q_mark { \q_stop \prg_map_break: } , + \l_str_tmpa_tl \s_tl { \q_stop \prg_map_break: } \s_tl } } -\cs_new_protected_nopar:Npn \str_convert_to_native_aux:w #1\q_mark #2, +\cs_new_protected_nopar:Npn \str_convert_to_native_aux:w #1 \s_tl #2 \s_tl { \use_none_delimit_by_q_stop:w #2 \q_stop \str_aux_convert_to_native_step:n {#2} @@ -2294,6 +2303,8 @@ { Overlong~UTF-8~byte~sequence~for~code~point~#1. } \msg_kernel_new:nnn { str } { utf8-premature-end } { Incomplete~last~UTF-8~character. } +\msg_kernel_new:nnn { str } { utf8-overflow } + { Code~point~#1~too~large~(UTF-8). } \msg_kernel_new:nnn { str } { utf32-overflow } { Code~point~too~large~(UTF-32~`#1'). } \msg_kernel_new:nnn { str } { utf32-truncated } @@ -2584,7 +2595,7 @@ % \begin{variable}{\c_str_convert_output_name_not_str} % For each byte, test whether it should % be output as is, or be \enquote{hash-encoded}. -% Roughly, bytes outside the range from |"2A| to |"7E| +% Roughly, bytes outside the range $[\hexnum{2A},\hexnum{7E}]$ % are hash-encoded. We keep two lists of exceptions: % characters in \cs{c_str_convert_output_name_not_str} % are not hash-encoded, and characters in the @@ -2767,22 +2778,21 @@ % \begin{macro}[aux, rEXP] % { % \str_aux_from_utf_viii:N, -% \str_aux_from_utf_viii:wwNN, -% \str_aux_from_utf_viii:wNNww +% \str_aux_from_utf_viii:wwN, +% \str_aux_from_utf_viii:wNnnwN % } % \begin{macro}[aux, rEXP] % { -% \str_aux_from_utf_viii_error:w, -% \str_aux_from_utf_viii_error:NwNN, +% \str_aux_from_utf_viii_overflow:w , % \str_aux_from_utf_viii_error: % } -% In the \textsc{utf-8} encoding, bytes in the range $[0,127]$ stand -% for themselves, bytes in the range $[128,191]$ are continuation bytes, +% In the \textsc{utf-8} encoding, bytes in the range +% $[0,\hexnum{7F}]$ stand for themselves, bytes in the range +% $[\hexnum{80},\hexnum{BF}]$ are continuation bytes, % and larger bytes must be followed by a number of continuation bytes. % Documentation to come. % ^^A todo: refuse "D800 -- "DFFF? % ^^A todo: document -% ^^A todo: what error recovery for code points > "10FFFF? % \begin{macrocode} \cs_new_protected_nopar:cpn { str_convert_from_utf8: } { @@ -2797,7 +2807,7 @@ { #1 \if_num:w `#1 < "C0 \exp_stop_f: - \exp_not:N \q_mark + \s_tl \if_num:w `#1 < "80 \exp_stop_f: \int_value:w `#1 \else: @@ -2807,71 +2817,73 @@ \fi: \else: \if_num:w `#1 < "F5 \exp_stop_f: - \exp_after:wN \str_aux_from_utf_viii:wwNN - \int_use:N \int_eval:w `#1 - "C0 + \exp_after:wN \str_aux_from_utf_viii:wwN + \int_value:w \int_eval:n { `#1 - "C0 } \else: - \exp_not:N \q_mark + \s_tl \msg_expandable_kernel_error:nnn { str } { utf8-invalid-byte } {#1} \int_use:N \c_str_replacement_char_int \fi: \fi: - , - \use_none_delimit_by_q_stop:w - "80 , "800 , "10000 , "10FFFF , - \q_stop + \s_tl + \use_none_delimit_by_q_stop:w {"80} {"800} {"10000} {"110000} \q_stop \str_aux_from_utf_viii:N } -\cs_new_nopar:Npn \str_aux_from_utf_viii:wwNN #1 , #2 \q_stop #3#4 +\cs_new_nopar:Npn \str_aux_from_utf_viii:wwN + #1 \s_tl #2 \str_aux_from_utf_viii:N #3 { - \use_none:n #4 - \exp_after:wN \str_aux_from_utf_viii:wNNww - \int_use:N \int_eval:w #1 * "40 + `#4 - "80 , - #4 - #2 - \q_stop #3 + \use_none:n #3 + \if_num:w \int_eval:w ( `#3 + "20 ) / "40 = \c_three + #3 + \exp_after:wN \str_aux_from_utf_viii:wNnnwN + \int_value:w \int_eval:n { #1 * "40 + `#3 - "80 } + \else: + \s_tl + \msg_expandable_kernel_error:nnn + { str } { utf8-missing-conti } {#3} + \int_use:N \c_str_replacement_char_int + \fi: + \s_tl + #2 + \str_aux_from_utf_viii:N #3 } -\cs_new_nopar:Npn \str_aux_from_utf_viii:wNNww #1, #2 #3#4, #5, +\cs_new_nopar:Npn \str_aux_from_utf_viii:wNnnwN + #1 \s_tl #2#3#4 #5 \str_aux_from_utf_viii:N #6 { - \if_num:w \int_eval:w ( `#2 + \c_thirty_two ) / "40 = \c_three - \if_num:w #1 < #4 \exp_stop_f: - \str_aux_from_utf_viii_error:w #1 , + \if_num:w #1 < #4 + \s_tl + \if_num:w #1 < #3 \exp_stop_f: + \msg_expandable_kernel_error:nnn + { str } { utf8-overlong } {#1} + \int_use:N \c_str_replacement_char_int \else: - #2 - \if_num:w #1 < #5 \exp_stop_f: - \exp_not:N \q_mark #1 - \else: - \exp_after:wN \str_aux_from_utf_viii:wwNN - \int_use:N \int_eval:w #1 - #5 - \fi: + #1 \fi: \else: - \exp_after:wN \str_aux_from_utf_viii_error:NwN - \exp_after:wN #2 + \if_meaning:w \q_stop #5 + \str_aux_from_utf_viii_overflow:w #1 + \fi: + \exp_after:wN \str_aux_from_utf_viii:wwN + \int_value:w \int_eval:n { #1 - #4 } \fi: - , #3#5, - } -\cs_new_nopar:Npn \str_aux_from_utf_viii_error:w #1 , - { - \exp_not:N \q_mark - \msg_expandable_kernel_error:nnn - { str } { utf8-overlong } {#1} - \c_str_replacement_char_int + \s_tl + #2 {#4} #5 + \str_aux_from_utf_viii:N } -\cs_new_nopar:Npn \str_aux_from_utf_viii_error:NwN #1 #2 \q_stop #3 +\cs_new_nopar:Npn \str_aux_from_utf_viii_overflow:w #1 \fi: #2 \fi: { - \exp_not:N \q_mark + \fi: \fi: \msg_expandable_kernel_error:nnn - { str } { utf8-missing-conti } {#1} - \c_str_replacement_char_int , - #3#1 + { str } { utf8-overflow } {#1} + \int_use:N \c_str_replacement_char_int } \cs_new_nopar:Npn \str_aux_from_utf_viii_error: { - \exp_not:N \q_mark + \s_tl \msg_expandable_kernel_error:nn { str } { utf8-premature-end } - \c_str_replacement_char_int , + \int_use:N \c_str_replacement_char_int \s_tl \prg_map_break: } % \end{macrocode} @@ -3018,19 +3030,32 @@ % \end{macro} % % \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi:NN} -% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi:wNNN} -% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_error:wNNw} +% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi:NNwNNN} % \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_end:Nw} -% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_end:wNNww} +% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_error:NNw} +% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_error:NNN} % When \cs{str_aux_from_utf_xvi:NN} is called, \cs{str_tmp:w} % is such that |\str_tmp:w #1#2| expands to |`#1| if the string % is big-endian, and |`#2| if the string is little-endian. -% If the most significant byte is between |"D8| and |"DB| -% inclusive, then the pair of bytes is a high surrogate, -% and we fetch the following pair, which must be a low surrogate, -% \emph{i.e.}, between |"DC| and |"DF| inclusive. -% Unpaired surrogates are errors, handled with -% \cs{str_aux_from_utf_xvi_error:wNNw}. +% If the code unit (a pair of characters) does not represent +% a surrogate, the \cs{if_case:w} construction expands to nothing, +% and we leave in the input stream \meta{characters} \cs{s_tl} +% \meta{char code} \cs{s_tl}, and move on to the next code unit. +% When the end is found, we check for the case of an odd-length +% string (an error) and end the mapping. +% +% Surrogates are characterized by a most significant byte in the +% ranges $[\hexnum{D8},\hexnum{DB}]$ and $[\hexnum{DC},\hexnum{DF}]$, +% detected by dividing the character code by $4$ (\eTeX{} rounds to +% nearest, ties away from zero). Here, a trail surrogate is invalid, +% replaced by the replacement character. Lead surrogates are treated by +% \cs{str_aux_from_utf_xvi:NNwNNN}, which grabs the next two +% characters, checks for a premature end, and checks whether +% the code unit is a trailing surrogate. If it is, leave +% \meta{characters} \cs{s_tl} \meta{char code} \cs{s_tl} in the +% input stream and move on to the next code unit. Otherwise, raise +% an error, and continue parsing, replacing only the first code unit +% (the leading surrogate) by the replacement character. % \begin{macrocode} \cs_new:Npn \str_aux_from_utf_xvi:NN #1#2 { @@ -3039,60 +3064,63 @@ \fi: \if_case:w \int_eval:w ( \str_tmp:w #1#2 - "D6 ) / \c_four \int_eval_end: - \or: #1#2 \exp_after:wN \str_aux_from_utf_xvi:wNNN - \or: \exp_after:wN \str_aux_from_utf_xvi_error:wNNw - \else: #1#2 \exp_not:N \q_mark \int_use:N \int_eval:w + \or: \exp_after:wN \str_aux_from_utf_xvi:NNwNNN + \or: \exp_after:wN \str_aux_from_utf_xvi_error:NNw \fi: - "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 , + #1#2 \s_tl + \int_eval:n { "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 } \s_tl \str_aux_from_utf_xvi:NN } - \cs_new:Npn \str_aux_from_utf_xvi:wNNN #1, #2#3#4 + \cs_new:Npn \str_aux_from_utf_xvi:NNwNNN #1#2 \s_tl #3 \s_tl #4#5#6 { - \if_meaning:w \scan_stop: #4 - \str_aux_from_utf_xvi_end:wNNww #1 , #3 + \if_meaning:w \scan_stop: #6 + \str_aux_from_utf_xvi_error:NNN #1#2#5 \fi: - \if_num:w \int_eval:w ( \str_tmp:w #3#4 - "DA ) / \c_four = \c_one - #3 #4 - \int_use:N \int_eval:w - ( #1 - "D800 ) * "400 - + \str_tmp:w #3#4 * "100 + \str_tmp:w #4#3 - "DC00 , + \if_num:w \int_eval:w ( \str_tmp:w #5#6 - "DA ) / \c_four = \c_one + #1 #2 #5 #6 \s_tl + \int_eval:n + { + ( "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 - "D800 ) * "400 + + "100 * \str_tmp:w #4#5 + \str_tmp:w #5#4 - "DC00 + } + \s_tl \exp_after:wN \use_i:nnn \else: - \str_aux_from_utf_xvi_error:wNNw #1 , + \str_aux_from_utf_xvi_error:NNw #1#2 \s_tl \s_tl \fi: - #2#3#4 - } - \cs_new:Npn \str_aux_from_utf_xvi_error:wNNw #1 \str_tmp:w #2#3 #4, - { - \exp_not:N \q_mark - \msg_expandable_kernel_error:nnn - { str } { utf16-surrogate } { #2#3 } - \c_str_replacement_char_int , + #4#5#6 } \cs_new:Npn \str_aux_from_utf_xvi_end:Nw #1 \fi: { \fi: \if_meaning:w \scan_stop: #1 \else: - #1 \exp_not:N \q_mark + #1 \s_tl \msg_expandable_kernel_error:nnn { str } { utf16-odd } {#1} - \int_use:N \c_str_replacement_char_int , + \int_use:N \c_str_replacement_char_int \s_tl \fi: \prg_map_break: } - \cs_new:Npn \str_aux_from_utf_xvi_end:wNNww #1 `#2#3 #4, #5 \fi: + \cs_new:Npn \str_aux_from_utf_xvi_error:NNw #1#2 \s_tl #3 \s_tl { - \fi: - \exp_not:N \q_mark - \if_meaning:w \scan_stop: #5 + #1 #2 \s_tl + \msg_expandable_kernel_error:nnn + { str } { utf16-surrogate } { #1#2 } + \int_use:N \c_str_replacement_char_int \s_tl + } + \cs_new:Npn \str_aux_from_utf_xvi_error:NNN #1#2#3 + { + \if_meaning:w \scan_stop: #3 + #1#2 \s_tl \msg_expandable_kernel_error:nnn - { str } { utf16-surrogate } { #2#3 } + { str } { utf16-surrogate } { #1#2 } \else: + #1#2#3 \s_tl \msg_expandable_kernel_error:nnn - { str } { utf16-odd } { #2#3#5 } + { str } { utf16-odd } { #1#2#3 } \fi: - \int_use:N \c_str_replacement_char_int , + \int_use:N \c_str_replacement_char_int \s_tl \prg_map_break: } % \end{macrocode} @@ -3180,7 +3208,7 @@ % \begin{macro}[aux, rEXP] % { % \str_aux_from_utf_xxxii:NNNN, -% \str_aux_from_utf_xxxii_end:ww +% \str_aux_from_utf_xxxii_end:w % } % See the conversion functions from \textsc{utf-16} encodings. % The \cs{str_aux_from_utf_xxxii:No} auxiliary defines @@ -3226,7 +3254,7 @@ \if_meaning:w \scan_stop: #4 \exp_after:wN \str_aux_from_utf_xxxii_end:w \fi: - #1#2#3#4 \exp_not:N \q_mark + #1#2#3#4 \s_tl \if_num:w \int_eval:w \str_tmp:w #1#4 * \c_two_hundred_fifty_six + \str_tmp:w #2#3 @@ -3235,12 +3263,10 @@ { str } { utf32-overflow } { #1#2#3#4 } \int_use:N \c_str_replacement_char_int \else: - \int_use:N \int_eval:w - \str_tmp:w #2#3*"10000 - + \str_tmp:w #3#2 * \c_two_hundred_fifty_six - + \str_tmp:w #4#1 + \int_eval:n + { \str_tmp:w #2#3*"10000 + \str_tmp:w #3#2*"100 + \str_tmp:w #4#1 } \fi: - , + \s_tl \str_aux_from_utf_xxxii:NNNN } \cs_new:Npn \str_aux_from_utf_xxxii_end:w #1 \scan_stop: @@ -3249,8 +3275,8 @@ { \msg_expandable_kernel_error:nnn { str } { utf32-truncated } { #1 } - #1 \exp_not:N \q_mark - \int_use:N \c_str_replacement_char_int , + #1 \s_tl + \int_use:N \c_str_replacement_char_int \s_tl } \prg_map_break: } diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx index c3310545ac0..ca9ac5399b0 100644 --- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx @@ -1,6 +1,6 @@ % \iffalse meta-comment % -%% File: l3tl-analysis.dtx Copyright (C) 2011 The LaTeX3 Project +%% File: l3tl-analysis.dtx Copyright (C) 2011-2012 The LaTeX3 Project %% %% It may be distributed and/or modified under the conditions of the %% LaTeX Project Public License (LPPL), either version 1.3c of this @@ -88,22 +88,83 @@ % % \subsection{Internal} % +% \begin{variable}{\s_tl} +% The format used to store token lists internally uses +% the scan mark \cs{s_tl} as a delimiter. +% \end{variable} +% +% The task of the \pkg{l3tl-analysis} module is to convert +% token lists to an internal format which allows us to extract +% all the relevant information about individual tokens +% (category code, character code), as well as reconstruct +% the token list quickly. This internal format is used in +% \pkg{l3regex} where we need to support arbitrary tokens, +% and it is used in conversion functions in \pkg{l3str}, +% where we wish to support clusters of characters instead +% of single characters. +% +% The internal format thus takes the form of a succession +% of items of the form +% \begin{quote} +% \meta{tokens} \cs{s_tl} \meta{catcode} \meta{char code} \cs{s_tl} +% \end{quote} +% Here, the \meta{tokens} \texttt{o}- and \texttt{x}-expand +% to the original token in the token list (even special +% category codes) or to the cluster of tokens corresponding +% to one Unicode character in the given encoding. +% The \meta{catcode} is given as a single hexadecimal digit, +% $0$ for control sequences. The \meta{char code} is given +% as a decimal number, $-1$ for control sequences. +% +% Using delimited arguments lets us build the \meta{tokens} +% progressively when doing an encoding conversion in \pkg{l3str}. +% On the other hand, the delimiter \cs{s_tl} may not appear +% unbraced in \meta{tokens}. This is not a problem because we +% are careful to wrap control sequences in braces (as an argument +% to \cs{exp_not:n}) when converting from a general token list +% to the internal format. +% % \begin{function}{\tl_set_analysis:Nn} % \begin{syntax} -% \cs{tl_analysis:Nn} \meta{tl var} \Arg{token list} +% \cs{tl_set_analysis:Nn} \meta{tl var} \Arg{token list} % \end{syntax} -% This function analysiss each token in the \meta{token list}, -% converting each \meta{token} to -% \begin{quote} -% \Arg{tokens} \cs{q_mark} \meta{catcode} \meta{char code} |,| -% \end{quote} -% where \meta{tokens} expands in one step to the \meta{token}, -% and \meta{catcode} and \meta{char code} are the category code -% and character code of the token if it is a character token, -% and otherwise are $0$ and $-1$ respectively. -% The result of the conversion is then stored in \meta{tl var}. +% This function analyzes each \meta{token} in the \meta{token list}, +% converting each \meta{token} to one item in the internal format. +% The result of the analysis is then stored in \meta{tl var}. +% The following transformation is performed: +% \begin{itemize} +% \item a control sequence |\cs| becomes +% |\exp_not:n { \cs } \s_tl 0 -1 \s_tl|; +% \item a begin-group character |{| becomes +% |\exp_after:wN { \if_false: } \fi: \s_tl 1| \meta{char code} \cs{s_tl}; +% \item an end-group character |}| becomes +% |\if_false: { \fi: } \s_tl 2| \meta{char code} \cs{s_tl}; +% \item a character with any other category code becomes +% |\exp_not:n| \Arg{character} \cs{s_tl} \meta{hex catcode} +% \meta{char code} \cs{s_tl}. +% \end{itemize} +% We can note that in every case the part before \cs{s_tl} +% both \texttt{o}-expands and \texttt{x}-expands to the token +% which is represented. % \end{function} % +% \begin{function}{\tl_set_analysis_from_str:Nn} +% \begin{syntax} +% \cs{tl_set_analysis:Nn} \meta{tl var} \Arg{token list} +% \end{syntax} +% This function analyzes each \meta{token} in the \meta{token list}, +% after conversion to a string. +% The result of the analysis is then stored in \meta{tl var}. +% \begin{itemize} +% \item Spaces become \verb*| |\cs{s_tl} |A32| \cs{s_tl}. +% \item Other characters become +% \meta{character} \cs{s_tl} |C| \meta{char code} \cs{s_tl}. +% \end{itemize} +% \end{function} +% +% ^^A todo: ask LuaTeX list for an \ifx\undefined +% ^^A which does not add the in memory. +% % \end{documentation} % % \begin{implementation} @@ -122,6 +183,33 @@ % % \subsection{Variables and helper functions} % +% \begin{variable}{\s_tl} +% The scan mark \cs{s_tl} is used as a delimiter in the internal +% format. This is more practical than using a quark, because we +% would then need to control expansion much more carefully: compare +% \cs{int_value:w} |`#1| \cs{s_tl} with +% \cs{int_value:w} |`#1| \cs{exp_stop_f:} \cs{exp_not:N} \cs{q_mark} +% to extract a character code followed by the delimiter in an +% \texttt{x}-expansion. +% \begin{macrocode} +\scan_new:N \s_tl +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_tl_analysis_token} +% \begin{variable}{\l_tl_analysis_char_token} +% The tokens in the token list are probed with the \TeX{} +% primitive \tn{futurelet}. We use \cs{l_tl_analysis_token} +% in that construction. In some cases, we convert the +% following token to a string before probing it: then the +% token variable used is \cs{l_tl_analysis_char_token}. +% \begin{macrocode} +\cs_new_eq:NN \l_tl_analysis_token ? +\cs_new_eq:NN \l_tl_analysis_char_token ? +% \end{macrocode} +% \end{variable} +% \end{variable} +% % \begin{variable}{\l_tl_analysis_normal_int} % The number of normal (\texttt{N}-type argument) tokens % since the last special token. @@ -148,14 +236,6 @@ % \end{macrocode} % \end{variable} % -% \begin{variable}{\l_tl_analysis_char_int} -% Holds the character code of the token currently considered, -% in cases where it is a character. -% \begin{macrocode} -\int_new:N \l_tl_analysis_char_int -% \end{macrocode} -% \end{variable} -% % \begin{variable}{\l_tl_analysis_type_int} % When encountering special characters, we record their \enquote{type} % in this integer. @@ -168,36 +248,40 @@ % The result of the conversion is stored in this token list, % with a succession of items of the form % \begin{quote} -% \meta{tokens} \cs{q_mark} \meta{catcode} \meta{char code} |,| +% \meta{tokens} \cs{s_tl} \meta{catcode} \meta{char code} \cs{s_tl} % \end{quote} % \begin{macrocode} \tl_new:N \g_tl_analysis_result_tl % \end{macrocode} % \end{variable} % -% \begin{macro}[int]{\tl_analysis_extract_charcode:} -% \begin{macro}[aux]{\tl_analysis_extract_charcode_aux:w} -% Extracting the character code from the meaning of \cs{l_tl_analysis_token}. -% This has no error checking, and should only be assumed to work -% for begin-group and end-group tokens. +% \begin{macro}[int, EXP]{\tl_analysis_extract_charcode:} +% \begin{macro}[aux, EXP]{\tl_analysis_extract_charcode_aux:w} +% Extracting the character code from the meaning of +% \cs{l_tl_analysis_token}. This has no error checking, +% and should only be assumed to work for begin-group +% and end-group character tokens. It produces a number +% in the form |`|\meta{char}. % \begin{macrocode} -\cs_new_protected_nopar:Npn \tl_analysis_extract_charcode: +\cs_new_nopar:Npn \tl_analysis_extract_charcode: { \exp_after:wN \tl_analysis_extract_charcode_aux:w \token_to_meaning:N \l_tl_analysis_token } -\cs_new_protected_nopar:Npn \tl_analysis_extract_charcode_aux:w #1 ~ #2 ~ { ` } +\cs_new:Npn \tl_analysis_extract_charcode_aux:w #1 ~ #2 ~ { ` } % \end{macrocode} % \end{macro} % \end{macro} % -% \begin{macro}[int]{\tl_analysis_cs_space_count:NN} -% \begin{macro}[aux]{\tl_analysis_cs_space_count:w} -% \begin{macro}[aux]{\tl_analysis_cs_space_count_end:w} +% \begin{macro}[int, EXP]{\tl_analysis_cs_space_count:NN} +% \begin{macro}[aux, EXP]{\tl_analysis_cs_space_count:w} +% \begin{macro}[aux, EXP]{\tl_analysis_cs_space_count_end:w} % Counts the number of spaces in the string representation of its % second argument, as well as the number of characters following % the last space in that representation, and feeds the two numbers % as semicolon-delimited arguments to the first argument. +% When this function is used, the escape character is printable +% and non-space. % \begin{macrocode} \cs_new:Npn \tl_analysis_cs_space_count:NN #1 #2 { @@ -224,10 +308,10 @@ % % Our goal is to produce a token list of the form roughly % \begin{quote} -% \Arg{token 1} \cs{q_mark} \meta{catcode 1} \meta{char code 1} |,| \\ -% \Arg{token 2} \cs{q_mark} \meta{catcode 2} \meta{char code 2} |,| \\ +% \meta{token 1} \cs{s_tl} \meta{catcode 1} \meta{char code 1} \cs{s_tl} \\ +% \meta{token 2} \cs{s_tl} \meta{catcode 2} \meta{char code 2} \cs{s_tl} \\ % \ldots{} -% \Arg{token N} \cs{q_mark} \meta{catcode N} \meta{char code N} |,| +% \meta{token N} \cs{s_tl} \meta{catcode N} \meta{char code N} \cs{s_tl} % \end{quote} % Most but not all tokens can be grabbed as an undelimited % (\texttt{N}-type) argument by \TeX{}. @@ -239,12 +323,12 @@ % registers, and removed from the input stream by some means. % % To ease the difficult first pass, we first do some setup -% with \cs{tl_analysis_i_setup:n}. +% with \cs{tl_analysis_setup:n}. % Active characters set equal to non-active characters % cause trouble, so we disable all active characters by -% setting them equal to \texttt{undefined}. +% setting them equal to \texttt{undefined} locally. % We also set there the escape character to be printable -% (backslash, but this later changes to a forward slash): +% (backslash, but this later oscillates between solidus and backslash): % this makes it possible to distinguish characters from % control sequences. % @@ -259,10 +343,6 @@ % \item an end-group token (category code $2$), % either space (character code $32$), or non-space; % \item a space token (category code $10$, character code $32$); -%^^A \item a character with category code $3$, $4$, $6$, $7$, $8$, $11$, -%^^A or $12$, or with category code $10$ but character code $\neq 32$; -%^^A \item a non-character (primitive, macro, register, font, -%^^A undefined, \emph{etc.}). % \item anything else (then the token is always an \texttt{N}-type argument). % \end{itemize} % The token itself can \enquote{look like} one of the following @@ -270,19 +350,18 @@ % \item a non-active character, in which case its meaning % is automatically that associated to its character code % and category code, we call it \enquote{true} character; -% \item an active character; +% \item an active character (we eliminate those in the setup step); % \item a control sequence. % \end{itemize} % The only tokens which are not valid \texttt{N}-type arguments are % true begin-group characters, true end-group characters, and true spaces. % We will detect those characters by scanning ahead with \tn{futurelet}, -% then distinguishing true characters from tokens set equal to them -% using the \tn{string} representation. +% then distinguishing true characters from control sequences set equal +% to them using the \tn{string} representation. % % The second pass is a simple exercise in expandable loops. % % \begin{macro}[int]{\tl_analysis:n} -% \begin{macro}[int]{\tl_set_analysis:Nn} % Everything is done within a group, and all definitions will be local. % We use \cs{group_align_safe_begin/end:} to avoid problems in case % \cs{tl_analysis:n} is used within an alignment and its argument @@ -298,6 +377,13 @@ \group_align_safe_end: \group_end: } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int]{\tl_set_analysis:Nn} +% This function is used in \pkg{l3regex}. Simply copy the result +% of \cs{tl_analysis:n} into the given token list. +% \begin{macrocode} \cs_new_protected:Npn \tl_set_analysis:Nn #1#2 { \tl_analysis:n {#2} @@ -305,6 +391,41 @@ } % \end{macrocode} % \end{macro} +% +% \begin{macro}[int]{\tl_set_analysis_from_str:Nn} +% \begin{macro}[aux, EXP]{\tl_analysis_from_str_loop:N} +% This function, used in \pkg{l3str} could be emulated by passing the +% token list through \cs{tl_to_str:n} and using +% \cs{tl_set_analysis:Nn}, but this would be terribly inefficient. +% Instead, convert to a string with spaces of category code other, +% and for each character output one item of the analysis: +% \meta{character} \cs{s_tl} \meta{catcode} \meta{char code} \cs{s_tl} +% where the \meta{catcode} is either |A| (for spaces) or |C| +% (for other characters). +% \begin{macrocode} +\cs_new_protected:Npn \tl_set_analysis_from_str:Nn #1#2 + { + \str_gset_other:Nn \g_tl_analysis_result_tl {#2} + \tl_set:Nx #1 + { + \exp_after:wN \tl_analysis_from_str_loop:N + \g_tl_analysis_result_tl \prg_map_break: + \prg_break_point:n { } + } + } +\group_begin: + \char_set_catcode_other:N A + \char_set_catcode_other:N C + \cs_new_protected:Npn \tl_analysis_from_str_loop:N #1 + { + #1 \s_tl + \if_charcode:w #1 ~ A \else: C \fi: + \int_value:w `#1 \s_tl + \tl_analysis_from_str_loop:N + } +\group_end: +% \end{macrocode} +% \end{macro} % \end{macro} % % \subsection{Setup} @@ -316,20 +437,22 @@ % \texttt{undefined}. Since Unicode contains too many characters % to loop over all of them, we instead loop over the input token % list as a string: any active character in the token list -% must appear in its string representation. -% ^^A todo: ask LuaTeX list for an \ifx\undefined -% ^^A which does not add the in memory. +% must appear in its string representation. The string is shortened +% a little by making the escape character unprintable. The active +% space must be disabled separately (the loop skips over it otherwise), +% and we end the loop by feeding an odd non-\texttt{N}-type +% argument to the looping macro. % \begin{macrocode} \cs_new_protected:Npn \tl_analysis_setup:n #1 { - \int_set:Nn \tex_escapechar:D { 92 } + \int_set_eq:NN \tex_escapechar:D \c_minus_one \exp_after:wN \tl_analysis_disable_loop:N \tl_to_str:n {#1} { ~ } { ? ~ \prg_map_break: } \prg_break_point:n { } } \group_begin: \char_set_catcode_active:N \^^@ - \cs_new_protected_nopar:Npn \tl_analysis_disable_loop:N #1 + \cs_new_protected:Npn \tl_analysis_disable_loop:N #1 { \tex_lccode:D \c_zero `#1 ~ \tl_to_lowercase:n { \tex_let:D ^^@ } \c_undefined:D @@ -349,22 +472,22 @@ % % After the setup step, we have $11$ types of tokens: % \begin{itemize} -% \item[1] a true non-space begin-group character; -% \item[2] a true space begin-group character; -% \item[3] a true non-space end-group character; -% \item[4] a true space end-group character; -% \item[5] a true space blank space character; -% \item[6] an undefined active character; -% \item[7] any other true character; -% \item[8] a control sequence equal to a begin-group token (category code $1$); -% \item[9] a control sequence equal to an end-group token (category code $2$); -% \item[10] a control sequence equal to a space token +% \item[1.] a true non-space begin-group character; +% \item[2.] a true space begin-group character; +% \item[3.] a true non-space end-group character; +% \item[4.] a true space end-group character; +% \item[5.] a true space blank space character; +% \item[6.] an undefined active character; +% \item[7.] any other true character; +% \item[8.] a control sequence equal to a begin-group token (category code $1$); +% \item[9.] a control sequence equal to an end-group token (category code $2$); +% \item[10.] a control sequence equal to a space token % (character code $32$, category code $10$); -% \item[11] any other control sequence. +% \item[11.] any other control sequence. % \end{itemize} % Our first tool is \tn{futurelet}. This cannot distinguish -% cases $1$ and $2$ from $8$, nor cases $3$ and $4$ from $9$, -% nor case $5$ from case $10$. Those cases will be distinguished +% case $8$ from $1$ or $2$, nor case $9$ from $3$ or $4$, +% nor case $10$ from case $5$. Those cases will be distinguished % by applying the \tn{string} primitive to the following token, % after possibly changing the escape character to ensure that % a control sequence's string representation cannot be mistaken @@ -376,7 +499,7 @@ % \cs{str_tail:n} \Arg{token} is non-empty, because the % escape character is printable. % -% \begin{macro}{\tl_analysis_i:n} +% \begin{macro}[int]{\tl_analysis_i:n} % We read tokens one by one using \tn{futurelet}. % While performing the loop, we keep track of the number of % true begin-group characters minus the number of @@ -385,6 +508,7 @@ % \begin{macrocode} \cs_new_protected:Npn \tl_analysis_i:n #1 { + \int_set:Nn \tex_escapechar:D { 92 } \int_zero:N \l_tl_analysis_normal_int \int_zero:N \l_tl_analysis_index_int \int_zero:N \l_tl_analysis_nesting_int @@ -403,17 +527,21 @@ % \end{macro} % % \begin{macro}[int]{\tl_analysis_i_type:w} -% At this point, \cs{l_tl_analysis_token} holds the meaning of the following -% token. We store in \cs{l_tl_analysis_type_int} the meaning of the token ahead: +% At this point, \cs{l_tl_analysis_token} holds the meaning +% of the following token. We store in \cs{l_tl_analysis_type_int} +% the meaning of the token ahead: % \begin{itemize} -% \item[0] space token; -% \item[1] begin-group token; -% \item[-1] end-group token; -% \item[2] other. +% \item 0 space token; +% \item 1 begin-group token; +% \item -1 end-group token; +% \item 2 other. % \end{itemize} -% The values $0$, $1$, $-1$ correspond to how much a true such character -% changes the nesting level ($2$ is used only here, and is irrelevant later). -% Then call the auxiliary for each case. +% The values $0$, $1$, $-1$ correspond to how much a true such +% character changes the nesting level ($2$ is used only here, +% and is irrelevant later). Then call the auxiliary for each case. +% Note that nesting conditionals here is safe because we only skip +% over \cs{l_tl_analysis_token} if it matches with one of the +% character tokens (hence is not a primitive conditional). % \begin{macrocode} \cs_new_protected_nopar:Npn \tl_analysis_i_type:w { @@ -441,65 +569,38 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}[int]{\tl_analysis_i_safe:N} -% \begin{macro}[aux]{\tl_analysis_i_cs:ww} -% The upcoming token is safe: we can simply grab it in a second pass. -% Since other branches of the code must pass their tokens through -% \tn{string}, we do it here as well. It's been optimized in an evil way, -% sorry. -%^^A todo: document -% \begin{macrocode} -\cs_new_protected:Npn \tl_analysis_i_safe:N #1 - { - \if_charcode:w - \scan_stop: - \exp_after:wN \use_none:n \token_to_str:N #1 \prg_do_nothing: - \scan_stop: - \int_incr:N \l_tl_analysis_normal_int - \else: - \tl_analysis_cs_space_count:NN \tl_analysis_i_cs:ww #1 - \fi: - \tl_analysis_i_loop:w - } -\cs_new_protected:Npn \tl_analysis_i_cs:ww #1; #2; - { - \if_num:w #1 > \c_zero - \tex_skip:D \l_tl_analysis_index_int - = \int_eval:w \l_tl_analysis_normal_int + \c_one sp \scan_stop: - \tex_advance:D \l_tl_analysis_index_int #1 \exp_stop_f: - \l_tl_analysis_normal_int #2 \exp_stop_f: - \else: - \tex_advance:D \l_tl_analysis_normal_int #2 \exp_stop_f: - \fi: - } -% \end{macrocode} -% \end{macro} -% \end{macro} -% -% \begin{macro}[aux]{\tl_analysis_i_space:w} +% \begin{macro}[int]{\tl_analysis_i_space:w} % \begin{macro}[aux]{\tl_analysis_i_space_test:w} % In this branch, the following token's meaning is a blank space. % Apply \tn{string} to that token: if it is a control sequence % the result starts with the escape character; otherwise it is % a true blank space, whose string representation is also a blank space. % We test for that in \cs{tl_analysis_i_space_test:w}, -% after grabbing the first character of the string representation -% as \cs{l_tl_analysis_token}. +% after grabbing as \cs{l_tl_analysis_char_token} the first character +% of the string representation. % Also, since \cs{tl_analysis_i_store:} expects the special token to be % stored in the relevant \tn{toks} register, we do that. The extra % \cs{exp_not:n} is unnecessary of course, but it makes the treatment % of all tokens more homogeneous. +% If we discover that the next token was actually a control sequence +% instead of a true space, then we step the counter of normal tokens. +% We now have in front of us the whole string representation of +% the control sequence, including potential spaces; those will appear +% to be true spaces later in this pass. Hence, all other branches of +% the code in this first pass need to consider the string representation, +% so that the second pass does not need to test the meaning of tokens, +% only strings. % \begin{macrocode} \cs_new_protected_nopar:Npn \tl_analysis_i_space:w { \tex_afterassignment:D \tl_analysis_i_space_test:w \exp_after:wN \cs_set_eq:NN - \exp_after:wN \l_tl_analysis_token + \exp_after:wN \l_tl_analysis_char_token \token_to_str:N } \cs_new_protected_nopar:Npn \tl_analysis_i_space_test:w { - \if_meaning:w \l_tl_analysis_token \c_space_token + \if_meaning:w \l_tl_analysis_char_token \c_space_token \tex_toks:D \l_tl_analysis_index_int { \exp_not:n { ~ } } \tl_analysis_i_store: \else: @@ -511,17 +612,20 @@ % \end{macro} % \end{macro} % -% \begin{macro}[int]{\tl_analysis_i_bgroup:w} -% \begin{macro}[int]{\tl_analysis_i_egroup:w} +% \begin{macro}[int]{\tl_analysis_i_bgroup:w, \tl_analysis_i_egroup:w} % \begin{macro}[aux]{\tl_analysis_i_group:nw} % \begin{macro}[aux]{\tl_analysis_i_group_test:w} % The token might be either a true character token with % catcode $1$ or $2$, or it could be a control sequence. % The only tricky case is if the character code happens % to be equal to the escape character: then we change -% the escape character, so that the string representation -% of the true character and of a control sequence set equal -% to it start differently. +% the escape character from backslash to solidus or back, +% so that the string representation of the true character +% and of a control sequence set equal to it start differently. +% Then probe what the first character of that string +% representation is: this is the place where we need +% \cs{l_tl_analysis_char_token} to be a separate control +% sequence from \cs{l_tl_analysis_token}, to compare them. % \begin{macrocode} \group_begin: \char_set_catcode_group_begin:N \^^@ @@ -533,22 +637,21 @@ \cs_new_protected_nopar:Npn \tl_analysis_i_egroup:w { \tl_analysis_i_group:nw { \if_false: ^^B \fi: ^^@ } } \group_end: -\cs_new_protected_nopar:Npn \tl_analysis_i_group:nw #1 +\cs_new_protected:Npn \tl_analysis_i_group:nw #1 { - \l_tl_analysis_char_int = \tl_analysis_extract_charcode: \scan_stop: - \tex_lccode:D \c_zero \l_tl_analysis_char_int + \tex_lccode:D \c_zero = \tl_analysis_extract_charcode: \scan_stop: \tl_to_lowercase:n { \tex_toks:D \l_tl_analysis_index_int {#1} } - \if_num:w \l_tl_analysis_char_int = \tex_escapechar:D + \if_num:w \tex_lccode:D \c_zero = \tex_escapechar:D \int_set:Nn \tex_escapechar:D { 139 - \tex_escapechar:D } \fi: \tex_afterassignment:D \tl_analysis_i_group_test:w \exp_after:wN \cs_set_eq:NN - \exp_after:wN \l_tl_analysis_token + \exp_after:wN \l_tl_analysis_char_token \token_to_str:N } \cs_new_protected_nopar:Npn \tl_analysis_i_group_test:w { - \if_num:w \tl_analysis_extract_charcode: = \l_tl_analysis_char_int + \if_charcode:w \l_tl_analysis_token \l_tl_analysis_char_token \tl_analysis_i_store: \else: \int_incr:N \l_tl_analysis_normal_int @@ -559,37 +662,41 @@ % \end{macro} % \end{macro} % \end{macro} -% \end{macro} % % \begin{macro}[int]{\tl_analysis_i_store:} -% This function is called each time we meet a special token, -% and the value of \cs{l_tl_analysis_type_int} indicates which case +% This function is called each time we meet a special token; +% at this point, the \tn{toks} register \cs{l_tl_analysis_index_int} +% holds a token list which expands to the given special token. +% Also, the value of \cs{l_tl_analysis_type_int} indicates which case % we are in: % \begin{itemize} -% \item[-1] end-group character; -% \item[0] space character; -% \item[1] begin-group character. +% \item -1 end-group character; +% \item 0 space character; +% \item 1 begin-group character. % \end{itemize} % We need to distinguish further the case of a space character % (code $32$) from other character codes, because those will -% behave differently in the second pass. Namely, we change the -% cases above to +% behave differently in the second pass. Namely, after testing +% the \tn{lccode} of $0$ (which holds the present character code) +% we change the cases above to % \begin{itemize} -% \item[-2] space end-group character; -% \item[-1] non-space end-group character; -% \item[0] space blank space character; -% \item[1] non-space begin-group character; -% \item[2] space begin-group character. +% \item -2 space end-group character; +% \item -1 non-space end-group character; +% \item 0 space blank space character; +% \item 1 non-space begin-group character; +% \item 2 space begin-group character. % \end{itemize} % This has the property that non-space characters correspond to odd % values of \cs{l_tl_analysis_type_int}. -% Also, the \tn{toks} register number \cs{l_tl_analysis_index_int} -% holds a token list which expands to the given special token. +% The number of normal tokens, and the type of special token, +% are packed into a \tn{skip} register. +% Finally, we check whether we reached the last closing brace, in which +% case we stop by disabling the looping function (locally). % \begin{macrocode} \cs_new_protected_nopar:Npn \tl_analysis_i_store: { \tex_advance:D \l_tl_analysis_nesting_int \l_tl_analysis_type_int - \if_num:w \l_tl_analysis_char_int = \c_thirty_two + \if_num:w \tex_lccode:D \c_zero = \c_thirty_two \tex_multiply:D \l_tl_analysis_type_int \c_two \fi: \tex_skip:D \l_tl_analysis_index_int @@ -603,12 +710,61 @@ % \end{macrocode} % \end{macro} % +% \begin{macro}[int]{\tl_analysis_i_safe:N} +% \begin{macro}[aux]{\tl_analysis_i_cs:ww} +% This should be the simplest case: since the upcoming token is safe, +% we can simply grab it in a second pass. However, other branches of +% the code must pass their tokens through \tn{string}, hence we do it +% here as well, with some optimizations. If the token is a single +% character (including space), the \cs{if_charcode:w} test yields +% true, and we simply count one \enquote{normal} token. On the other +% hand, if the token is a control sequence, we should replace it by +% its string representation for compatibility with other code +% branches. Instead of slowly looping through the characters with +% the main code, we use the knowledge of how the second pass works: +% if the control sequence name contains no space, count that token +% as a number of normal tokens equal to its string length. If the +% control sequence contains spaces, they should be registered as +% special characters by increasing \cs{l_tl_analysis_index_int} +% (no need to carefully count character between each space), and +% all characters after the last space should be counted in the +% following sequence of \enquote{normal} tokens. +% \begin{macrocode} +\cs_new_protected:Npn \tl_analysis_i_safe:N #1 + { + \if_charcode:w + \scan_stop: + \exp_after:wN \use_none:n \token_to_str:N #1 \prg_do_nothing: + \scan_stop: + \int_incr:N \l_tl_analysis_normal_int + \else: + \tl_analysis_cs_space_count:NN \tl_analysis_i_cs:ww #1 + \fi: + \tl_analysis_i_loop:w + } +\cs_new_protected:Npn \tl_analysis_i_cs:ww #1; #2; + { + \if_num:w #1 > \c_zero + \tex_skip:D \l_tl_analysis_index_int + = \int_eval:w \l_tl_analysis_normal_int + \c_one sp \scan_stop: + \tex_advance:D \l_tl_analysis_index_int #1 \exp_stop_f: + \l_tl_analysis_normal_int #2 \exp_stop_f: + \else: + \tex_advance:D \l_tl_analysis_normal_int #2 \exp_stop_f: + \fi: + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% % \subsection{Second pass} % % The second pass is an exercise in expandable loops. +% All the necessary information is stored in \tn{skip} +% and \tn{toks} registers. % % \begin{macro}[int]{\tl_analysis_ii:n} -% \begin{macro}[aux]{\tl_analysis_ii_loop:w} +% \begin{macro}[int, EXP]{\tl_analysis_ii_loop:w} % Start the loop with the index $0$. No need for an end-marker: % the loop will stop by itself when the last index is read. % We will repeatedly oscillate between reading long stretches @@ -631,16 +787,18 @@ % \end{macro} % \end{macro} % -% \begin{macro}[aux]{\tl_analysis_ii_normals:ww} -% \begin{macro}[aux]{\tl_analysis_ii_normal:wwN} +% \begin{macro}[int, EXP]{\tl_analysis_ii_normals:ww} +% \begin{macro}[aux, EXP]{\tl_analysis_ii_normal:wwN} % The first argument is the number of normal tokens which remain % to be read, and the second argument is the index in the array % produced in the first step. % A character's string representation is always one character long, % while a control sequence is always longer (we have set the escape % character to a printable value). In both cases, we leave -% \cs{exp_not:n} \Arg{token} \cs{q_mark} in the input stream -% (after \texttt{x}-expansion). +% \cs{exp_not:n} \Arg{token} \cs{s_tl} in the input stream +% (after \texttt{x}-expansion). Here, \cs{exp_not:n} is used +% rather than \cs{exp_not:N} because |#3| could be \cs{s_tl}, +% hence must be hidden behind braces in the result. % \begin{macrocode} \cs_new:Npn \tl_analysis_ii_normals:ww #1; { @@ -651,7 +809,7 @@ } \cs_new:Npn \tl_analysis_ii_normal:wwN #1; #2; #3 { - \exp_not:n { \exp_not:n { #3 } } \exp_not:N \q_mark + \exp_not:n { \exp_not:n { #3 } } \s_tl \if_charcode:w \scan_stop: \exp_after:wN \use_none:n \token_to_str:N #3 \prg_do_nothing: @@ -666,41 +824,53 @@ % \end{macro} % \end{macro} % -% \begin{macro}[aux]{\tl_analysis_ii_char:Nww} +% \begin{macro}[int, EXP]{\tl_analysis_ii_char:Nww} % If the normal token we grab is a character, leave -% \meta{catcode} \meta{charcode} followed by a comma +% \meta{catcode} \meta{charcode} followed by \cs{s_tl} % in the input stream, and call \cs{tl_analysis_ii_normals:ww} % with its first argument decremented. % \begin{macrocode} -\cs_new:Npn \tl_analysis_ii_char:Nww #1 - { - \if_meaning:w #1 \c_undefined:D D \else: - \if_catcode:w #1 \c_catcode_other_token C \else: - \if_catcode:w #1 \c_catcode_letter_token B \else: - \if_catcode:w #1 \c_math_toggle_token 3 \else: - \if_catcode:w #1 \c_alignment_token 4 \else: - \if_catcode:w #1 \c_math_superscript_token 7 \else: - \if_catcode:w #1 \c_math_subscript_token 8 \else: - 6 - \fi: \fi: \fi: \fi: \fi: \fi: \fi: - \int_value:w `#1 , - \exp_after:wN \tl_analysis_ii_normals:ww - \int_use:N \int_eval:w \c_minus_one + - } +\group_begin: + \char_set_catcode_other:N A + \char_set_catcode_other:N B + \char_set_catcode_other:N C + \char_set_uccode:nn { `? } { `D } + \tl_to_uppercase:n + { + \cs_new:Npn \tl_analysis_ii_char:Nww #1 + { + \if_meaning:w #1 \c_undefined:D ? \else: + \if_catcode:w #1 \c_catcode_other_token C \else: + \if_catcode:w #1 \c_catcode_letter_token B \else: + \if_catcode:w #1 \c_math_toggle_token 3 \else: + \if_catcode:w #1 \c_alignment_token 4 \else: + \if_catcode:w #1 \c_math_superscript_token 7 \else: + \if_catcode:w #1 \c_math_subscript_token 8 \else: + \if_catcode:w #1 \c_space_token A \else: + 6 + \fi: \fi: \fi: \fi: \fi: \fi: \fi: \fi: + \int_value:w `#1 \s_tl + \exp_after:wN \tl_analysis_ii_normals:ww + \int_use:N \int_eval:w \c_minus_one + + } + } +\group_end: % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\tl_analysis_ii_cs:Nww} +% \begin{macro}[int, EXP]{\tl_analysis_ii_cs:Nww} +% \begin{macro}[aux, EXP]{\tl_analysis_ii_cs_test:ww} % If the token we grab is a control sequence, leave -% |0-1,| (as category code and character code) in the input stream, +% |0 -1| (as category code and character code) in the input stream, +% followed by \cs{s_tl}, % and call \cs{tl_analysis_ii_normals:ww} with updated arguments. % \begin{macrocode} \cs_new:Npn \tl_analysis_ii_cs:Nww #1 { - 0 -1 , + 0 -1 \s_tl \tl_analysis_cs_space_count:NN \tl_analysis_ii_cs_test:ww #1 } -\cs_new:Npn \tl_analysis_ii_cs_test:ww #1 ; #2 ; #3; #4; +\cs_new:Npn \tl_analysis_ii_cs_test:ww #1 ; #2 ; #3 ; #4 ; { \exp_after:wN \tl_analysis_ii_normals:ww \int_use:N \int_eval:w @@ -711,16 +881,17 @@ \fi: - #2 \exp_after:wN ; - \int_use:N \int_eval:w #4 + #1; + \int_use:N \int_eval:w #4 + #1 ; } % \end{macrocode} % \end{macro} +% \end{macro} % -% \begin{macro}[aux]{\tl_analysis_ii_special:w} -% \begin{macro}[aux]{\tl_analysis_ii_special_char:wN} -% \begin{macro}[aux]{\tl_analysis_ii_special_space:w} +% \begin{macro}[int, EXP]{\tl_analysis_ii_special:w} +% \begin{macro}[aux, EXP]{\tl_analysis_ii_special_char:wN} +% \begin{macro}[aux, EXP]{\tl_analysis_ii_special_space:w} % Here, |#1| is the current index in the array built in the first pass. -% Check whether we reached the end (we shouldn't insert the trailing +% Check now whether we reached the end (we shouldn't keep the trailing % end-group character that marked the end of the token list in the % first pass). % Unpack the \tn{toks} register: when \texttt{x}-expanding again, @@ -728,35 +899,39 @@ % Then leave the category code in the input stream, followed by % the character code, and call \cs{tl_analysis_ii_loop:w} with the next index. % \begin{macrocode} -\cs_new:Npn \tl_analysis_ii_special:w \fi: \tl_analysis_ii_normal:wwN 0; #1; - { - \fi: - \if_num:w #1 = \l_tl_analysis_index_int - \exp_after:wN \prg_map_break: - \fi: - \tex_the:D \tex_toks:D #1 \exp_stop_f: \exp_not:N \q_mark - \if_case:w \etex_gluestretch:D \tex_skip:D #1 \exp_stop_f: - A - \or: 1 - \or: 1 - \else: 2 - \fi: - \if_int_odd:w \etex_gluestretch:D \tex_skip:D #1 \exp_stop_f: - \exp_after:wN \tl_analysis_ii_special_char:wN \int_use:N - \else: - \exp_after:wN \tl_analysis_ii_special_space:w \int_use:N - \fi: - \int_eval:w \c_one + #1 \exp_after:wN ; - \token_to_str:N - } +\group_begin: + \char_set_catcode_other:N A + \cs_new:Npn \tl_analysis_ii_special:w + \fi: \tl_analysis_ii_normal:wwN 0 ; #1 ; + { + \fi: + \if_num:w #1 = \l_tl_analysis_index_int + \exp_after:wN \prg_map_break: + \fi: + \tex_the:D \tex_toks:D #1 \s_tl + \if_case:w \etex_gluestretch:D \tex_skip:D #1 \exp_stop_f: + A + \or: 1 + \or: 1 + \else: 2 + \fi: + \if_int_odd:w \etex_gluestretch:D \tex_skip:D #1 \exp_stop_f: + \exp_after:wN \tl_analysis_ii_special_char:wN \int_use:N + \else: + \exp_after:wN \tl_analysis_ii_special_space:w \int_use:N + \fi: + \int_eval:w \c_one + #1 \exp_after:wN ; + \token_to_str:N + } +\group_end: \cs_new:Npn \tl_analysis_ii_special_char:wN #1 ; #2 { - \int_value:w `#2 , + \int_value:w `#2 \s_tl \tl_analysis_ii_loop:w #1 ; } \cs_new:Npn \tl_analysis_ii_special_space:w #1 ; ~ { - 32 , + 32 \s_tl \tl_analysis_ii_loop:w #1 ; } % \end{macrocode} @@ -777,7 +952,7 @@ { tl-analysis } { \exp_after:wN \tl_show_analysis_loop:wNw \g_tl_analysis_result_tl - \q_mark { ? \prg_map_break: } , + \s_tl { ? \prg_map_break: } \s_tl \prg_break_point:n { } } } @@ -790,16 +965,16 @@ % \end{macro} % \end{macro} % -% \begin{macro}[aux, EXP]{\tl_show_analysis_loop:wNw} -% Here, |#1| \texttt{o}-expands to the token; -% |#2| is the category code (one hexadecimal digit), +% \begin{macro}[aux, rEXP]{\tl_show_analysis_loop:wNw} +% Here, |#1| \texttt{o}- and \texttt{x}-expands to the token; +% |#2| is the category code (one uppercase hexadecimal digit), % $0$ for control sequences; % |#3| is the character code, which we ignore. % In the cases of control sequences and active characters, % the meaning may overflow one line, and we want to truncate % it. Those cases are thus separated out. % \begin{macrocode} -\cs_new:Npn \tl_show_analysis_loop:wNw #1 \q_mark #2 #3, +\cs_new:Npn \tl_show_analysis_loop:wNw #1 \s_tl #2 #3 \s_tl { \use_none:n #2 \iow_newline: > \c_space_tl \c_space_tl @@ -820,9 +995,9 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\tl_show_analysis_normal:n} +% \begin{macro}[aux, rEXP]{\tl_show_analysis_normal:n} % Non-active characters are a simple matter of printing -% the character, and its meaning. One can check that +% the character, and its meaning. Our test suite checks that % begin-group and end-group characters do not mess up % \TeX{}'s alignment status. % \begin{macrocode} @@ -834,10 +1009,10 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\tl_show_analysis_cs:n} -% \begin{macro}[aux]{\tl_show_analysis_active:n} -% \begin{macro}[aux]{\tl_show_analysis_long:nn} -% \begin{macro}[aux]{\tl_show_analysis_long_aux:nnn} +% \begin{macro}[aux, rEXP]{\tl_show_analysis_cs:n} +% \begin{macro}[aux, rEXP]{\tl_show_analysis_active:n} +% \begin{macro}[aux, rEXP]{\tl_show_analysis_long:nn} +% \begin{macro}[aux, rEXP]{\tl_show_analysis_long_aux:nnn} % Control sequences and active characters are printed in the same way, % making sure not to go beyond the \cs{l_iow_line_length_int}. In case % of an overflow, we replace the last characters by @@ -853,17 +1028,20 @@ { \token_to_str:N #1 } { \token_to_meaning:N #1 } } -\cs_new_nopar:Npn \tl_show_analysis_long_aux:nnn #1#2#3 +\cs_new:Npn \tl_show_analysis_long_aux:nnn #1#2#3 { \int_compare:nNnTF - { \str_length:n { >>> #1 ~ ( #3 #2 ) } } - > { \l_iow_line_length_int } + { \str_length:n { #1 ~ ( #3 #2 ) } } + > { \l_iow_line_length_int - \c_three } { - \str_substr:nnn { >>> #1 ~ ( #3 #2 ) } \c_three - { \l_iow_line_length_int - \str_length:N \c_tl_show_analysis_etc_str } + \str_substr:nnn { #1 ~ ( #3 #2 ) } \c_zero + { + \l_iow_line_length_int - \c_three + - \str_length:N \c_tl_show_analysis_etc_str + } \c_tl_show_analysis_etc_str } - {#1~(#3#2)} + { #1 ~ ( #3 #2 ) } } % \end{macrocode} % \end{macro} @@ -871,21 +1049,23 @@ % \end{macro} % \end{macro} % +% \subsection{Messages} +% % \begin{variable}{\c_tl_show_analysis_etc_str} % When a control sequence (or active character) % and its meaning are too long to fit in one line % of the terminal, the end is replaced by this token list. % \begin{macrocode} -\str_const:Nx \c_tl_show_analysis_etc_str % ( - { \iow_char:N \\ETC. ) } +\tl_const:Nx \c_tl_show_analysis_etc_str % ( + { \token_to_str:N \ETC.) } % \end{macrocode} % \end{variable} % % \begin{macrocode} \msg_kernel_new:nnn { tl-analysis } { show } { - Token~list~ - \str_if_eq:nnF {#1} { \l_tl_tmpa_tl } { \token_to_str:N #1~} + The~token~list~ + \str_if_eq:nnF {#1} { \l_tl_tmpa_tl } { \token_to_str:N #1 ~ } \tl_if_empty:NTF #1 { is~empty } { contains~the~tokens: } diff --git a/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-test.tex b/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-test.tex new file mode 100644 index 00000000000..1ede1770c08 --- /dev/null +++ b/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-test.tex @@ -0,0 +1,380 @@ + +\documentclass{article} + +\usepackage[T1]{fontenc} +\usepackage{times,multicol} +\usepackage{xcoffins} + +\usepackage{color} +\newcommand\cbox[2][.8]{{\setlength\fboxsep{0pt}\colorbox[gray]{#1}{#2}}} + +\newcommand\hrulebox [2]{\setbox#1\hbox to#2{{\scriptsize\itshape + \color{blue}\hrulefill #2\hrulefill}}} +\newcommand\vrulebox [2]{\setbox#1\vbox to#2{{\hsize 1pt\centering\scriptsize + \itshape\color{blue}\leaders\vrule\vfill + \hbox to0pt{\hss #2\hss}\leaders\vrule\vfill\par}}} + + +\addtolength\textwidth{10pt} + +\showboxdepth 9999 +\showboxbreadth 9999 +\tracingonline 1 + + +\scrollmode + +\newbox\zzz + + +\begin{document} + + +\title{Test file for coffins} +\author{FMi} +\maketitle + +First we add a few test coffins: +\begin{verbatim} + \newcoffin \aaa + \newcoffin \bbb + \newcoffin \ccc + \newcoffin \ddd +\end{verbatim} + \newcoffin \aaa + \newcoffin \bbb + \newcoffin \ccc + \newcoffin \ddd +and some boxes not set up as ordinary boxes (without extra poles): +\begin{verbatim} + \newbox \xxx + \newbox \yyy +\end{verbatim} + \newbox \xxx + \newbox \yyy + + +If a coffin receives data a set of ``natural'' default poles are automatically added. +\begin{verbatim} +\sbox\aaa{\fbox{\begin{tabular}[b]{l}123\\4\\5\end{tabular}}} +\showcoffindata \aaa +\end{verbatim} +\sbox\aaa{\fbox{\begin{tabular}[b]{l}123\\4\\5\end{tabular}}} +\showcoffindata \aaa + + +In contrast, boxes not declared as coffins have no poles defined. However, if +used as coffins at least the natural default poles can be used without +further problems, i.e.,they are changed to coffins automatically on use. +\begin{verbatim} +\sbox\xxx{\fbox{Some very looooonggg Caption Text}} +\showpoles \xxx +\end{verbatim} +\sbox\xxx{\fbox{Some very looooonggg Caption Text}} +\showcoffindata \xxx + + +Using \verb|\setvcoffin| instead of \verb|\sbox| gives us the codes with +\texttt{T} and \texttt{B}. Similar to \verb|\parbox| this command requires to +specify a target width of the box: +\begin{verbatim} +\setvcoffin \aaa {1.5cm} {\centering + \fbox{\begin{tabular}[b]{l}123\\4\\5\end{tabular}}} +\showcoffindata \aaa +\end{verbatim} +\setvcoffin \aaa {1.5cm} {\centering + \fbox{\begin{tabular}[b]{l}123\\4\\5\end{tabular}}} +\showcoffindata \aaa + +There are a dozen natural poles per box (some with multiple names); you can +move each of the poles to someplace else (either to an absolute position in +``box space'' or relative to its previous position). The coordinates can +refer to box dimensions (using \verb|\height|, \verb|\depth|, + and \verb|width|). First we move some pole and add one +additional one: +\begin{verbatim} +\setcoffinpole \aaa [h]{t}(\height -3pt) +\setcoffinpole \aaa [v]{l}(0pt) +\setcoffinpole \aaa [h]{mybottom}(\depth +2pt) +\showcoffindata \aaa +\end{verbatim} +\setcoffinpole \aaa [h]{t}(\height -3pt) +\setcoffinpole \aaa [v]{l}(0pt) +\setcoffinpole \aaa [h]{mybottom}(\depth +2pt) +\showcoffindata \aaa + + +More interesting in many cases is the relative move of poles. For this the +poles better exist or one gets an error and \texttt{0pt,0pt} is used: +\begin{verbatim} +\adjustcoffinpole \aaa [h]{H}(1pt) +\adjustcoffinpole \aaa [v]{l}(1pt) +\adjustcoffinpole \aaa [h]{undefined}(1pt) +\showcoffindata \aaa +\end{verbatim} +\adjustcoffinpole \aaa [h]{H}(1pt) +\adjustcoffinpole \aaa [v]{l}(1pt) +\adjustcoffinpole \aaa [h]{undefined}(1pt) +\showcoffindata \aaa + + + +\sbox \aaa {\fontsize{14.4}{5.5pc minus .5pc}\sffamily C\,H\,A\,P\,T\,E\,R + \fontsize{36}{40pt}\selectfont 2} + +\setvcoffin \bbb {11cm} + {\raggedleft\normalfont\fontsize{36}{38pt}\bfseries + The Structure of a \LaTeX{} Document} + +\setvcoffin \ccc {13cm} + {Some sample blind text to produce paragraph data after a heading. + Some sample blind text to produce paragraph data after a heading. + Some sample blind text to produce paragraph data after a heading. + Some sample blind text to produce paragraph data after a heading. + \endgraf + Some more sample blind text to produce paragraph data after a heading. + Some more sample blind text to produce paragraph data after a heading. +} + + +Suppose you have the following blocks of text: + +\bigskip + +\noindent\cbox{\usebox\aaa} + +\begin{multicols}{2} +\ttfamily\tiny +\ExplSyntaxOn + \noindent \coffin_print_pole_values:Nn \aaa \\ +\ExplSyntaxOff +\end{multicols} + +\medskip + +\noindent\cbox{\usebox\bbb} + +\medskip + +\noindent\cbox{\usebox\ccc} + +\bigskip + +produced by: +\begin{verbatim} +\sbox \aaa {\fontsize{14.4}{5.5pc minus .5pc}\sffamily + C\,H\,A\,P\,T\,E\,R + \fontsize{36}{40pt}\selectfont 2} +\setvcoffin \bbb {11cm} + {\raggedleft\normalfont\fontsize{36}{38pt}\bfseries + The Structure of a \LaTeX{} Document} +\setvcoffin \ccc {13cm} + {Some sample blind text to produce paragraph + data after a heading. ... } +\end{verbatim} + +How do you turn this into a heading of TLC2? + + +\newbox\RBi +\newbox\RBii +\newbox\RBiii + +\hrulebox\RBi{62pt} +\vrulebox\RBii{90pt} +\vrulebox\RBiii{60pt} + +\aligncoffins \bbb [T,r] \aaa [H,r](-62pt,60pt) +\aligncoffins \ccc[T,r] \bbb[B,r](62pt,90pt) + + \aligncoffins * \ccc[\bbb-T,\bbb-hc] \RBiii[b,hc](28pt,0pt) + \aligncoffins * \ccc[\bbb-b,\bbb-r] \RBi[H,r](0pt,-5pt) + \aligncoffins * \ccc[\bbb-B,\bbb-hc] \RBii[t,r](14pt,0pt) + +\newpage + +How do you best define/describe the following design? + +\medskip + +\noindent\cbox{\usebox\ccc} + + + + + + +\newpage \pagestyle{empty} + +\newcoffin \eee +\newcoffin \fff +\newcoffin \ggg + +\sbox \aaa {\small\scshape les vases communicants} +\sbox \bbb {\scshape comunicating} +\sbox \ccc {\fontsize{70pt}{60pt} \bfseries Ve\S els} +\sbox \ddd {\scshape andr\'e breton} +\setvcoffin \eee {4.7cm}{\noindent Translated by Mary Ann Caws \&\break + Geoffrey T.\,Harris, with notes \&\break + introduction by Mary Ann Caws\parfillskip=0pt\relax + } +\sbox \fff {University of Nebraska Press: Lincoln \& London} + +\setvcoffin \ggg {100mm}{\noindent\fbox{\parbox{97mm}{\leavevmode\vspace*{228mm}}}} + + +\aligncoffins \ccc [H,r] \ddd [H,r](0pt,-12pt) +\aligncoffins \ccc [t,l] \bbb (0pt,6pt) +\aligncoffins \ccc [H,r] \aaa [H,r](0pt,138pt) +\aligncoffins \eee [T,l] \ccc (0pt,22pc) +\aligncoffins \fff \eee [B,l](0pt,4pc) +\aligncoffins \ggg [b,l] \fff [B,l](54pt,11pc) + + +\vspace*{-2cm} +\noindent\cbox{\usebox\ggg} + + +\newpage + +\section{Rotation} + +\sbox \aaa {\tabcolsep0pt\begin{tabular}[c]{|c|}\hline a\\b\\cccccccccccccc\\d + \\e\\e\\e\\e\\\hline\end{tabular}} +\sbox \bbb {\fbox{A sample Text}} + +% just for the sake of it: +\adjustcoffinpole \aaa {T}(24pt) +\displaycoffinpoles \aaa {black} +x\cbox{\usebox \aaa }x +% +\rotatecoffin \aaa {45} +\displaycoffinhandle \aaa {b}{hc} {blue} +\displaycoffinhandle \aaa {b}{l} {blue} +\displaycoffinhandle \aaa {t}{r} {blue} +\displaycoffinhandle \aaa {vc}{r} {blue} +\displaycoffinhandle \aaa {vc}{l} {blue} +\quad +x\cbox{\usebox \aaa }x +\rotatecoffin \aaa {45} +\displaycoffinhandle \aaa {b}{r} {red} +\displaycoffinhandle \aaa {vc}{l} {red} +\quad +x\cbox{\usebox \aaa }x + +\vspace{1cm} + +\rotatecoffin \aaa {45} +\displaycoffinhandle \aaa {b}{l} {yellow} +% +x\cbox{\usebox \aaa }x +% + +\vspace{1cm} + +x\cbox{\usebox \bbb }x +% +\rotatecoffin \bbb {45} +\displaycoffinhandle \bbb {B}{hc} {yellow} +\quad +x\cbox{\usebox \bbb }x + + +\section{Rotation + alignment} + +\aligncoffins \aaa [b,l] \bbb[B,hc](30pt,0pt) +x\cbox{\usebox \aaa }x + + +After we have aligned a roated box with some other box we need to decide about +the bounding box of the new box. Right now this become the enclosing box and +we do not maintain information about the inner boxes. So when we rotate that +new box there seems to be unnecessary space in the enclosing bounding box, +even though that is correct if you think of the aligned box being unrotated. + +There is at least the possibility to refer to the handles of the inner boxes +as one can see by two of the green handles + +\medskip + +\rotatecoffin \aaa {45} +\displaycoffinhandle \aaa {vc}{hc} {green} +\displaycoffinhandle \aaa {H}{l} {green} +\displaycoffinhandle \aaa {\aaa-T}{\aaa-r} {green} +\displaycoffinhandle \aaa {\bbb-H}{\bbb-r} {green} +%\quad +x\cbox{\usebox \aaa }x + +We could do better, if we want to, by actually checking for max and +min of all inner bounding box corners and construct the result BB box from +that---but is it worth it? + +Perhaps it is! After all, the current implementation shows different results +depending on when you align boxes and when you rotate, e.g., aligning first +gives totally different bounding box results. + +\newpage + +Aligning first and then 135 + 45 rotation gives this: + +\sbox \aaa {\tabcolsep0pt\begin{tabular}[c]{|c|}\hline a\\b\\cccccccccccccc\\d + \\e\\e\\e\\e\\\hline\end{tabular}} +\sbox \bbb {\fbox{A sample Text}} +\rotatecoffin \bbb {90} + +\aligncoffins \aaa [b,l] \bbb[B,hc](-30pt,0pt) +x\cbox{\usebox \aaa }x +\qquad +\rotatecoffin \aaa {135} +x\cbox{\usebox \aaa }x +\qquad +\rotatecoffin \aaa {45} +x\cbox{\usebox \aaa }x + + + + +\newpage + +\section{Rotation by small amounts} + +\sbox \aaa {\tabcolsep0pt\begin{tabular}[c]{|c|}\hline a\\b\\ccccccccc\\d + \\e\\e\\e\\\hline\end{tabular}} + +\subsection{30 + 30 +30 } +\rotatecoffin \aaa {30} +x\cbox{\usebox \aaa }x +\rotatecoffin \aaa {30} +x\cbox{\usebox \aaa }x +\rotatecoffin \aaa {30} +x\cbox{\usebox \aaa }x + +\subsection{6 * 10 + 45 + 45 + 30} + +\rotatecoffin \aaa {10} +x\cbox{\usebox \aaa }x +\rotatecoffin \aaa {10} +x\cbox{\usebox \aaa }x +\rotatecoffin \aaa {10} +x\cbox{\usebox \aaa }x + +\vspace{1cm} +\rotatecoffin \aaa {10} +x\cbox{\usebox \aaa }x +\rotatecoffin \aaa {10} +x\cbox{\usebox \aaa }x +\rotatecoffin \aaa {10} +x\cbox{\usebox \aaa }x + +\vspace{1cm} +\rotatecoffin \aaa {45} +x\cbox{\usebox \aaa }x +\rotatecoffin \aaa {45} +x\cbox{\usebox \aaa }x +\rotatecoffin \aaa {30} +x\cbox{\usebox \aaa }x + + +\end{document} + + diff --git a/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example-keyval.tex b/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example-keyval.tex new file mode 100644 index 00000000000..e83c0b490b6 --- /dev/null +++ b/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example-keyval.tex @@ -0,0 +1,281 @@ + +\documentclass{article} + +\usepackage[a4paper,margin=5pt]{geometry} + +\usepackage[T1]{fontenc} +\usepackage{times,multicol,graphicx} +\usepackage{xcoffins} + +\usepackage{times,color} +\newcommand\cbox[2][.8]{{\setlength\fboxsep{0pt}\colorbox[gray]{#1}{#2}}} + +\ExplSyntaxOn + +\ExplSyntaxOff + + + +%\scrollmode + +\pagestyle{empty} + +\begin{document} + + \NewCoffin \result + \NewCoffin \aaa + \NewCoffin \bbb + \NewCoffin \ccc + \NewCoffin \ddd + \NewCoffin \eee + \NewCoffin \fff + \NewCoffin \rulei + \NewCoffin \ruleii + \NewCoffin \ruleiii + +\SetHorizontalCoffin \result {} +\SetHorizontalCoffin \aaa {\fontsize{52}{50}\sffamily\bfseries mitteilungen} +\SetHorizontalCoffin \bbb {\fontsize{52}{50}\sffamily\bfseries typographische} +\SetHorizontalCoffin \ccc {\fontsize{12}{10}\sffamily + \quad zeitschrift des bildungsverbandes der + deutschen buchdrucker leipzig + \textbullet{} oktoberheft 1925} +\SetHorizontalCoffin \ddd {\fontsize{28}{20}\sffamily sonderheft} +\SetVerticalCoffin \eee {180pt} + {\raggedleft\fontsize{31}{36}\sffamily\bfseries + elementare\\ + typographie} +\SetVerticalCoffin \fff {140pt} + {\raggedright \fontsize{13}{14}\sffamily\bfseries + natan altman \\ + otto baumberger \\ + herbert mayer \\ + max burchartz \\ + el lissitzky \\ + ladislaus moholy-nagy \\ + moln\'ar f.~farkas \\ + johannes molzahn \\ + kurt schwitters \\ + mart stam \\ + ivan tschichold} + +\RotateCoffin \bbb {90} +\RotateCoffin \ccc {270} + +\SetHorizontalCoffin \rulei {\color{red}\rule{6.5in}{1pc}} +\SetHorizontalCoffin \ruleii {\color{red}\rule{1pc}{23.5cm}} +\SetHorizontalCoffin \ruleiii{\color{black}\rule{10pt}{152pt}} + + +\JoinCoffins \result \aaa +\JoinCoffins [ + coffin1-hpole = \aaa-t , + coffin1-vpole = \aaa-r , + coffin2-hpole = b , + coffin2-vpole = r , + voffset = 2 mm , +] \result \rulei +\JoinCoffins [ + coffin1-hpole = \aaa-b , + coffin1-vpole = \aaa-l , + coffin2-hpole = B , + coffin2-vpole = r , + hoffset = 2 pt , +] \result \bbb +\JoinCoffins [ + coffin1-hpole = \bbb-t , + coffin1-vpole = \bbb-r , + coffin2-hpole = t , + coffin2-vpole = r , + hoffset = -2 mm , +] \result \ruleii +\JoinCoffins [ + coffin1-hpole = \aaa-B , + coffin1-vpole = \aaa-r , + coffin2-hpole = B , + coffin2-vpole = l , + hoffset = 66 pt , + voffset = 14 pc , +] \result \ccc +\JoinCoffins [ + coffin1-hpole = \bbb-l , + coffin1-vpole = \ccc-B , + coffin2-hpole = t , + coffin2-vpole = r , + hoffset = -2 mm , +] \result \fff +\JoinCoffins [ + coffin1-hpole = \fff-b , + coffin1-vpole = \fff-r , + coffin2-hpole = b , + coffin2-vpole = l , + hoffset = 2 mm , +] \result \ruleiii +\JoinCoffins [ + coffin1-hpole = \ccc-r , + coffin1-vpole = \fff-l , + coffin2-hpole = B , + coffin2-vpole = r , +] \result \eee +\JoinCoffins [ + coffin1-hpole = \eee-T , + coffin1-vpole = \eee-r , + coffin2-hpole = B , + coffin2-vpole = r , + voffset = 4 pc , +] \result \ddd + + +\vspace*{3cm} +\begin{center} + {\Large Title page of ``elementare typographie'' by Ivan Tschichold\par} + +\large +\vspace*{1cm} + + 1. first the scanned original from 1925 + +\vspace*{6mm} + + 2. then the recreated \TeX{} version from 2010 using coffins---not + attempting\\ to match the fonts and size but the structure + +\vspace*{6mm} + + 3. and finally the source code used. + + This document uses the new + implementation by Joseph in \texttt{l3coffins-new}. + +\end{center} + +\newpage + + +\begin{center} + \null + \fbox{\includegraphics[scale=.95]{elementare-typographie-title.jpg}} +\end{center} + +\newpage + +\TypesetCoffin \result + +\newpage + +\vspace*{3cm} +\begin{center} + {\Large Code used: \par} +\vspace*{1cm} + + +\begin{minipage}{14cm} +\begin{verbatim} +\SetHorizontalCoffin\result{} +\SetHorizontalCoffin \aaa {\fontsize{52}{50}\sffamily\bfseries mitteilungen} +\SetHorizontalCoffin \bbb {\fontsize{52}{50}\sffamily\bfseries typographische} +\SetHorizontalCoffin \ccc {\fontsize{12}{10}\sffamily + \quad zeitschrift des bildungsverbandes der + deutschen buchdrucker leipzig + \textbullet{} oktoberheft 1925} +\SetHorizontalCoffin \ddd {\fontsize{28}{20}\sffamily sonderheft} +\SetVerticalCoffin \eee {180pt} + {\raggedleft\fontsize{31}{36}\sffamily\bfseries + elementare\\ + typographie} +\SetVerticalCoffin \fff {140pt} + {\raggedright \fontsize{13}{14}\sffamily\bfseries + natan altman \\ + otto baumberger \\ + herbert mayer \\ + max burchartz \\ + el lissitzky \\ + ladislaus moholy-nagy \\ + moln\'ar f.~farkas \\ + jahannes molzahn \\ + kurt schwitters \\ + mart stam \\ + ivan tschichold} + +\RotateCoffin \bbb {90} +\RotateCoffin \ccc {270} + +\SetHorizontalCoffin \rulei {\color{red}\rule{6.5in}{1pc}} +\SetHorizontalCoffin \ruleii {\color{red}\rule{1pc}{23.5cm}} +\SetHorizontalCoffin \ruleiii{\color{black}\rule{10pt}{152pt}} + +\JoinCoffins \result \aaa +\JoinCoffins [ + coffin1-hpole = \aaa-t , + coffin1-vpole = \aaa-r , + coffin2-hpole = b , + coffin2-vpole = r , + voffset = 2 mm , +] \result \rulei +\JoinCoffins [ + coffin1-hpole = \aaa-b , + coffin1-vpole = \aaa-l , + coffin2-hpole = B , + coffin2-vpole = r , + hoffset = 2 pt , +] \result \bbb +\JoinCoffins [ + coffin1-hpole = \bbb-t , + coffin1-vpole = \bbb-r , + coffin2-hpole = t , + coffin2-vpole = r , + hoffset = -2 mm , +] \result \ruleii +\JoinCoffins [ + coffin1-hpole = \aaa-B , + coffin1-vpole = \aaa-r , + coffin2-hpole = B , + coffin2-vpole = l , + hoffset = 66 pt , + voffset = 14 pc , +] \result \ccc +\JoinCoffins [ + coffin1-hpole = \bbb-l , + coffin1-vpole = \ccc-B , + coffin2-hpole = t , + coffin2-vpole = r , + hoffset = -2 mm , +] \result \fff +\JoinCoffins [ + coffin1-hpole = \fff-b , + coffin1-vpole = \fff-r , + coffin2-hpole = b , + coffin2-vpole = l , + hoffset = 2 mm , +] \result \ruleiii +\JoinCoffins [ + coffin1-hpole = \ccc-r , + coffin1-vpole = \fff-l , + coffin2-hpole = B , + coffin2-vpole = r , +] \result \eee +\JoinCoffins [ + coffin1-hpole = \eee-T , + coffin1-vpole = \eee-r , + coffin2-hpole = B , + coffin2-vpole = r , + voffset = 4 pc , +] \result \ddd +\SetHorizontalCoffin\result{} +\SetHorizontalCoffin \aaa {\fontsize{52}{50}\sffamily\bfseries + mitteilungen} +\SetHorizontalCoffin \bbb {\fontsize{52}{50}\rotatebox{90}{\sffamily\bfseries + typographische}} + +\TypesetCoffin \result +\end{verbatim} + +This is not necessarily the final syntax but for now it does its job. For +example, flexible support for adding ornaments (lines, \ldots) is still +missing, so above the rules got added as predefined individual coffins. + +\end{minipage} +\end{center} + +\end{document} + diff --git a/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example.tex b/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example.tex new file mode 100644 index 00000000000..343c9d7e29f --- /dev/null +++ b/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example.tex @@ -0,0 +1,185 @@ + +\documentclass{article} + +\usepackage[a4paper,margin=5pt]{geometry} + +\usepackage[T1]{fontenc} +\usepackage{times,multicol,graphicx} +\usepackage{xcoffins} + +\usepackage{times,color} +\newcommand\cbox[2][.8]{{\setlength\fboxsep{0pt}\colorbox[gray]{#1}{#2}}} + +\ExplSyntaxOn + +\ExplSyntaxOff + + + +%\scrollmode + +\pagestyle{empty} + +\begin{document} + + \NewCoffin \result + \NewCoffin \aaa + \NewCoffin \bbb + \NewCoffin \ccc + \NewCoffin \ddd + \NewCoffin \eee + \NewCoffin \fff + \NewCoffin \rulei + \NewCoffin \ruleii + \NewCoffin \ruleiii + +\SetHorizontalCoffin \result {} +\SetHorizontalCoffin \aaa {\fontsize{52}{50}\sffamily\bfseries mitteilungen} +\SetHorizontalCoffin \bbb {\fontsize{52}{50}\sffamily\bfseries typographische} +\SetHorizontalCoffin \ccc {\fontsize{12}{10}\sffamily + \quad zeitschrift des bildungsverbandes der + deutschen buchdrucker leipzig + \textbullet{} oktoberheft 1925} +\SetHorizontalCoffin \ddd {\fontsize{28}{20}\sffamily sonderheft} +\SetVerticalCoffin \eee {180pt} + {\raggedleft\fontsize{31}{36}\sffamily\bfseries + elementare\\ + typographie} +\SetVerticalCoffin \fff {140pt} + {\raggedright \fontsize{13}{14}\sffamily\bfseries + natan altman \\ + otto baumberger \\ + herbert mayer \\ + max burchartz \\ + el lissitzky \\ + ladislaus moholy-nagy \\ + moln\'ar f.~farkas \\ + johannes molzahn \\ + kurt schwitters \\ + mart stam \\ + ivan tschichold} + +\RotateCoffin \bbb {90} +\RotateCoffin \ccc {270} + +\SetHorizontalCoffin \rulei {\color{red}\rule{6.5in}{1pc}} +\SetHorizontalCoffin \ruleii {\color{red}\rule{1pc}{23.5cm}} +\SetHorizontalCoffin \ruleiii{\color{black}\rule{10pt}{152pt}} + + +\JoinCoffins \result \aaa +\JoinCoffins \result[\aaa-t,\aaa-r] \rulei [b,r](0pt,2mm) +\JoinCoffins \result[\aaa-b,\aaa-l] \bbb [B,r](2pt,0pt) +\JoinCoffins \result[\bbb-t,\bbb-r] \ruleii [t,r](-2mm,0pt) +\JoinCoffins \result[\aaa-B,\aaa-r] \ccc [B,l](66pt,14pc) +\JoinCoffins \result[\bbb-l,\ccc-B] \fff [t,r](-2mm,0pt) +\JoinCoffins \result[\fff-b,\fff-r] \ruleiii [b,l](2mm,0pt) +\JoinCoffins \result[\ccc-r,\fff-l] \eee [B,r] +\JoinCoffins \result[\eee-T,\eee-r] \ddd [B,r](0pt,4pc) + + +\vspace*{3cm} +\begin{center} + {\Large Title page of ``elementare typographie'' by Ivan Tschichold\par} + +\large +\vspace*{1cm} + + 1. first the scanned original from 1925 + +\vspace*{6mm} + + 2. then the recreated \TeX{} version from 2010 using coffins---not + attempting\\ to match the fonts and size but the structure + +\vspace*{6mm} + + 3. and finally the source code used. + + This document uses the new + implementation by Joseph in \texttt{l3coffins-new}. + +\end{center} + +\newpage + + +\begin{center} + \null + \fbox{\includegraphics[scale=.95]{elementare-typographie-title.jpg}} +\end{center} + +\newpage + +\TypesetCoffin \result + +\newpage + +\vspace*{3cm} +\begin{center} + {\Large Code used: \par} +\vspace*{1cm} + + +\begin{minipage}{14cm} +\begin{verbatim} +\SetHorizontalCoffin\result{} +\SetHorizontalCoffin \aaa {\fontsize{52}{50}\sffamily\bfseries mitteilungen} +\SetHorizontalCoffin \bbb {\fontsize{52}{50}\sffamily\bfseries typographische} +\SetHorizontalCoffin \ccc {\fontsize{12}{10}\sffamily + \quad zeitschrift des bildungsverbandes der + deutschen buchdrucker leipzig + \textbullet{} oktoberheft 1925} +\SetHorizontalCoffin \ddd {\fontsize{28}{20}\sffamily sonderheft} +\SetVerticalCoffin \eee {180pt} + {\raggedleft\fontsize{31}{36}\sffamily\bfseries + elementare\\ + typographie} +\SetVerticalCoffin \fff {140pt} + {\raggedright \fontsize{13}{14}\sffamily\bfseries + natan altman \\ + otto baumberger \\ + herbert mayer \\ + max burchartz \\ + el lissitzky \\ + ladislaus moholy-nagy \\ + moln\'ar f.~farkas \\ + jahannes molzahn \\ + kurt schwitters \\ + mart stam \\ + ivan tschichold} + +\RotateCoffin \bbb {90} +\RotateCoffin \ccc {270} + +\SetHorizontalCoffin \rulei {\color{red}\rule{6.5in}{1pc}} +\SetHorizontalCoffin \ruleii {\color{red}\rule{1pc}{23.5cm}} +\SetHorizontalCoffin \ruleiii{\color{black}\rule{10pt}{152pt}} + +\JoinCoffins \result \aaa +\JoinCoffins \result[\aaa-t,\aaa-r] \rulei [b,r](0pt,2mm) +\JoinCoffins \result[\aaa-b,\aaa-l] \bbb [B,r](2pt,0pt) +\JoinCoffins \result[\bbb-t,\bbb-r] \ruleii [t,r](-2mm,0pt) +\JoinCoffins \result[\aaa-B,\aaa-r] \ccc [B,l](66pt,14pc) +\JoinCoffins \result[\bbb-l,\ccc-B] \fff [t,r](-2mm,0pt) +\JoinCoffins \result[\fff-b,\fff-r] \ruleiii [b,l](2mm,0pt) +\JoinCoffins \result[\ccc-r,\fff-l] \eee [B,r] +\JoinCoffins \result[\eee-T,\eee-r] \ddd [B,r](0pt,4pc) +\SetHorizontalCoffin\result{} +\SetHorizontalCoffin \aaa {\fontsize{52}{50}\sffamily\bfseries + mitteilungen} +\SetHorizontalCoffin \bbb {\fontsize{52}{50}\rotatebox{90}{\sffamily\bfseries + typographische}} + +\TypesetCoffin \result +\end{verbatim} + +This is not necessarily the final syntax but for now it does its job. For +example, flexible support for adding ornaments (lines, \ldots) is still +missing, so above the rules got added as predefined individual coffins. + +\end{minipage} +\end{center} + +\end{document} + diff --git a/Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx b/Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx index 0d3fdafcdf8..dc6279914af 100644 --- a/Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx +++ b/Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx @@ -37,7 +37,7 @@ % %<*driver|package> \RequirePackage{l3names} -\GetIdInfo$Id: l3galley.dtx 3071 2011-12-20 20:14:00Z bruno $ +\GetIdInfo$Id: l3galley.dtx 3205 2012-01-18 22:09:05Z joseph $ {L3 Experimental galley code} % %<*driver> @@ -372,7 +372,7 @@ % % \begin{variable}{\l_galley_final_hyphen_demerits_int} % Extra demerit charge if the second last line is hyphenated. -% \end{variable} +% \end{variable} % % \begin{variable}{\l_galley_linebreak_badness_int} % Boundary that if exceeded will cause \TeX{} to report an @@ -388,12 +388,12 @@ % this penalty higher \TeX{} will try harder to produce compact % paragraphs. % \end{variable} -% +% % \begin{variable}{\l_galley_linebreak_pretolerance_int} % Maximum tolerance allowed for individual lines to break the % paragraph without attempting hyphenation. % \end{variable} -% +% % \begin{variable}{\l_galley_linebreak_tolerance_int} % Maximum tolerance allowed for individual lines when breaking a % paragraph while attempting hyphenation (if this limit can't be @@ -416,7 +416,7 @@ % Boundary that if exceeded will cause \TeX{} to report an % underfull vertical box. % \end{variable} -% +% % \begin{variable}{\l_galley_parbreak_fuzz_dim} % Boundary below which overfull vertical boxes are not reported. % \end{variable} @@ -432,7 +432,7 @@ % \begin{variable}{\l_galley_post_display_penalty_int} % Penalty for breaking between immediately after display math material. % \end{variable} -% +% % \begin{function} % { % \galley_set_club_penalties:n, @@ -468,7 +468,7 @@ % In all cases, these penalties apply in addition to the general interline % penalty or to any \enquote{special} line penalties. % \end{function} -% +% % \begin{function}{\galley_set_interline_penalty:n} % \begin{syntax} % \cs{galley_set_interline_penalty:n} \Arg{penalty} @@ -476,7 +476,7 @@ % Sets the standard interline penalty applied between lines of a paragraph. % This value is added to any (display) club or widow penalty in force. % \end{function} -% +% % \begin{function} % { % \galley_set_interline_penalties:n, @@ -490,7 +490,7 @@ % The \meta{penalties} apply to the first, second, third, \emph{etc.}~line % of the paragraph. % \end{function} -% +% % \begin{function} % { % \galley_save_club_penalties:N, @@ -502,10 +502,10 @@ % \begin{syntax} % \cs{galley_save_club_penalties:N} \Arg{comma list} % \end{syntax} -% These functions save the current value of the appropriate to the +% These functions save the current value of the appropriate to the % comma list specified, within the current \TeX{} group. % \end{function} -% +% % \begin{function}[EXP]{\galley_interline_penalty:} % \begin{syntax} % \cs{galley_interline_penalty:} @@ -524,7 +524,7 @@ % \begin{variable}{\g_galley_par_end_hook_tl} % Token list inserted at the end of every paragraph in horizontal mode. % \end{variable} -% +% % \begin{variable}{\g_galley_par_after_hook_tl} % Token list inserted after each paragraph. This is used for resetting % galley parameters, and is therefore cleared after use. @@ -539,7 +539,7 @@ % Token list for whatsits to be inserted at the very end of the last % paragraph started. % \end{variable} -% +% % \section{Additional effects} % % \begin{function}{\galley_end_par:n} @@ -663,7 +663,7 @@ % The standard mode to leave vertical mode, starting a paragraph. % \begin{macrocode} \cs_new_protected_nopar:Npn \galley_leave_vmode: - { \tex_unhbox:D \c_empty_box } + { \hbox_unpack:N \c_empty_box } % \end{macrocode} % \end{macro} % @@ -815,7 +815,7 @@ \tl_new:N \l_galley_par_end_hook_tl % \end{macrocode} % \end{variable} -% +% % \begin{variable} % {\g_galley_par_after_hook_tl, \l_galley_par_after_hook_tl} % This one is used by the galley: it happens \enquote{after} the current @@ -1082,7 +1082,7 @@ { \tex_par:D } % \end{macrocode} % In horizontal mode, the paragraph shape is set \enquote{just in time} -% before inserting \cs{tex_par:D}. The \cs{tex_par:D} is inside a +% before inserting \cs{tex_par:D}. The \cs{tex_par:D} is inside a % group to preserve some dynamic settings (for example % \cs{etex_interlinepenalties}). Once the % paragraph has been typeset, the number of lines is \emph{added} to the @@ -1112,7 +1112,7 @@ % \end{macro} % \end{macro} % \end{macro} -% +% % \begin{macro}{\galley_end_par:n} % Inserts tokens such that they are appended to the end of the last % paragraph, using the paragraph-omitting system. @@ -1830,7 +1830,7 @@ \cs_new_eq:NN \l_galley_pre_display_penalty_int \tex_predisplaypenalty:D % \end{macrocode} % \end{variable} -% +% % \begin{macro}{\l_galley_club_penalties_clist, \l_galley_line_penalties_clist} % These are used to keep a track of information which cannot be % extracted out of the primitives due to the overlapping nature of @@ -1840,7 +1840,7 @@ \clist_new:N \l_galley_line_penalties_clist % \end{macrocode} % \end{macro} -% +% % \begin{macro} % { % \galley_set_display_widow_penalties:n, @@ -1878,7 +1878,7 @@ % \end{macrocode} % \end{macro} % \end{macro} -% +% % \begin{macro} % { % \galley_set_club_penalties:n, @@ -1907,7 +1907,7 @@ \cs_generate_variant:Nn \galley_set_interline_penalties:n { V , v } % \end{macrocode} % \end{macro} -% +% % \begin{macro} % { % \galley_set_display_club_penalties:n, @@ -1928,7 +1928,7 @@ \cs_generate_variant:Nn \galley_set_display_club_penalties:n { V , v } % \end{macrocode} % \end{macro} -% +% % \begin{macro}{\galley_set_interline_penalty:n} % \begin{macro}[aux]{\galley_set_interline_penalty_aux:nn} % \begin{macro}[aux] @@ -1957,17 +1957,17 @@ \etex_interlinepenalties:D ##1 - \etex_interlinepenalties:D \etex_interlinepenalties:D \c_zero + #1 - \int_eval_end: + \int_eval_end: } \exp_args:Nf \galley_set_interline_penalty_aux:nn { \clist_length:N \l_galley_line_penalties_clist } {#1} } } \cs_new_protected_nopar:Npn \galley_set_interline_penalty_aux:nn #1#2 - { + { \etex_interlinepenalties:D \etex_interlinepenalties:D \c_zero - \prg_stepwise_function:nnnN \c_one \c_one {#1} + \prg_stepwise_function:nnnN \c_one \c_one {#1} \galley_set_interline_penalty_aux_i:n \prg_stepwise_function:nnnN { #1 + \c_one } \c_one { \etex_interlinepenalties:D \c_zero - \c_one } @@ -1981,7 +1981,7 @@ % \end{macro} % \end{macro} % \end{macro} -% +% % \begin{macro}[int]{\galley_calc_interline_penalties:} % \begin{macro}[aux]{\galley_calc_interline_penalties_aux:nn} % \begin{macro}[aux] @@ -2012,7 +2012,7 @@ { \clist_length:N \l_galley_line_penalties_clist + \c_one - } + } { \etex_clubpenalties:D \c_zero } } } @@ -2049,7 +2049,7 @@ #1 { \int_eval:w - \etex_interlinepenalties:D \etex_interlinepenalties:D \c_zero + \etex_interlinepenalties:D \etex_interlinepenalties:D \c_zero + 0 \clist_item:Nn \l_galley_club_penalties_clist { #1 - \c_one } - \etex_clubpenalties:D #1 ~ @@ -2059,7 +2059,7 @@ % \end{macro} % \end{macro} % \end{macro} -% +% % \begin{macro} % { % \galley_save_club_penalties:N, @@ -2130,10 +2130,9 @@ % \end{macro} % \end{macro} % \end{macro} -% % % \subsection{\LaTeXe{} functions} -% +% % \begin{macrocode} %<*package> % \end{macrocode} @@ -2174,7 +2173,7 @@ % Purely for testing, some internal \LaTeXe{} functions are altered to work % with the mechanism here. This material is not comprehensive: additions are % made as-needed for test purposes. -% +% % \begin{macro}{\@@par} % The primitive is moved as otherwise the clever skipping code will fail. % \begin{macrocode} -- cgit v1.2.3