From 5730d1c679f7a725b75ce80066cc6a6987aee888 Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Sat, 13 Aug 2011 22:19:08 +0000 Subject: l3kernel 2564 (13aug11) git-svn-id: svn://tug.org/texlive/trunk@23540 c570f23f-e606-0410-a88d-b1316a301751 --- Master/texmf-dist/source/latex/l3kernel/expl3.dtx | 12 +- .../source/latex/l3kernel/l3bootstrap.dtx | 6 +- Master/texmf-dist/source/latex/l3kernel/l3doc.dtx | 6 +- .../texmf-dist/source/latex/l3kernel/l3expan.dtx | 19 +- Master/texmf-dist/source/latex/l3kernel/l3int.dtx | 6 +- .../texmf-dist/source/latex/l3kernel/l3luatex.dtx | 29 +- Master/texmf-dist/source/latex/l3kernel/l3msg.dtx | 45 +- .../texmf-dist/source/latex/l3kernel/l3names.dtx | 9 +- Master/texmf-dist/source/latex/l3kernel/l3prg.dtx | 9 +- .../texmf-dist/source/latex/l3kernel/l3quark.dtx | 11 +- Master/texmf-dist/source/latex/l3kernel/l3seq.dtx | 39 +- Master/texmf-dist/source/latex/l3kernel/l3tl.dtx | 1133 ++++++++++++++------ 12 files changed, 914 insertions(+), 410 deletions(-) (limited to 'Master/texmf-dist/source/latex/l3kernel') diff --git a/Master/texmf-dist/source/latex/l3kernel/expl3.dtx b/Master/texmf-dist/source/latex/l3kernel/expl3.dtx index 9f7a8a698aa..48ef3aa0a54 100644 --- a/Master/texmf-dist/source/latex/l3kernel/expl3.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/expl3.dtx @@ -37,8 +37,8 @@ \RequirePackage{l3names} \def\ExplFileName{expl3} \def\ExplFileDescription{L3 Experimental code bundle wrapper} -\def\ExplFileDate{2011/08/10} -\def\ExplFileVersion{2535} +\def\ExplFileDate{2011/08/13} +\def\ExplFileVersion{2564} % %<*driver> \documentclass[full]{l3doc} @@ -298,7 +298,7 @@ % of course goes hand in hand with when to use |l_| and |g_| % variable prefixes. % - %\subsubsection{Variables: scope and type} +% \subsubsection{Variables: scope and type} % % The \meta{scope} part of the name describes how the variable can be % accessed. Variables are classified as local, global or constant. @@ -367,12 +367,6 @@ % % \subsubsection{Variables: guidance} % -% Although \TeX{} provides token registers for certain programming -% tasks, \LaTeX3 is written so that token list variables can be used -% for all stored lists of tokens. The \cs{toks_\ldots} functions are -% therefore for internal use within \LaTeX3 and for supporting -% \LaTeXe, and should not be needed in new \LaTeX3-based code. -% % Both comma lists and sequences both have similar characteristics. % They both use special delimiters to mark out one entry from the % next, and are both accessible at both ends. In general, it is diff --git a/Master/texmf-dist/source/latex/l3kernel/l3bootstrap.dtx b/Master/texmf-dist/source/latex/l3kernel/l3bootstrap.dtx index ee971e3b09b..509f3b1ad7f 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3bootstrap.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3bootstrap.dtx @@ -83,7 +83,7 @@ {\def\ExplFileDate{#2}} \protected\def\GetIdInfoAuxSVN#1\relax#2-#3-#4\relax#5Z\relax {\def\ExplFileDate{#2/#3/#4}} -\GetIdInfo$Id: l3bootstrap.dtx 2519 2011-08-08 09:35:54Z mittelba $ +\GetIdInfo$Id: l3bootstrap.dtx 2558 2011-08-13 06:32:00Z joseph $ {L3 experimental bootstrap code} % %<*driver> @@ -605,9 +605,9 @@ \protected\edef\ExplSyntaxNamesOff {% \expandafter\noexpand - \csname\detokenize{char_set_catcode_math_subscript:n}\endcsname{58}% + \csname\detokenize{char_set_catcode_other:n}\endcsname{58}% \expandafter\noexpand - \csname\detokenize{char_set_catcode_other:n}\endcsname{95}% + \csname\detokenize{char_set_catcode_math_subscript:n}\endcsname{95}% } % \end{macrocode} % \end{macro} diff --git a/Master/texmf-dist/source/latex/l3kernel/l3doc.dtx b/Master/texmf-dist/source/latex/l3kernel/l3doc.dtx index 5039b8c0f62..e3dd82e210e 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3doc.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3doc.dtx @@ -78,7 +78,7 @@ Do not distribute a modified version of this file. % \end{macrocode} % %<*driver|class> -\GetIdInfo$Id: l3doc.dtx 2478 2011-06-19 21:34:23Z joseph $ +\GetIdInfo$Id: l3doc.dtx 2540 2011-08-10 23:39:33Z bruno $ {L3 Experimental documentation class} % % @@ -343,12 +343,12 @@ Do not distribute a modified version of this file. % If you are documenting an auxiliary macro, it's generally not necessary % to highlight it as much and you also don't need to check it for, say, % having a test function and having a documentation chunk earlier in a "function" -% environment. In this case, write "\begin[aux]{macro}" and it will be +% environment. In this case, write "\begin{macro}[aux]" and it will be % marked as such; its margin call-out will be printed in grey. % % Similarly, an internal package function still requires documentation % but usually will not be documented for users to see; these can be marked -% as such with "\begin[internal]{macro}". +% as such with "\begin{macro}[internal]". % % For documenting \pkg{expl3}-type conditionals, you may also pass this % environment a "TF" option (and omit it from the function name) to denote that diff --git a/Master/texmf-dist/source/latex/l3kernel/l3expan.dtx b/Master/texmf-dist/source/latex/l3kernel/l3expan.dtx index 5b37fd49aaf..915e48dec15 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3expan.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3expan.dtx @@ -35,7 +35,7 @@ % %<*driver|package> \RequirePackage{l3names} -\GetIdInfo$Id: l3expan.dtx 2480 2011-06-20 17:25:23Z bruno $ +\GetIdInfo$Id: l3expan.dtx 2545 2011-08-11 13:47:19Z bruno $ {L3 Experimental argument expansion} % %<*driver> @@ -845,17 +845,18 @@ % result is an error message looking like this: % \begin{verbatim} % ! Undefined control sequence. -% \exp_eval_error_msg:w ...erroneous variable used! -% +% \LaTeX3 error: +% Erroneous variable used! % l.55 \tl_set:Nv \l_tmpa_tl {undefined_tl} % \end{verbatim} % \begin{macrocode} -\group_begin:% -\tex_catcode:D`\!=11\scan_stop:% -\tex_catcode:D`\ =11\scan_stop:% -\cs_new:Npn\exp_eval_error_msg:w#1\tex_the:D#2{% -\fi:\fi:\exp_after:wN\c_zero\erroneous variable used!}% -\group_end:% +\cs_new:Npn \exp_eval_error_msg:w #1 \tex_the:D #2 + { + \fi: + \fi: + \msg_expandable_error:n { Erroneous ~ variable ~ #2 used! } + \c_zero + } % \end{macrocode} % \end{macro} % \end{macro} diff --git a/Master/texmf-dist/source/latex/l3kernel/l3int.dtx b/Master/texmf-dist/source/latex/l3kernel/l3int.dtx index 91dc8e798ce..89695862b2f 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3int.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3int.dtx @@ -35,7 +35,7 @@ % %<*driver|package> \RequirePackage{l3names} -\GetIdInfo$Id: l3int.dtx 2526 2011-08-09 08:58:42Z joseph $ +\GetIdInfo$Id: l3int.dtx 2551 2011-08-11 22:40:53Z bruno $ {L3 Experimental integers} % %<*driver> @@ -1803,7 +1803,7 @@ % \begin{macrocode} \cs_new:Npn \int_get_sign_and_digits_aux:nNNN #1#2#3#4 { - \tl_if_head_eq_charcode:fNTF {#1} - + \exp_args:Nf \tl_if_head_eq_charcode:nNTF {#1} - { \bool_if:NTF #2 { @@ -1816,7 +1816,7 @@ } } { - \tl_if_head_eq_charcode:fNTF {#1} + + \exp_args:Nf \tl_if_head_eq_charcode:nNTF {#1} + { \int_get_sign_and_digits_aux:oNNN { \use_none:n #1 } #2#3#4 } { \bool_if:NT #3 { \bool_if:NF #2 - } diff --git a/Master/texmf-dist/source/latex/l3kernel/l3luatex.dtx b/Master/texmf-dist/source/latex/l3kernel/l3luatex.dtx index 70259184340..a0e90de31fc 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3luatex.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3luatex.dtx @@ -35,7 +35,7 @@ % %<*driver|package> \RequirePackage{l3names} -\GetIdInfo$Id: l3luatex.dtx 2478 2011-06-19 21:34:23Z joseph $ +\GetIdInfo$Id: l3luatex.dtx 2545 2011-08-11 13:47:19Z bruno $ {L3 Experimental LuaTeX-specific functions} % %<*driver> @@ -235,12 +235,11 @@ % \begin{macro}{\lua_now:n, \lua_now:x} % \begin{macro}{\lua_shipout_x:n, \lua_shipout_x:x} % \begin{macro}{\lua_shipout:n, \lua_shipout:x} -% \begin{macro}[aux]{\lua_wrong_engine:} % When \LuaTeX{} is in use, this is all a question of primitives with new % names. On the other hand, for \pdfTeX{} and \XeTeX{} the argument should -% be removed from the input stream before issuing an error. This needs -% to be expandable, so the same idea is used as for \texttt{V}-type -% expansion, with an appropriately-named but undefined function. +% be removed from the input stream before issuing an error. This is +% expandable, using \cs{msg_expandable_error:n} as for \texttt{V}-type +% expansion. % \begin{macrocode} \luatex_if_engine:TF { @@ -248,8 +247,16 @@ \cs_new_eq:NN \lua_shipout_x:n \luatex_latelua:D } { - \cs_new:Npn \lua_now:x #1 { \lua_wrong_engine: } - \cs_new_protected:Npn \lua_shipout_x:n #1 { \lua_wrong_engine: } + \cs_new:Npn \lua_now:x #1 + { + \msg_expandable_error:n + { LuaTeX~ engine~ not~ in~ use!~ Ignoring~ \lua_now:x. } + } + \cs_new_protected:Npn \lua_shipout_x:n #1 + { + \msg_expandable_error:n + { LuaTeX~ engine~ not~ in~ use!~ Ignoring~ \lua_shipout_x:n. } + } } \cs_new:Npn \lua_now:n #1 { \lua_now:x { \exp_not:n {#1} } } @@ -257,18 +264,10 @@ \cs_new_protected:Npn \lua_shipout:n #1 { \lua_shipout_x:n { \exp_not:n {#1} } } \cs_generate_variant:Nn \lua_shipout:n { x } -\group_begin: -\char_set_catcode_letter:N\! -\char_set_catcode_letter:N\ % -\cs_gset:Npn\lua_wrong_engine:{% -\LuaTeX engine not in use!% -}% -\group_end:% % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} -% \end{macro} % % \subsection{Category code tables} % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3msg.dtx b/Master/texmf-dist/source/latex/l3kernel/l3msg.dtx index eceadf3c1a8..7e23d67b838 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3msg.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3msg.dtx @@ -35,7 +35,7 @@ % %<*driver|package> \RequirePackage{l3names} -\GetIdInfo$Id: l3msg.dtx 2478 2011-06-19 21:34:23Z joseph $ +\GetIdInfo$Id: l3msg.dtx 2551 2011-08-11 22:40:53Z bruno $ {L3 Experimental messages} % %<*driver> @@ -610,6 +610,22 @@ % The information text will be added to the log file. % \end{function} % +% \section{Expandable errors} +% +% In a few places, the \LaTeX3 kernel needs to produce errors in an +% expansion only context. This must be handled very differently from +% normal error messages, as none of the tools to print to the terminal +% or the log file are expandable. +% +% \begin{function}{\msg_expandable_error:n} +% \begin{syntax} +% \cs{msg_expandable_error:n} \Arg{error message} +% \end{syntax} +% Issues an \enquote{Undefined error} message from \TeX{} itself, +% and prints the \meta{error message}. The \meta{error message} +% must be short: it is cropped at the end of one line. +% \end{function} +% % \end{documentation} % % \begin{implementation} @@ -1521,6 +1537,13 @@ LaTeX~has~been~asked~to~recover~an~entry~from~a~sequence~that~ has~no~content:~that~cannot~happen! } +\msg_kernel_new:nnnn { tl } { empty-search-pattern } + { Empty~search~pattern. } + { + \c_msg_coding_error_text_tl + LaTeX~has~been~asked~to~replace~an~empty~pattern~by~'#1':~that~% + would~lead~to~an~infinite~loop! + } % \end{macrocode} % % \begin{macro}{\msg_kernel_bug:x} @@ -1548,6 +1571,26 @@ % \end{variable} % \end{macro} % +% \subsection{Expandable errors} +% +% \begin{macro}{\msg_expandable_error:n} +% +% \begin{macrocode} +\group_begin: +\cs_new:Npx \msg_expandable_error:n #1 + { + \exp_not:n + { + \exp_after:wN \exp_after:wN + \exp_after:wN \use_none_delimit_by_q_stop:w + } + \exp_not:N \use:n { \exp_not:c { LaTeX3 ~ error : } #1 } + \exp_not:N \q_stop + } +\group_end: +% \end{macrocode} +% \end{macro} +% % \subsection{Deprecated functions} % % Deprecated on 2011-05-27, for removal by 2011-08-31. diff --git a/Master/texmf-dist/source/latex/l3kernel/l3names.dtx b/Master/texmf-dist/source/latex/l3kernel/l3names.dtx index d8a33196962..46f7734e2d4 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3names.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3names.dtx @@ -35,7 +35,7 @@ % %<*package> \RequirePackage{l3bootstrap} -\GetIdInfo$Id: l3names.dtx 2478 2011-06-19 21:34:23Z joseph $ +\GetIdInfo$Id: l3names.dtx 2538 2011-08-10 19:37:10Z joseph $ {L3 Experimental namespace for primitives} % %<*driver> @@ -559,7 +559,7 @@ % In the case of the \pdfTeX{} primitives, we retain |pdf| at the start % of the names \emph{only} for directly PDF-related primitives, as % there are a lot of \pdfTeX{} primitives that start \cs{pdf\ldots} but -% are not related to PDF output. These ones re +% are not related to PDF output. These ones related to PDF output. % \begin{macrocode} \name_primitive:NN \pdfcreationdate \pdftex_pdfcreationdate:D \name_primitive:NN \pdfcolorstack \pdftex_pdfcolorstack:D @@ -582,11 +582,12 @@ % \begin{macrocode} \name_primitive:NN \pdfstrcmp \pdftex_strcmp:D % \end{macrocode} -% \XeTeX{}-specific primitives. +% \XeTeX{}-specific primitives. Note that \XeTeX{}'s \cs{strcmp} is +% handled earlier and is \enquote{rolled up} into \cs{pdfstrcmp}. % \begin{macrocode} \name_primitive:NN \XeTeXversion \xetex_XeTeXversion:D % \end{macrocode} -% Primitives from \LuaTeX. +% Primitives from \LuaTeX{}. % \begin{macrocode} \name_primitive:NN \catcodetable \luatex_catcodetable:D \name_primitive:NN \directlua \luatex_directlua:D diff --git a/Master/texmf-dist/source/latex/l3kernel/l3prg.dtx b/Master/texmf-dist/source/latex/l3kernel/l3prg.dtx index 58175b05561..386b98b32dd 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3prg.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3prg.dtx @@ -35,7 +35,7 @@ % %<*driver|package> \RequirePackage{l3names} -\GetIdInfo$Id: l3prg.dtx 2509 2011-08-07 11:21:26Z bruno $ +\GetIdInfo$Id: l3prg.dtx 2553 2011-08-11 23:55:20Z bruno $ {L3 Experimental control structures} % %<*driver> @@ -1488,7 +1488,7 @@ { \str_if_eq:xxTF {#1} {#2} { \prg_case_end:nw {#3} } - { \prg_case_str_aux:nw {#1} } + { \prg_case_str_x_aux:nw {#1} } } % \end{macrocode} % \end{macro} @@ -1797,9 +1797,12 @@ % find its way to the output. The following functions help with this % by using code documented only in Appendix~D of % \emph{The \TeX{}book}\dots +% We place the \cs{if_false:} |{| \cs{fi:} part at that place so +% that the successive expansions of \cs{group_align_safe_begin/end:} +% are always brace balanced. % \begin{macrocode} \cs_new_nopar:Npn \group_align_safe_begin: - { \if_false: { \fi: \if_int_compare:w `} = \c_zero \fi: } + { \if_int_compare:w \if_false: { \fi: `} = \c_zero \fi: } \cs_new_nopar:Npn \group_align_safe_end: { \if_int_compare:w `{ = \c_zero } \fi: } % \end{macrocode} diff --git a/Master/texmf-dist/source/latex/l3kernel/l3quark.dtx b/Master/texmf-dist/source/latex/l3kernel/l3quark.dtx index 2326fd4519d..11f4b0805d1 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3quark.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3quark.dtx @@ -35,7 +35,7 @@ % %<*driver|package> \RequirePackage{l3names} -\GetIdInfo$Id: l3quark.dtx 2478 2011-06-19 21:34:23Z joseph $ +\GetIdInfo$Id: l3quark.dtx 2549 2011-08-11 21:06:39Z bruno $ {L3 Experimental quarks} % %<*driver> @@ -475,6 +475,15 @@ % \end{macro} % \end{macro} % +% \begin{variable}{\q_tl_act_mark,\q_tl_act_stop} +% These private quarks are needed by \pkg{l3tl}, but that is loaded +% before the quark module, hence their definition is deferred. +% \begin{macrocode} +\quark_new:N \q_tl_act_mark +\quark_new:N \q_tl_act_stop +% \end{macrocode} +% \end{variable} +% % \begin{macrocode} % % \end{macrocode} diff --git a/Master/texmf-dist/source/latex/l3kernel/l3seq.dtx b/Master/texmf-dist/source/latex/l3kernel/l3seq.dtx index 088165a02fc..0babc835ec9 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3seq.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3seq.dtx @@ -35,7 +35,7 @@ % %<*driver|package> \RequirePackage{l3names} -\GetIdInfo$Id: l3seq.dtx 2478 2011-06-19 21:34:23Z joseph $ +\GetIdInfo$Id: l3seq.dtx 2556 2011-08-12 14:00:43Z bruno $ {L3 Experimental sequences and stacks} % %<*driver> @@ -713,6 +713,15 @@ % \meta{comma-list}. % \end{function} % +% \begin{function}{\seq_set_reverse:N,\seq_gset_reverse:N} +% \begin{syntax} +% \cs{seq_set_reverse:N} \meta{sequence} +% \end{syntax} +% Reverses the order of items in the \meta{sequence}, and +% assigns the result to \meta{sequence}, locally or globally +% according to the variant chosen. +% \end{function} +% % \section{Internal sequence functions} % % \begin{function}{\seq_if_empty_err_break:N} @@ -823,7 +832,7 @@ % \begin{macrocode} \cs_new:Npn \seq_item:n { - \seq_use_error: + \msg_expandable_error:n { A~sequence~was~used~incorrectly. } \use_none:n } % \end{macrocode} @@ -1781,6 +1790,32 @@ % \end{macro} % \end{macro} % +% \begin{macro}{\seq_set_reverse:N,\seq_gset_reverse:N} +% Define \cs{seq_item:n} to place its argument after a marker, +% \cs{prg_do_nothing:}. Then x-expand the sequence. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \seq_tmp:w + { \msg_expandable_error:n { There~is~a~bug~in~LaTeX3! } } +\cs_new_protected_nopar:Npn \seq_set_reverse:N + { \seq_reverse_aux:NN \tl_set:Nx } +\cs_new_protected_nopar:Npn \seq_gset_reverse:N + { \seq_reverse_aux:NN \tl_gset:Nx } +\cs_new_protected_nopar:Npn \seq_reverse_aux:NN #1 #2 + { + \cs_set_eq:NN \seq_tmp:w \seq_item:n + \cs_set_eq:NN \seq_item:n \seq_reverse_aux_item:w + #1 #2 { #2 \prg_do_nothing: } + \cs_set_eq:NN \seq_item:n \seq_tmp:w + } +\cs_new:Npn \seq_reverse_aux_item:w #1 #2 \prg_do_nothing: + { + #2 + \prg_do_nothing: + \exp_not:n { \seq_item:n {#1} } + } +% \end{macrocode} +% \end{macro} +% % \subsection{Deprecated interfaces} % % A few functions which are no longer documented: these were moved diff --git a/Master/texmf-dist/source/latex/l3kernel/l3tl.dtx b/Master/texmf-dist/source/latex/l3kernel/l3tl.dtx index 149f4a487ba..3db13f9b400 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3tl.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3tl.dtx @@ -35,7 +35,7 @@ % %<*driver|package> \RequirePackage{l3names} -\GetIdInfo$Id: l3tl.dtx 2534 2011-08-10 18:21:44Z joseph $ +\GetIdInfo$Id: l3tl.dtx 2563 2011-08-13 12:23:59Z joseph $ {L3 Experimental token lists} % %<*driver> @@ -79,6 +79,15 @@ % braces. Many of the functions for token lists and token list variables % are very similar, and so are grouped together here. % +% A token list can be seen either as a list of \enquote{items}, +% or a list of \enquote{tokens}. An item is whatever \cs{use_none:n} +% grabs as its argument: either a single token or a brace group, +% with optional leading explicit space characters (each item is thus +% itself a token list). A token is either a normal \texttt{N} argument, +% or | |, |{|, or |}| (assuming normal \TeX{} category codes). +% Functions which act on items are often faster than their analog acting +% directly on tokens. +% % \section{Creating and initialising token list variables} % % \begin{function}{\tl_new:N, \tl_new:c} @@ -274,7 +283,10 @@ % Replaces all occurrences of \meta{old tokens} in the % \meta{tl~var} with \meta{new tokens}. \meta{Old tokens} % cannot contain |{|, |}| or |#| -% (assuming normal \TeX{} category codes). The assignment is +% (assuming normal \TeX{} category codes). As this function +% operates from left to right, the pattern \meta{old tokens} +% may remain after the replacement (see \cs{tl_remove_all:Nn} +% for an example). The assignment is % restricted to the current \TeX{} group. % \end{function} % @@ -286,8 +298,10 @@ % Replaces all occurrences of \meta{old tokens} in the % \meta{tl~var} with \meta{new tokens}. \meta{Old tokens} % cannot contain |{|, |}| or |#| -% (assuming normal \TeX{} category codes). The assignment is -% applied globally. +% (assuming normal \TeX{} category codes). As this function +% operates from left to right, the pattern \meta{old tokens} +% may remain after the replacement (see \cs{tl_remove_all:Nn} +% for an example). The assignment is applied globally. % \end{function} % % \begin{function}{\tl_remove_once:Nn, \tl_remove_once:cn} @@ -316,8 +330,15 @@ % \end{syntax} % Removes all occurrences of \meta{tokens} from the % \meta{tl~var}. \meta{Tokens} cannot contain |{|, |}| or -% |#| (assuming normal \TeX{} category codes). The assignment is -% restricted to the current \TeX{} group. +% |#| (assuming normal \TeX{} category codes). As this function +% operates from left to right, the pattern \meta{tokens} +% may remain after the removal, for instance, +% \begin{quote} +% \cs{tl_set:Nn} \cs{l_tmpa_tl} |{abbccd}| +% \cs{tl_remove_all:Nn} \cs{l_tmpa_tl} |{bc}| +% \end{quote} +% will result in \cs{l_tmpa_tl} containing \texttt{abcd}. +% The assignment is restricted to the current \TeX{} group. % \end{function} % % \begin{function}{\tl_gremove_all:Nn, \tl_gremove_all:cn} @@ -326,7 +347,10 @@ % \end{syntax} % Removes all occurrences of \meta{tokens} from the % \meta{tl~var}. \meta{Tokens} cannot contain |{|, |}| or -% |#| (assuming normal \TeX{} category codes). The assignment is +% |#| (assuming normal \TeX{} category codes). As this function +% operates from left to right, the pattern \meta{tokens} +% may remain after the removal (see \cs{tl_remove_all:Nn} +% for an example). The assignment is % applied globally. % \end{function} % @@ -342,7 +366,7 @@ % ~~\Arg{tokens} % \end{syntax} % Sets \meta{tl~var} to contain \meta{tokens}, -% applying the category code regim{\'e} specified in the +% applying the category code r\'egime specified in the % \meta{setup} before carrying out the assignment. This allows the % \meta{tl~var} to contain material with category codes % other than those that apply when \meta{tokens} are absorbed. The @@ -360,7 +384,7 @@ % ~~\Arg{tokens} % \end{syntax} % Sets \meta{tl~var} to contain \meta{tokens}, -% applying the category code regim{\'e} specified in the +% applying the category code r\'egime specified in the % \meta{setup} before carrying out the assignment. This allows the % \meta{tl~var} to contain material with category codes % other than those that apply when \meta{tokens} are absorbed. The @@ -371,7 +395,7 @@ % \begin{syntax} % \cs{tl_rescan:nn} \Arg{setup} \Arg{tokens} % \end{syntax} -% Rescans \meta{tokens} applying the category code regim{\'e} specified +% Rescans \meta{tokens} applying the category code r\'egime specified % in the \meta{setup}, and leaves the resulting tokens in the input % stream. See also \cs{tl_set_rescan:Nnn}. % \end{function} @@ -404,7 +428,7 @@ % unchanged. This process does not alter the category code assigned % to the \meta{tokens}. % \begin{texnote} -% This is the \TeX{} primitive \cs{uppercase} renamed.. +% This is the \TeX{} primitive \cs{uppercase} renamed. % As a result, this function takes place on execution and % not on expansion. % \end{texnote} @@ -417,11 +441,11 @@ % \cs{tl_if_blank_p:n} \Arg{token list} % \cs{tl_if_blank:nTF} \Arg{token list} \Arg{true code} \Arg{false code} % \end{syntax} -% Tests if the \meta{token list} consists only of blank spaces. The -% test is \texttt{true} if \meta{token list} is empty or consists -% entirely of explicit tokens of character code $32$ -% and category code $10$, and is \texttt{false} -% otherwise. The branching versions then leave either \meta{true code} +% Tests if the \meta{token list} consists only of blank spaces +% (\emph{i.e.}~contains no item). The test is \texttt{true} if +% \meta{token list} is zero or more explicit tokens of character code~$32$ +% and category code~$10$, and is \texttt{false} otherwise. +% The branching versions then leave either \meta{true code} % or \meta{false code} in the input stream, as appropriate to the % truth of the test and the variant of the function chosen. The logical % truth of the test is left in the input stream by the predicate @@ -520,14 +544,13 @@ % \begin{function}[EXP,pTF]{\tl_if_single:N, \tl_if_single:c} % \begin{syntax} % \cs{tl_if_single_p:N} \Arg{tl~var} -% \cs{tl_if_single:nTF} \Arg{tl~var} \Arg{true code} \Arg{false code} -% \end{syntax} -% Tests if the content of the \meta{tl~var} consists of a -% single token or token group. The test is \texttt{true} if -% \meta{token list} contains exactly one token, if it consists -% entirely of explicit tokens of character code $32$ and category -% code $10$, or if it contains one braced token group (optionally -% preceeded by spaces), and it is \texttt{false} otherwise. The branching +% \cs{tl_if_single:NTF} \Arg{tl~var} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the content of the \meta{tl~var} consists of a single item, +% \emph{i.e.}~is either a single normal token (excluding spaces, +% and brace tokens) or a single brace group, surrounded by optional +% spaces on both sides. In other words, such a token list has length +% $1$ according to \cs{tl_length:N}. The branching % versions then leave either \meta{true code} or \meta{false code} % in the input stream, as appropriate to the truth of the test and the % variant of the function chosen. The logical truth of the test is left @@ -537,19 +560,32 @@ % \begin{function}[EXP,pTF]{\tl_if_single:n} % \begin{syntax} % \cs{tl_if_single_p:n} \Arg{token list} -% \cs{tl_if_single:nTF} \Arg{token list} \Arg{true code} \Arg{false code} -% \end{syntax} -% Tests if the \meta{token list} consists of a single token -% or token group. The test is \texttt{true} if \meta{token list} -% contains exactly one token, or if it consists entirely of -% explicit tokens of character code $32$ and category code $10$, -% or if it contains a braced token group (optionally preceeded by blank -% spaces), and it is -% \texttt{false} otherwise. The branching versions then leave -% either \meta{true code} or \meta{false code} in the input stream, as -% appropriate to the truth of the test and the variant of the function -% chosen. The logical truth of the test is left in the input stream by -% the predicate version. +% \cs{tl_if_single:nTF} \Arg{token list} +% ~~\Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the token list has exactly one item, \emph{i.e.}~is either +% a single normal token or a single brace group, surrounded by +% optional spaces on both sides. In other words, such a token list +% has length $1$ according to \cs{tl_length:n}. +% The branching versions leave either \meta{true code} or \meta{false code} +% in the input stream, as appropriate to the truth of the test and +% the variant of the function chosen. The logical truth of the test +% is left in the input stream by the predicate. +% \end{function} +% +% \begin{function}[EXP,pTF]{\tl_if_single_token:n} +% \begin{syntax} +% \cs{tl_if_single_token_p:n} \Arg{token list} +% \cs{tl_if_single_token:nTF} \Arg{token list} +% ~~\Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the token list consists of exactly one token, \emph{i.e.}~is +% either a single space character or a single \enquote{normal} token. +% Token groups (|{|\ldots|}|) are not single tokens. +% The branching versions leave either \meta{true code} or \meta{false code} +% in the input stream, as appropriate to the truth of the test and +% the variant of the function chosen. The logical truth of the test +% is left in the input stream by the predicate. % \end{function} % % \section{Mapping to token lists} @@ -558,10 +594,9 @@ % \begin{syntax} % \cs{tl_map_function:NN} \meta{tl~var} \meta{function} % \end{syntax} -% Applies \meta{function} to every \meta{token group} stored in the -% \meta{tl~var}. The \meta{function} -% will receive one argument for each iteration. This may be a number -% of tokens if the \meta{token group} was an item stored within +% Applies \meta{function} to every \meta{item} in the \meta{tl~var}. +% The \meta{function} will receive one argument for each iteration. +% This may be a number of tokens if the \meta{item} was stored within % braces. Hence the \meta{function} should anticipate receiving % \texttt{n}-type arguments. See also \cs{tl_map_function:nN}. % \end{function} @@ -570,10 +605,9 @@ % \begin{syntax} % \cs{tl_map_function:nN} \meta{token list} \meta{function} % \end{syntax} -% Applies \meta{function} to every \meta{token group} in the -% \meta{token list}. The \meta{function} -% will receive one argument for each iteration. This may be a number -% of tokens if the \meta{token group} was an item stored within +% Applies \meta{function} to every \meta{item} in the \meta{token list}, +% The \meta{function} will receive one argument for each iteration. +% This may be a number of tokens if the \meta{item} was stored within % braces. Hence the \meta{function} should anticipate receiving % \texttt{n}-type arguments. See also \cs{tl_map_function:NN}. % \end{function} @@ -582,31 +616,29 @@ % \begin{syntax} % \cs{tl_map_inline:Nn} \meta{tl~var} \Arg{inline function} % \end{syntax} -% Applies the \meta{inline function} to every \meta{token group} stored -% within the \meta{tl~var}. The \meta{inline function} -% should consist of code which will receive the \meta{token group} as -% |#1|. One in line mapping can be nested inside another. See also -% \cs{tl_map_function:Nn}. +% Applies the \meta{inline function} to every \meta{item} stored within the +% \meta{tl~var}. The \meta{inline function} should consist of code which +% will receive the \meta{item} as |#1|. One in line mapping can be nested +% inside another. See also \cs{tl_map_function:Nn}. % \end{function} % % \begin{function}{\tl_map_inline:nn} % \begin{syntax} % \cs{tl_map_inline:nn} \meta{token list} \Arg{inline function} % \end{syntax} -% Applies the \meta{inline function} to every \meta{token group} stored -% within the \meta{token list}. The \meta{inline function} should -% consist of code which will receive the \meta{token group} as -% |#1|. One in line mapping can be nested inside another. See also -% \cs{tl_map_function:nn}. +% Applies the \meta{inline function} to every \meta{item} stored within the +% \meta{token list}. The \meta{inline function} should consist of code which +% will receive the \meta{item} as |#1|. One in line mapping can be nested +% inside another. See also \cs{tl_map_function:nn}. % \end{function} % % \begin{function}{\tl_map_variable:NNn, \tl_map_variable:cNn} % \begin{syntax} % \cs{tl_map_variable:NNn} \meta{tl~var} \meta{variable} \Arg{function} % \end{syntax} -% Applies the \meta{function} to every \meta{token group} stored +% Applies the \meta{function} to every \meta{item} stored % within the \meta{tl~var}. The \meta{function} should consist of code -% which will receive the \meta{token group} stored in the \meta{variable}. +% which will receive the \meta{item} stored in the \meta{variable}. % One variable mapping can be nested inside another. See also % \cs{tl_map_inline:Nn}. % \end{function} @@ -615,9 +647,9 @@ % \begin{syntax} % \cs{tl_map_variable:nNn} \meta{token list} \meta{variable} \Arg{function} % \end{syntax} -% Applies the \meta{function} to every \meta{token group} stored +% Applies the \meta{function} to every \meta{item} stored % within the \meta{token list}. The \meta{function} should consist of code -% which will receive the \meta{token group} stored in the \meta{variable}. +% which will receive the \meta{item} stored in the \meta{variable}. % One variable mapping can be nested inside another. See also % \cs{tl_map_inline:nn}. % \end{function} @@ -681,11 +713,12 @@ % % \section{Working with the content of token lists} % -% \begin{function}[EXP]{\tl_length:n, \tl_length:V, \tl_length:o} +% \begin{function}[EXP] +% {\tl_length:n, \tl_length:V, \tl_length:o} % \begin{syntax} % \cs{tl_length:n} \Arg{tokens} % \end{syntax} -% Counts the number of token groups in \meta{tokens} and leaves this +% Counts the number of \meta{items} in \meta{tokens} and leaves this % information in the input stream. Unbraced tokens count as one % element as do each token group (|{|\ldots|}|). This process will % ignore any unprotected spaces within \meta{tokens}. See also @@ -707,14 +740,16 @@ % % \begin{function}[EXP]{\tl_reverse:n, \tl_reverse:V, \tl_reverse:o} % \begin{syntax} -% \cs{tl_reverse:n} \Arg{tokens} +% \cs{tl_reverse:n} \Arg{token list} % \end{syntax} -% Reverses the order of the \meta{tokens}, so that -% \meta{token1}\meta{token2}\meta{token3} \ldots \meta{token$_n$} -% becomes \meta{token$_n$}\ldots -% \meta{token3}\meta{token2}\meta{token1}. This process will remove -% any unprotected space within the \meta{tokens}. Tokens are not reversed -% within braced token groups, which lose their outer set of braces. +% Reverses the order of the \meta{items} in the \meta{token list}, +% so that \meta{item1}\meta{item2}\meta{item3} \ldots \meta{item$_n$} +% becomes \meta{item$_n$}\ldots \meta{item3}\meta{item2}\meta{item1}. +% This process will preserve unprotected space within the +% \meta{token list}. Tokens are not reversed within braced token +% groups, which keep their outer set of braces. +% In situations where performance is important, +% consider \cs{tl_reverse_items:n}. % See also \cs{tl_reverse:N}. % \end{function} % @@ -722,18 +757,31 @@ % \begin{syntax} % \cs{tl_reverse:N} \Arg{tl~var} % \end{syntax} -% Reverses the order of the \meta{tokens} stored in -% \meta{tl~var}, so that -% \meta{token1}\meta{token2}\meta{token3} \ldots \meta{token$_n$} -% becomes \meta{token$_n$}\ldots -% \meta{token3}\meta{token2}\meta{token1}. This process will -% remove any unprotected spaces within the +% Reverses the order of the \meta{items} stored in \meta{tl~var}, so +% that \meta{item1}\meta{item2}\meta{item3} \ldots \meta{item$_n$} +% becomes \meta{item$_n$}\ldots \meta{item3}\meta{item2}\meta{item1}. +% This process will preserve unprotected spaces within the % \meta{token list variable}. Braced token groups are copied without -% reversing the order of tokens, and the outer set of braces is removed. +% reversing the order of tokens, but keep the outer set of braces. % The reversal is local to the current % \TeX{} group. See also \cs{tl_reverse:n}. % \end{function} % +% \begin{function}[EXP]{\tl_reverse_items:n} +% \begin{syntax} +% \cs{tl_reverse_items:n} \Arg{token list} +% \end{syntax} +% Reverses the order of the \meta{items} stored in \meta{tl~var}, +% so that \Arg{item1}\Arg{item2}\Arg{item3} \ldots \Arg{item$_n$} +% becomes \Arg{item$_n$} \ldots{} \Arg{item3}\Arg{item2}\Arg{item1}. +% This process will remove any unprotected space within the +% \meta{token list}. Braced token groups are copied without +% reversing the order of tokens, and keep the outer set of braces. +% Items which are initially not braced are copied with braces in +% the result. In cases where preserving spaces is important, +% consider \cs{tl_reverse:n} or \cs{tl_reverse_tokens:n}. +% \end{function} +% % \begin{function}[EXP]{\tl_trim_spaces:n} % \begin{syntax} % \cs{tl_trim_spaces:n} \meta{token list} @@ -761,7 +809,7 @@ % % \section{The first token from a token list} % -% Functions which deal with either onlt the very first token of a +% Functions which deal with either only the very first token of a % token list or everything except the first token. % % \begin{function}[EXP]{\tl_head:n, \tl_head:V, \tl_head:v, \tl_head:f} @@ -798,17 +846,6 @@ % is not critical. % \end{function} % -% \begin{function}[EXP]{\str_head:n} -% \begin{syntax} -% \cs{str_head:n} \Arg{tokens} -% \end{syntax} -% Converts the \meta{tokens} into a string, as described for -% \cs{tl_to_str:n}, and then leaves the first character of this string -% in the input stream; the first character may be a space. If the -% \meta{tokens} argument is entirely empty, \cs{scan_stop:} will be left in -% the input stream. -% \end{function} -% % \begin{function}[EXP]{\tl_tail:n, \tl_tail:V, \tl_tail:v, \tl_tail:f} % \begin{syntax} % \cs{tl_tail:n} \Arg{tokens} @@ -845,6 +882,20 @@ % is not critical. % \end{function} % +% \begin{function}[EXP]{\str_head:n,\str_tail:n} +% \begin{syntax} +% \cs{str_head:n} \Arg{tokens} +% \cs{str_tail:n} \Arg{tokens} +% \end{syntax} +% Converts the \meta{tokens} into a string, as described for +% \cs{tl_to_str:n}. The \cs{str_head:n} function then leaves +% the first character of this string in the input stream. +% The \cs{str_tail:n} function leaves all characters except +% the first in the input stream. The first character may be +% a space. If the \meta{tokens} argument is entirely empty, +% nothing is left in the input stream. +% \end{function} +% % \begin{function}[EXP,pTF]{\tl_if_head_eq_catcode:nN} % \begin{syntax} % \cs{tl_if_head_eq_catcode_p:nN} \Arg{token list} \meta{test token} @@ -852,7 +903,10 @@ % ~~\Arg{true code} \Arg{false code} % \end{syntax} % Tests if the first \meta{token} in the \meta{token list} has the same -% category code as the \meta{test token}. The branching versions then +% category code as the \meta{test token}. In the case where +% \meta{token list} is empty, its head is considered to be \cs{q_nil}, +% and the test will be true if \meta{test token} is a control sequence. +% The branching versions then % leave either \meta{true code} or \meta{false code} in the input % stream, as appropriate to the truth of the test and the variant of % the function chosen. The logical truth of the test is left in the @@ -868,7 +922,10 @@ % ~~\Arg{true code} \Arg{false code} % \end{syntax} % Tests if the first \meta{token} in the \meta{token list} has the same -% character code as the \meta{test token}. The branching versions then +% character code as the \meta{test token}. In the case where +% \meta{token list} is empty, its head is considered to be \cs{q_nil}, +% and the test will be true if \meta{test token} is a control sequence. +% The branching versions then % leave either \meta{true code} or \meta{false code} in the input % stream, as appropriate to the truth of the test and the variant of % the function chosen. The logical truth of the test is left in the @@ -883,13 +940,86 @@ % ~~\Arg{true code} \Arg{false code} % \end{syntax} % Tests if the first \meta{token} in the \meta{token list} has the same -% meaning as the \meta{test token}. The branching versions then +% meaning as the \meta{test token}. In the case where \meta{token list} +% is empty, its head is considered to be \cs{q_nil}, and the test will +% be true if \meta{test token} has the same meaning as \cs{q_nil}. +% The branching versions then % leave either \meta{true code} or \meta{false code} in the input % stream, as appropriate to the truth of the test and the variant of % the function chosen. The logical truth of the test is left in the % input stream by the predicate % version. % \end{function} +% +% \begin{function}[EXP,pTF,int]{\tl_if_head_group:n} +% \begin{syntax} +% \cs{tl_if_head_group_p:n} \Arg{token list} +% \cs{tl_if_head_group:nTF} \Arg{token list} +% ~~\Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the first \meta{token} in the \meta{token list} +% is an explicit begin-group character (with category code~1 +% and any character code), in other words, if the \meta{token list} +% starts with a brace group. In particular, the test is false +% if the \meta{token list} starts with an implicit token such as +% \cs{c_group_begin_token}, or if it empty. +% This function is useful to implement actions on token lists on +% a token by token basis. +% The branching versions leave either \meta{true code} or \meta{false code} +% in the input stream, as appropriate to the truth of the test and +% the variant of the function chosen. The logical truth of the test +% is left in the input stream by the predicate. +% \end{function} +% +% \begin{function}[EXP,pTF,int]{\tl_if_head_N_type:n} +% \begin{syntax} +% \cs{tl_if_head_N_type_p:n} \Arg{token list} +% \cs{tl_if_head_N_type:nTF} \Arg{token list} +% ~~\Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the first \meta{token} in the \meta{token list} +% is a normal \texttt{N}-type argument. In other words, +% it is neither an explicit space character (with category code~10 +% and character code~32) nor an explicit begin-group character +% (with category code~1 and any character code). An empty +% argument yields false, as it does not have a \enquote{normal} +% first token. +% This function is useful to implement actions on token lists on +% a token by token basis. +% The branching versions leave either \meta{true code} or \meta{false code} +% in the input stream, as appropriate to the truth of the test and +% the variant of the function chosen. The logical truth of the test +% is left in the input stream by the predicate. +% \end{function} +% +% \begin{function}[EXP,pTF,int]{\tl_if_head_space:n} +% \begin{syntax} +% \cs{tl_if_head_space_p:n} \Arg{token list} +% \cs{tl_if_head_space:nTF} \Arg{token list} +% ~~\Arg{true code} \Arg{false code} +% \end{syntax} +%^^A We need to add a discussion of +%^^A explicit vs implicit tokens somewhere in the doc. +% Tests if the first \meta{token} in the \meta{token list} +% is an explicit space character (with category code~$10$ +% and character code~$32$). If \meta{token list} starts with +% an implicit token such as \cs{c_space_token}, the test +% will yield false, as well as if the argument is empty. +% This function is useful to implement actions on token lists on +% a token by token basis. +% The branching versions leave either \meta{true code} or +% \meta{false code} in the input stream, as appropriate to the truth +% of the test and the variant of the function chosen. The logical +% truth of the test is left in the input stream by the predicate. +% \begin{texnote} +% When \TeX{} reads a character of category code $10$ for the +% first time, it is converted to an explicit space token, with +% character code $32$, regardless of the initial character code. +% \enquote{Funny} spaces with a different category code, can be produced +% using \cs{tex_lowercase:D}. Explicit spaces are also produced +% as a result of \cs{token_to_str:N}, \cs{tl_to_str:n}, etc. +% \end{texnote} +% \end{function} % % \section{Viewing token lists} % @@ -950,82 +1080,56 @@ % % \section{Experimental token list functions} % -% \begin{function}[EXP,pTF]{\tl_if_single_item:n} +% \begin{function}[EXP]{\tl_reverse_tokens:n} % \begin{syntax} -% \cs{tl_if_single_item_p:n} \Arg{token list} -% \cs{tl_if_single_item:nTF} \Arg{token list} -% ~~\Arg{true code} \Arg{false code} +% \cs{tl_reverse_tokens:n} \Arg{tokens} % \end{syntax} -% Tests if the token list has exactly one item, i.e., is either -% a single normal token or a single brace group, surrounded by -% optional spaces on both sides. In other words, such a token list -% has length $1$ according to \cs{tl_length:n}. The branching -% versions then leave either \meta{true code} or \meta{false code} -% in the input stream, as appropriate to the truth of the test and -% the variant of the function chosen. The logical truth of the test -% is left in the input stream by the predicate. +% This function, which works directly on \TeX{} tokens, reverses +% the order of the \meta{tokens}: the first will be the last and +% the last will become first. Spaces are preserved. The reversal +% also operates within brace groups, but the braces themselves +% are not exchanged, as this would lead to an unbalanced token +% list. For instance, \cs{tl_reverse_tokens:n} |{a~{b()}}| +% leaves |{)(b}~a| in the input stream. This function requires +% two steps of expansion. % \end{function} % -% \begin{function}[EXP,pTF]{\tl_if_head_begin_group:n} +% \begin{function}[EXP]{\tl_length_tokens:n} % \begin{syntax} -% \cs{tl_if_head_begin_group_p:n} \Arg{token list} -% \cs{tl_if_head_begin_group:nTF} \Arg{token list} -% ~~\Arg{true code} \Arg{false code} +% \cs{tl_length_tokens:n} \Arg{tokens} % \end{syntax} -% Tests if the first \meta{token} in the \meta{token list} -% has catcode~1, i.e., is a begin-group character. This conditional -% is mainly intended to be used in combination with -% \cs{tl_if_head_eq_space:n} to check if grabbing an undelimited -% argument from the token list is safe. The branching -% versions leave either \meta{true code} or \meta{false code} -% in the input stream, as appropriate to the truth of the test and -% the variant of the function chosen. The logical truth of the test -% is left in the input stream by the predicate. +% Counts the number of \TeX{} tokens in the \meta{tokens} and leaves +% this information in the input stream. Every token, including spaces and +% braces, contributes one to the total; thus for instance, the length of +% |a~{bc}| is $6$. +% This function requires three expansions, +% giving an \meta{integer denotation}. % \end{function} % -% \begin{function}[EXP,pTF]{\tl_if_head_eq_space:n} +% \begin{function}[EXP]{\tl_expandable_uppercase:n,\tl_expandable_lowercase:n} % \begin{syntax} -% \cs{tl_if_head_eq_space_p:n} \Arg{token list} -% \cs{tl_if_head_eq_space:nTF} \Arg{token list} -% ~~\Arg{true code} \Arg{false code} +% \cs{tl_expandable_uppercase:n} \Arg{tokens} +% \cs{tl_expandable_lowercase:n} \Arg{tokens} % \end{syntax} -%^^A Needs rewriting. We need to add a discussion of -%^^A explicit vs implicit tokens somewhere in the doc. -% Tests if the first token of the token list is an explicit space, -% i.e., a token which matches \verb*+ + both in character code and -% in category code. In the case of an implicit space token, such as -% \cs{c_space_token}, the test will return \meta{false}. -% This conditional is mainly intended to be used in combination with -% \cs{tl_if_head_begin_group:n} to test if grabbing an undelimited -% argument is safe (see \cs{tl_if_single_token:n} for an example. -% The branching versions leave either \meta{true code} or -% \meta{false code} in the input stream, as appropriate to the truth -% of the test and the variant of the function chosen. The logical -% truth of the test is left in the input stream by the predicate. +% The \cs{tl_expandable_uppercase:n} function works through all of +% the \meta{tokens}, replacing characters in the range |a|--|z| +% (with arbitrary category code) by the corresponding letter +% in the range |A|--|Z|, with category code $11$ (letter). Similarly, +% \cs{tl_expandable_lowercase:n} replaces characters in the range +% |A|--|Z| by letters in the range |a|--|z|, and leaves other tokens +% unchanged. This function requires two steps of expansion. % \begin{texnote} -% When \TeX{}'s reads a character of category code $10$ for the -% first time, it is converted to an explicit space token, with -% character code $32$, regardless of the initial character code. -% ``Funny'' spaces with a different category code, can be produced -% using \cs{tex_lowercase:D}. Explicit spaces are also produced -% as a result of \cs{token_to_str:N}, \cs{tl_to_str:n}, etc. +% Begin-group and end-group characters are normalized and become +% |{| and |}|, respectively. % \end{texnote} % \end{function} -% -% \begin{function}[EXP,pTF]{\tl_if_single_token:n} -% \begin{syntax} -% \cs{tl_if_single_token_p:n} \Arg{token list} -% \cs{tl_if_single_token:nTF} \Arg{token list} -% ~~\Arg{true code} \Arg{false code} -% \end{syntax} -% Tests if the token list consists of exactly one token, i.e., -% is either a single space character or a single \enquote{normal} token. -% Token groups are not single tokens. The branching -% versions leaves either \meta{true code} or \meta{false code} -% in the input stream, as appropriate to the truth of the test and -% the variant of the function chosen. The logical truth of the test -% is left in the input stream by the predicate. -% \end{function} +% +% \section{Internal functions} +% +% \begin{variable}{\q_tl_act_mark,\q_tl_act_stop} +% Quarks which are only used for the particular purposes of +% \cs{tl_act_...} functions. +% \end{variable} % % \end{documentation} % @@ -1302,8 +1406,10 @@ % within the token list which is scanned: two |@| symbols with different % category codes. The rescanned token list cannot contain the end marker, % because all |@| present in the token list are read with the same category -% code. The setting of \cs{tex_endlinechar:D} -% is needed to avoid introducing an extraneous space in the result. +% code. As every character with charcode \cs{tex_newlinechar:D} is replaced +% by the \cs{tex_endlinechar:D}, and an extra \cs{tex_endlinechar:D} is +% added at the end, we need to set both of those to $-1$, +% \enquote{unprintable}. % \begin{macrocode} \cs_new_protected_nopar:Npn \tl_set_rescan:Nnn { \tl_set_rescan_aux:NNnn \tl_set:Nn } @@ -1314,6 +1420,7 @@ \group_begin: \exp_args:No \etex_everyeof:D { \c_tl_rescan_marker_tl } \tex_endlinechar:D \c_minus_one + \tex_newlinechar:D \c_minus_one #3 \tl_clear:N \l_tl_rescan_tl \exp_after:wN \tl_rescan_aux:w \etex_scantokens:D {#4} @@ -1352,6 +1459,7 @@ \group_begin: \etex_everyeof:D { \exp_not:N } \tex_endlinechar:D \c_minus_one + \tex_newlinechar:D \c_minus_one #3 \tl_set:Nx \l_tl_rescan_tl { \etex_scantokens:D {#4} } \exp_args:NNNo \group_end: @@ -1372,6 +1480,7 @@ \group_begin: \exp_args:No \etex_everyeof:D { \c_tl_rescan_marker_tl } \tex_endlinechar:D \c_minus_one + \tex_newlinechar:D \c_minus_one #1 \exp_after:wN \tl_rescan_aux:w \etex_scantokens:D {#2} \exp_args:No \group_end: @@ -1401,86 +1510,106 @@ % \end{macrocode} % \end{variable} % +% +% \begin{macro}{\tl_replace_all:Nnn, \tl_replace_all:cnn} +% \begin{macro}{\tl_greplace_all:Nnn, \tl_greplace_all:cnn} % \begin{macro}{\tl_replace_once:Nnn, \tl_replace_once:cnn} % \begin{macro}{\tl_greplace_once:Nnn, \tl_greplace_once:cnn} -% \begin{macro}[aux]{\tl_replace_once_aux:NNnn} -% The concept here is that only the first occurrence should be -% replaced. The first step is to define an auxiliary which will -% match the appropriate item, with a trailing marker. If the last token -% is the marker there is nothing to do, otherwise replace the token -% and clean up (hence the second use of \cs{tl_tmp:w}). To prevent -% loosing braces or spaces there are a couple of empty groups and -% the strange-looking \cs{use:n}. There is a \cs{q_nil} between the -% original and the search input, to prevent cases where -% the end of the original and the start of the search run together -% to give an erroneous test result. +% \begin{macro}[aux]{\tl_replace_aux:NNNnn, \tl_replace_aux_ii:w} +% \begin{macro}[aux]{\tl_replace_all_aux:, \tl_replace_once_aux:} +% \begin{macro}[aux]{\tl_replace_once_aux_end:w} +% All of the replace functions are based on \cs{tl_replace_aux:NNNnn}, +% whose arguments are: \meta{function}, \cs{tl_(g)set:Nx}, \meta{tl~var}, +% \meta{search tokens}, \meta{replacement tokens}. % \begin{macrocode} \cs_new_protected_nopar:Npn \tl_replace_once:Nnn - { \tl_replace_once_aux:NNnn \tl_set_eq:NN } + { \tl_replace_aux:NNNnn \tl_replace_once_aux: \tl_set:Nx } \cs_new_protected_nopar:Npn \tl_greplace_once:Nnn - { \tl_replace_once_aux:NNnn \tl_gset_eq:NN } -\cs_new_protected:Npn \tl_replace_once_aux:NNnn #1#2#3#4 + { \tl_replace_aux:NNNnn \tl_replace_once_aux: \tl_gset:Nx } +\cs_new_protected_nopar:Npn \tl_replace_all:Nnn + { \tl_replace_aux:NNNnn \tl_replace_all_aux: \tl_set:Nx } +\cs_new_protected_nopar:Npn \tl_greplace_all:Nnn + { \tl_replace_aux:NNNnn \tl_replace_all_aux: \tl_gset:Nx } +\cs_generate_variant:Nn \tl_replace_once:Nnn { c } +\cs_generate_variant:Nn \tl_greplace_once:Nnn { c } +\cs_generate_variant:Nn \tl_replace_all:Nnn { c } +\cs_generate_variant:Nn \tl_greplace_all:Nnn { c } +% \end{macrocode} +% The idea is easier to understand by considering the case of +% \cs{tl_replace_all:Nnn}. The replacement happens within an +% \texttt{x}-type expansion. We use an auxiliary function \cs{tl_tmp:w}, +% which essentially replaces the next \meta{search tokens} by +% \meta{replacement tokens}. To avoid runaway arguments, +% we expand something like \cs{tl_tmp:w} \meta{token list} \cs{q_mark} +% \meta{search tokens} \cs{q_stop}, repeating until the end. How do we +% detect that we have reached the last occurrence of \meta{search tokens}? +% The last replacement is characterized by the fact that the argument of +% \cs{tl_tmp:w} contains \cs{q_mark}. In the code below, +% \cs{tl_replace_aux_ii:w} takes an argument delimited by \cs{q_mark}, +% and removes the following token. Before we reach the end, this gobbles +% \cs{q_mark} \cs{use_none_delimit_by_q_stop:w} which appear in the +% definition of \cs{tl_tmp:w}, and leaves the \meta{replacement tokens}, +% passed to \cs{exp_not:n}, to be included in the \texttt{x}-expanding +% definition. At the end, the first \cs{q_mark} is within the argument +% of \cs{tl_tmp:w}, and \cs{tl_replace_aux_ii:w} gobbles the second +% \cs{q_mark} as well, leaving \cs{use_none_delimit_by_q_stop:w}, +% which ends the recursion cleanly. +% \begin{macrocode} +\cs_new_protected:Npn \tl_replace_aux:NNNnn #1#2#3#4#5 { - \cs_set_protected:Npx \tl_tmp:w ##1 #3 ##2 \q_stop + \tl_if_empty:nTF {#4} { - \exp_not:N \quark_if_no_value:nF {##2} + \msg_kernel_error:nnx { tl } { empty-search-pattern } + { \tl_to_str:n {#5} } + } + { + \cs_set:Npx \tl_tmp:w ##1##2 #4 + { + ##2 + \exp_not:N \q_mark + \exp_not:N \use_none_delimit_by_q_stop:w + \exp_not:n { \exp_not:n {#5} } + ##1 + } + #2 #3 { - \tl_set:No \exp_not:N \l_tl_replace_tl { ##1 \exp_not:n{#4} } - \exp_not:n - { - \cs_set_protected:Npn \tl_tmp:w ##1 \q_nil #3 \q_no_value - { \tl_put_right:No \l_tl_replace_tl {##1} } - } - \exp_not:n { \tl_tmp:w \prg_do_nothing: } ##2 - \exp_not:n { #1 #2 \l_tl_replace_tl } + \exp_after:wN #1 + #3 \q_mark #4 \q_stop } } - \exp_after:wN \tl_tmp:w \exp_after:wN \prg_do_nothing: - #2 \q_nil #3 \q_no_value \q_stop } -\cs_generate_variant:Nn \tl_replace_once:Nnn { c } -\cs_generate_variant:Nn \tl_greplace_once:Nnn { c } +\cs_new:Npn \tl_replace_aux_ii:w #1 \q_mark #2 { \exp_not:o {#1} } +% \end{macrocode} +% The first argument of \cs{tl_tmp:w} is responsible for repeating +% the replacement in the case of \texttt{replace_all}, and stopping +% it early for \texttt{replace_once}. Note also that we build +% \cs{tl_tmp:w} within an \texttt{x}-expansion so that the +% \meta{replacement tokens} can contain |#|. The second +% \cs{exp_not:n} ensures that the \meta{replacement tokens} +% are not expanded by \cs{tl_(g)set:Nx}. +% +% Now on to the difference between \enquote{once} and \enquote{all}. +% The \cs{prg_do_nothing:} and accompanying \texttt{o}-expansion +% ensure that we don't lose braces in case the tokens between two +% occurrences of the \meta{search tokens} form a brace group. +% \begin{macrocode} +\cs_new:Npn \tl_replace_all_aux: + { + \exp_after:wN \tl_replace_aux_ii:w + \tl_tmp:w \tl_replace_all_aux: \prg_do_nothing: + } +\cs_new_nopar:Npn \tl_replace_once_aux: + { + \exp_after:wN \tl_replace_aux_ii:w + \tl_tmp:w { \tl_replace_once_aux_end:w \prg_do_nothing: } \prg_do_nothing: + } +\cs_new:Npn \tl_replace_once_aux_end:w #1 \q_mark #2 \q_stop + { \exp_not:o {#1} } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} -% -% \begin{macro}{\tl_replace_all:Nnn, \tl_replace_all:cnn} -% \begin{macro}{\tl_greplace_all:Nnn, \tl_greplace_all:cnn} -% \begin{macro}[aux]{\tl_replace_all_aux:NNnn} -% A similar approach here but with a loop built in. -% \begin{macrocode} -\cs_new_protected_nopar:Npn \tl_replace_all:Nnn - { \tl_replace_all_aux:NNnn \tl_set_eq:NN } -\cs_new_protected_nopar:Npn \tl_greplace_all:Nnn - { \tl_replace_all_aux:NNnn \tl_gset_eq:NN } -\cs_new_protected:Npn \tl_replace_all_aux:NNnn #1#2#3#4 - { - \tl_clear:N \l_tl_replace_tl - \cs_set_protected:Npx \tl_tmp:w ##1 #3 ##2 \q_stop - { - \exp_not:N \quark_if_no_value:nTF {##2} - { - \exp_not:n - { - \cs_set_protected:Npn \tl_tmp:w ##1 \q_nil ##2 \q_stop - { \tl_put_right:No \l_tl_replace_tl {##1} } - } - \exp_not:N \tl_tmp:w ##1 \exp_not:N \q_stop - } - { - \exp_not:n { \tl_put_right:No \l_tl_replace_tl } - { ##1 \exp_not:n{#4} } - \exp_not:n { \tl_tmp:w \prg_do_nothing: } ##2 \exp_not:N \q_stop - } - } - \exp_after:wN \tl_tmp:w \exp_after:wN \prg_do_nothing: - #2 \q_nil #3 \q_no_value \q_stop - #1 #2 \l_tl_replace_tl - } -\cs_generate_variant:Nn \tl_replace_all:Nnn { c } -\cs_generate_variant:Nn \tl_greplace_all:Nnn { c } -% \end{macrocode} +% \end{macro} % \end{macro} % \end{macro} % \end{macro} @@ -1575,13 +1704,14 @@ % that. This way the test will even accept |\q_nil| as the first % token. % \begin{macrocode} -\prg_new_conditional:Npnn \tl_if_empty:n #1 { p,TF,T,F} { - \exp_after:wN \if_meaning:w \exp_after:wN \q_nil \tl_to_str:n {#1} \q_nil - \prg_return_true: - \else: - \prg_return_false: - \fi: -} +\prg_new_conditional:Npnn \tl_if_empty:n #1 { p , TF , T , F } + { + \exp_after:wN \if_meaning:w \exp_after:wN \q_nil \tl_to_str:n {#1} \q_nil + \prg_return_true: + \else: + \prg_return_false: + \fi: + } \cs_generate_variant:Nn \tl_if_empty_p:n { V } \cs_generate_variant:Nn \tl_if_empty:nTF { V } \cs_generate_variant:Nn \tl_if_empty:nT { V } @@ -1595,27 +1725,28 @@ % in conditionals on token lists, which mostly reduce to testing % if a given token list is empty after applying a simple function % to it. -% The test for emptyness is based on \cs{tl_if_empty:n(TF)}, but +% The test for emptiness is based on \cs{tl_if_empty:n(TF)}, but % the expansion is hard-coded for efficiency, as this auxiliary % function is used in many places. % Note that this works because \cs{tl_to_str:n} expands tokens % that follow until reading a catcode $1$ (begin-group) token. % \begin{macrocode} -\cs_new:Npn \tl_if_empty_return:o #1 { - \exp_after:wN \if_meaning:w \exp_after:wN \q_nil - \tl_to_str:n \exp_after:wN {#1} \q_nil - \prg_return_true: - \else: - \prg_return_false: - \fi: -} -\prg_new_conditional:Npnn \tl_if_empty:o #1 { p,TF,T,F} +\cs_new:Npn \tl_if_empty_return:o #1 + { + \exp_after:wN \if_meaning:w \exp_after:wN \q_nil + \tl_to_str:n \exp_after:wN {#1} \q_nil + \prg_return_true: + \else: + \prg_return_false: + \fi: + } +\prg_new_conditional:Npnn \tl_if_empty:o #1 { p , TF , T , F } { \tl_if_empty_return:o {#1} } % \end{macrocode} % \end{macro} % \end{macro} % -% \begin{macro}[pTF]{\tl_if_eq:NN,\tl_if_eq:Nc,\tl_if_eq:cN,\tl_if_eq:cc} +% \begin{macro}[pTF]{\tl_if_eq:NN, \tl_if_eq:Nc, \tl_if_eq:cN, \tl_if_eq:cc} % Returns \cs{c_true_bool} if and only if the two token list variables are % equal. % \begin{macrocode} @@ -1703,30 +1834,6 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}[pTF]{\tl_if_single:n} -% \begin{macro}[aux]{\tl_if_single_aux:n} -% If the argument is a single token, or a single brace group -% preceeded by optional (explicit) spaces, or a non-zero number -% of spaces. -% \begin{macrocode} -\prg_new_conditional:Npnn \tl_if_single:n #1 { p , T , F , TF } - { - \tl_if_blank:nTF {#1} - { \tl_if_empty:nTF {#1} { \prg_return_false: } { \prg_return_true: } } - { \tl_if_single_aux:n {#1} } - } -\prg_new_conditional:Npnn \tl_if_single:N #1 { p , T , F , TF } - { - \tl_if_blank:oTF #1 - { \tl_if_empty:NTF #1 { \prg_return_false: } { \prg_return_true: } } - { \exp_args:No \tl_if_single_aux:n {#1} } - } -\cs_new:Npn \tl_if_single_aux:n #1 - { \tl_if_empty_return:o { \use_none:n #1 } } -% \end{macrocode} -% \end{macro} -% \end{macro} -% % \subsection{Mapping to token lists} % % \begin{macro}{\tl_map_function:nN} @@ -1851,14 +1958,9 @@ \if_cs_exist:w #1 \cs_end: \cs:w #1 \exp_after:wN \cs_end: \else: - \exp_after:wN \tl_error_message: + \msg_expandable_error:n { Undefined~variable~name~'#1'! } \fi: } -\group_begin: -\tex_catcode:D `\! = 11 \scan_stop: -\tex_catcode:D `\ = 11\scan_stop:% -\cs_gset_nopar:Npn\tl_error_message:{\undefined variable name!}% -\group_end:% % \end{macrocode} % \end{macro} % \end{macro} @@ -1892,33 +1994,23 @@ % \end{macro} % \end{macro} % -% \begin{macro}{\tl_reverse:n, \tl_reverse:V, \tl_reverse:o} -% \begin{macro}[aux]{\tl_reverse_aux:nN} -% Reversal of a token list is done by taking one token at a time -% and putting it in front of the ones before it. +% \begin{macro}{\tl_reverse_items:n} +% \begin{macro}[aux]{\tl_reverse_items_aux:nN} +% Reversal of a token list is done by taking one item at a time +% and putting it after \cs{q_recursion_stop}. % \begin{macrocode} -\cs_new:Npn \tl_reverse:n #1 - { \tl_reverse_aux:nN { } #1 \q_recursion_tail \q_recursion_stop } -\cs_new:Npn \tl_reverse_aux:nN #1#2 +\cs_new:Npn \tl_reverse_items:n #1 + { \tl_reverse_items_aux:nw #1 \q_recursion_tail \q_recursion_stop } +\cs_new:Npn \tl_reverse_items_aux:nw #1 #2 \q_recursion_stop { - \quark_if_recursion_tail_stop_do:nn {#2} {#1} - \tl_reverse_aux:nN { #2 #1 } + \quark_if_recursion_tail_stop_do:nn {#1} { \use_none:n } + \tl_reverse_items_aux:nw #2 \q_recursion_stop + {#1} } -\cs_generate_variant:Nn \tl_reverse:n {V,o} % \end{macrocode} % \end{macro} % \end{macro} % -% \begin{macro}{\tl_reverse:N, \tl_reverse:c} -% This reverses the list, leaving |\exp_stop_f:| in front, which in turn -% is removed by the |f| expansion which comes to a halt. -% \begin{macrocode} -\cs_new_protected_nopar:Npn \tl_reverse:N #1 - { \tl_set:Nf #1 { \tl_reverse:o { #1 \exp_stop_f: } } } -\cs_generate_variant:Nn \tl_reverse:N { c } -% \end{macrocode} -% \end{macro} -% % \begin{macro}{\tl_trim_spaces:n} % \begin{macro} % { @@ -1966,15 +2058,16 @@ % \begin{macro}{\tl_tail:n, \tl_tail:V, \tl_tail:v, \tl_tail:f} % \begin{macro}{\tl_tail:w} % These functions pick up either the head or the tail of a list. The -% \texttt{n}-type implementation needs a test in order to avoid run-away -% arguments occurring with blank input. +% empty brace groups in \cs{tl_head:n} and \cs{tl_tail:n} ensure that +% a blank argument gives an empty result. % \begin{macrocode} \cs_new:Npn \tl_head:w #1#2 \q_stop {#1} \cs_new:Npn \tl_tail:w #1#2 \q_stop {#2} \cs_new:Npn \tl_head:n #1 - { \tl_if_blank:nF {#1} { \tl_head:w #1 \q_stop } } + { \tl_head:w #1 { } \q_stop } \cs_new:Npn \tl_tail:n #1 - { \tl_if_blank:nF {#1} { \tl_tail:w #1 \q_stop } } + { \tl_tail_aux:w #1 \q_mark { } \q_mark \q_stop } +\cs_new:Npn \tl_tail_aux:w #1 #2 \q_mark #3 \q_stop { #2 } \cs_generate_variant:Nn \tl_head:n { V , v , f } \cs_generate_variant:Nn \tl_tail:n { V , v , f } % \end{macrocode} @@ -1983,7 +2076,7 @@ % \end{macro} % \end{macro} % -% \begin{macro}{\str_head:n} +% \begin{macro}{\str_head:n, \str_tail:n} % \begin{macro}[aux]{\str_head_aux:w} % After \cs{tl_to_str:n}, we have a list of character tokens, % all with category code 12, except the space, which has category @@ -1993,17 +2086,34 @@ % space, then the argument of \cs{str_head_aux:w} is empty, and % the function correctly returns a space character. Otherwise, % it returns the first token of |#1|, which is the first token -% of the string. If the string is empty, we return -% \cs{scan_stop:}. +% of the string. If the string is empty, we return an empty result. +% +% To remove the first character of \cs{tl_to_str:n} |{#1}|, +% we test it using \cs{if_charcode:w} \cs{scan_stop:}, +% always false for characters. If the argument was non-empty, +% then \cs{str_tail_aux:w} returns everything until the first +% \texttt{X} (with category code letter, no risk of confusing +% with the user input). If the argument was empty, the first +% \texttt{X} is taken by \cs{if_charcode:w}, and nothing +% is returned. We use \texttt{X} as a \meta{marker}, rather than +% a quark because the test \cs{if_charcode:w} \cs{scan_stop:} +% \meta{marker} has to be false. % \begin{macrocode} \cs_new:Npn \str_head:n #1 { \exp_after:wN \str_head_aux:w \tl_to_str:n {#1} - { \scan_stop: } ~ \q_stop + { { } } ~ \q_stop } \cs_new_nopar:Npn \str_head_aux:w #1 ~ % { \tl_head:w #1 { ~ } } +\cs_new:Npn \str_tail:n #1 + { + \exp_after:wN \str_tail_aux:w + \reverse_if:N \if_charcode:w + \scan_stop: \tl_to_str:n {#1} X X \q_stop + } +\cs_new_nopar:Npn \str_tail_aux:w #1 X #2 \q_stop { \fi: #1 } % \end{macrocode} % \end{macro} % \end{macro} @@ -2013,8 +2123,8 @@ % \begin{macro}[pTF]{\tl_if_head_eq_charcode:fN} % \begin{macro}[pTF]{\tl_if_head_eq_catcode:nN} % Accessing the first token of a token list is tricky in two cases: -% when it has category code 1 (begin-group token), or when it is -% an explicit space, with category code 10 and character code 32. +% when it has category code $1$ (begin-group token), or when it is +% an explicit space, with category code $10$ and character code $32$. % % Forgetting temporarily about this issue we would use the % following test in \cs{tl_if_head_eq_charcode:nN}. Here, @@ -2025,8 +2135,8 @@ % \exp_after:wN \exp_not:N \tl_head:w #1 \q_nil \q_stop % \exp_not:N #2 % \end{verbatim} -% The special cases are detected by comparing |#1| with -% \cs{use:n}~|#1| (the extra |?| takes care of empty arguments). +% The special cases are detected using \cs{tl_if_head_N_type:n} +% (the extra |?| takes care of empty arguments). % In those cases, the first token is a character, and % since we only care about its character code, we can % use \cs{str_head:n} to access it (this works even if @@ -2036,7 +2146,7 @@ { \if_charcode:w \exp_not:N #2 - \str_if_eq:onTF { \use:n #1 ? } { #1 ? } + \tl_if_head_N_type:nTF { #1 ? } { \exp_after:wN \exp_not:N \tl_head:w #1 \q_nil \q_stop } { \str_head:n {#1} } \prg_return_true: @@ -2050,7 +2160,7 @@ \cs_generate_variant:Nn \tl_if_head_eq_charcode:nNF { f } % \end{macrocode} % For \cs{tl_if_head_eq_catcode:nN}, again we detect special -% cases with a \cs{str_if_eq:onTF} test. Then we need to test +% cases with a \cs{tl_if_head_N_type}. Then we need to test % if the first token is a begin-group token or an explicit % space token, and produce the relevant token, either % \cs{c_group_begin_token} or \cs{c_space_token}. @@ -2059,10 +2169,10 @@ { \if_catcode:w \exp_not:N #2 - \str_if_eq:onTF { \use:n #1 ? } { #1 ? } + \tl_if_head_N_type:nTF { #1 ? } { \exp_after:wN \exp_not:N \tl_head:w #1 \q_nil \q_stop } { - \tl_if_head_begin_group:nTF {#1} + \tl_if_head_group:nTF {#1} { \c_group_begin_token } { \c_space_token } } @@ -2081,7 +2191,7 @@ % \begin{macrocode} \prg_new_conditional:Npnn \tl_if_head_eq_meaning:nN #1#2 { p , T , F , TF } { - \str_if_eq:onTF { \use:n #1 ? } { #1 ? } + \tl_if_head_N_type:nTF { #1 ? } { \tl_if_head_eq_meaning_aux_normal:nN } { \tl_if_head_eq_meaning_aux_special:nN } {#1} #2 @@ -2104,7 +2214,7 @@ \fi: { \if_catcode:w \exp_not:N #2 - \tl_if_head_begin_group:nTF {#1} + \tl_if_head_group:nTF {#1} { \c_group_begin_token } { \c_space_token } \prg_return_true: @@ -2119,6 +2229,78 @@ % \end{macro} % \end{macro} % +% \begin{macro}[pTF]{\tl_if_head_N_type:n} +% The first token of a token list can be either an N-type argument, +% a begin-group token (catcode 1), or an explicit space token +% (catcode 10 and charcode 32). These two cases are characterized +% by the fact that \cs{use:n} removes some tokens from |#1|, hence +% changing its string representation (no token can have an empty +% string representation). The extra brace group covers the case of +% an empty argument, whose head is not \enquote{normal}. +% \begin{macrocode} +\prg_new_conditional:Npnn \tl_if_head_N_type:n #1 { p , T , F , TF } + { \str_if_eq_return:on { \use:n #1 { } } { #1 { } } } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[EXP,pTF]{\tl_if_head_group:n} +% Pass the first token of |#1| through \cs{token_to_str:N}, +% then check for the brace balance. The extra \texttt{?} +% caters for an empty argument.\footnote{Bruno: this could +% be made faster, but we don't: if we hope to ever have +% an e-type argument, we need all brace \enquote{tricks} +% to happen in one step of expansion, keeping the token +% list brace balanced at all times.} +% \begin{macrocode} +\prg_new_conditional:Npnn \tl_if_head_group:n #1 { p , T , F , TF } + { + \if_predicate:w + \exp_after:wN \use_none:n + \exp_after:wN { + \exp_after:wN { + \token_to_str:N #1 ? + } + \c_false_bool + } + \c_true_bool + \prg_return_false: + \else: + \prg_return_true: + \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[EXP,pTF]{\tl_if_head_space:n} +% \begin{macro}[EXP,aux]{\tl_if_head_space_aux:w} +% If the first token of the token list is an explicit space, i.e., +% a character token with character code $32$ and category code $10$, +% then this test will be \meta{true}. It is \meta{false} if the token +% list is empty, if the first token is an implicit space token, +% such as \cs{c_space_token}, or any token other than an explicit space. +% \begin{macrocode} +\prg_new_conditional:Npnn \tl_if_head_space:n #1 { p , T , F , TF } + { + \if_int_compare:w + \pdftex_strcmp:D + { } + { \tl_if_head_space_aux:w \prg_do_nothing: #1 ? ~ } + = \c_zero + \prg_return_true: + \else: + \prg_return_false: + \fi: + } +\cs_new:Npn \tl_if_head_space_aux:w #1 ~ % + { + \exp_not:o {#1} + \if_false: { \fi: } + \exp_after:wN \use_none:n \exp_after:wN { \if_false: } \fi: + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% % \subsection{Viewing token lists} % % \begin{macro}{\tl_show:N, \tl_show:c} @@ -2142,7 +2324,9 @@ % \begin{variable}{\c_job_name_tl} % Inherited from the \LaTeX3 name for the primitive: this needs to % actually contain the text of the job name rather than the name of -% the primitive, of course. +% the primitive, of course. \LuaTeX{} does not quote file names containing +% spaces, whereas \pdfTeX{} and \XeTeX{} do. So there may be a correction to +% make in the \LuaTeX{} case. % \begin{macrocode} %<*package> \tl_const:Nx \c_job_name_tl { \tex_jobname:D } @@ -2202,7 +2386,36 @@ % % \subsection{Experimental functions} % -% \begin{macro}[EXP,pTF]{\tl_if_single_item:n} +% \begin{macro}[EXP,pTF]{\str_if_eq_return:on} +% It turns out that we often need to compare a token list +% with the result of applying some function to it, and +% return with \cs{prg_return_true/false:}. This test is +% similar to \cs{str_if_eq:nnTF}, but hard-coded for speed. +% \begin{macrocode} +\cs_new:Npn \str_if_eq_return:on #1 #2 + { + \if_int_compare:w + \pdftex_strcmp:D { \exp_not:o {#1} } { \exp_not:n {#2} } + = \c_zero + \prg_return_true: + \else: + \prg_return_false: + \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[EXP,pTF]{\tl_if_single:N} +% Expand the token list and feed it to \cs{tl_if_single:n}. +% \begin{macrocode} +\cs_new:Npn \tl_if_single_p:N { \exp_args:No \tl_if_single_p:n } +\cs_new:Npn \tl_if_single:NT { \exp_args:No \tl_if_single:nT } +\cs_new:Npn \tl_if_single:NF { \exp_args:No \tl_if_single:nF } +\cs_new:Npn \tl_if_single:NTF { \exp_args:No \tl_if_single:nTF } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[EXP,pTF]{\tl_if_single:n} % A token list has exactly one item if it is either a single % token surrounded by optional explicit spaces, or a single brace % group surrounded by optional explicit spaces. The naive @@ -2211,82 +2424,288 @@ % the token list is empty. Furthermore, it does not allow optional % trailing spaces. % \begin{macrocode} -\cs_new:Npn \tl_if_single_item:nTF #1 - { \str_if_eq:onTF { \use_ii:nn #1 {?} ? } {??} } -\cs_new:Npn \tl_if_single_item:nT #1 - { \str_if_eq:onT { \use_ii:nn #1 {?} ? } {??} } -\cs_new:Npn \tl_if_single_item:nF #1 - { \str_if_eq:onF { \use_ii:nn #1 {?} ? } {??} } -\cs_new:Npn \tl_if_single_item_p:n #1 - { \str_if_eq_p:on { \use_ii:nn #1 {?} ? } {??} } +\prg_new_conditional:Npnn \tl_if_single:n #1 { p , T , F , TF } + { \str_if_eq_return:on { \use_none:nn #1 ?? } {?} } % \end{macrocode} % \end{macro} % -% \begin{macro}[EXP,pTF]{\tl_if_head_begin_group:n} +% \begin{macro}[EXP,pTF]{\tl_if_single_token:n} +% There are four cases: empty token list, token list starting with +% a normal token, with a brace group, or with a space token. +% If the token list starts with a normal token, remove it +% and check for emptyness. Otherwise, compare with a single +% space, only case where we have a single token. % \begin{macrocode} -\prg_new_conditional:Npnn \tl_if_head_begin_group:n #1 { p , T , F , TF } +\prg_new_conditional:Npnn \tl_if_single_token:n #1 { p , T , F , TF } + { + \tl_if_head_N_type:nTF {#1} + { \str_if_eq_return:on { \use_none:n #1 } { } } + { \str_if_eq_return:on { ~ } { #1 } } + } +% \end{macrocode} +% \end{macro} +% +% \begin{variable}{\q_tl_act_mark,\q_tl_act_stop} +% The \cs{tl_act} functions may be applied to any token list. +% Hence, we use two private quarks, to allow any token, even quarks, +% in the token list.^^A in particular critical for future \::e. +% Only \cs{q_tl_act_mark} and \cs{q_tl_act_stop} may not appear +% in the token lists manipulated by \cs{tl_act} functions. The quarks +% are effectively defined in \pkg{l3quark}. +% \end{variable} +% +% \begin{macro}[EXP]{\tl_act:NNNnn,\tl_act_aux:NNNnn} +% \begin{macro}[EXP]{\tl_act_output:n,\tl_act_reverse_output:n, +% \tl_act_group_recurse:Nnn} +% \begin{macro}[EXP,aux]{\tl_act_loop:w,\tl_act_normal:NwnNNN, +% \tl_act_group:nwnNNN,\tl_act_space:wwnNNN,\tl_act_end:w} +% To help control the expansion, \cs{tl_act:NNNnn} starts with +% \cs{tex_romannumeral:D} and ends by producing \cs{c_zero} +% once the result has been obtained. Then loop over tokens, +% groups, and spaces in |#5|. The marker \cs{q_tl_act_mark} +% is used both to avoid losing outer braces and to detect the +% end of the token list more easily. The result is stored +% as an argument for the dummy function \cs{tl_act_result:n}. +% \begin{macrocode} +\cs_new:Npn \tl_act:NNNnn { \tex_romannumeral:D \tl_act_aux:NNNnn } +\cs_new:Npn \tl_act_aux:NNNnn #1 #2 #3 #4 #5 { - \exp_after:wN \use_none:n - \exp_after:wN { - \exp_after:wN { - \token_to_str:N #1 . - } - \prg_return_true: \exp_after:wN \use_none:nn \token_to_str:N + \tl_act_loop:w #5 \q_tl_act_mark \q_tl_act_stop + {#4} #1 #2 #3 + \tl_act_result:n { } + } +% \end{macrocode} +% In the loop, we check how the token list begins and act +% accordingly. In the \enquote{normal} case, we may have +% reached \cs{q_tl_act_mark}, the end of the list. Then +% leave \cs{c_zero} and the result in the input stream, +% to terminate the expansion of \cs{tex_romannumeral:D}. +% Otherwise, apply the relevant function to the +% \enquote{arguments}, |#3| +% and to the head of the token list. Then repeat the loop. +% The scheme is the same if the token list starts with a +% group or with a space. Some extra work is needed to +% make \cs{tl_act_space:wwnNNN} gobble the space. +% \begin{macrocode} +\cs_new:Npn \tl_act_loop:w #1 \q_tl_act_stop + { + \tl_if_head_N_type:nTF {#1} + { \tl_act_normal:NwnNNN } + { + \tl_if_head_group:nTF {#1} + { \tl_act_group:nwnNNN } + { \tl_act_space:wwnNNN } } - \prg_return_false: + #1 \q_tl_act_stop } +\cs_new:Npn \tl_act_normal:NwnNNN #1 #2 \q_tl_act_stop #3#4 + { + \if_meaning:w \q_tl_act_mark #1 + \exp_after:wN \tl_act_end:wn + \fi: + #4 {#3} #1 + \tl_act_loop:w #2 \q_tl_act_stop + {#3} #4 + } +\cs_new:Npn \tl_act_end:wn #1 \tl_act_result:n #2 { \c_zero #2 } +\cs_new:Npn \tl_act_group:nwnNNN #1 #2 \q_tl_act_stop #3#4#5 + { + #5 {#3} {#1} + \tl_act_loop:w #2 \q_tl_act_stop + {#3} #4 #5 + } +\exp_last_unbraced:NNo + \cs_new:Npn \tl_act_space:wwnNNN \c_space_tl #1 \q_tl_act_stop #2#3#4#5 + { + #5 {#2} + \tl_act_loop:w #1 \q_tl_act_stop + {#2} #3 #4 #5 + } +% \end{macrocode} +% Typically, the output is done to the right of what was already output, +% using \cs{tl_act_output:n}, but for the \cs{tl_act_reverse} functions, +% it should be done to the left. +% \begin{macrocode} +\cs_new:Npn \tl_act_output:n #1 #2 \tl_act_result:n #3 + { #2 \tl_act_result:n { #3 #1 } } +\cs_new:Npn \tl_act_reverse_output:n #1 #2 \tl_act_result:n #3 + { #2 \tl_act_result:n { #1 #3 } } % \end{macrocode} +% In many applications of \cs{tl_act:NNNnn}, we need to recursively +% apply some transformation within brace groups, then output. In this +% code, |#1| is the output function, |#2| is the transformation, +% which should expand in two steps, and |#3| is the group. +% \begin{macrocode} +\cs_new:Npn \tl_act_group_recurse:Nnn #1#2#3 + { + \exp_args:Nf #1 + { \exp_after:wN \exp_after:wN \exp_after:wN { #2 {#3} } } + } +% \end{macrocode} +% \end{macro} +% \end{macro} % \end{macro} % -% \begin{macro}[EXP,pTF]{\tl_if_head_eq_space:n} -% \begin{macro}[EXP,aux]{\tl_if_head_eq_space_aux:w} -% If the first token of the token list is an explicit space, i.e., -% a character token with character code $32$ and category code $10$, -% then this test will be \meta{true}. It is \meta{false} if the token -% list is empty, if the first token is an implicit space token, -% such as \cs{c_space_token}, or any token other than an explicit space. +% \begin{macro}[EXP]{\tl_reverse_tokens:n} +% \begin{macro}[EXP,aux]{\tl_act_reverse_normal:nN, +% \tl_act_reverse_group:nn, \tl_act_reverse_space:n} +% The goal is to reverse a token list. This is done by feeding +% \cs{tl_act_aux:NNNnn} three functions, an empty fourth argument +% (we don't use it for \cs{tl_act_reverse_tokens:n}), and as +% a fifth argument the token list to be reversed. +% Spaces and normal tokens are output to the left of the current +% output. For groups, we must recursively apply +% \cs{tl_act_reverse_tokens:n} to the group, and output, still +% on the left. Note that in all three cases, we throw one argument +% away: this \meta{parameter} is where for instance the +% upper/lowercasing action stores the information of whether it +% is uppercasing or lowercasing. % \begin{macrocode} -\prg_new_conditional:Npnn \tl_if_head_eq_space:n #1 { p , T , F , TF } +\cs_new:Npn \tl_reverse_tokens:n { - \if_false: { \fi: - \tl_if_head_eq_space_aux:w \prg_do_nothing: #1 ? ~ } + \tex_romannumeral:D + \tl_act_aux:NNNnn + \tl_act_reverse_normal:nN + \tl_act_reverse_group:nn + \tl_act_reverse_space:n + { } } -\cs_new:Npn \tl_if_head_eq_space_aux:w #1 ~ % +\cs_new:Npn \tl_act_reverse_space:n #1 + { \tl_act_reverse_output:n {~} } +\cs_new:Npn \tl_act_reverse_normal:nN #1 #2 + { \tl_act_reverse_output:n {#2} } +\cs_new:Npn \tl_act_reverse_group:nn #1 { - \tl_if_empty_return:o { #1 } - \exp_after:wN \use_none:n \exp_after:wN { \if_false: } \fi: + \tl_act_group_recurse:Nnn + \tl_act_reverse_output:n + { \tl_reverse_tokens:n } } % \end{macrocode} % \end{macro} % \end{macro} -% -% \begin{macro}[EXP,pTF]{\tl_if_single_token:n} -% If the token list starts with a space, then it is -% a single token if and only if its string representation -% is exactly one space character. Otherwise, we check if removing -% the first token group makes the token list empty. If it doesn't, -% the token list is not a single token. If it does, then the token -% list is either a single token or a single brace group. This is -% tested for using \cs{tl_if_head_begin_group:n}. +% +% \begin{macro}[EXP]{\tl_reverse:n,\tl_reverse:o,\tl_reverse:V} +% \begin{macro}[EXP,aux]{\tl_reverse_group_preserve:nn} +% The goal here is to reverse without losing spaces nor braces. +% The only difference with \cs{tl_reverse_tokens:n} is that +% we now simply output groups without entering them. % \begin{macrocode} -\prg_new_conditional:Npnn \tl_if_single_token:n #1 { p , T , F , TF } +\cs_new:Npn \tl_reverse:n + { + \tex_romannumeral:D + \tl_act_aux:NNNnn + \tl_act_reverse_normal:nN + \tl_act_reverse_group_preserve:nn + \tl_act_reverse_space:n + { } + } +\cs_new:Npn \tl_act_reverse_group_preserve:nn #1 #2 + { \tl_act_reverse_output:n { {#2} } } +\cs_generate_variant:Nn \tl_reverse:n { o , V } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}{\tl_reverse:N, \tl_reverse:c} +% This reverses the list, leaving |{}| in front, which in turn +% is removed by the \cs{etex_unexpanded:D} primitive. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \tl_reverse:N #1 + { \tl_set:No #1 { \etex_unexpanded:D \tl_reverse:o { #1 { } } } } +\cs_generate_variant:Nn \tl_reverse:N { c } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[EXP]{\tl_length_tokens:n} +% \begin{macro}[EXP,aux]{\tl_act_length_normal:nN, +% \tl_act_length_group:nn,\tl_act_length_space:n} +% The length is computed through an \cs{int_eval:n} construction. +% Each \texttt{1+} is output to the \emph{left}, into the integer +% expression, and the sum is ended by the \cs{c_zero} inserted by +% \cs{tl_act_end:wn}. Somewhat a hack. +% \begin{macrocode} +\cs_new:Npn \tl_length_tokens:n #1 { - \tl_if_head_eq_space:nTF {#1} + \int_eval:n { - \str_if_eq:nnTF {#1} {~} - \prg_return_true: \prg_return_false: + \tl_act_aux:NNNnn + \tl_act_length_normal:nN + \tl_act_length_group:nn + \tl_act_length_space:n + { } + {#1} } + } +\cs_new:Npn \tl_act_length_normal:nN #1 #2 { 1 + } +\cs_new:Npn \tl_act_length_space:n #1 { 1 + } +\cs_new:Npn \tl_act_length_group:nn #1 #2 + { 2 + \tl_length_tokens:n {#2} + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{variable}{\c_tl_act_uppercase_tl, \c_tl_act_lowercase_tl} +% These constants contain the correspondance between lowercase +% and uppercase letters, in the form |aAbBcC...| and |AaBbCc...| +% respectively. +% \begin{macrocode} +\tl_const:Nn \c_tl_act_uppercase_tl + { + aA bB cC dD eE fF gG hH iI jJ kK lL mM + nN oO pP qQ rR sS tT uU vV wW xX yY zZ + } +\tl_const:Nn \c_tl_act_lowercase_tl + { + Aa Bb Cc Dd Ee Ff Gg Hh Ii Jj Kk Ll Mm + Nn Oo Pp Qq Rr Ss Tt Uu Vv Ww Xx Yy Zz + } +% \end{macrocode} +% \end{variable} +% +% \begin{macro}[EXP]{\tl_expandable_uppercase:n,\tl_expandable_lowercase:n} +% \begin{macro}[EXP,aux]{\tl_act_case_normal:nN, +% \tl_act_case_group:nn,\tl_act_case_space:n} +% The only difference between uppercasing and lowercasing is +% the table of correspondance that is used. As for other +% token list actions, we feed \cs{tl_act_aux:NNNnn} three +% functions, and this time, we use the \meta{parameters} +% argument to carry which case-changing we are applying. +% A space is simply output. A normal token is compared +% to each letter in the alphabet using \cs{str_if_eq:nn} +% tests, and converted if necessary to upper/lowercase, +% before being output. For a group, we must perform the +% conversion within the group (the \cs{exp_after:wN} trigger +% \cs{tex_romannumeral:D}, which expands fully to give the +% converted group), then output. +% \begin{macrocode} +\cs_new:Npn \tl_expandable_uppercase:n + { \tex_romannumeral:D \tl_act_case_aux:nn { \c_tl_act_uppercase_tl } } +\cs_new:Npn \tl_expandable_lowercase:n + { \tex_romannumeral:D \tl_act_case_aux:nn { \c_tl_act_lowercase_tl } } +\cs_new:Npn \tl_act_case_aux:nn + { + \tl_act_aux:NNNnn + \tl_act_case_normal:nN + \tl_act_case_group:nn + \tl_act_case_space:n + } +\cs_new:Npn \tl_act_case_space:n #1 { \tl_act_output:n {~} } +\cs_new:Npn \tl_act_case_normal:nN #1 #2 + { + \exp_args:Nf \tl_act_output:n { - \str_if_eq:onTF { \use_none:n #1 ? } {?} - { - \tl_if_head_begin_group:nTF {#1} - \prg_return_false: \prg_return_true: - } - \prg_return_false: + \exp_args:NNo \prg_case_str:nnn #2 {#1} + { \exp_stop_f: #2 } } } +\cs_new:Npn \tl_act_case_group:nn #1 #2 + { + \exp_after:wN \tl_act_output:n \exp_after:wN + { \exp_after:wN { \tex_romannumeral:D \tl_act_case_aux:nn {#1} {#2} } } + } % \end{macrocode} % \end{macro} +% \end{macro} % % \subsection{Deprecated functions} % -- cgit v1.2.3