summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/source/latex/l3kernel/l3tl.dtx
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/source/latex/l3kernel/l3tl.dtx')
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3tl.dtx432
1 files changed, 240 insertions, 192 deletions
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3tl.dtx b/Master/texmf-dist/source/latex/l3kernel/l3tl.dtx
index c639989d793..b4ae4118b33 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3tl.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3tl.dtx
@@ -35,7 +35,7 @@
%
%<*driver|package>
\RequirePackage{l3bootstrap}
-\GetIdInfo$Id: l3tl.dtx 3991 2012-07-16 19:00:35Z joseph $
+\GetIdInfo$Id: l3tl.dtx 4201 2012-09-05 08:00:54Z joseph $
{L3 Token lists}
%</driver|package>
%<*driver>
@@ -68,14 +68,14 @@
% \begin{documentation}
%
% \TeX{} works with tokens, and \LaTeX3 therefore provides a number of
-% functions to deal with token lists. Token lists may be present directly in
-% the argument to a function:
+% functions to deal with lists of tokens. Token lists may be present
+% directly in the argument to a function:
% \begin{verbatim}
% \foo:n { a collection of \tokens }
% \end{verbatim}
-% or may be stored for processing in a so-called \enquote{token list
-% variable}, which have the suffix \texttt{tl}:
-% the argument to a function:
+% or may be stored in a so-called \enquote{token list variable}, which
+% have the suffix \texttt{tl}: a token list variable can also be used as
+% the argument to a function, for example
% \begin{verbatim}
% \foo:N \l_some_tl
% \end{verbatim}
@@ -86,12 +86,13 @@
% of tokens: the two \enquote{views} of a token list are therefore collected
% together here.
%
-% A token list can be seen either as a list of \enquote{items},
-% or a list of \enquote{tokens}. An item is whatever \cs{use_none:n}
-% grabs as its argument: either a single token or a brace group,
+% A token list (explicit, or stored in a variable) can be seen either
+% as a list of \enquote{items},
+% or a list of \enquote{tokens}. An item is whatever \cs{use:n} would
+% grab as its argument: a single non-space token or a brace group,
% with optional leading explicit space characters (each item is thus
% itself a token list). A token is either a normal \texttt{N} argument,
-% or | |, |{|, or |}| (assuming normal \TeX{} category codes).
+% or \verb*| |, |{|, or |}| (assuming normal \TeX{} category codes).
% Thus for example
% \begin{verbatim}
% { Hello } ~ world
@@ -103,6 +104,30 @@
% Functions which act on items are often faster than their analogue acting
% directly on tokens.
%
+% ^^A todo: perhaps move to another module, l3token or l3basics?
+% \begin{texnote}
+% When \TeX{} fetches an undelimited argument from the input stream,
+% explicit character tokens with character code $32$ (space) and
+% category code $10$ (space), which we here call \enquote{explicit
+% space characters}, are ignored. If the following token is an
+% explicit character token with category code $1$ (begin-group) and an
+% arbitrary character code, then \TeX{} scans ahead to obtain an equal
+% number of explicit character tokens with category code $1$
+% (begin-group) and $2$ (end-group), and the resulting list of tokens
+% (with outer braces removed) becomes the argument. Otherwise, a
+% single token is taken as the argument for the macro: we call such
+% single tokens \enquote{N-type}, as they are suitable to be used as
+% an argument for a function with the signature~\texttt{:N}.
+%
+% When \TeX{} reads a character of category code $10$ for the first
+% time, it is converted to an explicit space character, with character
+% code $32$, regardless of the initial character code.
+% \enquote{Funny} spaces with a different category code, can be
+% produced using \cs{tl_to_lowercase:n} or \cs{tl_to_uppercase:n}.
+% Explicit space characters are also produced as a result of
+% \cs{token_to_str:N}, \cs{tl_to_str:n}, etc.
+% \end{texnote}
+%
% \section{Creating and initialising token list variables}
%
% \begin{function}{\tl_new:N, \tl_new:c}
@@ -181,7 +206,7 @@
% \begin{function}
% {
% \tl_set:Nn, \tl_set:NV, \tl_set:Nv, \tl_set:No, \tl_set:Nf, \tl_set:Nx,
-% \tl_set:cn, \tl_set:NV, \tl_set:Nv, \tl_set:co, \tl_set:cf, \tl_set:cx,
+% \tl_set:cn, \tl_set:cV, \tl_set:cv, \tl_set:co, \tl_set:cf, \tl_set:cx,
% \tl_gset:Nn, \tl_gset:NV, \tl_gset:Nv,
% \tl_gset:No, \tl_gset:Nf, \tl_gset:Nx,
% \tl_gset:cn, \tl_gset:cV, \tl_gset:cv,
@@ -237,7 +262,8 @@
% Replaces the first (leftmost) occurrence of \meta{old tokens} in the
% \meta{tl~var} with \meta{new tokens}. \meta{Old tokens}
% cannot contain |{|, |}| or |#|
-% (assuming normal \TeX{} category codes).
+% (more precisely, explicit character tokens with category code $1$
+% (begin-group) or $2$ (end-group), and tokens with category code $6$).
% \end{function}
%
% \begin{function}[updated = 2011-08-11]
@@ -251,11 +277,12 @@
% Replaces all occurrences of \meta{old tokens} in the
% \meta{tl~var} with \meta{new tokens}. \meta{Old tokens}
% cannot contain |{|, |}| or |#|
-% (assuming normal \TeX{} category codes). As this function
+% (more precisely, explicit character tokens with category code $1$
+% (begin-group) or $2$ (end-group), and tokens with category code $6$).
+% As this function
% operates from left to right, the pattern \meta{old tokens}
% may remain after the replacement (see \cs{tl_remove_all:Nn}
-% for an example). The assignment is
-% restricted to the current \TeX{} group.
+% for an example).
% \end{function}
%
% \begin{function}[updated = 2011-08-11]
@@ -267,8 +294,9 @@
% \cs{tl_remove_once:Nn} \meta{tl~var} \Arg{tokens}
% \end{syntax}
% Removes the first (leftmost) occurrence of \meta{tokens} from the
-% \meta{tl~var}. \meta{Tokens} cannot contain |{|, |}| or
-% |#| (assuming normal \TeX{} category codes).
+% \meta{tl~var}. \meta{Tokens} cannot contain |{|, |}| or |#|
+% (more precisely, explicit character tokens with category code $1$
+% (begin-group) or $2$ (end-group), and tokens with category code $6$).
% \end{function}
%
% \begin{function}[updated = 2011-08-11]
@@ -280,8 +308,10 @@
% \cs{tl_remove_all:Nn} \meta{tl~var} \Arg{tokens}
% \end{syntax}
% Removes all occurrences of \meta{tokens} from the
-% \meta{tl~var}. \meta{Tokens} cannot contain |{|, |}| or
-% |#| (assuming normal \TeX{} category codes). As this function
+% \meta{tl~var}. \meta{Tokens} cannot contain |{|, |}| or |#|
+% (more precisely, explicit character tokens with category code $1$
+% (begin-group) or $2$ (end-group), and tokens with category code $6$).
+% As this function
% operates from left to right, the pattern \meta{tokens}
% may remain after the removal, for instance,
% \begin{quote}
@@ -332,9 +362,9 @@
% unchanged. This process does not alter the category code assigned
% to the \meta{tokens}.
% \begin{texnote}
-% This is the \TeX{} primitive \tn{lowercase} renamed.
-% As a result, this function takes place on execution and
-% not on expansion.
+% This is the \TeX{} primitive \tn{lowercase} renamed. As a result,
+% its argument \emph{must} be given within braces, and this function
+% takes place on execution and not on expansion.
% \end{texnote}
% \end{function}
%
@@ -348,9 +378,9 @@
% unchanged. This process does not alter the category code assigned
% to the \meta{tokens}.
% \begin{texnote}
-% This is the \TeX{} primitive \tn{uppercase} renamed.
-% As a result, this function takes place on execution and
-% not on expansion.
+% This is the \TeX{} primitive \tn{uppercase} renamed. As a result,
+% its argument \emph{must} be given within braces, and this function
+% takes place on execution and not on expansion.
% \end{texnote}
% \end{function}
%
@@ -363,14 +393,15 @@
% \end{syntax}
% Tests if the \meta{token list} consists only of blank spaces
% (\emph{i.e.}~contains no item). The test is \texttt{true} if
-% \meta{token list} is zero or more explicit tokens of character code~$32$
-% and category code~$10$, and is \texttt{false} otherwise.
+% \meta{token list} is zero or more explicit space characters
+% (explicit tokens with character code~$32$ and category code~$10$),
+% and is \texttt{false} otherwise.
% \end{function}
%
% \begin{function}[EXP,pTF]{\tl_if_empty:N, \tl_if_empty:c}
% \begin{syntax}
-% \cs{tl_if_empty_p:N} \meta{tl var}
-% \cs{tl_if_empty:NTF} \meta{tl var} \Arg{true code} \Arg{false code}
+% \cs{tl_if_empty_p:N} \meta{tl~var}
+% \cs{tl_if_empty:NTF} \meta{tl~var} \Arg{true code} \Arg{false code}
% \end{syntax}
% Tests if the \meta{token list variable} is entirely empty
% (\emph{i.e.}~contains no tokens at all).
@@ -389,8 +420,8 @@
% \begin{function}[EXP,pTF]
% {\tl_if_eq:NN, \tl_if_eq:Nc, \tl_if_eq:cN, \tl_if_eq:cc}
% \begin{syntax}
-% \cs{tl_if_eq_p:NN} \Arg{tl var_1} \Arg{tl var_2}
-% \cs{tl_if_eq:NNTF} \Arg{tl var_1} \Arg{tl var_2} \Arg{true code} \Arg{false code}
+% \cs{tl_if_eq_p:NN} \Arg{tl~var_1} \Arg{tl~var_2}
+% \cs{tl_if_eq:NNTF} \Arg{tl~var_1} \Arg{tl~var_2} \Arg{true code} \Arg{false code}
% \end{syntax}
% Compares the content of two \meta{token list variables} and
% is logically \texttt{true} if the two contain the same list of
@@ -399,17 +430,18 @@
% \begin{verbatim}
% \tl_set:Nn \l_tmpa_tl { abc }
% \tl_set:Nx \l_tmpb_tl { \tl_to_str:n { abc } }
-% \tl_if_eq_p:NN \l_tmpa_tl \l_tmpb_tl
+% \tl_if_eq:NNTF \l_tmpa_tl \l_tmpb_tl { true } { false }
% \end{verbatim}
-% is logically \texttt{false}.
+% yields \texttt{false}.
% \end{function}
%
% \begin{function}[TF]{\tl_if_eq:nn}
% \begin{syntax}
% \cs{tl_if_eq:nnTF} \meta{token list_1} \Arg{token list_2} \Arg{true code} \Arg{false code}
% \end{syntax}
-% Tests if \meta{token list_1} and \meta{token list_2} are equal, both in
-% respect of character codes and category codes.
+% Tests if \meta{token list_1} and \meta{token list_2} contain the
+% same list of tokens, both in respect of character codes and category
+% codes.
% \end{function}
%
% \begin{function}[TF]{\tl_if_in:Nn, \tl_if_in:cn}
@@ -417,9 +449,10 @@
% \cs{tl_if_in:NnTF} \meta{tl~var} \Arg{token list} \Arg{true code} \Arg{false code}
% \end{syntax}
% Tests if the \meta{token list} is found in the content of the
-% \meta{token list variable}. The \meta{token list} cannot contain
-% the tokens |{|, |}| or |#| (assuming the usual \TeX{} category
-% codes apply).
+% \meta{tl~var}. The \meta{token list} cannot contain
+% the tokens |{|, |}| or |#|
+% (more precisely, explicit character tokens with category code $1$
+% (begin-group) or $2$ (end-group), and tokens with category code $6$).
% \end{function}
%
% \begin{function}[TF]
@@ -427,9 +460,10 @@
% \begin{syntax}
% \cs{tl_if_in:nnTF} \Arg{token list_1} \Arg{token list_2} \Arg{true code} \Arg{false code}
% \end{syntax}
-% Tests if <token list_2> is found inside <token list_1>.
-% The \meta{token list} cannot contain the tokens |{|, |}| or |#|
-% (assuming the usual \TeX{} category codes apply).
+% Tests if \meta{token list_2} is found inside \meta{token list_1}.
+% The \meta{token list_2} cannot contain the tokens |{|, |}| or |#|
+% (more precisely, explicit character tokens with category code $1$
+% (begin-group) or $2$ (end-group), and tokens with category code $6$).
% \end{function}
%
% \begin{function}[updated = 2011-08-13, EXP,pTF]
@@ -439,10 +473,10 @@
% \cs{tl_if_single:NTF} \Arg{tl~var} \Arg{true code} \Arg{false code}
% \end{syntax}
% Tests if the content of the \meta{tl~var} consists of a single item,
-% \emph{i.e.}~is either a single normal token (excluding spaces,
-% and brace tokens) or a single brace group, surrounded by optional
-% spaces on both sides. In other words, such a token list has token count
-% $1$ according to \cs{tl_count:N}.
+% \emph{i.e.}~is a single normal token (neither an explicit space
+% character nor a begin-group character) or a single brace group,
+% surrounded by optional spaces on both sides. In other words, such a
+% token list has token count $1$ according to \cs{tl_count:N}.
% \end{function}
%
% \begin{function}[updated = 2011-08-13, EXP,pTF]{\tl_if_single:n}
@@ -450,10 +484,11 @@
% \cs{tl_if_single_p:n} \Arg{token list}
% \cs{tl_if_single:nTF} \Arg{token list} \Arg{true code} \Arg{false code}
% \end{syntax}
-% Tests if the token list has exactly one item, \emph{i.e.}~is either
-% a single normal token or a single brace group, surrounded by
-% optional spaces on both sides. In other words, such a token list
-% has token count $1$ according to \cs{tl_count:n}.
+% Tests if the \meta{token list} has exactly one item, \emph{i.e.}~is
+% a single normal token (neither an explicit space character nor a
+% begin-group character) or a single brace group, surrounded by
+% optional spaces on both sides. In other words, such a token list has
+% token count $1$ according to \cs{tl_count:n}.
% \end{function}
%
% \begin{function}[added = 2012-06-03, EXP]{\tl_case:Nnn, \tl_case:cnn}
@@ -469,8 +504,7 @@
% \end{syntax}
% This function compares the \meta{test token list variable} in turn
% with each of the \meta{token list variable cases}. If the two
-% are equal (as described for
-% \cs{tl_if_eq:nnTF}
+% are equal (as described for \cs{tl_if_eq:NNTF})
% then the associated \meta{code} is left in the input
% stream. If none of the tests are \texttt{true} then the
% \texttt{else code} will be left in the input stream.
@@ -509,7 +543,7 @@
% Applies the \meta{inline function} to every \meta{item} stored within the
% \meta{tl~var}. The \meta{inline function} should consist of code which
% will receive the \meta{item} as |#1|. One in line mapping can be nested
-% inside another. See also \cs{tl_map_function:Nn}.
+% inside another. See also \cs{tl_map_function:NN}.
% \end{function}
%
% \begin{function}[updated = 2012-06-29]{\tl_map_inline:nn}
@@ -519,7 +553,7 @@
% Applies the \meta{inline function} to every \meta{item} stored within the
% \meta{token list}. The \meta{inline function} should consist of code which
% will receive the \meta{item} as |#1|. One in line mapping can be nested
-% inside another. See also \cs{tl_map_function:nn}.
+% inside another. See also \cs{tl_map_function:nN}.
% \end{function}
%
% \begin{function}[updated = 2012-06-29]
@@ -555,14 +589,12 @@
% \begin{verbatim}
% \tl_map_inline:Nn \l_my_tl
% {
-% \str_if_eq:nnTF { #1 } { bingo }
-% { \tl_map_break: }
-% {
-% % Do something useful
-% }
+% \str_if_eq:nnT { #1 } { bingo } { \tl_map_break: }
+% % Do something useful
% }
% \end{verbatim}
-% Use outside of a \cs{tl_map_\ldots} scenario will lead low
+% See also \cs{tl_map_break:n}.
+% Use outside of a \cs{tl_map_\ldots} scenario will lead to low
% level \TeX{} errors.
% \begin{texnote}
% When the mapping is broken, additional tokens may be inserted by the
@@ -583,11 +615,9 @@
% \begin{verbatim}
% \tl_map_inline:Nn \l_my_tl
% {
-% \str_if_eq:nnTF { #1 } { bingo }
+% \str_if_eq:nnT { #1 } { bingo }
% { \tl_map_break:n { <tokens> } }
-% {
-% % Do something useful
-% }
+% % Do something useful
% }
% \end{verbatim}
% Use outside of a \cs{tl_map_\ldots} scenario will lead to low
@@ -659,7 +689,7 @@
% Counts the number of token groups in the \meta{tl~var}
% and leaves this information in the input stream. Unbraced tokens
% count as one element as do each token group (|{|\ldots|}|). This
-% process will ignore any unprotected spaces within \meta{tokens}.
+% process will ignore any unprotected spaces within the \meta{tl~var}.
% See also \cs{tl_count:n}. This function requires three expansions,
% giving an \meta{integer denotation}.
% \end{function}
@@ -679,9 +709,8 @@
% consider \cs{tl_reverse_items:n}.
% See also \cs{tl_reverse:N}.
% \begin{texnote}
-% The result is returned within the \tn{unexpanded}
-% primitive (\cs{exp_not:n}), which means that the token
-% list will not expand further when appearing in an x-type
+% The result is returned within \tn{exp_not:n}, which means that the token
+% list will not expand further when appearing in an \texttt{x}-type
% argument expansion.
% \end{texnote}
% \end{function}
@@ -697,7 +726,8 @@
% This process will preserve unprotected spaces within the
% \meta{token list variable}. Braced token groups are copied without
% reversing the order of tokens, but keep the outer set of braces.
-% See also \cs{tl_reverse:n}.
+% See also \cs{tl_reverse:n}, and, for improved performance,
+% \cs{tl_reverse_items:n}.
% \end{function}
%
% \begin{function}[added = 2012-01-08, EXP]{\tl_reverse_items:n}
@@ -712,11 +742,10 @@
% reversing the order of tokens, and keep the outer set of braces.
% Items which are initially not braced are copied with braces in
% the result. In cases where preserving spaces is important,
-% consider \cs{tl_reverse:n} or \cs{tl_reverse_tokens:n}.
+% consider the slower function \cs{tl_reverse:n}.
% \begin{texnote}
-% The result is returned within the \tn{unexpanded}
-% primitive (\cs{exp_not:n}), which means that the token
-% list will not expand further when appearing in an x-type
+% The result is returned within \tn{exp_not:n}, which means that the token
+% list will not expand further when appearing in an \texttt{x}-type
% argument expansion.
% \end{texnote}
% \end{function}
@@ -724,15 +753,15 @@
% \begin{function}[added = 2011-07-09, updated = 2012-06-25, EXP]
% {\tl_trim_spaces:n}
% \begin{syntax}
-% \cs{tl_trim_spaces:n} \meta{token list}
+% \cs{tl_trim_spaces:n} \Arg{token list}
% \end{syntax}
% Removes any leading and trailing explicit space characters
+% (explicit tokens with character code~$32$ and category code~$10$)
% from the \meta{token list} and leaves the result in the input
% stream.
% \begin{texnote}
-% The result is returned within the \tn{unexpanded}
-% primitive (\cs{exp_not:n}), which means that the token
-% list will not expand further when appearing in an x-type
+% The result is returned within \tn{exp_not:n}, which means that the token
+% list will not expand further when appearing in an \texttt{x}-type
% argument expansion.
% \end{texnote}
% \end{function}
@@ -746,22 +775,25 @@
% \cs{tl_trim_spaces:N} \meta{tl~var}
% \end{syntax}
% Removes any leading and trailing explicit space characters
+% (explicit tokens with character code~$32$ and category code~$10$)
% from the content of the \meta{tl~var}.
% \end{function}
%
% \section{The first token from a token list}
%
-% Functions which deal with either only the very first token of a
-% token list or everything except the first token.
+% Functions which deal with either only the very first item (balanced
+% text or single normal token) in a token list, or the remaining tokens.
%
% \begin{function}[updated = 2012-02-08, EXP]
% {\tl_head:N, \tl_head:n, \tl_head:V, \tl_head:v, \tl_head:f}
% \begin{syntax}
-% \cs{tl_head:n} \Arg{tokens}
+% \cs{tl_head:n} \Arg{token list}
% \end{syntax}
-% Leaves in the input stream the first non-space token from the
-% \meta{tokens}. Any leading space tokens will be discarded, and thus for
-% example
+% Leaves in the input stream the first \meta{item} in the
+% \meta{token list}, discarding the rest of the \meta{token list}.
+% All leading explicit space characters
+% (explicit tokens with character code~$32$ and category code~$10$)
+% are discarded; for example
% \begin{verbatim}
% \tl_head:n { abc }
% \end{verbatim}
@@ -769,86 +801,79 @@
% \begin{verbatim}
% \tl_head:n { ~ abc }
% \end{verbatim}
-% will both leave |a| in the input stream.
-% An empty list of \meta{tokens} or one which consists
-% only of space (category code $10$) tokens will result in \cs{tl_head:n}
-% leaving nothing in the input stream.
+% will both leave |a| in the input stream. If the \enquote{head} is a
+% brace group, rather than a single token, the braces will be removed, and
+% so
+% \begin{verbatim}
+% \tl_head:n { ~ { ~ ab } c }
+% \end{verbatim}
+% yields \verb*| ab|.
+% A blank \meta{token list} (see \cs{tl_if_blank:nTF}) will result in
+% \cs{tl_head:n} leaving nothing in the input stream. The \meta{token list}
+% should not contain the marker \cs{q_stop}.
% \begin{texnote}
-% The result is returned within the \tn{unexpanded}
-% primitive (\cs{exp_not:n}), which means that the token
-% list will not expand further when appearing in an x-type
+% The result is returned within \cs{exp_not:n}, which means that the token
+% list will not expand further when appearing in an \texttt{x}-type
% argument expansion.
% \end{texnote}
% \end{function}
%
% \begin{function}[EXP]{\tl_head:w}
% \begin{syntax}
-% \cs{tl_head:w} \meta{tokens} \cs{q_stop}
+% \cs{tl_head:w} \meta{token list} | { } | \cs{q_stop}
% \end{syntax}
-% Leaves in the input stream the first non-space token from the
-% \meta{tokens}. An empty list of \meta{tokens} or one which consists
-% only of space (category code $10$) tokens will result in an error, and
-% thus \meta{tokens} must \emph{not} be \enquote{blank} as determined by
-% \cs{tl_if_blank:n(TF)}. This function requires only a single expansion,
-% and thus is suitable for use within an \texttt{o}-type expansion. In
-% general, \cs{tl_head:n} should be preferred if the number of expansions
-% is not critical.
+% Leaves in the input stream the first \meta{item} in the
+% \meta{token list}, discarding the rest of the \meta{token list}.
+% All leading explicit space characters
+% (explicit tokens with character code~$32$ and category code~$10$)
+% are discarded.
+% A blank \meta{token list} (which consists only of space characters)
+% will result in a low-level \TeX{} error, which may be avoided by the
+% inclusion of an empty group in the input (as shown), without the need
+% for an explicit test. Alternatively, \cs{tl_if_blank:nF} may be used to
+% avoid using the function with a \enquote{blank} argument.
+% This function requires only a single expansion, and thus is suitable for
+% use within an \texttt{o}-type expansion. In general, \cs{tl_head:n} should
+% be preferred if the number of expansions is not critical.
% \end{function}
%
-% \begin{function}[updated = 2012-02-08, EXP]
+% \begin{function}[updated = 2012-09-01, EXP]
% {\tl_tail:N, \tl_tail:n, \tl_tail:V, \tl_tail:v, \tl_tail:f}
% \begin{syntax}
-% \cs{tl_tail:n} \Arg{tokens}
+% \cs{tl_tail:n} \Arg{token list}
% \end{syntax}
-% Discards the all leading space tokens and the first non-space token
-% in the \meta{tokens}, and leaves the remaining tokens in the
-% input stream. Thus for example
+% Discards all leading explicit space characters
+% (explicit tokens with character code~$32$ and category code~$10$)
+% and the first \meta{item} in the \meta{token list}, and leaves the
+% remaining tokens in the input stream. Thus for example
% \begin{verbatim}
-% \tl_tail:n { abc }
+% \tl_tail:n { a ~ {bc} d }
% \end{verbatim}
% and
% \begin{verbatim}
-% \tl_tail:n { ~ abc }
+% \tl_tail:n { ~ a ~ {bc} d }
% \end{verbatim}
-% will both leave |bc| in the input stream.
-% An empty list of \meta{tokens} or one which consists
-% only of space (category code $10$) tokens will result in \cs{tl_tail:n}
-% leaving nothing in the input stream.
+% will both leave \verb*| {bc}d| in the input stream. A blank
+% \meta{token list} (see \cs{tl_if_blank:nTF}) will result
+% in \cs{tl_tail:n} leaving nothing in the input stream.
% \begin{texnote}
-% The result is returned within the \tn{unexpanded}
-% primitive (\cs{exp_not:n}), which means that the token
-% list will not expand further when appearing in an x-type
+% The result is returned within \cs{exp_not:n}, which means that the
+% token list will not expand further when appearing in an \texttt{x}-type
% argument expansion.
% \end{texnote}
% \end{function}
%
-% \begin{function}[EXP]{\tl_tail:w}
-% \begin{syntax}
-% \cs{tl_tail:w} \Arg{tokens} \cs{q_stop}
-% \end{syntax}
-% Discards the all leading space tokens and the first non-space token
-% in the \meta{tokens}, and leaves the remaining tokens in the
-% input stream.
-% An empty list of \meta{tokens} or one which consists
-% only of space (category code $10$) tokens will result in an error, and
-% thus \meta{tokens} must \emph{not} be \enquote{blank} as determined by
-% \cs{tl_if_blank:n(TF)}. This function requires only a single expansion,
-% and thus is suitable for use within an \texttt{o}-type expansion. In
-% general, \cs{tl_tail:n} should be preferred if the number of expansions
-% is not critical.
-% \end{function}
-%
% \begin{function}[added = 2011-08-10, EXP]{\str_head:n,\str_tail:n}
% \begin{syntax}
-% \cs{str_head:n} \Arg{tokens}
-% \cs{str_tail:n} \Arg{tokens}
+% \cs{str_head:n} \Arg{token list}
+% \cs{str_tail:n} \Arg{token list}
% \end{syntax}
-% Converts the \meta{tokens} into a string, as described for
+% Converts the \meta{token list} into a string, as described for
% \cs{tl_to_str:n}. The \cs{str_head:n} function then leaves
% the first character of this string in the input stream.
% The \cs{str_tail:n} function leaves all characters except
% the first in the input stream. The first character may be
-% a space. If the \meta{tokens} argument is entirely empty,
+% a space. If the \meta{token list} argument is entirely empty,
% nothing is left in the input stream.
% \end{function}
%
@@ -892,11 +917,11 @@
% \cs{tl_if_head_is_group:nTF} \Arg{token list} \Arg{true code} \Arg{false code}
% \end{syntax}
% Tests if the first \meta{token} in the \meta{token list}
-% is an explicit begin-group character (with category code~1
+% is an explicit begin-group character (with category code~$1$
% and any character code), in other words, if the \meta{token list}
% starts with a brace group. In particular, the test is \texttt{false}
% if the \meta{token list} starts with an implicit token such as
-% \cs{c_group_begin_token}, or if it empty.
+% \cs{c_group_begin_token}, or if it is empty.
% This function is useful to implement actions on token lists on
% a token by token basis.
% \end{function}
@@ -908,8 +933,9 @@
% \end{syntax}
% Tests if the first \meta{token} in the \meta{token list}
% is a normal \texttt{N}-type argument. In other words,
-% it is neither an explicit space character (with category code~10
-% and character code~32) nor an explicit begin-group character
+% it is neither an explicit space character
+% (explicit token with character code~$32$ and category code~$10$)
+% nor an explicit begin-group character
% (with category code~1 and any character code). An empty
% argument yields \texttt{false}, as it does not have a \enquote{normal}
% first token.
@@ -922,23 +948,14 @@
% \cs{tl_if_head_is_space_p:n} \Arg{token list}
% \cs{tl_if_head_is_space:nTF} \Arg{token list} \Arg{true code} \Arg{false code}
% \end{syntax}
-%^^A We need to add a discussion of
-%^^A explicit vs implicit tokens somewhere in the doc.
% Tests if the first \meta{token} in the \meta{token list}
-% is an explicit space character (with category code~$10$
-% and character code~$32$). If \meta{token list} starts with
-% an implicit token such as \cs{c_space_token}, the test
-% will yield \texttt{false}, as well as if the argument is empty.
+% is an explicit space character
+% (explicit token with character code~$12$ and category code~$10$).
+% In particular, the test is \texttt{false} if the \meta{token list}
+% starts with an implicit token such as \cs{c_space_token}, or if it
+% is empty.
% This function is useful to implement actions on token lists on
% a token by token basis.
-% \begin{texnote}
-% When \TeX{} reads a character of category code $10$ for the
-% first time, it is converted to an explicit space token, with
-% character code $32$, regardless of the initial character code.
-% \enquote{Funny} spaces with a different category code, can be produced
-% using \tn{lowercase}. Explicit spaces are also produced
-% as a result of \cs{token_to_str:N}, \cs{tl_to_str:n}, etc.
-% \end{texnote}
% \end{function}
%
% \section{Viewing token lists}
@@ -960,25 +977,26 @@
% Displays the \meta{token list} on the terminal.
% \begin{texnote}
% \cs{tl_show:n} is the \eTeX{} primitive \tn{showtokens}.
+% Hence its argument \emph{must} be given within braces.
% \end{texnote}
% \end{function}
%
% \section{Constant token lists}
%
+% \begin{variable}{\c_empty_tl}
+% Constant that is always empty.
+% \end{variable}
+%
% \begin{variable}[updated = 2011-08-18]{\c_job_name_tl}
% Constant that gets the \enquote{job name} assigned when \TeX{} starts.
% \begin{texnote}
-% This is the new name for the primitive \tn{jobname}. It is a constant
+% This copies the contents of the primitive \tn{jobname}. It is a constant
% that is set by \TeX{} and should not be overwritten by the package.
% \end{texnote}
% \end{variable}
%
-% \begin{variable}{\c_empty_tl}
-% Constant that is always empty.
-% \end{variable}
-%
% \begin{variable}{\c_space_tl}
-% A space token contained in a token list (compare this with
+% An explicit space character contained in a token list (compare this with
% \cs{c_space_token}). For use where an explicit space is required.
% \end{variable}
%
@@ -1011,7 +1029,7 @@
% instance, \cs{tl_trim_spaces:n} is implemented by taking the
% \meta{continuation} to be \cs{exp_not:o}, and the \texttt{o}-type
% expansion removes the \cs{q_mark}. This function is also used in
-% \pkg{l3clist}.
+% \pkg{l3clist} and \pkg{l3candidates}.
% \end{function}
%
% \end{documentation}
@@ -1044,7 +1062,7 @@
%
% \begin{macro}{\tl_new:N, \tl_new:c}
% Creating new token list variables is a case of checking for an
-% existing definition and if free doing the definition.
+% existing definition and doing the definition.
% \begin{macrocode}
\cs_new_protected:Npn \tl_new:N #1
{
@@ -1073,14 +1091,6 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{variable}{\c_empty_tl}
-% Never full. We need to define that constant early
-% for \cs{tl_new:N} to work properly.
-% \begin{macrocode}
-\tl_const:Nn \c_empty_tl { }
-% \end{macrocode}
-% \end{variable}
-%
% \begin{macro}{\tl_clear:N, \tl_clear:c}
% \begin{macro}{\tl_gclear:N, \tl_gclear:c}
% Clearing a token list variable means setting it to an empty value.
@@ -1157,6 +1167,13 @@
%
% \subsection{Constant token lists}
%
+% \begin{variable}{\c_empty_tl}
+% Never full. We need to define that constant before using \cs{tl_new:N}.
+% \begin{macrocode}
+\tl_const:Nn \c_empty_tl { }
+% \end{macrocode}
+% \end{variable}
+%
% \begin{variable}{\c_job_name_tl}
% Inherited from the \LaTeX3 name for the primitive: this needs to
% actually contain the text of the job name rather than the name of
@@ -1165,17 +1182,24 @@
% make in the \LuaTeX{} case.
% \begin{macrocode}
%<*initex>
-\tex_everyjob:D \exp_after:wN
+\luatex_if_engine:T
{
- \tex_the:D \tex_everyjob:D
- \luatex_if_engine:T
+ \tex_everyjob:D \exp_after:wN
{
- \lua_now:x
+ \tex_the:D \tex_everyjob:D
+ \lua_now_x:n
{ dofile ( assert ( kpse.find_file ("lualatexquotejobname.lua" ) ) ) }
}
}
+\tex_everyjob:D \exp_after:wN
+ {
+ \tex_the:D \tex_everyjob:D
+ \tl_const:Nx \c_job_name_tl { \tex_jobname:D }
+ }
%</initex>
+%<*package>
\tl_const:Nx \c_job_name_tl { \tex_jobname:D }
+%</package>
% \end{macrocode}
% \end{variable}
%
@@ -1191,13 +1215,13 @@
% \begin{macro}
% {
% \tl_set:Nn, \tl_set:NV, \tl_set:Nv, \tl_set:No, \tl_set:Nf, \tl_set:Nx,
-% \tl_set:cn, \tl_set:NV, \tl_set:Nv, \tl_set:co, \tl_set:cf, \tl_set:cx
+% \tl_set:cn, \tl_set:cV, \tl_set:cv, \tl_set:co, \tl_set:cf, \tl_set:cx
% }
% \begin{macro}
% {
% \tl_gset:Nn, \tl_gset:NV, \tl_gset:Nv,
% \tl_gset:No, \tl_gset:Nf, \tl_gset:Nx,
-% \tl_gset:cn, \tl_gset:NV, \tl_gset:Nv,
+% \tl_gset:cn, \tl_gset:cV, \tl_gset:cv,
% \tl_gset:co, \tl_gset:cf, \tl_gset:cx
% }
% By using \cs{exp_not:n} token list variables can contain |#| tokens,
@@ -1745,9 +1769,9 @@
% \end{macro}
%
% \begin{macro}[EXP,pTF]{\tl_if_single:n}
-% A token list has exactly one item if it is either a single
-% token surrounded by optional explicit spaces, or a single brace
-% group surrounded by optional explicit spaces. The naive
+% A token list has exactly one item if it is a single token or a
+% single brace group, surrounded by optional explicit spaces.
+% The naive
% version of this test would do \cs{use_none:n} |#1|, and
% test if the result is empty. However, this will fail when
% the token list is empty. Furthermore, it does not allow optional
@@ -2200,33 +2224,48 @@
%
% \subsection{The first token from a token list}
%
-% \begin{macro}{\tl_head:N, \tl_head:n, \tl_head:V, \tl_head:v, \tl_head:f}
% \begin{macro}{\tl_head:w}
+% \begin{macro}{\tl_head:N, \tl_head:n, \tl_head:V, \tl_head:v, \tl_head:f}
% \begin{macro}{\tl_tail:N, \tl_tail:n, \tl_tail:V, \tl_tail:v, \tl_tail:f}
-% \begin{macro}{\tl_tail:w}
-% \begin{macro}[aux]{\@@_tail:w}
-% These functions pick up either the head or the tail of a list. The
-% empty brace groups in \cs{tl_head:n} and \cs{tl_tail:n} ensure that
-% a blank argument gives an empty result. The result is returned within
-% the \tn{unexpanded} primitive.
+% Finding the head of a token list expandably will always strip braces, which
+% is fine as this is consistent with for example mapping to a list. The
+% empty brace groups in \cs{tl_head:n} ensure that a blank argument gives an
+% empty result. The result is returned within the \tn{unexpanded} primitive.
% \begin{macrocode}
\cs_new:Npn \tl_head:w #1#2 \q_stop {#1}
-\cs_new:Npn \tl_tail:w #1#2 \q_stop {#2}
\cs_new:Npn \tl_head:n #1
{ \etex_unexpanded:D \exp_after:wN { \tl_head:w #1 { } \q_stop } }
-\cs_new:Npn \tl_tail:n #1
- { \etex_unexpanded:D \@@_tail:w #1 \q_mark { } \q_mark \q_stop }
-\cs_new:Npn \@@_tail:w #1 #2 \q_mark #3 \q_stop { {#2} }
-\cs_new_nopar:Npn \tl_head:N { \exp_args:No \tl_head:n }
\cs_generate_variant:Nn \tl_head:n { V , v , f }
-\cs_new_nopar:Npn \tl_tail:N { \exp_args:No \tl_tail:n }
+\cs_new_nopar:Npn \tl_head:N { \exp_args:No \tl_head:n }
+% \end{macrocode}
+% To corrected leave the tail of a token list, it's important \emph{not} to
+% absorb any of the tail part as an argument. For example, the simple
+% definition
+% \begin{verbatim}
+% \cs_new:Npn \tl_tail:n #1 { \tl_tail:w #1 \q_stop }
+% \cs_new:Npn \tl_tail:w #1#2 \q_stop
+% \end{verbatim}
+% will give the wrong result for |\tl_tail:n { a { bc } }| (the braces will
+% be stripped). Thus the only safe way to proceed is to first check that
+% there is an item to grab (\emph{i.e.}~that the argument is not blank) and
+% assuming there is to dispose of the first item. As with \cs{tl_head:n},
+% the result is protected from further expansion by \tn{etex_unexpanded:D}.
+% While we could optimise the test here, this would leave some tokens
+% \enquote{banned} in the input, which we do not have with this definition.
+% \begin{macrocode}
+\cs_new:Npn \tl_tail:n #1
+ {
+ \etex_unexpanded:D
+ \tl_if_blank:nTF {#1}
+ { { } }
+ { \exp_after:wN { \use_none:n #1 } }
+ }
\cs_generate_variant:Nn \tl_tail:n { V , v , f }
+\cs_new_nopar:Npn \tl_tail:N { \exp_args:No \tl_tail:n }
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
-% \end{macro}
%
% \begin{macro}{\str_head:n, \str_tail:n}
% \begin{macro}[aux]{\__str_head:w}
@@ -2404,7 +2443,7 @@
% \begin{macro}[pTF]{\tl_if_head_is_N_type:n}
% The first token of a token list can be either an N-type argument,
% a begin-group token (catcode 1), or an explicit space token
-% (catcode 10 and charcode 32). These two cases are characterized
+% (catcode 10 and charcode 32). The latter two cases are characterized
% by the fact that \cs{use:n} removes some tokens from |#1|, hence
% changing its string representation (no token can have an empty
% string representation). The extra brace group covers the case of
@@ -2658,7 +2697,7 @@
{ \__str_if_eq_x_return:nn { } {#1} }
% \end{macrocode}
% \end{macro}
-%
+%
% Deprecated 2012-07-08 for removal after 2012-10-31.
%
% \begin{macro}[EXP, pTF]{\tl_if_head_group:n}
@@ -2676,6 +2715,15 @@
% \end{macro}
% \end{macro}
%
+% \begin{macro}{\tl_tail:w}
+% Deprecated 2012-09-01 for removal after 2012-12-31. This is broken as
+% it will strip braces from a case such as |a{bc}|.
+% \begin{macrocode}
+\cs_new:Npn \tl_tail:w #1#2 \q_stop {#2}
+% \end{macrocode}
+% \end{macro}
+%
+%
% \begin{macrocode}
%</initex|package>
% \end{macrocode}