summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/source/latex/l3kernel/l3token.dtx')
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3token.dtx226
1 files changed, 168 insertions, 58 deletions
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
index ac7ed290c36..98aedaa038e 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
@@ -37,7 +37,7 @@
\documentclass[full]{l3doc}
%</driver>
%<*driver|package>
-\GetIdInfo$Id: l3token.dtx 6470 2016-04-03 11:29:18Z joseph $
+\GetIdInfo$Id: l3token.dtx 6695 2016-09-01 15:38:50Z bruno $
{L3 Experimental token manipulation}
%</driver|package>
%<*driver>
@@ -82,28 +82,32 @@
% categories: |\token_| for anything that deals with tokens and
% |\peek_| for looking ahead in the token stream.
%
-% Most of the time we will be using the term \enquote{token} but most of the
-% time the function we're describing can equally well by used on a
-% control sequence as such one is one token as well.
+% Most functions we will describe here can be used on control sequences,
+% as those are tokens as well.
%
-% \section{All possible tokens}
-%
-% Let us start by reviewing every case that a given token can fall into.
-% It is very important to distinguish two aspects of a token: its meaning,
-% and what it looks like.
+% It is important to distinguish two aspects of a token: its
+% \enquote{shape} (for lack of a better word), which affects the
+% matching of delimited arguments and the comparison of token lists
+% containing this token, and its \enquote{meaning}, which affects
+% whether the token expands or what operation it performs. One can have
+% tokens of different shapes with the same meaning, but not the
+% converse.
%
% For instance, \cs{if:w}, \cs{if_charcode:w}, and \cs{tex_if:D} are
-% three names for the same internal operation of \TeX{}, namely the primitive
-% testing the next two characters for equality of their character code.
-% They behave identically in many situations. However, \TeX{}
-% distinguishes them when searching for a delimited argument. Namely, the
-% example function |\show_until_if:w| defined below will take everything
-% until \cs{if:w} as an argument, despite the presence of other copies of
-% \cs{if:w} under different names.
+% three names for the same internal operation of \TeX{}, namely the
+% primitive testing the next two characters for equality of their
+% character code. They have the same meaning hence behave identically
+% in many situations. However, \TeX{} distinguishes them when searching
+% for a delimited argument. Namely, the example function
+% |\show_until_if:w| defined below will take everything until \cs{if:w}
+% as an argument, despite the presence of other copies of \cs{if:w}
+% under different names.
% \begin{verbatim}
% \cs_new:Npn \show_until_if:w #1 \if:w { \tl_show:n {#1} }
% \show_until_if:w \tex_if:D \if_charcode:w \if:w
% \end{verbatim}
+% A list of all possible shapes and a list of all possible meanings are
+% given in section~\ref{sec:l3token:all-tokens}.
%
% \section{Creating character tokens}
%
@@ -416,11 +420,6 @@
% This will be an implicit representation of \meta{token_2}.
% \end{function}
%
-% ^^A Because it's late I can't figure out a better way to handle this:
-% \ExplSyntaxOn
-% \cs_set_eq:NN \c_alignment_token @
-% \cs_set_eq:NN \c_parameter_token @
-% \ExplSyntaxOff
% \begin{variable}
% {
% \c_group_begin_token,
@@ -436,10 +435,6 @@
% by their name. They are used internally for test purposes but
% are also available to the programmer for other uses.
% \end{variable}
-% \ExplSyntaxOn
-% \cs_set_eq:NN \c_alignment_token &
-% \cs_set_eq:NN \c_parameter_token #
-% \ExplSyntaxOff
%
% \begin{variable}
% {
@@ -1007,6 +1002,117 @@
% not a macro then \cs{scan_stop:} will be left in the input stream
% \end{function}
%
+% \section{Description of all possible tokens}
+% \label{sec:l3token:all-tokens}
+%
+% Let us end by reviewing every case that a given token can fall into.
+% This section is quite technical and some details are only meant for
+% completeness. We distinguish the meaning of the token, which controls
+% the expansion of the token and its effect on \TeX{}'s state, and its
+% shape, which is used when comparing token lists such as for delimited
+% arguments. Two tokens of the same shape must have the same meaning,
+% but the converse does not hold.
+%
+% A token has one of the following shapes.
+% \begin{itemize}
+% \item A control sequence, characterized by the sequence of
+% characters that constitute its name: for instance, \cs{use:n} is a
+% five-letter control sequence.
+% \item An active character token, characterized by its character code
+% (between $0$ and $1114111$ for \LuaTeX{} and \XeTeX{} and less for
+% other engines) and category code~$13$.
+% \item A character token, characterized by its character code and
+% category code (one of $1$, $2$, $3$, $4$, $6$, $7$, $8$, $10$,
+% $11$ or~$12$ whose meaning is described below).\footnote{In
+% \LuaTeX{}, there is also the case of \enquote{bytes}, which behave as
+% character tokens of category code $12$~(other) and character code
+% between $1114112$ and~$1114366$. They are used to output
+% individual bytes to files, rather than UTF-8.}
+% \end{itemize}
+% There are also a few internal tokens. The following list may be
+% incomplete in some engines.
+% \begin{itemize}
+% \item Expanding \tn{the}\th{font} results in a token that looks
+% identical to the command that was used to select the current font
+% (such as \tn{tenrm}) but it differs from it in shape.
+% \item A \enquote{frozen} |\relax|, which differs from the primitive in
+% both shape and meaning, is inserted when the closing \tn{fi} of a
+% conditional is encountered before the conditional is evaluated.
+% \item An |\endtemplate| (which expands to |\outer endtemplate:|)
+% can be enountered when peeking ahead at the next token.
+% \item Tricky programming might access a frozen |\endwrite|.
+% \item Some frozen tokens can only be accessed in interactive
+% sessions: |\cr|, |\right|, |\endgroup|, |\fi|, |\inaccessible|.
+% \end{itemize}
+%
+% The meaning of a (non-active) character token is fixed by its category
+% code (and character code) and cannot be changed. We will call these
+% tokens \emph{explicit} character tokens. Category codes that a
+% character token can have are listed below by giving a sample output of
+% the \TeX{} primitive \tn{meaning}, together with their \LaTeX3 names
+% and most common example:
+% \begin{itemize}
+% \item[1] begin-group character (|group_begin|, often |{|),
+% \item[2] end-group character (|group_end|, often |}|),
+% \item[3] math shift character (|math_toggle|, often |$|),
+% \item[4] alignment tab character (|alignment|, often |&|),
+% \item[6] macro parameter character (|parameter|, often |#|),
+% \item[7] superscript character (|math_superscript|, often |^|),
+% \item[8] subscript character (|math_subscript|, often |_|),
+% \item[10] blank space (|space|, often character code~$32$),
+% \item[11] the letter (|letter|, such as |A|),
+% \item[12] the character (|other|, such as |0|).
+% \end{itemize}
+% Category code~$13$ (|active|) is discussed below. Input characters
+% can also have several other category codes which do not lead to
+% character tokens for later processing: $0$~(|escape|),
+% $5$~(|end_line|), $9$~(|ignore|), $14$~(|comment|), and
+% $15$~(|invalid|).
+%
+% The meaning of a control sequence or active character can be identical
+% to that of any character token listed above (with any character code),
+% and we will call such tokens \emph{implicit} character tokens. The
+% meaning is otherwise in the following list:
+% \begin{itemize}
+% \item a macro, used in \LaTeX3 for most functions and some variables
+% (|tl|, |fp|, |seq|, \ldots{}),
+% \item a primitive such as \tn{def} or \tn{topmark}, used in \LaTeX3
+% for some functions,
+% \item a register such as \tn{count}|123|, used in \LaTeX3{} for the
+% implementation of some variables (|int|, |dim|, \ldots{}),
+% \item a constant integer such as \tn{char}|"56| or \tn{mathchar}|"121|,
+% \item a font selection command,
+% \item undefined.
+% \end{itemize}
+% Macros be \tn{protected} or not, \tn{long} or not (the opposite of
+% what \LaTeX3 calls |nopar|), and \tn{outer} or not (unused in
+% \LaTeX3). Their \tn{meaning} takes the form
+% \begin{quote}
+% \meta{properties} |macro:|\meta{parameters}|->|\meta{replacement}
+% \end{quote}
+% where \meta{properties} is among \tn{protected}\tn{long}\tn{outer},
+% \meta{parameters} describes parameters that the macro expects, such as
+% |#1#2#3|, and \meta{replacement} describes how the parameters are
+% manipulated, such as~|#2/#1/#3|.
+%
+% ^^A todo Bruno: discuss here some other subtleties of space tokens? when looking for numbers, when looking for equal signs in let, in expressions, etc.
+%
+% Now is perhaps a good time to mention some subtleties relating to
+% tokens with category code $10$ (space). Any input character with this
+% category code (normally, space and tab characters) becomes a normal
+% space, with character code~$32$ and category code~$10$.
+%
+% When a macro takes an undelimited argument, explicit space characters
+% (with character code $32$ and category code $10$) are ignored. If the
+% following token is an explicit character token with category code $1$
+% (begin-group) and an arbitrary character code, then \TeX{} scans ahead
+% to obtain an equal number of explicit character tokens with category
+% code $1$ (begin-group) and $2$ (end-group), and the resulting list of
+% tokens (with outer braces removed) becomes the argument. Otherwise, a
+% single token is taken as the argument for the macro: we call such
+% single tokens \enquote{N-type}, as they are suitable to be used as an
+% argument for a function with the signature~\texttt{:N}.
+%
% \section{Internal functions}
%
% \begin{function}[EXP, added = 2016-03-25]{\__char_generate:nn}
@@ -1016,7 +1122,8 @@
% This function is identical in operation to the public
% \cs{char_generate:nn} but omits various sanity tests. In particular, this
% means it is used in certain places where engine variations need to be
-% accounted for by the kernel.
+% accounted for by the kernel. The \meta{catcode} must give an explicit
+% integer after a single expansion.
% \end{function}
%
% \end{documentation}
@@ -1033,7 +1140,7 @@
%<@@=char>
% \end{macrocode}
%
-% \section{Manipulating and interrogating character tokens}
+% \subsection{Manipulating and interrogating character tokens}
%
% \begin{macro}{\char_set_catcode:nn}
% \begin{macro}{\char_value_catcode:n}
@@ -1245,10 +1352,10 @@
% \end{macrocode}
% \end{variable}
%
-% \section{Creating character tokens}
+% \subsection{Creating character tokens}
%
% \begin{macro}
-% {^^A
+% {
% \char_set_active_eq:NN, \char_gset_active_eq:NN,
% \char_set_active_eq:Nc, \char_gset_active_eq:Nc,
% \char_set_active_eq:nN, \char_gset_active_eq:nN,
@@ -1256,6 +1363,7 @@
% }
% Four simple functions with very similar definitions, so set up using
% an auxiliary.
+% These are similar to \LuaTeX{}'s \tn{letcharcode} primitive.
% \begin{macrocode}
\group_begin:
\char_set_catcode_active:N \^^@
@@ -1303,7 +1411,7 @@
\cs_new:Npn \__char_generate:nn #1#2
{
\exp:w \exp_after:wN
- \@@_generate_aux:nnw \exp_after:wN
+ \@@_generate_aux:nnw \exp_after:wN
{ \__int_value:w \__int_eval:w #1 \exp_after:wN }
{#2} \exp_end:
}
@@ -1341,7 +1449,7 @@
\if_int_compare:w #1 < \c_zero 1 \fi:
\if_int_compare:w #1 > \c_@@_max_int 1 \fi: \exp_stop_f:
\__msg_kernel_expandable_error:nn { kernel }
- { char-out-of-range }
+ { char-out-of-range }
\else:
\@@_generate_aux:nnw {#1} {#2}
\fi:
@@ -1362,7 +1470,7 @@
\group_begin:
%<*package>
\char_set_catcode_active:N \^^L
- \cs_set_nopar:Npn ^^L { }
+ \cs_set:Npn ^^L { }
%</package>
\char_set_catcode_other:n { 0 }
\if_int_odd:w 0
@@ -1396,11 +1504,7 @@
% low-level conditional. There are a few things to notice here.
% As |^^L| is |\outer| we need to locally set it to avoid a problem.
% To get open/close braces into the list, they are set up using |\if_false:|
-% pairing here and will later be |x|-type expanded into the desired form.
-% For making spaces, there needs to be an |o|-type expansion of a |\use:n|
-% (or some other tokenization) to avoid dropping the space. We also
-% set up active tokens although they are (currently) filtered out by the
-% interface layer (\tn{Ucharcat} cannot make active tokens).
+% pairing and are then |x|-type expanded together into the desired form.
% \begin{macrocode}
\int_const:Nn \c_@@_max_int { 255 }
\tl_set:Nn \l_@@_tmp_tl { \exp_not:N \or: }
@@ -1434,6 +1538,12 @@
\char_set_catcode_math_subscript:n { 0 }
\tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
\tl_put_right:Nn \l_@@_tmp_tl { \or: }
+% \end{macrocode}
+% For making spaces, there needs to be an |o|-type expansion of a |\use:n|
+% (or some other tokenization) to avoid dropping the space. We also
+% set up active tokens although they are (currently) filtered out by the
+% interface layer (\tn{Ucharcat} cannot make active tokens).
+% \begin{macrocode}
\char_set_catcode_space:n { 0 }
\tl_put_right:No \l_@@_tmp_tl { \use:n { \or: } ^^@ }
\char_set_catcode_letter:n { 0 }
@@ -1445,10 +1555,10 @@
% \end{macrocode}
% Convert the above temporary list into a series of constant token
% lists, one for each character code, using \tn{tex_lowercase:D} to
-% convert |^^@| in each case. The \texttt{x}-type expansion deals with
-% the |\if_false:| stuff introduced earlier. This is done in three parts
-% as |^^L| is awkward. Notice that at this stage |^^@| is active. In
-% format mode this is not needed.
+% convert |^^@| in each case. The \texttt{x}-type expansion ensures
+% that \tn{tex_lowercase:D} receives the contents of the token list.
+% In package mode, |^^L| is awkward hence this is done in three parts.
+% Notice that at this stage |^^@| is active.
% \begin{macrocode}
\cs_set_protected:Npn \@@_tmp:n #1
{
@@ -1748,7 +1858,7 @@
% When a token is a macro, \cs{token_to_meaning:N} will always output
% something like |\long macro:#1->#1| so we could naively check to
% see if the meaning contains |->|. However, this can fail the five
-% \tn{...mark} primitives, whose meaning has the form
+% \tn[no-index]{...mark} primitives, whose meaning has the form
% |...mark:|\meta{user material}. The problem is that the
% \meta{user material} can contain |->|.
%
@@ -2132,9 +2242,9 @@
% {\@@_true:w, \@@_true_aux:w, \@@_false:w, \@@_tmp:w}
% Functions used by the branching and space-stripping code.
% \begin{macrocode}
-\cs_new_nopar:Npn \@@_true:w { }
-\cs_new_nopar:Npn \@@_true_aux:w { }
-\cs_new_nopar:Npn \@@_false:w { }
+\cs_new:Npn \@@_true:w { }
+\cs_new:Npn \@@_true_aux:w { }
+\cs_new:Npn \@@_false:w { }
\cs_new:Npn \@@_tmp:w { }
% \end{macrocode}
% \end{macro}
@@ -2144,9 +2254,9 @@
% Simple wrappers for \tn{futurelet}: no arguments absorbed
% here.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \peek_after:Nw
+\cs_new_protected:Npn \peek_after:Nw
{ \tex_futurelet:D \l_peek_token }
-\cs_new_protected_nopar:Npn \peek_gafter:Nw
+\cs_new_protected:Npn \peek_gafter:Nw
{ \tex_global:D \tex_futurelet:D \g_peek_token }
% \end{macrocode}
% \end{macro}
@@ -2174,12 +2284,12 @@
{
\cs_set_eq:NN \l_@@_search_token #2
\tl_set:Nn \l_@@_search_tl {#2}
- \cs_set_nopar:Npx \@@_true:w
+ \cs_set:Npx \@@_true:w
{
\exp_not:N \group_align_safe_end:
\exp_not:n {#3}
}
- \cs_set_nopar:Npx \@@_false:w
+ \cs_set:Npx \@@_false:w
{
\exp_not:N \group_align_safe_end:
\exp_not:n {#4}
@@ -2203,8 +2313,8 @@
\cs_set_eq:NN \l_@@_search_token #2
\tl_set:Nn \l_@@_search_tl {#2}
\cs_set_eq:NN \@@_true:w \@@_true_remove:w
- \cs_set_nopar:Npx \@@_true_aux:w { \exp_not:n {#3} }
- \cs_set_nopar:Npx \@@_false:w
+ \cs_set:Npx \@@_true_aux:w { \exp_not:n {#3} }
+ \cs_set:Npx \@@_false:w
{
\exp_not:N \group_align_safe_end:
\exp_not:n {#4}
@@ -2222,7 +2332,7 @@
% \begin{macro}{\@@_execute_branches_meaning:}
% The meaning test is straight forward.
% \begin{macrocode}
-\cs_new_nopar:Npn \@@_execute_branches_meaning:
+\cs_new:Npn \@@_execute_branches_meaning:
{
\if_meaning:w \l_peek_token \l_@@_search_token
\exp_after:wN \@@_true:w
@@ -2266,11 +2376,11 @@
% search token may be of any of the three types above, hence the need
% to use the explicit token that was given to the peek function.
% \begin{macrocode}
-\cs_new_nopar:Npn \@@_execute_branches_catcode:
+\cs_new:Npn \@@_execute_branches_catcode:
{ \if_catcode:w \@@_execute_branches_catcode_aux: }
-\cs_new_nopar:Npn \@@_execute_branches_charcode:
+\cs_new:Npn \@@_execute_branches_charcode:
{ \if_charcode:w \@@_execute_branches_catcode_aux: }
-\cs_new_nopar:Npn \@@_execute_branches_catcode_aux:
+\cs_new:Npn \@@_execute_branches_catcode_aux:
{
\if_catcode:w \exp_not:N \l_peek_token \scan_stop:
\exp_after:wN \exp_after:wN
@@ -2290,7 +2400,7 @@
\fi:
#1
}
-\cs_new_nopar:Npn \@@_execute_branches_catcode_auxiii:
+\cs_new:Npn \@@_execute_branches_catcode_auxiii:
{
\exp_not:N \l_peek_token
\exp_after:wN \exp_not:N \l_@@_search_tl
@@ -2312,7 +2422,7 @@
% using a side-effect of \texttt{f}-expansion:
% |\exp:w \exp_end_continue_f:w| removes one space.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \@@_ignore_spaces_execute_branches:
+\cs_new_protected:Npn \@@_ignore_spaces_execute_branches:
{
\if_meaning:w \l_peek_token \c_space_token
\exp_after:wN \peek_after:Nw
@@ -2341,7 +2451,7 @@
}
\cs_set:Npn \@@_def:nnnnn #1#2#3#4#5
{
- \cs_new_protected_nopar:cpx { #1 #5 }
+ \cs_new_protected:cpx { #1 #5 }
{
\tl_if_empty:nF {#2}
{ \exp_not:n { \cs_set_eq:NN \@@_execute_branches: #2 } }