diff options
Diffstat (limited to 'Master/texmf-dist/source/latex/l3kernel/l3token.dtx')
-rw-r--r-- | Master/texmf-dist/source/latex/l3kernel/l3token.dtx | 226 |
1 files changed, 168 insertions, 58 deletions
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx index ac7ed290c36..98aedaa038e 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx @@ -37,7 +37,7 @@ \documentclass[full]{l3doc} %</driver> %<*driver|package> -\GetIdInfo$Id: l3token.dtx 6470 2016-04-03 11:29:18Z joseph $ +\GetIdInfo$Id: l3token.dtx 6695 2016-09-01 15:38:50Z bruno $ {L3 Experimental token manipulation} %</driver|package> %<*driver> @@ -82,28 +82,32 @@ % categories: |\token_| for anything that deals with tokens and % |\peek_| for looking ahead in the token stream. % -% Most of the time we will be using the term \enquote{token} but most of the -% time the function we're describing can equally well by used on a -% control sequence as such one is one token as well. +% Most functions we will describe here can be used on control sequences, +% as those are tokens as well. % -% \section{All possible tokens} -% -% Let us start by reviewing every case that a given token can fall into. -% It is very important to distinguish two aspects of a token: its meaning, -% and what it looks like. +% It is important to distinguish two aspects of a token: its +% \enquote{shape} (for lack of a better word), which affects the +% matching of delimited arguments and the comparison of token lists +% containing this token, and its \enquote{meaning}, which affects +% whether the token expands or what operation it performs. One can have +% tokens of different shapes with the same meaning, but not the +% converse. % % For instance, \cs{if:w}, \cs{if_charcode:w}, and \cs{tex_if:D} are -% three names for the same internal operation of \TeX{}, namely the primitive -% testing the next two characters for equality of their character code. -% They behave identically in many situations. However, \TeX{} -% distinguishes them when searching for a delimited argument. Namely, the -% example function |\show_until_if:w| defined below will take everything -% until \cs{if:w} as an argument, despite the presence of other copies of -% \cs{if:w} under different names. +% three names for the same internal operation of \TeX{}, namely the +% primitive testing the next two characters for equality of their +% character code. They have the same meaning hence behave identically +% in many situations. However, \TeX{} distinguishes them when searching +% for a delimited argument. Namely, the example function +% |\show_until_if:w| defined below will take everything until \cs{if:w} +% as an argument, despite the presence of other copies of \cs{if:w} +% under different names. % \begin{verbatim} % \cs_new:Npn \show_until_if:w #1 \if:w { \tl_show:n {#1} } % \show_until_if:w \tex_if:D \if_charcode:w \if:w % \end{verbatim} +% A list of all possible shapes and a list of all possible meanings are +% given in section~\ref{sec:l3token:all-tokens}. % % \section{Creating character tokens} % @@ -416,11 +420,6 @@ % This will be an implicit representation of \meta{token_2}. % \end{function} % -% ^^A Because it's late I can't figure out a better way to handle this: -% \ExplSyntaxOn -% \cs_set_eq:NN \c_alignment_token @ -% \cs_set_eq:NN \c_parameter_token @ -% \ExplSyntaxOff % \begin{variable} % { % \c_group_begin_token, @@ -436,10 +435,6 @@ % by their name. They are used internally for test purposes but % are also available to the programmer for other uses. % \end{variable} -% \ExplSyntaxOn -% \cs_set_eq:NN \c_alignment_token & -% \cs_set_eq:NN \c_parameter_token # -% \ExplSyntaxOff % % \begin{variable} % { @@ -1007,6 +1002,117 @@ % not a macro then \cs{scan_stop:} will be left in the input stream % \end{function} % +% \section{Description of all possible tokens} +% \label{sec:l3token:all-tokens} +% +% Let us end by reviewing every case that a given token can fall into. +% This section is quite technical and some details are only meant for +% completeness. We distinguish the meaning of the token, which controls +% the expansion of the token and its effect on \TeX{}'s state, and its +% shape, which is used when comparing token lists such as for delimited +% arguments. Two tokens of the same shape must have the same meaning, +% but the converse does not hold. +% +% A token has one of the following shapes. +% \begin{itemize} +% \item A control sequence, characterized by the sequence of +% characters that constitute its name: for instance, \cs{use:n} is a +% five-letter control sequence. +% \item An active character token, characterized by its character code +% (between $0$ and $1114111$ for \LuaTeX{} and \XeTeX{} and less for +% other engines) and category code~$13$. +% \item A character token, characterized by its character code and +% category code (one of $1$, $2$, $3$, $4$, $6$, $7$, $8$, $10$, +% $11$ or~$12$ whose meaning is described below).\footnote{In +% \LuaTeX{}, there is also the case of \enquote{bytes}, which behave as +% character tokens of category code $12$~(other) and character code +% between $1114112$ and~$1114366$. They are used to output +% individual bytes to files, rather than UTF-8.} +% \end{itemize} +% There are also a few internal tokens. The following list may be +% incomplete in some engines. +% \begin{itemize} +% \item Expanding \tn{the}\th{font} results in a token that looks +% identical to the command that was used to select the current font +% (such as \tn{tenrm}) but it differs from it in shape. +% \item A \enquote{frozen} |\relax|, which differs from the primitive in +% both shape and meaning, is inserted when the closing \tn{fi} of a +% conditional is encountered before the conditional is evaluated. +% \item An |\endtemplate| (which expands to |\outer endtemplate:|) +% can be enountered when peeking ahead at the next token. +% \item Tricky programming might access a frozen |\endwrite|. +% \item Some frozen tokens can only be accessed in interactive +% sessions: |\cr|, |\right|, |\endgroup|, |\fi|, |\inaccessible|. +% \end{itemize} +% +% The meaning of a (non-active) character token is fixed by its category +% code (and character code) and cannot be changed. We will call these +% tokens \emph{explicit} character tokens. Category codes that a +% character token can have are listed below by giving a sample output of +% the \TeX{} primitive \tn{meaning}, together with their \LaTeX3 names +% and most common example: +% \begin{itemize} +% \item[1] begin-group character (|group_begin|, often |{|), +% \item[2] end-group character (|group_end|, often |}|), +% \item[3] math shift character (|math_toggle|, often |$|), +% \item[4] alignment tab character (|alignment|, often |&|), +% \item[6] macro parameter character (|parameter|, often |#|), +% \item[7] superscript character (|math_superscript|, often |^|), +% \item[8] subscript character (|math_subscript|, often |_|), +% \item[10] blank space (|space|, often character code~$32$), +% \item[11] the letter (|letter|, such as |A|), +% \item[12] the character (|other|, such as |0|). +% \end{itemize} +% Category code~$13$ (|active|) is discussed below. Input characters +% can also have several other category codes which do not lead to +% character tokens for later processing: $0$~(|escape|), +% $5$~(|end_line|), $9$~(|ignore|), $14$~(|comment|), and +% $15$~(|invalid|). +% +% The meaning of a control sequence or active character can be identical +% to that of any character token listed above (with any character code), +% and we will call such tokens \emph{implicit} character tokens. The +% meaning is otherwise in the following list: +% \begin{itemize} +% \item a macro, used in \LaTeX3 for most functions and some variables +% (|tl|, |fp|, |seq|, \ldots{}), +% \item a primitive such as \tn{def} or \tn{topmark}, used in \LaTeX3 +% for some functions, +% \item a register such as \tn{count}|123|, used in \LaTeX3{} for the +% implementation of some variables (|int|, |dim|, \ldots{}), +% \item a constant integer such as \tn{char}|"56| or \tn{mathchar}|"121|, +% \item a font selection command, +% \item undefined. +% \end{itemize} +% Macros be \tn{protected} or not, \tn{long} or not (the opposite of +% what \LaTeX3 calls |nopar|), and \tn{outer} or not (unused in +% \LaTeX3). Their \tn{meaning} takes the form +% \begin{quote} +% \meta{properties} |macro:|\meta{parameters}|->|\meta{replacement} +% \end{quote} +% where \meta{properties} is among \tn{protected}\tn{long}\tn{outer}, +% \meta{parameters} describes parameters that the macro expects, such as +% |#1#2#3|, and \meta{replacement} describes how the parameters are +% manipulated, such as~|#2/#1/#3|. +% +% ^^A todo Bruno: discuss here some other subtleties of space tokens? when looking for numbers, when looking for equal signs in let, in expressions, etc. +% +% Now is perhaps a good time to mention some subtleties relating to +% tokens with category code $10$ (space). Any input character with this +% category code (normally, space and tab characters) becomes a normal +% space, with character code~$32$ and category code~$10$. +% +% When a macro takes an undelimited argument, explicit space characters +% (with character code $32$ and category code $10$) are ignored. If the +% following token is an explicit character token with category code $1$ +% (begin-group) and an arbitrary character code, then \TeX{} scans ahead +% to obtain an equal number of explicit character tokens with category +% code $1$ (begin-group) and $2$ (end-group), and the resulting list of +% tokens (with outer braces removed) becomes the argument. Otherwise, a +% single token is taken as the argument for the macro: we call such +% single tokens \enquote{N-type}, as they are suitable to be used as an +% argument for a function with the signature~\texttt{:N}. +% % \section{Internal functions} % % \begin{function}[EXP, added = 2016-03-25]{\__char_generate:nn} @@ -1016,7 +1122,8 @@ % This function is identical in operation to the public % \cs{char_generate:nn} but omits various sanity tests. In particular, this % means it is used in certain places where engine variations need to be -% accounted for by the kernel. +% accounted for by the kernel. The \meta{catcode} must give an explicit +% integer after a single expansion. % \end{function} % % \end{documentation} @@ -1033,7 +1140,7 @@ %<@@=char> % \end{macrocode} % -% \section{Manipulating and interrogating character tokens} +% \subsection{Manipulating and interrogating character tokens} % % \begin{macro}{\char_set_catcode:nn} % \begin{macro}{\char_value_catcode:n} @@ -1245,10 +1352,10 @@ % \end{macrocode} % \end{variable} % -% \section{Creating character tokens} +% \subsection{Creating character tokens} % % \begin{macro} -% {^^A +% { % \char_set_active_eq:NN, \char_gset_active_eq:NN, % \char_set_active_eq:Nc, \char_gset_active_eq:Nc, % \char_set_active_eq:nN, \char_gset_active_eq:nN, @@ -1256,6 +1363,7 @@ % } % Four simple functions with very similar definitions, so set up using % an auxiliary. +% These are similar to \LuaTeX{}'s \tn{letcharcode} primitive. % \begin{macrocode} \group_begin: \char_set_catcode_active:N \^^@ @@ -1303,7 +1411,7 @@ \cs_new:Npn \__char_generate:nn #1#2 { \exp:w \exp_after:wN - \@@_generate_aux:nnw \exp_after:wN + \@@_generate_aux:nnw \exp_after:wN { \__int_value:w \__int_eval:w #1 \exp_after:wN } {#2} \exp_end: } @@ -1341,7 +1449,7 @@ \if_int_compare:w #1 < \c_zero 1 \fi: \if_int_compare:w #1 > \c_@@_max_int 1 \fi: \exp_stop_f: \__msg_kernel_expandable_error:nn { kernel } - { char-out-of-range } + { char-out-of-range } \else: \@@_generate_aux:nnw {#1} {#2} \fi: @@ -1362,7 +1470,7 @@ \group_begin: %<*package> \char_set_catcode_active:N \^^L - \cs_set_nopar:Npn ^^L { } + \cs_set:Npn ^^L { } %</package> \char_set_catcode_other:n { 0 } \if_int_odd:w 0 @@ -1396,11 +1504,7 @@ % low-level conditional. There are a few things to notice here. % As |^^L| is |\outer| we need to locally set it to avoid a problem. % To get open/close braces into the list, they are set up using |\if_false:| -% pairing here and will later be |x|-type expanded into the desired form. -% For making spaces, there needs to be an |o|-type expansion of a |\use:n| -% (or some other tokenization) to avoid dropping the space. We also -% set up active tokens although they are (currently) filtered out by the -% interface layer (\tn{Ucharcat} cannot make active tokens). +% pairing and are then |x|-type expanded together into the desired form. % \begin{macrocode} \int_const:Nn \c_@@_max_int { 255 } \tl_set:Nn \l_@@_tmp_tl { \exp_not:N \or: } @@ -1434,6 +1538,12 @@ \char_set_catcode_math_subscript:n { 0 } \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } \tl_put_right:Nn \l_@@_tmp_tl { \or: } +% \end{macrocode} +% For making spaces, there needs to be an |o|-type expansion of a |\use:n| +% (or some other tokenization) to avoid dropping the space. We also +% set up active tokens although they are (currently) filtered out by the +% interface layer (\tn{Ucharcat} cannot make active tokens). +% \begin{macrocode} \char_set_catcode_space:n { 0 } \tl_put_right:No \l_@@_tmp_tl { \use:n { \or: } ^^@ } \char_set_catcode_letter:n { 0 } @@ -1445,10 +1555,10 @@ % \end{macrocode} % Convert the above temporary list into a series of constant token % lists, one for each character code, using \tn{tex_lowercase:D} to -% convert |^^@| in each case. The \texttt{x}-type expansion deals with -% the |\if_false:| stuff introduced earlier. This is done in three parts -% as |^^L| is awkward. Notice that at this stage |^^@| is active. In -% format mode this is not needed. +% convert |^^@| in each case. The \texttt{x}-type expansion ensures +% that \tn{tex_lowercase:D} receives the contents of the token list. +% In package mode, |^^L| is awkward hence this is done in three parts. +% Notice that at this stage |^^@| is active. % \begin{macrocode} \cs_set_protected:Npn \@@_tmp:n #1 { @@ -1748,7 +1858,7 @@ % When a token is a macro, \cs{token_to_meaning:N} will always output % something like |\long macro:#1->#1| so we could naively check to % see if the meaning contains |->|. However, this can fail the five -% \tn{...mark} primitives, whose meaning has the form +% \tn[no-index]{...mark} primitives, whose meaning has the form % |...mark:|\meta{user material}. The problem is that the % \meta{user material} can contain |->|. % @@ -2132,9 +2242,9 @@ % {\@@_true:w, \@@_true_aux:w, \@@_false:w, \@@_tmp:w} % Functions used by the branching and space-stripping code. % \begin{macrocode} -\cs_new_nopar:Npn \@@_true:w { } -\cs_new_nopar:Npn \@@_true_aux:w { } -\cs_new_nopar:Npn \@@_false:w { } +\cs_new:Npn \@@_true:w { } +\cs_new:Npn \@@_true_aux:w { } +\cs_new:Npn \@@_false:w { } \cs_new:Npn \@@_tmp:w { } % \end{macrocode} % \end{macro} @@ -2144,9 +2254,9 @@ % Simple wrappers for \tn{futurelet}: no arguments absorbed % here. % \begin{macrocode} -\cs_new_protected_nopar:Npn \peek_after:Nw +\cs_new_protected:Npn \peek_after:Nw { \tex_futurelet:D \l_peek_token } -\cs_new_protected_nopar:Npn \peek_gafter:Nw +\cs_new_protected:Npn \peek_gafter:Nw { \tex_global:D \tex_futurelet:D \g_peek_token } % \end{macrocode} % \end{macro} @@ -2174,12 +2284,12 @@ { \cs_set_eq:NN \l_@@_search_token #2 \tl_set:Nn \l_@@_search_tl {#2} - \cs_set_nopar:Npx \@@_true:w + \cs_set:Npx \@@_true:w { \exp_not:N \group_align_safe_end: \exp_not:n {#3} } - \cs_set_nopar:Npx \@@_false:w + \cs_set:Npx \@@_false:w { \exp_not:N \group_align_safe_end: \exp_not:n {#4} @@ -2203,8 +2313,8 @@ \cs_set_eq:NN \l_@@_search_token #2 \tl_set:Nn \l_@@_search_tl {#2} \cs_set_eq:NN \@@_true:w \@@_true_remove:w - \cs_set_nopar:Npx \@@_true_aux:w { \exp_not:n {#3} } - \cs_set_nopar:Npx \@@_false:w + \cs_set:Npx \@@_true_aux:w { \exp_not:n {#3} } + \cs_set:Npx \@@_false:w { \exp_not:N \group_align_safe_end: \exp_not:n {#4} @@ -2222,7 +2332,7 @@ % \begin{macro}{\@@_execute_branches_meaning:} % The meaning test is straight forward. % \begin{macrocode} -\cs_new_nopar:Npn \@@_execute_branches_meaning: +\cs_new:Npn \@@_execute_branches_meaning: { \if_meaning:w \l_peek_token \l_@@_search_token \exp_after:wN \@@_true:w @@ -2266,11 +2376,11 @@ % search token may be of any of the three types above, hence the need % to use the explicit token that was given to the peek function. % \begin{macrocode} -\cs_new_nopar:Npn \@@_execute_branches_catcode: +\cs_new:Npn \@@_execute_branches_catcode: { \if_catcode:w \@@_execute_branches_catcode_aux: } -\cs_new_nopar:Npn \@@_execute_branches_charcode: +\cs_new:Npn \@@_execute_branches_charcode: { \if_charcode:w \@@_execute_branches_catcode_aux: } -\cs_new_nopar:Npn \@@_execute_branches_catcode_aux: +\cs_new:Npn \@@_execute_branches_catcode_aux: { \if_catcode:w \exp_not:N \l_peek_token \scan_stop: \exp_after:wN \exp_after:wN @@ -2290,7 +2400,7 @@ \fi: #1 } -\cs_new_nopar:Npn \@@_execute_branches_catcode_auxiii: +\cs_new:Npn \@@_execute_branches_catcode_auxiii: { \exp_not:N \l_peek_token \exp_after:wN \exp_not:N \l_@@_search_tl @@ -2312,7 +2422,7 @@ % using a side-effect of \texttt{f}-expansion: % |\exp:w \exp_end_continue_f:w| removes one space. % \begin{macrocode} -\cs_new_protected_nopar:Npn \@@_ignore_spaces_execute_branches: +\cs_new_protected:Npn \@@_ignore_spaces_execute_branches: { \if_meaning:w \l_peek_token \c_space_token \exp_after:wN \peek_after:Nw @@ -2341,7 +2451,7 @@ } \cs_set:Npn \@@_def:nnnnn #1#2#3#4#5 { - \cs_new_protected_nopar:cpx { #1 #5 } + \cs_new_protected:cpx { #1 #5 } { \tl_if_empty:nF {#2} { \exp_not:n { \cs_set_eq:NN \@@_execute_branches: #2 } } |