diff options
author | Norbert Preining <norbert@preining.info> | 2019-09-02 13:46:59 +0900 |
---|---|---|
committer | Norbert Preining <norbert@preining.info> | 2019-09-02 13:46:59 +0900 |
commit | e0c6872cf40896c7be36b11dcc744620f10adf1d (patch) | |
tree | 60335e10d2f4354b0674ec22d7b53f0f8abee672 /macros/latex/contrib/l3kernel/l3token.dtx |
Initial commit
Diffstat (limited to 'macros/latex/contrib/l3kernel/l3token.dtx')
-rw-r--r-- | macros/latex/contrib/l3kernel/l3token.dtx | 2505 |
1 files changed, 2505 insertions, 0 deletions
diff --git a/macros/latex/contrib/l3kernel/l3token.dtx b/macros/latex/contrib/l3kernel/l3token.dtx new file mode 100644 index 0000000000..9b9946a4f8 --- /dev/null +++ b/macros/latex/contrib/l3kernel/l3token.dtx @@ -0,0 +1,2505 @@ +% \iffalse meta-comment +% +%% File: l3token.dtx +% +% Copyright (C) 2005-2019 The LaTeX3 Project +% +% It may be distributed and/or modified under the conditions of the +% LaTeX Project Public License (LPPL), either version 1.3c of this +% license or (at your option) any later version. The latest version +% of this license is in the file +% +% https://www.latex-project.org/lppl.txt +% +% This file is part of the "l3kernel bundle" (The Work in LPPL) +% and all files in that bundle must be distributed together. +% +% ----------------------------------------------------------------------- +% +% The development version of the bundle can be found at +% +% https://github.com/latex3/latex3 +% +% for those people who are interested. +% +%<*driver> +\documentclass[full,kernel]{l3doc} +\begin{document} + \DocInput{\jobname.dtx} +\end{document} +%</driver> +% \fi +% +% \title{^^A +% The \pkg{l3token} package\\ Token manipulation^^A +% } +% +% \author{^^A +% The \LaTeX3 Project\thanks +% {^^A +% E-mail: +% \href{mailto:latex-team@latex-project.org} +% {latex-team@latex-project.org}^^A +% }^^A +% } +% +% \date{Released 2019-08-25} +% +% \maketitle +% +% \begin{documentation} +% +% This module deals with tokens. Now this is perhaps not the most +% precise description so let's try with a better description: When +% programming in \TeX{}, it is often desirable to know just what a +% certain token is: is it a control sequence or something +% else. Similarly one often needs to know if a control sequence is +% expandable or not, a macro or a primitive, how many arguments it +% takes etc. Another thing of great importance (especially when it +% comes to document commands) is looking ahead in the token stream to +% see if a certain character is present and maybe even remove it or +% disregard other tokens while scanning. This module provides +% functions for both and as such has two primary function +% categories: |\token_| for anything that deals with tokens and +% |\peek_| for looking ahead in the token stream. +% +% Most functions we describe here can be used on control sequences, +% as those are tokens as well. +% +% It is important to distinguish two aspects of a token: its +% \enquote{shape} (for lack of a better word), which affects the +% matching of delimited arguments and the comparison of token lists +% containing this token, and its \enquote{meaning}, which affects +% whether the token expands or what operation it performs. One can have +% tokens of different shapes with the same meaning, but not the +% converse. +% +% For instance, \cs{if:w}, \cs{if_charcode:w}, and \cs{tex_if:D} are +% three names for the same internal operation of \TeX{}, namely the +% primitive testing the next two characters for equality of their +% character code. They have the same meaning hence behave identically +% in many situations. However, \TeX{} distinguishes them when searching +% for a delimited argument. Namely, the example function +% |\show_until_if:w| defined below takes everything until \cs{if:w} +% as an argument, despite the presence of other copies of \cs{if:w} +% under different names. +% \begin{verbatim} +% \cs_new:Npn \show_until_if:w #1 \if:w { \tl_show:n {#1} } +% \show_until_if:w \tex_if:D \if_charcode:w \if:w +% \end{verbatim} +% A list of all possible shapes and a list of all possible meanings are +% given in section~\ref{sec:l3token:all-tokens}. +% +% \section{Creating character tokens} +% +% \begin{function}[updated = 2015-11-12] +% { +% \char_set_active_eq:NN, \char_set_active_eq:Nc, +% \char_gset_active_eq:NN, \char_gset_active_eq:Nc +% } +% \begin{syntax} +% \cs{char_set_active_eq:NN} \meta{char} \meta{function} +% \end{syntax} +% Sets the behaviour of the \meta{char} in situations where it is +% active (category code $13$) to be equivalent to that of the +% \meta{function}. The category code of the \meta{char} is +% \emph{unchanged} by this process. The \meta{function} may itself +% be an active character. +% \end{function} +% +% \begin{function}[added = 2015-11-12] +% { +% \char_set_active_eq:nN, \char_set_active_eq:nc, +% \char_gset_active_eq:nN, \char_gset_active_eq:nc +% } +% \begin{syntax} +% \cs{char_set_active_eq:nN} \Arg{integer expression} \meta{function} +% \end{syntax} +% Sets the behaviour of the \meta{char} which has character +% code as given by the \meta{integer expression} in situations +% where it is active (category code $13$) to be equivalent to that of the +% \meta{function}. The category code of the \meta{char} is +% \emph{unchanged} by this process. The \meta{function} may itself +% be an active character. +% \end{function} +% +% \begin{function}[EXP, added = 2015-09-09, updated = 2019-01-16] +% {\char_generate:nn} +% \begin{syntax} +% \cs{char_generate:nn} \Arg{charcode} \Arg{catcode} +% \end{syntax} +% Generates a character token of the given \meta{charcode} and \meta{catcode} +% (both of which may be integer expressions). The \meta{catcode} may be +% one of +% \begin{itemize} +% \item $1$ (begin group) +% \item $2$ (end group) +% \item $3$ (math toggle) +% \item $4$ (alignment) +% \item $6$ (parameter) +% \item $7$ (math superscript) +% \item $8$ (math subscript) +% \item $11$ (letter) +% \item $12$ (other) +% \item $13$ (active) +% \end{itemize} +% and other values raise an error. The \meta{charcode} may be any one valid +% for the engine in use. +% Active characters cannot be generated in older versions of \XeTeX{}. +% \begin{texnote} +% Exactly two expansions are needed to produce the character. +% \end{texnote} +% \end{function} +% +% \begin{variable}[added = 2011-09-05]{\c_catcode_other_space_tl} +% Token list containing one character with category code $12$, +% (\enquote{other}), and character code $32$ (space). +% \end{variable} +% +% \section{Manipulating and interrogating character tokens} +% +% \begin{function}[updated = 2015-11-11] +% { +% \char_set_catcode_escape:N , +% \char_set_catcode_group_begin:N , +% \char_set_catcode_group_end:N , +% \char_set_catcode_math_toggle:N , +% \char_set_catcode_alignment:N , +% \char_set_catcode_end_line:N , +% \char_set_catcode_parameter:N , +% \char_set_catcode_math_superscript:N , +% \char_set_catcode_math_subscript:N , +% \char_set_catcode_ignore:N , +% \char_set_catcode_space:N , +% \char_set_catcode_letter:N , +% \char_set_catcode_other:N , +% \char_set_catcode_active:N , +% \char_set_catcode_comment:N , +% \char_set_catcode_invalid:N +% } +% \begin{syntax} +% \cs{char_set_catcode_letter:N} \meta{character} +% \end{syntax} +% Sets the category code of the \meta{character} to that indicated in +% the function name. Depending on the current category code of the +% \meta{token} the escape token may also be needed: +% \begin{verbatim} +% \char_set_catcode_other:N \% +% \end{verbatim} +% The assignment is local. +% \end{function} +% +% \begin{function}[updated = 2015-11-11] +% { +% \char_set_catcode_escape:n , +% \char_set_catcode_group_begin:n , +% \char_set_catcode_group_end:n , +% \char_set_catcode_math_toggle:n , +% \char_set_catcode_alignment:n , +% \char_set_catcode_end_line:n , +% \char_set_catcode_parameter:n , +% \char_set_catcode_math_superscript:n , +% \char_set_catcode_math_subscript:n , +% \char_set_catcode_ignore:n , +% \char_set_catcode_space:n , +% \char_set_catcode_letter:n , +% \char_set_catcode_other:n , +% \char_set_catcode_active:n , +% \char_set_catcode_comment:n , +% \char_set_catcode_invalid:n +% } +% \begin{syntax} +% \cs{char_set_catcode_letter:n} \Arg{integer expression} +% \end{syntax} +% Sets the category code of the \meta{character} which has character +% code as given by the \meta{integer expression}. This version can be +% used to set up characters which cannot otherwise be given +% (\emph{cf.}~the \texttt{N}-type variants). The assignment is local. +% \end{function} +% +% \begin{function}[updated = 2015-11-11]{\char_set_catcode:nn} +% \begin{syntax} +% \cs{char_set_catcode:nn} \Arg{intexpr_1} \Arg{intexpr_2} +% \end{syntax} +% These functions set the category code of the \meta{character} which +% has character code as given by the \meta{integer expression}. +% The first \meta{integer expression} +% is the character code and the second is the category code to apply. +% The setting applies within the current \TeX{} group. In general, the +% symbolic functions \cs[no-index]{char_set_catcode_\meta{type}} should be preferred, +% but there are cases where these lower-level functions may be useful. +% \end{function} +% +% \begin{function}[EXP]{\char_value_catcode:n} +% \begin{syntax} +% \cs{char_value_catcode:n} \Arg{integer expression} +% \end{syntax} +% Expands to the current category code of the \meta{character} with +% character code given by the +% \meta{integer expression}. +% \end{function} +% +% \begin{function}{\char_show_value_catcode:n} +% \begin{syntax} +% \cs{char_show_value_catcode:n} \Arg{integer expression} +% \end{syntax} +% Displays the current category code of the \meta{character} with +% character code given by the \meta{integer expression} on the +% terminal. +% \end{function} +% +% \begin{function}[updated = 2015-08-06]{\char_set_lccode:nn} +% \begin{syntax} +% \cs{char_set_lccode:nn} \Arg{intexpr_1} \Arg{intexpr_2} +% \end{syntax} +% Sets up the behaviour of the \meta{character} when +% found inside \cs{tl_lower_case:n}, such that \meta{character_1} +% will be converted into \meta{character_2}. The two \meta{characters} +% may be specified using an \meta{integer expression} for the character code +% concerned. This may include the \TeX{} |`|\meta{character} +% method for converting a single character into its character +% code: +% \begin{verbatim} +% \char_set_lccode:nn { `\A } { `\a } % Standard behaviour +% \char_set_lccode:nn { `\A } { `\A + 32 } +% \char_set_lccode:nn { 50 } { 60 } +% \end{verbatim} +% The setting applies within the current \TeX{} group. +% \end{function} +% +% \begin{function}[EXP]{\char_value_lccode:n} +% \begin{syntax} +% \cs{char_value_lccode:n} \Arg{integer expression} +% \end{syntax} +% Expands to the current lower case code of the \meta{character} with +% character code given by the +% \meta{integer expression}. +% \end{function} +% +% \begin{function}{\char_show_value_lccode:n} +% \begin{syntax} +% \cs{char_show_value_lccode:n} \Arg{integer expression} +% \end{syntax} +% Displays the current lower case code of the \meta{character} with +% character code given by the \meta{integer expression} on the +% terminal. +% \end{function} +% +% \begin{function}[updated = 2015-08-06]{\char_set_uccode:nn} +% \begin{syntax} +% \cs{char_set_uccode:nn} \Arg{intexpr_1} \Arg{intexpr_2} +% \end{syntax} +% Sets up the behaviour of the \meta{character} when +% found inside \cs{tl_upper_case:n}, such that \meta{character_1} +% will be converted into \meta{character_2}. The two \meta{characters} +% may be specified using an \meta{integer expression} for the character code +% concerned. This may include the \TeX{} |`|\meta{character} +% method for converting a single character into its character +% code: +% \begin{verbatim} +% \char_set_uccode:nn { `\a } { `\A } % Standard behaviour +% \char_set_uccode:nn { `\A } { `\A - 32 } +% \char_set_uccode:nn { 60 } { 50 } +% \end{verbatim} +% The setting applies within the current \TeX{} group. +% \end{function} +% +% \begin{function}[EXP]{\char_value_uccode:n} +% \begin{syntax} +% \cs{char_value_uccode:n} \Arg{integer expression} +% \end{syntax} +% Expands to the current upper case code of the \meta{character} with +% character code given by the +% \meta{integer expression}. +% \end{function} +% +% \begin{function}{\char_show_value_uccode:n} +% \begin{syntax} +% \cs{char_show_value_uccode:n} \Arg{integer expression} +% \end{syntax} +% Displays the current upper case code of the \meta{character} with +% character code given by the \meta{integer expression} on the +% terminal. +% \end{function} +% +% \begin{function}[updated = 2015-08-06]{\char_set_mathcode:nn} +% \begin{syntax} +% \cs{char_set_mathcode:nn} \Arg{intexpr_1} \Arg{intexpr_2} +% \end{syntax} +% This function sets up the math code of \meta{character}. +% The \meta{character} is specified as +% an \meta{integer expression} which will be used as the character +% code of the relevant character. The setting applies within the +% current \TeX{} group. +% \end{function} +% +% \begin{function}[EXP]{\char_value_mathcode:n} +% \begin{syntax} +% \cs{char_value_mathcode:n} \Arg{integer expression} +% \end{syntax} +% Expands to the current math code of the \meta{character} with +% character code given by the +% \meta{integer expression}. +% \end{function} +% +% \begin{function}{\char_show_value_mathcode:n} +% \begin{syntax} +% \cs{char_show_value_mathcode:n} \Arg{integer expression} +% \end{syntax} +% Displays the current math code of the \meta{character} with +% character code given by the \meta{integer expression} on the +% terminal. +% \end{function} +% +% \begin{function}[updated = 2015-08-06]{\char_set_sfcode:nn} +% \begin{syntax} +% \cs{char_set_sfcode:nn} \Arg{intexpr_1} \Arg{intexpr_2} +% \end{syntax} +% This function sets up the space factor for the \meta{character}. +% The \meta{character} is specified as +% an \meta{integer expression} which will be used as the character +% code of the relevant character. The setting applies within the +% current \TeX{} group. +% \end{function} +% +% \begin{function}[EXP]{\char_value_sfcode:n} +% \begin{syntax} +% \cs{char_value_sfcode:n} \Arg{integer expression} +% \end{syntax} +% Expands to the current space factor for the \meta{character} with +% character code given by the +% \meta{integer expression}. +% \end{function} +% +% \begin{function}{\char_show_value_sfcode:n} +% \begin{syntax} +% \cs{char_show_value_sfcode:n} \Arg{integer expression} +% \end{syntax} +% Displays the current space factor for the \meta{character} with +% character code given by the \meta{integer expression} on the +% terminal. +% \end{function} +% +% \begin{variable}[added = 2012-01-23, updated = 2015-11-11]{\l_char_active_seq} +% Used to track which tokens may require special handling at the document +% level as they are (or have been at some point) +% of category \meta{active} (catcode~$13$). Each entry in +% the sequence consists of a single escaped token, for example |\~|. +% Active tokens should be added to the sequence when they are defined for +% general document use. +% \end{variable} +% +% \begin{variable}[added = 2012-01-23, updated = 2015-11-11]{\l_char_special_seq} +% Used to track which tokens will require special handling when working with +% verbatim-like material at the document level as they are not of categories +% \meta{letter} (catcode~$11$) or \meta{other} (catcode~$12$). Each entry in +% the sequence consists of a single escaped token, for example |\\| for the +% backslash or |\{| for an opening brace.^^A \} +% Escaped tokens should be added to the sequence when they are defined for +% general document use. +% \end{variable} +% +% \section{Generic tokens} +% +% \begin{variable} +% { +% \c_group_begin_token, +% \c_group_end_token, +% \c_math_toggle_token, +% \c_alignment_token, +% \c_parameter_token, +% \c_math_superscript_token, +% \c_math_subscript_token, +% \c_space_token +% } +% These are implicit tokens which have the category code described +% by their name. They are used internally for test purposes but +% are also available to the programmer for other uses. +% \end{variable} +% +% \begin{variable} +% { +% \c_catcode_letter_token, +% \c_catcode_other_token +% } +% These are implicit tokens which have the category code described +% by their name. They are used internally for test purposes and should +% not be used other than for category code tests. +% \end{variable} +% +% \begin{variable}{\c_catcode_active_tl} +% A token list containing an active token. This is used internally +% for test purposes and should not be used other than in +% appropriately-constructed category code tests. +% \end{variable} +% +% \section{Converting tokens} +% +% \begin{function}[EXP]{\token_to_meaning:N, \token_to_meaning:c} +% \begin{syntax} +% \cs{token_to_meaning:N} \meta{token} +% \end{syntax} +% Inserts the current meaning of the \meta{token} into the input +% stream as a series of characters of category code $12$ (other). +% This is the primitive \TeX{} description of the \meta{token}, +% thus for example both functions defined by \cs{cs_set_nopar:Npn} +% and token list variables defined using \cs{tl_new:N} are described +% as |macro|s. +% \begin{texnote} +% This is the \TeX{} primitive \tn{meaning}. +% The \meta{token} can thus be an explicit space tokens or an +% explicit begin-group or end-group character token (|{|~or~|}| when +% normal \TeX{} category codes apply) even though these are not +% valid \texttt{N}-type arguments. +% \end{texnote} +% \end{function} +% +% \begin{function}[EXP]{\token_to_str:N, \token_to_str:c} +% \begin{syntax} +% \cs{token_to_str:N} \meta{token} +% \end{syntax} +% Converts the given \meta{token} into a series of characters with +% category code $12$ (other). If the \meta{token} is a control +% sequence, this will start with the current escape character with +% category code $12$ (the escape character is part of the +% \meta{token}). This function requires only a single expansion. +% \begin{texnote} +% \cs{token_to_str:N} is the \TeX{} primitive \tn{string} renamed. +% The \meta{token} can thus be an explicit space tokens or an +% explicit begin-group or end-group character token (|{|~or~|}| when +% normal \TeX{} category codes apply) even though these are not +% valid \texttt{N}-type arguments. +% \end{texnote} +% \end{function} +% +% \section{Token conditionals} +% +% \begin{function}[EXP,pTF]{\token_if_group_begin:N} +% \begin{syntax} +% \cs{token_if_group_begin_p:N} \meta{token} \\ +% \cs{token_if_group_begin:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if \meta{token} has the category code of a begin group token +% (|{| when normal \TeX{} category codes are in ^^A } +% force). +% Note that an explicit begin group token cannot be tested in this way, +% as it is not a valid \texttt{N}-type argument. +% \end{function} +% +% \begin{function}[EXP,pTF]{\token_if_group_end:N} +% \begin{syntax} +% \cs{token_if_group_end_p:N} \meta{token} \\ +% \cs{token_if_group_end:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if \meta{token} has the category code of an end group token +% (^^A { +% |}| when normal \TeX{} category codes are in force). +% Note that an explicit end group token cannot be tested in this way, +% as it is not a valid \texttt{N}-type argument. +% \end{function} +% +% \begin{function}[EXP,pTF]{\token_if_math_toggle:N} +% \begin{syntax} +% \cs{token_if_math_toggle_p:N} \meta{token} \\ +% \cs{token_if_math_toggle:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if \meta{token} has the category code of a math shift token +% (|$| when normal \TeX{} category codes are in force). +% \end{function} +% +% \begin{function}[EXP,pTF]{\token_if_alignment:N} +% \begin{syntax} +% \cs{token_if_alignment_p:N} \meta{token} \\ +% \cs{token_if_alignment:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if \meta{token} has the category code of an alignment token +% (|&| when normal \TeX{} category codes are in force). +% \end{function} +% +% \begin{function}[EXP,pTF]{\token_if_parameter:N} +% \begin{syntax} +% \cs{token_if_parameter_p:N} \meta{token} \\ +% \cs{token_if_alignment:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if \meta{token} has the category code of a macro parameter token +% (|#| when normal \TeX{} category codes are in force). +% \end{function} +% +% \begin{function}[EXP,pTF]{\token_if_math_superscript:N} +% \begin{syntax} +% \cs{token_if_math_superscript_p:N} \meta{token} \\ +% \cs{token_if_math_superscript:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if \meta{token} has the category code of a superscript token +% (|^| when normal \TeX{} category codes are in force). +% \end{function} +% +% \begin{function}[EXP,pTF]{\token_if_math_subscript:N} +% \begin{syntax} +% \cs{token_if_math_subscript_p:N} \meta{token} \\ +% \cs{token_if_math_subscript:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if \meta{token} has the category code of a subscript token +% (|_| when normal \TeX{} category codes are in force). +% \end{function} +% +% \begin{function}[EXP,pTF]{\token_if_space:N} +% \begin{syntax} +% \cs{token_if_space_p:N} \meta{token} \\ +% \cs{token_if_space:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if \meta{token} has the category code of a space token. +% Note that an explicit space token with character code $32$ cannot +% be tested in this way, as it is not a valid \texttt{N}-type argument. +% \end{function} +% +% \begin{function}[EXP,pTF]{\token_if_letter:N} +% \begin{syntax} +% \cs{token_if_letter_p:N} \meta{token} \\ +% \cs{token_if_letter:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if \meta{token} has the category code of a letter token. +% \end{function} +% +% \begin{function}[EXP,pTF]{\token_if_other:N} +% \begin{syntax} +% \cs{token_if_other_p:N} \meta{token} \\ +% \cs{token_if_other:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if \meta{token} has the category code of an \enquote{other} +% token. +% \end{function} +% +% \begin{function}[EXP,pTF]{\token_if_active:N} +% \begin{syntax} +% \cs{token_if_active_p:N} \meta{token} \\ +% \cs{token_if_active:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if \meta{token} has the category code of an active character. +% \end{function} +% +% \begin{function}[EXP,pTF]{\token_if_eq_catcode:NN} +% \begin{syntax} +% \cs{token_if_eq_catcode_p:NN} \meta{token_1} \meta{token_2} \\ +% \cs{token_if_eq_catcode:NNTF} \meta{token_1} \meta{token_2} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the two \meta{tokens} have the same category code. +% \end{function} +% +% \begin{function}[EXP,pTF]{\token_if_eq_charcode:NN} +% \begin{syntax} +% \cs{token_if_eq_charcode_p:NN} \meta{token_1} \meta{token_2} \\ +% \cs{token_if_eq_charcode:NNTF} \meta{token_1} \meta{token_2} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the two \meta{tokens} have the same character code. +% \end{function} +% +% \begin{function}[EXP,pTF]{\token_if_eq_meaning:NN} +% \begin{syntax} +% \cs{token_if_eq_meaning_p:NN} \meta{token_1} \meta{token_2} \\ +% \cs{token_if_eq_meaning:NNTF} \meta{token_1} \meta{token_2} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the two \meta{tokens} have the same meaning when expanded. +% \end{function} +% +% \begin{function}[updated = 2011-05-23, EXP,pTF]{\token_if_macro:N} +% \begin{syntax} +% \cs{token_if_macro_p:N} \meta{token} \\ +% \cs{token_if_macro:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the \meta{token} is a \TeX{} macro. +% \end{function} +% +% \begin{function}[EXP,pTF]{\token_if_cs:N} +% \begin{syntax} +% \cs{token_if_cs_p:N} \meta{token} \\ +% \cs{token_if_cs:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the \meta{token} is a control sequence. +% \end{function} +% +% \begin{function}[EXP,pTF]{\token_if_expandable:N} +% \begin{syntax} +% \cs{token_if_expandable_p:N} \meta{token} \\ +% \cs{token_if_expandable:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the \meta{token} is expandable. This test returns \meta{false} +% for an undefined token. +% \end{function} +% +% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_long_macro:N} +% \begin{syntax} +% \cs{token_if_long_macro_p:N} \meta{token} \\ +% \cs{token_if_long_macro:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the \meta{token} is a long macro. +% \end{function} +% +% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_protected_macro:N} +% \begin{syntax} +% \cs{token_if_protected_macro_p:N} \meta{token} \\ +% \cs{token_if_protected_macro:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the \meta{token} is a protected macro: for a macro which +% is both protected and long this returns \texttt{false}. +% \end{function} +% +% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_protected_long_macro:N} +% \begin{syntax} +% \cs{token_if_protected_long_macro_p:N} \meta{token} \\ +% \cs{token_if_protected_long_macro:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the \meta{token} is a protected long macro. +% \end{function} +% +% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_chardef:N} +% \begin{syntax} +% \cs{token_if_chardef_p:N} \meta{token} \\ +% \cs{token_if_chardef:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the \meta{token} is defined to be a chardef. +% \begin{texnote} +% Booleans, boxes and small integer constants are implemented as +% \tn{chardef}s. +% \end{texnote} +% \end{function} +% +% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_mathchardef:N} +% \begin{syntax} +% \cs{token_if_mathchardef_p:N} \meta{token} \\ +% \cs{token_if_mathchardef:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the \meta{token} is defined to be a mathchardef. +% \end{function} +% +% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_dim_register:N} +% \begin{syntax} +% \cs{token_if_dim_register_p:N} \meta{token} \\ +% \cs{token_if_dim_register:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the \meta{token} is defined to be a dimension register. +% \end{function} +% +% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_int_register:N} +% \begin{syntax} +% \cs{token_if_int_register_p:N} \meta{token} \\ +% \cs{token_if_int_register:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the \meta{token} is defined to be a integer register. +% \begin{texnote} +% Constant integers may be implemented as integer registers, +% \tn{chardef}s, or \tn{mathchardef}s depending on their value. +% \end{texnote} +% \end{function} +% +% \begin{function}[EXP,pTF, added=2012-02-15]{\token_if_muskip_register:N} +% \begin{syntax} +% \cs{token_if_muskip_register_p:N} \meta{token} \\ +% \cs{token_if_muskip_register:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the \meta{token} is defined to be a muskip register. +% \end{function} +% +% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_skip_register:N} +% \begin{syntax} +% \cs{token_if_skip_register_p:N} \meta{token} \\ +% \cs{token_if_skip_register:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the \meta{token} is defined to be a skip register. +% \end{function} +% +% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_toks_register:N} +% \begin{syntax} +% \cs{token_if_toks_register_p:N} \meta{token} \\ +% \cs{token_if_toks_register:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the \meta{token} is defined to be a toks register +% (not used by \LaTeX3). +% \end{function} +% +% \begin{function}[updated = 2011-05-23, EXP,pTF]{\token_if_primitive:N} +% \begin{syntax} +% \cs{token_if_primitive_p:N} \meta{token} \\ +% \cs{token_if_primitive:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the \meta{token} is an engine primitive. +% \end{function} +% +% \section{Peeking ahead at the next token} +% +% There is often a need to look ahead at the next token in the input +% stream while leaving it in place. This is handled using the +% \enquote{peek} functions. The generic \cs{peek_after:Nw} is +% provided along with a family of predefined tests for common cases. +% As peeking ahead does \emph{not} skip spaces the predefined tests +% include both a space-respecting and space-skipping version. +% +% \begin{function}{\peek_after:Nw} +% \begin{syntax} +% \cs{peek_after:Nw} \meta{function} \meta{token} +% \end{syntax} +% Locally sets the test variable \cs{l_peek_token} equal to \meta{token} +% (as an implicit token, \emph{not} as a token list), and then +% expands the \meta{function}. The \meta{token} remains in +% the input stream as the next item after the \meta{function}. +% The \meta{token} here may be \verb*| |, |{| or |}| (assuming +% normal \TeX{} category codes), \emph{i.e.}~it is not necessarily the +% next argument which would be grabbed by a normal function. +% \end{function} +% +% \begin{function}{\peek_gafter:Nw} +% \begin{syntax} +% \cs{peek_gafter:Nw} \meta{function} \meta{token} +% \end{syntax} +% Globally sets the test variable \cs{g_peek_token} equal to \meta{token} +% (as an implicit token, \emph{not} as a token list), and then +% expands the \meta{function}. The \meta{token} remains in +% the input stream as the next item after the \meta{function}. +% The \meta{token} here may be \verb*| |, |{| or |}| (assuming +% normal \TeX{} category codes), \emph{i.e.}~it is not necessarily the +% next argument which would be grabbed by a normal function. +% \end{function} +% +% \begin{variable}{\l_peek_token} +% Token set by \cs{peek_after:Nw} and available for testing +% as described above. +% \end{variable} +% +% \begin{variable}{\g_peek_token} +% Token set by \cs{peek_gafter:Nw} and available for testing +% as described above. +% \end{variable} +% +% \begin{function}[updated = 2012-12-20, TF]{\peek_catcode:N} +% \begin{syntax} +% \cs{peek_catcode:NTF} \meta{test token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the next \meta{token} in the input stream has the same +% category code as the \meta{test token} (as defined by the test +% \cs{token_if_eq_catcode:NNTF}). Spaces are respected by the test +% and the \meta{token} is left in the input stream after +% the \meta{true code} or \meta{false code} (as appropriate to the +% result of the test). +% \end{function} +% +% \begin{function}[updated = 2012-12-20, TF]{\peek_catcode_ignore_spaces:N} +% \begin{syntax} +% \cs{peek_catcode_ignore_spaces:NTF} \meta{test token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the next non-space \meta{token} in the input stream has the +% same category code as the \meta{test token} (as defined by the test +% \cs{token_if_eq_catcode:NNTF}). Explicit and implicit space tokens +% (with character code 32 and category code 10) are ignored and +% removed by the test and the \meta{token} is left in the input +% stream after the \meta{true code} or \meta{false code} (as +% appropriate to the result of the test). +% \end{function} +% +% \begin{function}[updated = 2012-12-20, TF]{\peek_catcode_remove:N} +% \begin{syntax} +% \cs{peek_catcode_remove:NTF} \meta{test token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the next \meta{token} in the input stream has the same +% category code as the \meta{test token} (as defined by the test +% \cs{token_if_eq_catcode:NNTF}). Spaces are respected by the test +% and the \meta{token} is removed from the input stream if the +% test is true. The function then places either the +% \meta{true code} or \meta{false code} in the input stream (as +% appropriate to the result of the test). +% \end{function} +% +% \begin{function}[updated = 2012-12-20, TF] +% {\peek_catcode_remove_ignore_spaces:N} +% \begin{syntax} +% \cs{peek_catcode_remove_ignore_spaces:NTF} \meta{test token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the next non-space \meta{token} in the input stream has the +% same category code as the \meta{test token} (as defined by the test +% \cs{token_if_eq_catcode:NNTF}). Explicit and implicit space tokens +% (with character code 32 and category code 10) are ignored and +% removed by the test and the \meta{token} is removed from the +% input stream if the test is true. The function then places +% either the \meta{true code} or \meta{false code} in the input stream +% (as appropriate to the result of the test). +% \end{function} +% +% \begin{function}[updated = 2012-12-20, TF]{\peek_charcode:N} +% \begin{syntax} +% \cs{peek_charcode:NTF} \meta{test token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the next \meta{token} in the input stream has the same +% character code as the \meta{test token} (as defined by the test +% \cs{token_if_eq_charcode:NNTF}). Spaces are respected by the test +% and the \meta{token} is left in the input stream after +% the \meta{true code} or \meta{false code} (as appropriate to the +% result of the test). +% \end{function} +% +% \begin{function}[updated = 2012-12-20, TF]{\peek_charcode_ignore_spaces:N} +% \begin{syntax} +% \cs{peek_charcode_ignore_spaces:NTF} \meta{test token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the next non-space \meta{token} in the input stream has the +% same character code as the \meta{test token} (as defined by the test +% \cs{token_if_eq_charcode:NNTF}). Explicit and implicit space tokens +% (with character code 32 and category code 10) are ignored and removed by +% the test and the \meta{token} is left in the input stream after +% the \meta{true code} or \meta{false code} (as appropriate to the +% result of the test). +% \end{function} +% +% \begin{function}[updated = 2012-12-20, TF]{\peek_charcode_remove:N} +% \begin{syntax} +% \cs{peek_charcode_remove:NTF} \meta{test token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the next \meta{token} in the input stream has the same +% character code as the \meta{test token} (as defined by the test +% \cs{token_if_eq_charcode:NNTF}). Spaces are respected by the test +% and the \meta{token} is removed from the input stream if the +% test is true. The function then places either the +% \meta{true code} or \meta{false code} in the input stream (as +% appropriate to the result of the test). +% \end{function} +% +% \begin{function}[updated = 2012-12-20, TF] +% {\peek_charcode_remove_ignore_spaces:N} +% \begin{syntax} +% \cs{peek_charcode_remove_ignore_spaces:NTF} \meta{test token} +% ~~\Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the next non-space \meta{token} in the input stream has the +% same character code as the \meta{test token} (as defined by the test +% \cs{token_if_eq_charcode:NNTF}). Explicit and implicit space tokens +% (with character code 32 and category code 10) are ignored and +% removed by the test and the \meta{token} is removed from the +% input stream if the test is true. The function then places +% either the \meta{true code} or \meta{false code} in the input stream +% (as appropriate to the result of the test). +% \end{function} +% +% \begin{function}[updated = 2011-07-02, TF]{\peek_meaning:N} +% \begin{syntax} +% \cs{peek_meaning:NTF} \meta{test token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the next \meta{token} in the input stream has the same +% meaning as the \meta{test token} (as defined by the test +% \cs{token_if_eq_meaning:NNTF}). Spaces are respected by the test +% and the \meta{token} is left in the input stream after +% the \meta{true code} or \meta{false code} (as appropriate to the +% result of the test). +% \end{function} +% +% \begin{function}[updated = 2012-12-05, TF]{\peek_meaning_ignore_spaces:N} +% \begin{syntax} +% \cs{peek_meaning_ignore_spaces:NTF} \meta{test token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the next non-space \meta{token} in the input stream has the +% same meaning as the \meta{test token} (as defined by the test +% \cs{token_if_eq_meaning:NNTF}). Explicit and implicit space tokens +% (with character code 32 and category code 10) are ignored and +% removed by the test and the \meta{token} is left in the input +% stream after the \meta{true code} or \meta{false code} (as +% appropriate to the result of the test). +% \end{function} +% +% \begin{function}[updated = 2011-07-02, TF]{\peek_meaning_remove:N} +% \begin{syntax} +% \cs{peek_meaning_remove:NTF} \meta{test token} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the next \meta{token} in the input stream has the same +% meaning as the \meta{test token} (as defined by the test +% \cs{token_if_eq_meaning:NNTF}). Spaces are respected by the test +% and the \meta{token} is removed from the input stream if the +% test is true. The function then places either the +% \meta{true code} or \meta{false code} in the input stream (as +% appropriate to the result of the test). +% \end{function} +% +% \begin{function}[updated = 2012-12-05, TF] +% {\peek_meaning_remove_ignore_spaces:N} +% \begin{syntax} +% \cs{peek_meaning_remove_ignore_spaces:NTF} \meta{test token} +% ~~\Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the next non-space \meta{token} in the input stream has the +% same meaning as the \meta{test token} (as defined by the test +% \cs{token_if_eq_meaning:NNTF}). Explicit and implicit space tokens +% (with character code 32 and category code 10) are ignored and +% removed by the test and the \meta{token} is removed from the +% input stream if the test is true. The function then places +% either the \meta{true code} or \meta{false code} in the input stream +% (as appropriate to the result of the test). +% \end{function} +% +% \begin{function}[TF, updated = 2012-12-20]{\peek_N_type:} +% \begin{syntax} +% \cs{peek_N_type:TF} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the next \meta{token} in the input stream can be safely +% grabbed as an \texttt{N}-type argument. The test is \meta{false} +% if the next \meta{token} is either an explicit or implicit +% begin-group or end-group token (with any character code), or +% an explicit or implicit space character (with character code $32$ +% and category code $10$), or an outer token (never used in \LaTeX3) +% and \meta{true} in all other cases. +% Note that a \meta{true} result ensures that the next \meta{token} is +% a valid \texttt{N}-type argument. However, if the next \meta{token} +% is for instance \cs{c_space_token}, the test takes the +% \meta{false} branch, even though the next \meta{token} is in fact +% a valid \texttt{N}-type argument. The \meta{token} is left +% in the input stream after the \meta{true code} or \meta{false code} +% (as appropriate to the result of the test). +% \end{function} +% +% \section{Description of all possible tokens} +% \label{sec:l3token:all-tokens} +% +% Let us end by reviewing every case that a given token can fall into. +% This section is quite technical and some details are only meant for +% completeness. We distinguish the meaning of the token, which controls +% the expansion of the token and its effect on \TeX{}'s state, and its +% shape, which is used when comparing token lists such as for delimited +% arguments. Two tokens of the same shape must have the same meaning, +% but the converse does not hold. +% +% A token has one of the following shapes. +% \begin{itemize} +% \item A control sequence, characterized by the sequence of +% characters that constitute its name: for instance, \cs{use:n} is a +% five-letter control sequence. +% \item An active character token, characterized by its character code +% (between $0$ and $1114111$ for \LuaTeX{} and \XeTeX{} and less for +% other engines) and category code~$13$. +% \item A character token, characterized by its character code and +% category code (one of $1$, $2$, $3$, $4$, $6$, $7$, $8$, $10$, +% $11$ or~$12$ whose meaning is described below).\footnote{In +% \LuaTeX{}, there is also the case of \enquote{bytes}, which behave as +% character tokens of category code $12$~(other) and character code +% between $1114112$ and~$1114366$. They are used to output +% individual bytes to files, rather than UTF-8.} +% \end{itemize} +% There are also a few internal tokens. The following list may be +% incomplete in some engines. +% \begin{itemize} +% \item Expanding \tn{the}\tn{font} results in a token that looks +% identical to the command that was used to select the current font +% (such as \tn{tenrm}) but it differs from it in shape. +% \item A \enquote{frozen} |\relax|, which differs from the primitive in +% shape (but has the same meaning), is inserted when the closing \tn{fi} of a +% conditional is encountered before the conditional is evaluated. +% \item Expanding \tn{noexpand} \meta{token} (when the \meta{token} is +% expandable) results in an internal token, displayed (temporarily) +% as \cs{notexpanded: \meta{token}}, whose shape coincides with the +% \meta{token} and whose meaning differs from \tn{relax}. +% \item An |\outer endtemplate:| can be encountered when peeking ahead +% at the next token; this expands to another internal token, +% |end of alignment template|. +% \item Tricky programming might access a frozen |\endwrite|. +% \item Some frozen tokens can only be accessed in interactive +% sessions: |\cr|, |\right|, |\endgroup|, |\fi|, |\inaccessible|. +% \end{itemize} +% +% The meaning of a (non-active) character token is fixed by its category +% code (and character code) and cannot be changed. We call these +% tokens \emph{explicit} character tokens. Category codes that a +% character token can have are listed below by giving a sample output of +% the \TeX{} primitive \tn{meaning}, together with their \LaTeX3 names +% and most common example: +% \begin{itemize} +% \item[1] begin-group character (|group_begin|, often |{|), +% \item[2] end-group character (|group_end|, often |}|), +% \item[3] math shift character (|math_toggle|, often |$|), +% \item[4] alignment tab character (|alignment|, often |&|), +% \item[6] macro parameter character (|parameter|, often |#|), +% \item[7] superscript character (|math_superscript|, often |^|), +% \item[8] subscript character (|math_subscript|, often |_|), +% \item[10] blank space (|space|, often character code~$32$), +% \item[11] the letter (|letter|, such as |A|), +% \item[12] the character (|other|, such as |0|). +% \end{itemize} +% Category code~$13$ (|active|) is discussed below. Input characters +% can also have several other category codes which do not lead to +% character tokens for later processing: $0$~(|escape|), +% $5$~(|end_line|), $9$~(|ignore|), $14$~(|comment|), and +% $15$~(|invalid|). +% +% The meaning of a control sequence or active character can be identical +% to that of any character token listed above (with any character code), +% and we call such tokens \emph{implicit} character tokens. The +% meaning is otherwise in the following list: +% \begin{itemize} +% \item a macro, used in \LaTeX3 for most functions and some variables +% (|tl|, |fp|, |seq|, \ldots{}), +% \item a primitive such as \tn{def} or \tn{topmark}, used in \LaTeX3 +% for some functions, +% \item a register such as \tn{count}|123|, used in \LaTeX3{} for the +% implementation of some variables (|int|, |dim|, \ldots{}), +% \item a constant integer such as \tn{char}|"56| or \tn{mathchar}|"121|, +% \item a font selection command, +% \item undefined. +% \end{itemize} +% Macros be \tn{protected} or not, \tn{long} or not (the opposite of +% what \LaTeX3 calls |nopar|), and \tn{outer} or not (unused in +% \LaTeX3). Their \tn{meaning} takes the form +% \begin{quote} +% \meta{properties} |macro:|\meta{parameters}|->|\meta{replacement} +% \end{quote} +% where \meta{properties} is among \tn{protected}\tn{long}\tn{outer}, +% \meta{parameters} describes parameters that the macro expects, such as +% |#1#2#3|, and \meta{replacement} describes how the parameters are +% manipulated, such as~|#2/#1/#3|. +% +% ^^A todo Bruno: discuss here some other subtleties of space tokens? when looking for numbers, when looking for equal signs in let, in expressions, etc. +% +% Now is perhaps a good time to mention some subtleties relating to +% tokens with category code $10$ (space). Any input character with this +% category code (normally, space and tab characters) becomes a normal +% space, with character code~$32$ and category code~$10$. +% +% When a macro takes an undelimited argument, explicit space characters +% (with character code $32$ and category code $10$) are ignored. If the +% following token is an explicit character token with category code $1$ +% (begin-group) and an arbitrary character code, then \TeX{} scans ahead +% to obtain an equal number of explicit character tokens with category +% code $1$ (begin-group) and $2$ (end-group), and the resulting list of +% tokens (with outer braces removed) becomes the argument. Otherwise, a +% single token is taken as the argument for the macro: we call such +% single tokens \enquote{N-type}, as they are suitable to be used as an +% argument for a function with the signature~\texttt{:N}. +% +% \end{documentation} +% +% \begin{implementation} +% +% \section{\pkg{l3token} implementation} +% +% \begin{macrocode} +%<*initex|package> +% \end{macrocode} +% +% \begin{macrocode} +%<@@=char> +% \end{macrocode} +% +% \subsection{Manipulating and interrogating character tokens} +% +% \begin{macro}{\char_set_catcode:nn} +% \begin{macro}{\char_value_catcode:n} +% \begin{macro}{\char_show_value_catcode:n} +% Simple wrappers around the primitives. +% \begin{macrocode} +\cs_new_protected:Npn \char_set_catcode:nn #1#2 + { \tex_catcode:D \int_eval:n {#1} = \int_eval:n {#2} \exp_stop_f: } +\cs_new:Npn \char_value_catcode:n #1 + { \tex_the:D \tex_catcode:D \int_eval:n {#1} \exp_stop_f: } +\cs_new_protected:Npn \char_show_value_catcode:n #1 + { \exp_args:Nf \tl_show:n { \char_value_catcode:n {#1} } } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \begin{macro} +% { +% \char_set_catcode_escape:N , +% \char_set_catcode_group_begin:N , +% \char_set_catcode_group_end:N , +% \char_set_catcode_math_toggle:N , +% \char_set_catcode_alignment:N , +% \char_set_catcode_end_line:N , +% \char_set_catcode_parameter:N , +% \char_set_catcode_math_superscript:N , +% \char_set_catcode_math_subscript:N , +% \char_set_catcode_ignore:N , +% \char_set_catcode_space:N , +% \char_set_catcode_letter:N , +% \char_set_catcode_other:N , +% \char_set_catcode_active:N , +% \char_set_catcode_comment:N , +% \char_set_catcode_invalid:N +% } +% \begin{macrocode} +\cs_new_protected:Npn \char_set_catcode_escape:N #1 + { \char_set_catcode:nn { `#1 } { 0 } } +\cs_new_protected:Npn \char_set_catcode_group_begin:N #1 + { \char_set_catcode:nn { `#1 } { 1 } } +\cs_new_protected:Npn \char_set_catcode_group_end:N #1 + { \char_set_catcode:nn { `#1 } { 2 } } +\cs_new_protected:Npn \char_set_catcode_math_toggle:N #1 + { \char_set_catcode:nn { `#1 } { 3 } } +\cs_new_protected:Npn \char_set_catcode_alignment:N #1 + { \char_set_catcode:nn { `#1 } { 4 } } +\cs_new_protected:Npn \char_set_catcode_end_line:N #1 + { \char_set_catcode:nn { `#1 } { 5 } } +\cs_new_protected:Npn \char_set_catcode_parameter:N #1 + { \char_set_catcode:nn { `#1 } { 6 } } +\cs_new_protected:Npn \char_set_catcode_math_superscript:N #1 + { \char_set_catcode:nn { `#1 } { 7 } } +\cs_new_protected:Npn \char_set_catcode_math_subscript:N #1 + { \char_set_catcode:nn { `#1 } { 8 } } +\cs_new_protected:Npn \char_set_catcode_ignore:N #1 + { \char_set_catcode:nn { `#1 } { 9 } } +\cs_new_protected:Npn \char_set_catcode_space:N #1 + { \char_set_catcode:nn { `#1 } { 10 } } +\cs_new_protected:Npn \char_set_catcode_letter:N #1 + { \char_set_catcode:nn { `#1 } { 11 } } +\cs_new_protected:Npn \char_set_catcode_other:N #1 + { \char_set_catcode:nn { `#1 } { 12 } } +\cs_new_protected:Npn \char_set_catcode_active:N #1 + { \char_set_catcode:nn { `#1 } { 13 } } +\cs_new_protected:Npn \char_set_catcode_comment:N #1 + { \char_set_catcode:nn { `#1 } { 14 } } +\cs_new_protected:Npn \char_set_catcode_invalid:N #1 + { \char_set_catcode:nn { `#1 } { 15 } } +% \end{macrocode} +% \end{macro} +% +% \begin{macro} +% { +% \char_set_catcode_escape:n , +% \char_set_catcode_group_begin:n , +% \char_set_catcode_group_end:n , +% \char_set_catcode_math_toggle:n , +% \char_set_catcode_alignment:n , +% \char_set_catcode_end_line:n , +% \char_set_catcode_parameter:n , +% \char_set_catcode_math_superscript:n , +% \char_set_catcode_math_subscript:n , +% \char_set_catcode_ignore:n , +% \char_set_catcode_space:n , +% \char_set_catcode_letter:n , +% \char_set_catcode_other:n , +% \char_set_catcode_active:n , +% \char_set_catcode_comment:n , +% \char_set_catcode_invalid:n +% } +% \begin{macrocode} +\cs_new_protected:Npn \char_set_catcode_escape:n #1 + { \char_set_catcode:nn {#1} { 0 } } +\cs_new_protected:Npn \char_set_catcode_group_begin:n #1 + { \char_set_catcode:nn {#1} { 1 } } +\cs_new_protected:Npn \char_set_catcode_group_end:n #1 + { \char_set_catcode:nn {#1} { 2 } } +\cs_new_protected:Npn \char_set_catcode_math_toggle:n #1 + { \char_set_catcode:nn {#1} { 3 } } +\cs_new_protected:Npn \char_set_catcode_alignment:n #1 + { \char_set_catcode:nn {#1} { 4 } } +\cs_new_protected:Npn \char_set_catcode_end_line:n #1 + { \char_set_catcode:nn {#1} { 5 } } +\cs_new_protected:Npn \char_set_catcode_parameter:n #1 + { \char_set_catcode:nn {#1} { 6 } } +\cs_new_protected:Npn \char_set_catcode_math_superscript:n #1 + { \char_set_catcode:nn {#1} { 7 } } +\cs_new_protected:Npn \char_set_catcode_math_subscript:n #1 + { \char_set_catcode:nn {#1} { 8 } } +\cs_new_protected:Npn \char_set_catcode_ignore:n #1 + { \char_set_catcode:nn {#1} { 9 } } +\cs_new_protected:Npn \char_set_catcode_space:n #1 + { \char_set_catcode:nn {#1} { 10 } } +\cs_new_protected:Npn \char_set_catcode_letter:n #1 + { \char_set_catcode:nn {#1} { 11 } } +\cs_new_protected:Npn \char_set_catcode_other:n #1 + { \char_set_catcode:nn {#1} { 12 } } +\cs_new_protected:Npn \char_set_catcode_active:n #1 + { \char_set_catcode:nn {#1} { 13 } } +\cs_new_protected:Npn \char_set_catcode_comment:n #1 + { \char_set_catcode:nn {#1} { 14 } } +\cs_new_protected:Npn \char_set_catcode_invalid:n #1 + { \char_set_catcode:nn {#1} { 15 } } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\char_set_mathcode:nn} +% \begin{macro}{\char_value_mathcode:n} +% \begin{macro}{\char_show_value_mathcode:n} +% \begin{macro}{\char_set_lccode:nn} +% \begin{macro}{\char_value_lccode:n} +% \begin{macro}{\char_show_value_lccode:n} +% \begin{macro}{\char_set_uccode:nn} +% \begin{macro}{\char_value_uccode:n} +% \begin{macro}{\char_show_value_uccode:n} +% \begin{macro}{\char_set_sfcode:nn} +% \begin{macro}{\char_value_sfcode:n} +% \begin{macro}{\char_show_value_sfcode:n} +% Pretty repetitive, but necessary! +% \begin{macrocode} +\cs_new_protected:Npn \char_set_mathcode:nn #1#2 + { \tex_mathcode:D \int_eval:n {#1} = \int_eval:n {#2} \exp_stop_f: } +\cs_new:Npn \char_value_mathcode:n #1 + { \tex_the:D \tex_mathcode:D \int_eval:n {#1} \exp_stop_f: } +\cs_new_protected:Npn \char_show_value_mathcode:n #1 + { \exp_args:Nf \tl_show:n { \char_value_mathcode:n {#1} } } +\cs_new_protected:Npn \char_set_lccode:nn #1#2 + { \tex_lccode:D \int_eval:n {#1} = \int_eval:n {#2} \exp_stop_f: } +\cs_new:Npn \char_value_lccode:n #1 + { \tex_the:D \tex_lccode:D \int_eval:n {#1} \exp_stop_f: } +\cs_new_protected:Npn \char_show_value_lccode:n #1 + { \exp_args:Nf \tl_show:n { \char_value_lccode:n {#1} } } +\cs_new_protected:Npn \char_set_uccode:nn #1#2 + { \tex_uccode:D \int_eval:n {#1} = \int_eval:n {#2} \exp_stop_f: } +\cs_new:Npn \char_value_uccode:n #1 + { \tex_the:D \tex_uccode:D \int_eval:n {#1} \exp_stop_f: } +\cs_new_protected:Npn \char_show_value_uccode:n #1 + { \exp_args:Nf \tl_show:n { \char_value_uccode:n {#1} } } +\cs_new_protected:Npn \char_set_sfcode:nn #1#2 + { \tex_sfcode:D \int_eval:n {#1} = \int_eval:n {#2} \exp_stop_f: } +\cs_new:Npn \char_value_sfcode:n #1 + { \tex_the:D \tex_sfcode:D \int_eval:n {#1} \exp_stop_f: } +\cs_new_protected:Npn \char_show_value_sfcode:n #1 + { \exp_args:Nf \tl_show:n { \char_value_sfcode:n {#1} } } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \begin{variable}{\l_char_active_seq, \l_char_special_seq} +% Two sequences for dealing with special characters. The first is characters +% which may be active, the second longer list is for \enquote{special} +% characters more generally. Both lists are escaped so that for example +% bulk code assignments can be carried out. In both cases, the order is +% by \textsc{ascii} character code (as is done in for example +% \cs{ExplSyntaxOn}). +% \begin{macrocode} +\seq_new:N \l_char_special_seq +\seq_set_split:Nnn \l_char_special_seq { } + { \ \" \# \$ \% \& \\ \^ \_ \{ \} \~ } +\seq_new:N \l_char_active_seq +\seq_set_split:Nnn \l_char_active_seq { } + { \" \$ \& \^ \_ \~ } +% \end{macrocode} +% \end{variable} +% +% \subsection{Creating character tokens} +% +% \begin{macro} +% { +% \char_set_active_eq:NN, \char_gset_active_eq:NN, +% \char_set_active_eq:Nc, \char_gset_active_eq:Nc, +% \char_set_active_eq:nN, \char_gset_active_eq:nN, +% \char_set_active_eq:nc, \char_gset_active_eq:nc +% } +% Four simple functions with very similar definitions, so set up using +% an auxiliary. +% These are similar to \LuaTeX{}'s \tn{letcharcode} primitive. +% \begin{macrocode} +\group_begin: + \char_set_catcode_active:N \^^@ + \cs_set_protected:Npn \@@_tmp:nN #1#2 + { + \cs_new_protected:cpn { #1 :nN } ##1 + { + \group_begin: + \char_set_lccode:nn { `\^^@ } { ##1 } + \tex_lowercase:D { \group_end: #2 ^^@ } + } + \cs_new_protected:cpx { #1 :NN } ##1 + { \exp_not:c { #1 : nN } { `##1 } } + } + \@@_tmp:nN { char_set_active_eq } \cs_set_eq:NN + \@@_tmp:nN { char_gset_active_eq } \cs_gset_eq:NN +\group_end: +\cs_generate_variant:Nn \char_set_active_eq:NN { Nc } +\cs_generate_variant:Nn \char_gset_active_eq:NN { Nc } +\cs_generate_variant:Nn \char_set_active_eq:nN { nc } +\cs_generate_variant:Nn \char_gset_active_eq:nN { nc } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_int_to_roman:w} +% For efficiency in 8-bit engines, we use the faster primitive approach +% to making roman numerals. +% \begin{macrocode} +\cs_new_eq:NN \@@_int_to_roman:w \tex_romannumeral:D +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[EXP]{\char_generate:nn} +% \begin{macro}[EXP]{\@@_generate_aux:nn} +% \begin{macro}[EXP]{\@@_generate_aux:nnw, \@@_generate_auxii:nnw} +% \begin{variable}{\l_@@_tmp_tl} +% \begin{macro}[EXP]{\@@_generate_invalid_catcode:} +% The aim here is to generate characters of (broadly) arbitrary category +% code. Where possible, that is done using engine support (\XeTeX{}, +% \LuaTeX{}). There are though various issues which are covered below. At +% the interface layer, turn the two arguments into integers up-front so +% this is only done once. +% \begin{macrocode} +\cs_new:Npn \char_generate:nn #1#2 + { + \exp:w \exp_after:wN \@@_generate_aux:w + \int_value:w \int_eval:n {#1} \exp_after:wN ; + \int_value:w \int_eval:n {#2} ; + } +% \end{macrocode} +% Before doing any actual conversion, first some special case filtering. +% Spaces are out here as \LuaTeX{} emulation only makes normal (charcode +% $32$ spaces). However, |^^@| is filtered out separately as that can't be +% done with macro emulation either, so is flagged up separately. That +% done, hand off to the engine-dependent part. +% \begin{macrocode} +\cs_new:Npn \@@_generate_aux:w #1 ; #2 ; + { + \if_int_compare:w #2 = 10 \exp_stop_f: + \if_int_compare:w #1 = 0 \exp_stop_f: + \__kernel_msg_expandable_error:nn { kernel } { char-null-space } + \else: + \__kernel_msg_expandable_error:nn { kernel } { char-space } + \fi: + \else: + \if_int_odd:w 0 + \if_int_compare:w #2 < 1 \exp_stop_f: 1 \fi: + \if_int_compare:w #2 = 5 \exp_stop_f: 1 \fi: + \if_int_compare:w #2 = 9 \exp_stop_f: 1 \fi: + \if_int_compare:w #2 > 13 \exp_stop_f: 1 \fi: \exp_stop_f: + \__kernel_msg_expandable_error:nn { kernel } + { char-invalid-catcode } + \else: + \if_int_odd:w 0 + \if_int_compare:w #1 < 0 \exp_stop_f: 1 \fi: + \if_int_compare:w #1 > \c_max_char_int 1 \fi: \exp_stop_f: + \__kernel_msg_expandable_error:nn { kernel } + { char-out-of-range } + \else: + \@@_generate_aux:nnw {#1} {#2} + \fi: + \fi: + \fi: + \exp_end: + } +\tl_new:N \l_@@_tmp_tl +% \end{macrocode} +% Engine-dependent definitions are now needed for the implementation. For +% \LuaTeX{} and \XeTeX{} there is engine-level support. +% They can do cases that macro emulation can't. All of those are filtered +% out here using a primitive-based boolean expression to avoid fixing +% the category code of the null character used in the false branch +% (for 8-bit engines). +% The final level is the basic definition at the engine level: the arguments +% here are integers so there is no need to worry about them too much. +% Older versions of \XeTeX{} cannot generate active characters so we filter +% that: +% at some future stage that may change: the slightly odd ordering of +% auxiliaries reflects that. +% \begin{macrocode} +\group_begin: +%<*package> + \char_set_catcode_active:N \^^L + \cs_set:Npn ^^L { } +%</package> + \char_set_catcode_other:n { 0 } + \if_int_odd:w 0 + \sys_if_engine_luatex:T { 1 } + \sys_if_engine_xetex:T { 1 } \exp_stop_f: + \sys_if_engine_luatex:TF + { + \cs_new:Npn \@@_generate_aux:nnw #1#2#3 \exp_end: + { + #3 + \exp_after:wN \exp_after:wN \exp_after:wN \exp_end: + \lua_now:e { l3kernel.charcat(#1, #2) } + } + } + { + \cs_new:Npn \@@_generate_aux:nnw #1#2#3 \exp_end: + { + #3 + \exp_after:wN \exp_end: + \tex_Ucharcat:D #1 \exp_stop_f: #2 \exp_stop_f: + } + \cs_if_exist:NF \tex_expanded:D + { + \cs_new_eq:NN \@@_generate_auxii:nnw \@@_generate_aux:nnw + \cs_gset:Npn \@@_generate_aux:nnw #1#2#3 \exp_end: + { + #3 + \if_int_compare:w #2 = 13 \exp_stop_f: + \__kernel_msg_expandable_error:nn { kernel } { char-active } + \else: + \@@_generate_auxii:nnw {#1} {#2} + \fi: + \exp_end: + } + } + } + \else: +% \end{macrocode} +% For engines where \tn{Ucharcat} isn't available or emulated, we have +% to work in macros, and cover only the $8$-bit range. The first stage is +% to build up a |tl| containing |^^@| with each category code that can +% be accessed in this way, with an error set up for the other cases. This +% is all done such that it can be quickly accessed using a |\if_case:w| +% low-level conditional. There are a few things to notice here. +% As |^^L| is |\outer| we need to locally set it to avoid a problem. +% To get open/close braces into the list, they are set up using |\if_false:| +% pairing and are then |x|-type expanded together into the desired form. +% \begin{macrocode} + \tl_set:Nn \l_@@_tmp_tl { \exp_not:N \or: } + \char_set_catcode_group_begin:n { 0 } % { + \tl_put_right:Nn \l_@@_tmp_tl { ^^@ \if_false: } } + \char_set_catcode_group_end:n { 0 } + \tl_put_right:Nn \l_@@_tmp_tl { { \fi: \exp_not:N \or: ^^@ } % } + \tl_set:Nx \l_@@_tmp_tl { \l_@@_tmp_tl } + \char_set_catcode_math_toggle:n { 0 } + \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } + \char_set_catcode_alignment:n { 0 } + \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } + \tl_put_right:Nn \l_@@_tmp_tl { \or: } + \char_set_catcode_parameter:n { 0 } + \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } + \char_set_catcode_math_superscript:n { 0 } + \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } + \char_set_catcode_math_subscript:n { 0 } + \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } + \tl_put_right:Nn \l_@@_tmp_tl { \or: } +% \end{macrocode} +% For making spaces, there needs to be an |o|-type expansion of a |\use:n| +% (or some other tokenization) to avoid dropping the space. We also +% set up active tokens although they are (currently) filtered out by the +% interface layer (\tn{Ucharcat} cannot make active tokens). +% \begin{macrocode} + \char_set_catcode_space:n { 0 } + \tl_put_right:No \l_@@_tmp_tl { \use:n { \or: } ^^@ } + \char_set_catcode_letter:n { 0 } + \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } + \char_set_catcode_other:n { 0 } + \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } + \char_set_catcode_active:n { 0 } + \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } +% \end{macrocode} +% Convert the above temporary list into a series of constant token +% lists, one for each character code, using \tn{tex_lowercase:D} to +% convert |^^@| in each case. The \texttt{x}-type expansion ensures +% that \tn{tex_lowercase:D} receives the contents of the token list. +% In package mode, |^^L| is awkward hence this is done in three parts. +% Notice that at this stage |^^@| is active. +% \begin{macrocode} + \cs_set_protected:Npn \@@_tmp:n #1 + { + \char_set_lccode:nn { 0 } {#1} + \char_set_lccode:nn { 32 } {#1} + \exp_args:Nx \tex_lowercase:D + { + \tl_const:Nn + \exp_not:c { c_@@_ \@@_int_to_roman:w #1 _tl } + { \exp_not:o \l_@@_tmp_tl } + } + } +%<*package> + \int_step_function:nnN { 0 } { 11 } \@@_tmp:n + \group_begin: + \tl_replace_once:Nnn \l_@@_tmp_tl { ^^@ } { \ERROR } + \@@_tmp:n { 12 } + \group_end: + \int_step_function:nnN { 13 } { 255 } \@@_tmp:n +%</package> +%<*initex> + \int_step_function:nnN { 0 } { 255 } \@@_tmp:n +%</initex> +% \end{macrocode} +% As \TeX{} is very unhappy if if finds an alignment character inside +% a primitive \tn{halign} even when skipping false branches, some +% precautions are required. \TeX{} is happy if the token is hidden +% between braces within \cs{if_false:} \dots{} \cs{fi:}. +% \begin{macrocode} + \cs_new:Npn \@@_generate_aux:nnw #1#2#3 \exp_end: + { + #3 + \if_false: { \fi: + \exp_after:wN \exp_after:wN + \exp_after:wN \exp_end: + \exp_after:wN \exp_after:wN + \if_case:w #2 + \exp_last_unbraced:Nv \exp_stop_f: + { c_@@_ \@@_int_to_roman:w #1 _tl } + \or: } + \fi: + } + \fi: +\group_end: +% \end{macrocode} +% \end{macro} +% \end{variable} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \begin{macro}{\c_catcode_other_space_tl} +% Create a space with category code $12$: an \enquote{other} space. +% \begin{macrocode} +\tl_const:Nx \c_catcode_other_space_tl { \char_generate:nn { `\ } { 12 } } +% \end{macrocode} +% \end{macro} +% +% \subsection{Generic tokens} +% +% \begin{macrocode} +%<@@=token> +% \end{macrocode} +% +% \begin{macro}{\token_to_meaning:N, \token_to_meaning:c} +% \begin{macro}{\token_to_str:N, \token_to_str:c} +% These are all defined in \pkg{l3basics}, as they are needed +% \enquote{early}. This is just a reminder! +% \end{macro} +% \end{macro} +% +% \begin{macro} +% { +% \c_group_begin_token, +% \c_group_end_token, +% \c_math_toggle_token, +% \c_alignment_token, +% \c_parameter_token, +% \c_math_superscript_token, +% \c_math_subscript_token, +% \c_space_token, +% \c_catcode_letter_token, +% \c_catcode_other_token +% } +% We define these useful tokens. For the brace and space tokens things have +% to be done by hand: the formal argument spec.~for \cs{cs_new_eq:NN} does +% not cover them so we do things by hand. (As currently coded it would +% \emph{work} with \cs{cs_new_eq:NN} but that's not really a great idea to +% show off: we want people to stick to the defined interfaces and that +% includes us.) So that these few odd names go into the log when appropriate +% there is a need to hand-apply the \cs{__kernel_chk_if_free_cs:N} check. +% \begin{macrocode} +\group_begin: + \__kernel_chk_if_free_cs:N \c_group_begin_token + \tex_global:D \tex_let:D \c_group_begin_token { + \__kernel_chk_if_free_cs:N \c_group_end_token + \tex_global:D \tex_let:D \c_group_end_token } + \char_set_catcode_math_toggle:N \* + \cs_new_eq:NN \c_math_toggle_token * + \char_set_catcode_alignment:N \* + \cs_new_eq:NN \c_alignment_token * + \cs_new_eq:NN \c_parameter_token # + \cs_new_eq:NN \c_math_superscript_token ^ + \char_set_catcode_math_subscript:N \* + \cs_new_eq:NN \c_math_subscript_token * + \__kernel_chk_if_free_cs:N \c_space_token + \use:n { \tex_global:D \tex_let:D \c_space_token = ~ } ~ + \cs_new_eq:NN \c_catcode_letter_token a + \cs_new_eq:NN \c_catcode_other_token 1 +\group_end: +% \end{macrocode} +% \end{macro} +% +% \begin{variable}{\c_catcode_active_tl} +% Not an implicit token! +% \begin{macrocode} +\group_begin: + \char_set_catcode_active:N \* + \tl_const:Nn \c_catcode_active_tl { \exp_not:N * } +\group_end: +% \end{macrocode} +% \end{variable} +% +% \subsection{Token conditionals} +% +% \begin{macro}[pTF]{\token_if_group_begin:N} +% Check if token is a begin group token. We use the constant +% \cs{c_group_begin_token} for this. +% \begin{macrocode} +\prg_new_conditional:Npnn \token_if_group_begin:N #1 { p , T , F , TF } + { + \if_catcode:w \exp_not:N #1 \c_group_begin_token + \prg_return_true: \else: \prg_return_false: \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[pTF]{\token_if_group_end:N} +% Check if token is a end group token. We use the constant +% \cs{c_group_end_token} for this. +% \begin{macrocode} +\prg_new_conditional:Npnn \token_if_group_end:N #1 { p , T , F , TF } + { + \if_catcode:w \exp_not:N #1 \c_group_end_token + \prg_return_true: \else: \prg_return_false: \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[pTF]{\token_if_math_toggle:N} +% Check if token is a math shift token. We use the constant +% \cs{c_math_toggle_token} for this. +% \begin{macrocode} +\prg_new_conditional:Npnn \token_if_math_toggle:N #1 { p , T , F , TF } + { + \if_catcode:w \exp_not:N #1 \c_math_toggle_token + \prg_return_true: \else: \prg_return_false: \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[pTF]{\token_if_alignment:N} +% Check if token is an alignment tab token. We use the constant +% \cs{c_alignment_token} for this. +% \begin{macrocode} +\prg_new_conditional:Npnn \token_if_alignment:N #1 { p , T , F , TF } + { + \if_catcode:w \exp_not:N #1 \c_alignment_token + \prg_return_true: \else: \prg_return_false: \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[pTF]{\token_if_parameter:N} +% Check if token is a parameter token. We use the constant +% \cs{c_parameter_token} for this. We have to trick \TeX{} a bit to +% avoid an error message: within a group we prevent +% \cs{c_parameter_token} from behaving like a macro parameter character. +% The definitions of \cs{prg_new_conditional:Npnn} are global, so they +% remain after the group. +% \begin{macrocode} +\group_begin: +\cs_set_eq:NN \c_parameter_token \scan_stop: +\prg_new_conditional:Npnn \token_if_parameter:N #1 { p , T , F , TF } + { + \if_catcode:w \exp_not:N #1 \c_parameter_token + \prg_return_true: \else: \prg_return_false: \fi: + } +\group_end: +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[pTF]{\token_if_math_superscript:N} +% Check if token is a math superscript token. We use the constant +% \cs{c_math_superscript_token} for this. +% \begin{macrocode} +\prg_new_conditional:Npnn \token_if_math_superscript:N #1 + { p , T , F , TF } + { + \if_catcode:w \exp_not:N #1 \c_math_superscript_token + \prg_return_true: \else: \prg_return_false: \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[pTF]{\token_if_math_subscript:N} +% Check if token is a math subscript token. We use the constant +% \cs{c_math_subscript_token} for this. +% \begin{macrocode} +\prg_new_conditional:Npnn \token_if_math_subscript:N #1 { p , T , F , TF } + { + \if_catcode:w \exp_not:N #1 \c_math_subscript_token + \prg_return_true: \else: \prg_return_false: \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[pTF]{\token_if_space:N} +% Check if token is a space token. We use the constant +% \cs{c_space_token} for this. +% \begin{macrocode} +\prg_new_conditional:Npnn \token_if_space:N #1 { p , T , F , TF } + { + \if_catcode:w \exp_not:N #1 \c_space_token + \prg_return_true: \else: \prg_return_false: \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[pTF]{\token_if_letter:N} +% Check if token is a letter token. We use the constant +% \cs{c_catcode_letter_token} for this. +% \begin{macrocode} +\prg_new_conditional:Npnn \token_if_letter:N #1 { p , T , F , TF } + { + \if_catcode:w \exp_not:N #1 \c_catcode_letter_token + \prg_return_true: \else: \prg_return_false: \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[pTF]{\token_if_other:N} +% Check if token is an other char token. We use the constant +% \cs{c_catcode_other_token} for this. +% \begin{macrocode} +\prg_new_conditional:Npnn \token_if_other:N #1 { p , T , F , TF } + { + \if_catcode:w \exp_not:N #1 \c_catcode_other_token + \prg_return_true: \else: \prg_return_false: \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[pTF]{\token_if_active:N} +% Check if token is an active char token. We use the constant +% \cs{c_catcode_active_tl} for this. A technical point is that +% \cs{c_catcode_active_tl} is in fact a macro expanding to +% |\exp_not:N *|, where |*| is active. +% \begin{macrocode} +\prg_new_conditional:Npnn \token_if_active:N #1 { p , T , F , TF } + { + \if_catcode:w \exp_not:N #1 \c_catcode_active_tl + \prg_return_true: \else: \prg_return_false: \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[pTF]{\token_if_eq_meaning:NN} +% Check if the tokens |#1| and |#2| have same meaning. +% \begin{macrocode} +\prg_new_conditional:Npnn \token_if_eq_meaning:NN #1#2 { p , T , F , TF } + { + \if_meaning:w #1 #2 + \prg_return_true: \else: \prg_return_false: \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[pTF]{\token_if_eq_catcode:NN} +% Check if the tokens |#1| and |#2| have same category code. +% \begin{macrocode} +\prg_new_conditional:Npnn \token_if_eq_catcode:NN #1#2 { p , T , F , TF } + { + \if_catcode:w \exp_not:N #1 \exp_not:N #2 + \prg_return_true: \else: \prg_return_false: \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[pTF]{\token_if_eq_charcode:NN} +% Check if the tokens |#1| and |#2| have same character code. +% \begin{macrocode} +\prg_new_conditional:Npnn \token_if_eq_charcode:NN #1#2 { p , T , F , TF } + { + \if_charcode:w \exp_not:N #1 \exp_not:N #2 + \prg_return_true: \else: \prg_return_false: \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[pTF]{\token_if_macro:N} +% \begin{macro}{\@@_if_macro_p:w} +% When a token is a macro, \cs{token_to_meaning:N} always outputs +% something like |\long macro:#1->#1| so we could naively check to +% see if the meaning contains |->|. However, this can fail the five +% \tn[no-index]{...mark} primitives, whose meaning has the form +% |...mark:|\meta{user material}. The problem is that the +% \meta{user material} can contain |->|. +% +% However, only characters, macros, and marks can contain the colon +% character. The idea is thus to grab until the first |:|, and analyse +% what is left. However, macros can have any combination of |\long|, +% |\protected| or |\outer| (not used in \LaTeX3) before the string +% |macro:|. We thus only select the part of the meaning between +% the first |ma| and the first following |:|. If this string is +% |cro|, then we have a macro. If the string is |rk|, then we have +% a mark. The string can also be |cro parameter character | for a +% colon with a weird category code (namely the usual category code +% of |#|). Otherwise, it is empty. +% +% This relies on the fact that |\long|, |\protected|, |\outer| +% cannot contain |ma|, regardless of the escape character, even if +% the escape character is |m|\ldots{} +% +% Both |ma| and |:| must be of category code $12$ (other), so are +% detokenized. +% +% \begin{macrocode} +\use:x + { + \prg_new_conditional:Npnn \exp_not:N \token_if_macro:N ##1 + { p , T , F , TF } + { + \exp_not:N \exp_after:wN \exp_not:N \@@_if_macro_p:w + \exp_not:N \token_to_meaning:N ##1 \tl_to_str:n { ma : } + \exp_not:N \q_stop + } + \cs_new:Npn \exp_not:N \@@_if_macro_p:w + ##1 \tl_to_str:n { ma } ##2 \c_colon_str ##3 \exp_not:N \q_stop + } + { + \str_if_eq:nnTF { #2 } { cro } + { \prg_return_true: } + { \prg_return_false: } + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[pTF]{\token_if_cs:N} +% Check if token has same catcode as a control sequence. This +% follows the same pattern as for \cs{token_if_letter:N} \emph{etc.} +% We use \cs{scan_stop:} for this. +% \begin{macrocode} +\prg_new_conditional:Npnn \token_if_cs:N #1 { p , T , F , TF } + { + \if_catcode:w \exp_not:N #1 \scan_stop: + \prg_return_true: \else: \prg_return_false: \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[pTF]{\token_if_expandable:N} +% Check if token is expandable. We use the fact that \TeX{} +% temporarily converts \cs{exp_not:N} \meta{token} into \cs{scan_stop:} +% if \meta{token} is expandable. An \texttt{undefined} token is not +% considered as expandable. No problem nesting the conditionals, +% since the third |#1| is only skipped if it is non-expandable (hence +% not part of \TeX{}'s conditional apparatus). +% \begin{macrocode} +\prg_new_conditional:Npnn \token_if_expandable:N #1 { p , T , F , TF } + { + \exp_after:wN \if_meaning:w \exp_not:N #1 #1 + \prg_return_false: + \else: + \if_cs_exist:N #1 + \prg_return_true: + \else: + \prg_return_false: + \fi: + \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro} +% { +% \@@_delimit_by_char":w, +% \@@_delimit_by_count:w, +% \@@_delimit_by_dimen:w, +% \@@_delimit_by_macro:w, +% \@@_delimit_by_muskip:w, +% \@@_delimit_by_skip:w, +% \@@_delimit_by_toks:w, +% } +% These auxiliary functions are used below to define some +% conditionals which detect whether the \tn{meaning} of their +% argument begins with a particular string. Each auxiliary takes an +% argument delimited by a string, a second one delimited by +% \cs{q_stop}, and returns the first one and its delimiter. +% This result is eventually compared to another string. +% \begin{macrocode} +\group_begin: +\cs_set_protected:Npn \@@_tmp:w #1 + { + \use:x + { + \cs_new:Npn \exp_not:c { @@_delimit_by_ #1 :w } + ####1 \tl_to_str:n {#1} ####2 \exp_not:N \q_stop + { ####1 \tl_to_str:n {#1} } + } + } +\@@_tmp:w { char" } +\@@_tmp:w { count } +\@@_tmp:w { dimen } +\@@_tmp:w { macro } +\@@_tmp:w { muskip } +\@@_tmp:w { skip } +\@@_tmp:w { toks } +\group_end: +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[pTF] +% { +% \token_if_chardef:N, \token_if_mathchardef:N, +% \token_if_long_macro:N, +% \token_if_protected_macro:N, \token_if_protected_long_macro:N, +% \token_if_dim_register:N, \token_if_int_register:N, +% \token_if_muskip_register:N, +% \token_if_skip_register:N, \token_if_toks_register:N, +% } +% Each of these conditionals tests whether its argument's +% \tn{meaning} starts with a given string. This is essentially done +% by having an auxiliary grab an argument delimited by the string and +% testing whether the argument was empty. Of course, a copy of this +% string must first be added to the end of the \tn{meaning} to avoid +% a runaway argument in case it does not contain the string. Two +% complications arise. First, the escape character is not fixed, and +% cannot be included in the delimiter of the auxiliary function (this +% function cannot be defined on the fly because tests must remain +% expandable): instead the first argument of the auxiliary (plus the +% delimiter to avoid complications with trailing spaces) is compared +% using \cs{str_if_eq:eeTF} to the result of applying +% \cs{token_to_str:N} to a control sequence. Second, the +% \tn{meaning} of primitives such as \tn{dimen} or \tn{dimendef} +% starts in the same way as registers such as +% \tn{dimen}\texttt{123}, so they must be tested for. +% +% Characters used as delimiters must have catcode~$12$ +% and are obtained through \cs{tl_to_str:n}. This requires doing all +% definitions within \texttt{x}-expansion. The temporary function +% \cs{@@_tmp:w} used to define each conditional receives three +% arguments: the name of the conditional, the auxiliary's delimiter +% (also used to name the auxiliary), and the string to which one +% compares the auxiliary's result. Note that the \tn{meaning} of a +% protected long macro starts with |\protected\long macro|, with no +% space after |\protected| but a space after |\long|, hence the +% mixture of \cs{token_to_str:N} and \cs{tl_to_str:n}. +% +% For the first five conditionals, \cs{cs_if_exist:cT} turns out to +% be \texttt{false}, and the code boils down to a string comparison +% between the result of the auxiliary on the \tn{meaning} of the +% conditional's argument~|####1|, and~|#3|. Both are evaluated at +% run-time, as this is important to get the correct escape character. +% +% The other five conditionals have additional code that compares the +% argument~|####1| to two \TeX{} primitives which would wrongly be +% recognized as registers otherwise. Despite using \TeX{}'s +% primitive conditional construction, this does not break +% when~|####1| is itself a conditional, because branches of the +% conditionals are only skipped if |####1|~is one of the two +% primitives that are tested for (which are not \TeX{} conditionals). +% \begin{macrocode} +\group_begin: +\cs_set_protected:Npn \@@_tmp:w #1#2#3 + { + \use:x + { + \prg_new_conditional:Npnn \exp_not:c { token_if_ #1 :N } ####1 + { p , T , F , TF } + { + \cs_if_exist:cT { tex_ #2 :D } + { + \exp_not:N \if_meaning:w ####1 \exp_not:c { tex_ #2 :D } + \exp_not:N \prg_return_false: + \exp_not:N \else: + \exp_not:N \if_meaning:w ####1 \exp_not:c { tex_ #2 def:D } + \exp_not:N \prg_return_false: + \exp_not:N \else: + } + \exp_not:N \str_if_eq:eeTF + { + \exp_not:N \exp_after:wN + \exp_not:c { @@_delimit_by_ #2 :w } + \exp_not:N \token_to_meaning:N ####1 + ? \tl_to_str:n {#2} \exp_not:N \q_stop + } + { \exp_not:n {#3} } + { \exp_not:N \prg_return_true: } + { \exp_not:N \prg_return_false: } + \cs_if_exist:cT { tex_ #2 :D } + { + \exp_not:N \fi: + \exp_not:N \fi: + } + } + } + } +\@@_tmp:w { chardef } { char" } { \token_to_str:N \char" } +\@@_tmp:w { mathchardef } { char" } { \token_to_str:N \mathchar" } +\@@_tmp:w { long_macro } { macro } { \tl_to_str:n { \long } macro } +\@@_tmp:w { protected_macro } { macro } + { \tl_to_str:n { \protected } macro } +\@@_tmp:w { protected_long_macro } { macro } + { \token_to_str:N \protected \tl_to_str:n { \long } macro } +\@@_tmp:w { dim_register } { dimen } { \token_to_str:N \dimen } +\@@_tmp:w { int_register } { count } { \token_to_str:N \count } +\@@_tmp:w { muskip_register } { muskip } { \token_to_str:N \muskip } +\@@_tmp:w { skip_register } { skip } { \token_to_str:N \skip } +\@@_tmp:w { toks_register } { toks } { \token_to_str:N \toks } +\group_end: +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[pTF]{\token_if_primitive:N} +% \begin{macro}{\@@_if_primitive:NNw, +% \@@_if_primitive_space:w, +% \@@_if_primitive_nullfont:N, +% \@@_if_primitive_loop:N, +% \@@_if_primitive:Nw, +% \@@_if_primitive_undefined:N} +%^^A See http://groups.google.com/group/comp.text.tex/browse_thread/thread/0a72666873f8753d# +% +% We filter out macros first, because they cause endless trouble later +% otherwise. +% +% Primitives are almost distinguished by the fact that the result +% of \cs{token_to_meaning:N} is formed from letters only. Every other +% token has either a space (e.g., |the letter A|), a digit +% (e.g., |\count123|) or a double quote (e.g., |\char"A|). +% +% Ten exceptions: on the one hand, \cs{tex_undefined:D} is not a +% primitive, but its meaning is |undefined|, only letters; +% on the other hand, \tn{space}, \tn{italiccorr}, +% \tn{hyphen}, \tn{firstmark}, \tn{topmark}, +% \tn{botmark}, \tn{splitfirstmark}, \tn{splitbotmark}, +% and \tn{nullfont} are primitives, but have non-letters +% in their meaning. +% +% We start by removing the two first (non-space) characters from +% the meaning. This removes the escape character (which may be +% nonexistent depending on \tn{endlinechar}), and takes care +% of three of the exceptions: \tn{space}, \tn{italiccorr} +% and \tn{hyphen}, whose meaning is at most two characters. +% This leaves a string terminated by some |:|, and \cs{q_stop}. +% +% The meaning of each one of the five \tn[no-index]{...mark} primitives +% has the form \meta{letters}|:|\meta{user material}. In other words, +% the first non-letter is a colon. We remove everything after the first +% colon. +% +% We are now left with a string, which we must analyze. For primitives, +% it contains only letters. For non-primitives, it contains either +% |"|, or a space, or a digit. Two exceptions remain: \cs{tex_undefined:D}, +% which is not a primitive, and \tn{nullfont}, which is a primitive. +% +% Spaces cannot be grabbed in an undelimited way, so we check them +% separately. If there is a space, we test for \tn{nullfont}. +% Otherwise, we go through characters one by one, and stop at the +% first character less than |`A| (this is not quite a test for +% \enquote{only letters}, but is close enough to work in this context). +% If this first character is |:| then we have a primitive, or +% \cs{tex_undefined:D}, and if it is |"| or a digit, then the token +% is not a primitive. +% +% \begin{macrocode} +\tex_chardef:D \c_@@_A_int = `A ~ % +\use:x + { + \prg_new_conditional:Npnn \exp_not:N \token_if_primitive:N ##1 + { p , T , F , TF } + { + \exp_not:N \token_if_macro:NTF ##1 + \exp_not:N \prg_return_false: + { + \exp_not:N \exp_after:wN \exp_not:N \@@_if_primitive:NNw + \exp_not:N \token_to_meaning:N ##1 + \tl_to_str:n { : : : } \exp_not:N \q_stop ##1 + } + } + \cs_new:Npn \exp_not:N \@@_if_primitive:NNw + ##1##2 ##3 \c_colon_str ##4 \exp_not:N \q_stop + { + \exp_not:N \tl_if_empty:oTF + { \exp_not:N \@@_if_primitive_space:w ##3 ~ } + { + \exp_not:N \@@_if_primitive_loop:N ##3 + \c_colon_str \exp_not:N \q_stop + } + { \exp_not:N \@@_if_primitive_nullfont:N } + } + } +\cs_new:Npn \@@_if_primitive_space:w #1 ~ { } +\cs_new:Npn \@@_if_primitive_nullfont:N #1 + { + \if_meaning:w \tex_nullfont:D #1 + \prg_return_true: + \else: + \prg_return_false: + \fi: + } +\cs_new:Npn \@@_if_primitive_loop:N #1 + { + \if_int_compare:w `#1 < \c_@@_A_int % + \exp_after:wN \@@_if_primitive:Nw + \exp_after:wN #1 + \else: + \exp_after:wN \@@_if_primitive_loop:N + \fi: + } +\cs_new:Npn \@@_if_primitive:Nw #1 #2 \q_stop + { + \if:w : #1 + \exp_after:wN \@@_if_primitive_undefined:N + \else: + \prg_return_false: + \exp_after:wN \use_none:n + \fi: + } +\cs_new:Npn \@@_if_primitive_undefined:N #1 + { + \if_cs_exist:N #1 + \prg_return_true: + \else: + \prg_return_false: + \fi: + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \subsection{Peeking ahead at the next token} +% +% \begin{macrocode} +%<@@=peek> +% \end{macrocode} +% +% Peeking ahead is implemented using a two part mechanism. The +% outer level provides a defined interface to the lower level material. +% This allows a large amount of code to be shared. There are four +% cases: +% \begin{enumerate} +% \item peek at the next token; +% \item peek at the next non-space token; +% \item peek at the next token and remove it; +% \item peek at the next non-space token and remove it. +% \end{enumerate} +% +% \begin{variable}{\l_peek_token} +% \begin{variable}{\g_peek_token} +% Storage tokens which are publicly documented: the token peeked. +% \begin{macrocode} +\cs_new_eq:NN \l_peek_token ? +\cs_new_eq:NN \g_peek_token ? +% \end{macrocode} +% \end{variable} +% \end{variable} +% +% \begin{variable}{\l_@@_search_token} +% The token to search for as an implicit token: +% \emph{cf.}~\cs{l_@@_search_tl}. +% \begin{macrocode} +\cs_new_eq:NN \l_@@_search_token ? +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_@@_search_tl} +% The token to search for as an explicit token: +% \emph{cf.}~\cs{l_@@_search_token}. +% \begin{macrocode} +\tl_new:N \l_@@_search_tl +% \end{macrocode} +% \end{variable} +% +% \begin{macro} +% {\@@_true:w, \@@_true_aux:w, \@@_false:w, \@@_tmp:w} +% Functions used by the branching and space-stripping code. +% \begin{macrocode} +\cs_new:Npn \@@_true:w { } +\cs_new:Npn \@@_true_aux:w { } +\cs_new:Npn \@@_false:w { } +\cs_new:Npn \@@_tmp:w { } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\peek_after:Nw} +% \begin{macro}{\peek_gafter:Nw} +% Simple wrappers for \tn{futurelet}: no arguments absorbed +% here. +% \begin{macrocode} +\cs_new_protected:Npn \peek_after:Nw + { \tex_futurelet:D \l_peek_token } +\cs_new_protected:Npn \peek_gafter:Nw + { \tex_global:D \tex_futurelet:D \g_peek_token } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}{\@@_true_remove:w} +% A function to remove the next token and then regain control. +% \begin{macrocode} +\cs_new_protected:Npn \@@_true_remove:w + { + \tex_afterassignment:D \@@_true_aux:w + \cs_set_eq:NN \@@_tmp:w + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\peek_remove_spaces:n, \@@_remove_spaces:} +% Repeatedly use \cs{@@_true_remove:w} to remove a space and call +% \cs{@@_true_aux:w}. +% \begin{macrocode} +\cs_new_protected:Npn \peek_remove_spaces:n #1 + { + \cs_set:Npx \@@_false:w { \exp_not:n {#1} } + \group_align_safe_begin: + \cs_set:Npn \@@_true_aux:w { \peek_after:Nw \@@_remove_spaces: } + \@@_true_aux:w + } +\cs_new_protected:Npn \@@_remove_spaces: + { + \if_meaning:w \l_peek_token \c_space_token + \exp_after:wN \@@_true_remove:w + \else: + \group_align_safe_end: + \exp_after:wN \@@_false:w + \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_token_generic_aux:NNNTF} +% The generic functions store the test token in both implicit and +% explicit modes, and the \texttt{true} and \texttt{false} code as +% token lists, more or less. The two branches have to be absorbed here +% as the input stream needs to be cleared for the peek function itself. +% Here, |#1| is \cs{@@_true_remove:w} when removing the token and +% \cs{@@_true_aux:w} otherwise. +% \begin{macrocode} +\cs_new_protected:Npn \@@_token_generic_aux:NNNTF #1#2#3#4#5 + { + \group_align_safe_begin: + \cs_set_eq:NN \l_@@_search_token #3 + \tl_set:Nn \l_@@_search_tl {#3} + \cs_set:Npx \@@_true_aux:w + { + \exp_not:N \group_align_safe_end: + \exp_not:n {#4} + } + \cs_set_eq:NN \@@_true:w #1 + \cs_set:Npx \@@_false:w + { + \exp_not:N \group_align_safe_end: + \exp_not:n {#5} + } + \peek_after:Nw #2 + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[TF]{\@@_token_generic:NN, \@@_token_remove_generic:NN} +% For token removal there needs to be a call to the auxiliary +% function which does the work. +% \begin{macrocode} +\cs_new_protected:Npn \@@_token_generic:NNTF + { \@@_token_generic_aux:NNNTF \@@_true_aux:w } +\cs_new_protected:Npn \@@_token_generic:NNT #1#2#3 + { \@@_token_generic:NNTF #1 #2 {#3} { } } +\cs_new_protected:Npn \@@_token_generic:NNF #1#2#3 + { \@@_token_generic:NNTF #1 #2 { } {#3} } +\cs_new_protected:Npn \@@_token_remove_generic:NNTF + { \@@_token_generic_aux:NNNTF \@@_true_remove:w } +\cs_new_protected:Npn \@@_token_remove_generic:NNT #1#2#3 + { \@@_token_remove_generic:NNTF #1 #2 {#3} { } } +\cs_new_protected:Npn \@@_token_remove_generic:NNF #1#2#3 + { \@@_token_remove_generic:NNTF #1 #2 { } {#3} } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_execute_branches_meaning:} +% The meaning test is straight forward. +% \begin{macrocode} +\cs_new:Npn \@@_execute_branches_meaning: + { + \if_meaning:w \l_peek_token \l_@@_search_token + \exp_after:wN \@@_true:w + \else: + \exp_after:wN \@@_false:w + \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_execute_branches_catcode:, \@@_execute_branches_charcode:} +% \begin{macro} +% { +% \@@_execute_branches_catcode_aux: , +% \@@_execute_branches_catcode_auxii:N , +% \@@_execute_branches_catcode_auxiii: +% } +% The catcode and charcode tests are very similar, and in order to use +% the same auxiliaries we do something a little bit odd, firing +% \cs{if_catcode:w} and \cs{if_charcode:w} before finding the operands +% for those tests, which are only given in the |auxii:N| and +% |auxiii:| auxiliaries. For our purposes, three kinds of tokens may +% follow the peeking function: +% \begin{itemize} +% \item control sequences which are not equal to a non-active +% character token (\emph{e.g.}, macro, primitive); +% \item active characters which are not equal to a non-active +% character token (\emph{e.g.}, macro, primitive); +% \item explicit non-active character tokens, or control sequences +% or active characters set equal to a non-active character token. +% \end{itemize} +% The first two cases are not distinguishable simply using \TeX{}'s +% \tn{futurelet}, because we can only access the \tn{meaning} of +% tokens in that way. In those cases, detected thanks to a +% comparison with \cs{scan_stop:}, we grab the following token, and +% compare it explicitly with the explicit search token stored in +% \cs{l_@@_search_tl}. The \cs{exp_not:N} prevents outer macros +% (coming from non-\LaTeX3 code) from blowing up. In the third case, +% \cs{l_peek_token} is good enough for the test, and we compare it +% again with the explicit search token. Just like the peek token, the +% search token may be of any of the three types above, hence the need +% to use the explicit token that was given to the peek function. +% \begin{macrocode} +\cs_new:Npn \@@_execute_branches_catcode: + { \if_catcode:w \@@_execute_branches_catcode_aux: } +\cs_new:Npn \@@_execute_branches_charcode: + { \if_charcode:w \@@_execute_branches_catcode_aux: } +\cs_new:Npn \@@_execute_branches_catcode_aux: + { + \if_catcode:w \exp_not:N \l_peek_token \scan_stop: + \exp_after:wN \exp_after:wN + \exp_after:wN \@@_execute_branches_catcode_auxii:N + \exp_after:wN \exp_not:N + \else: + \exp_after:wN \@@_execute_branches_catcode_auxiii: + \fi: + } +\cs_new:Npn \@@_execute_branches_catcode_auxii:N #1 + { + \exp_not:N #1 + \exp_after:wN \exp_not:N \l_@@_search_tl + \exp_after:wN \@@_true:w + \else: + \exp_after:wN \@@_false:w + \fi: + #1 + } +\cs_new:Npn \@@_execute_branches_catcode_auxiii: + { + \exp_not:N \l_peek_token + \exp_after:wN \exp_not:N \l_@@_search_tl + \exp_after:wN \@@_true:w + \else: + \exp_after:wN \@@_false:w + \fi: + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[TF] +% { +% \peek_catcode:N, +% \peek_catcode_remove:N, +% \peek_charcode:N, +% \peek_charcode_remove:N, +% \peek_meaning:N, +% \peek_meaning_remove:N, +% } +% The public functions themselves cannot be defined using +% \cs{prg_new_conditional:Npnn}. Instead, the |TF|, |T|, |F| variants +% are defined in terms of corresponding variants of +% \cs{@@_token_generic:NNTF} or \cs{@@_token_remove_generic:NNTF}, +% with first argument one of \cs{@@_execute_branches_catcode:}, +% \cs{@@_execute_branches_charcode:}, or +% \cs{@@_execute_branches_meaning:}. +% \begin{macrocode} +\tl_map_inline:nn { { catcode } { charcode } { meaning } } + { + \tl_map_inline:nn { { } { _remove } } + { + \tl_map_inline:nn { { TF } { T } { F } } + { + \cs_new_protected:cpx { peek_ #1 ##1 :N ####1 } + { + \exp_not:c { @@_token ##1 _generic:NN ####1 } + \exp_not:c { @@_execute_branches_ #1 : } + } + } + } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[TF] +% { +% \peek_catcode_ignore_spaces:N, \peek_catcode_remove_ignore_spaces:N, +% \peek_charcode_ignore_spaces:N, \peek_charcode_remove_ignore_spaces:N, +% \peek_meaning_ignore_spaces:N, \peek_meaning_remove_ignore_spaces:N +% } +% To ignore spaces, remove them using \cs{peek_remove_spaces:n} +% before running the tests. +% \begin{macrocode} +\tl_map_inline:nn + { + { catcode } { catcode_remove } + { charcode } { charcode_remove } + { meaning } { meaning_remove } + } + { + \cs_new_protected:cpx { peek_#1_ignore_spaces:NTF } ##1##2##3 + { + \peek_remove_spaces:n + { \exp_not:c { peek_#1:NTF } ##1 {##2} {##3} } + } + \cs_new_protected:cpx { peek_#1_ignore_spaces:NT } ##1##2 + { + \peek_remove_spaces:n + { \exp_not:c { peek_#1:NT } ##1 {##2} } + } + \cs_new_protected:cpx { peek_#1_ignore_spaces:NF } ##1##2 + { + \peek_remove_spaces:n + { \exp_not:c { peek_#1:NF } ##1 {##2} } + } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[TF]{\peek_N_type:} +% \begin{macro} +% {\@@_execute_branches_N_type:, \@@_N_type:w, \@@_N_type_aux:nnw} +% All tokens are \texttt{N}-type tokens, except in four cases: +% begin-group tokens, end-group tokens, space tokens with character +% code~$32$, and outer tokens. Since \cs{l_peek_token} might be +% outer, we cannot use the convenient \cs{bool_if:nTF} function, and +% must resort to the old trick of using \tn{ifodd} to expand a set of +% tests. The \texttt{false} branch of this test is taken if the token +% is one of the first three kinds of non-\texttt{N}-type tokens +% (explicit or implicit), thus we call \cs{@@_false:w}. In the +% \texttt{true} branch, we must detect outer tokens, without impacting +% performance too much for non-outer tokens. The first filter is to +% search for \texttt{outer} in the \tn{meaning} of \cs{l_peek_token}. +% If that is absent, \cs{use_none_delimit_by_q_stop:w} cleans up, and +% we call \cs{@@_true:w}. Otherwise, the token can be a non-outer +% macro or a primitive mark whose parameter or replacement text +% contains \texttt{outer}, it can be the primitive \tn{outer}, or it +% can be an outer token. Macros and marks would have \texttt{ma} in +% the part before the first occurrence of \texttt{outer}; the meaning +% of \tn{outer} has nothing after \texttt{outer}, contrarily to outer +% macros; and that covers all cases, calling \cs{@@_true:w} or +% \cs{@@_false:w} as appropriate. Here, there is no \meta{search +% token}, so we feed a dummy \cs{scan_stop:} to the +% \cs{@@_token_generic:NNTF} function. +% \begin{macrocode} +\group_begin: + \cs_set_protected:Npn \@@_tmp:w #1 \q_stop + { + \cs_new_protected:Npn \@@_execute_branches_N_type: + { + \if_int_odd:w + \if_catcode:w \exp_not:N \l_peek_token { 0 \exp_stop_f: \fi: + \if_catcode:w \exp_not:N \l_peek_token } 0 \exp_stop_f: \fi: + \if_meaning:w \l_peek_token \c_space_token 0 \exp_stop_f: \fi: + 1 \exp_stop_f: + \exp_after:wN \@@_N_type:w + \token_to_meaning:N \l_peek_token + \q_mark \@@_N_type_aux:nnw + #1 \q_mark \use_none_delimit_by_q_stop:w + \q_stop + \exp_after:wN \@@_true:w + \else: + \exp_after:wN \@@_false:w + \fi: + } + \cs_new_protected:Npn \@@_N_type:w ##1 #1 ##2 \q_mark ##3 + { ##3 {##1} {##2} } + } + \exp_after:wN \@@_tmp:w \tl_to_str:n { outer } \q_stop +\group_end: +\cs_new_protected:Npn \@@_N_type_aux:nnw #1 #2 #3 \fi: + { + \fi: + \tl_if_in:noTF {#1} { \tl_to_str:n {ma} } + { \@@_true:w } + { \tl_if_empty:nTF {#2} { \@@_true:w } { \@@_false:w } } + } +\cs_new_protected:Npn \peek_N_type:TF + { + \@@_token_generic:NNTF + \@@_execute_branches_N_type: \scan_stop: + } +\cs_new_protected:Npn \peek_N_type:T + { \@@_token_generic:NNT \@@_execute_branches_N_type: \scan_stop: } +\cs_new_protected:Npn \peek_N_type:F + { \@@_token_generic:NNF \@@_execute_branches_N_type: \scan_stop: } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macrocode} +%</initex|package> +% \end{macrocode} +% +% \end{implementation} +% +% \PrintIndex |