summaryrefslogtreecommitdiff
path: root/macros/latex/contrib/l3kernel/l3token.dtx
diff options
context:
space:
mode:
authorNorbert Preining <norbert@preining.info>2019-09-02 13:46:59 +0900
committerNorbert Preining <norbert@preining.info>2019-09-02 13:46:59 +0900
commite0c6872cf40896c7be36b11dcc744620f10adf1d (patch)
tree60335e10d2f4354b0674ec22d7b53f0f8abee672 /macros/latex/contrib/l3kernel/l3token.dtx
Initial commit
Diffstat (limited to 'macros/latex/contrib/l3kernel/l3token.dtx')
-rw-r--r--macros/latex/contrib/l3kernel/l3token.dtx2505
1 files changed, 2505 insertions, 0 deletions
diff --git a/macros/latex/contrib/l3kernel/l3token.dtx b/macros/latex/contrib/l3kernel/l3token.dtx
new file mode 100644
index 0000000000..9b9946a4f8
--- /dev/null
+++ b/macros/latex/contrib/l3kernel/l3token.dtx
@@ -0,0 +1,2505 @@
+% \iffalse meta-comment
+%
+%% File: l3token.dtx
+%
+% Copyright (C) 2005-2019 The LaTeX3 Project
+%
+% It may be distributed and/or modified under the conditions of the
+% LaTeX Project Public License (LPPL), either version 1.3c of this
+% license or (at your option) any later version. The latest version
+% of this license is in the file
+%
+% https://www.latex-project.org/lppl.txt
+%
+% This file is part of the "l3kernel bundle" (The Work in LPPL)
+% and all files in that bundle must be distributed together.
+%
+% -----------------------------------------------------------------------
+%
+% The development version of the bundle can be found at
+%
+% https://github.com/latex3/latex3
+%
+% for those people who are interested.
+%
+%<*driver>
+\documentclass[full,kernel]{l3doc}
+\begin{document}
+ \DocInput{\jobname.dtx}
+\end{document}
+%</driver>
+% \fi
+%
+% \title{^^A
+% The \pkg{l3token} package\\ Token manipulation^^A
+% }
+%
+% \author{^^A
+% The \LaTeX3 Project\thanks
+% {^^A
+% E-mail:
+% \href{mailto:latex-team@latex-project.org}
+% {latex-team@latex-project.org}^^A
+% }^^A
+% }
+%
+% \date{Released 2019-08-25}
+%
+% \maketitle
+%
+% \begin{documentation}
+%
+% This module deals with tokens. Now this is perhaps not the most
+% precise description so let's try with a better description: When
+% programming in \TeX{}, it is often desirable to know just what a
+% certain token is: is it a control sequence or something
+% else. Similarly one often needs to know if a control sequence is
+% expandable or not, a macro or a primitive, how many arguments it
+% takes etc. Another thing of great importance (especially when it
+% comes to document commands) is looking ahead in the token stream to
+% see if a certain character is present and maybe even remove it or
+% disregard other tokens while scanning. This module provides
+% functions for both and as such has two primary function
+% categories: |\token_| for anything that deals with tokens and
+% |\peek_| for looking ahead in the token stream.
+%
+% Most functions we describe here can be used on control sequences,
+% as those are tokens as well.
+%
+% It is important to distinguish two aspects of a token: its
+% \enquote{shape} (for lack of a better word), which affects the
+% matching of delimited arguments and the comparison of token lists
+% containing this token, and its \enquote{meaning}, which affects
+% whether the token expands or what operation it performs. One can have
+% tokens of different shapes with the same meaning, but not the
+% converse.
+%
+% For instance, \cs{if:w}, \cs{if_charcode:w}, and \cs{tex_if:D} are
+% three names for the same internal operation of \TeX{}, namely the
+% primitive testing the next two characters for equality of their
+% character code. They have the same meaning hence behave identically
+% in many situations. However, \TeX{} distinguishes them when searching
+% for a delimited argument. Namely, the example function
+% |\show_until_if:w| defined below takes everything until \cs{if:w}
+% as an argument, despite the presence of other copies of \cs{if:w}
+% under different names.
+% \begin{verbatim}
+% \cs_new:Npn \show_until_if:w #1 \if:w { \tl_show:n {#1} }
+% \show_until_if:w \tex_if:D \if_charcode:w \if:w
+% \end{verbatim}
+% A list of all possible shapes and a list of all possible meanings are
+% given in section~\ref{sec:l3token:all-tokens}.
+%
+% \section{Creating character tokens}
+%
+% \begin{function}[updated = 2015-11-12]
+% {
+% \char_set_active_eq:NN, \char_set_active_eq:Nc,
+% \char_gset_active_eq:NN, \char_gset_active_eq:Nc
+% }
+% \begin{syntax}
+% \cs{char_set_active_eq:NN} \meta{char} \meta{function}
+% \end{syntax}
+% Sets the behaviour of the \meta{char} in situations where it is
+% active (category code $13$) to be equivalent to that of the
+% \meta{function}. The category code of the \meta{char} is
+% \emph{unchanged} by this process. The \meta{function} may itself
+% be an active character.
+% \end{function}
+%
+% \begin{function}[added = 2015-11-12]
+% {
+% \char_set_active_eq:nN, \char_set_active_eq:nc,
+% \char_gset_active_eq:nN, \char_gset_active_eq:nc
+% }
+% \begin{syntax}
+% \cs{char_set_active_eq:nN} \Arg{integer expression} \meta{function}
+% \end{syntax}
+% Sets the behaviour of the \meta{char} which has character
+% code as given by the \meta{integer expression} in situations
+% where it is active (category code $13$) to be equivalent to that of the
+% \meta{function}. The category code of the \meta{char} is
+% \emph{unchanged} by this process. The \meta{function} may itself
+% be an active character.
+% \end{function}
+%
+% \begin{function}[EXP, added = 2015-09-09, updated = 2019-01-16]
+% {\char_generate:nn}
+% \begin{syntax}
+% \cs{char_generate:nn} \Arg{charcode} \Arg{catcode}
+% \end{syntax}
+% Generates a character token of the given \meta{charcode} and \meta{catcode}
+% (both of which may be integer expressions). The \meta{catcode} may be
+% one of
+% \begin{itemize}
+% \item $1$ (begin group)
+% \item $2$ (end group)
+% \item $3$ (math toggle)
+% \item $4$ (alignment)
+% \item $6$ (parameter)
+% \item $7$ (math superscript)
+% \item $8$ (math subscript)
+% \item $11$ (letter)
+% \item $12$ (other)
+% \item $13$ (active)
+% \end{itemize}
+% and other values raise an error. The \meta{charcode} may be any one valid
+% for the engine in use.
+% Active characters cannot be generated in older versions of \XeTeX{}.
+% \begin{texnote}
+% Exactly two expansions are needed to produce the character.
+% \end{texnote}
+% \end{function}
+%
+% \begin{variable}[added = 2011-09-05]{\c_catcode_other_space_tl}
+% Token list containing one character with category code $12$,
+% (\enquote{other}), and character code $32$ (space).
+% \end{variable}
+%
+% \section{Manipulating and interrogating character tokens}
+%
+% \begin{function}[updated = 2015-11-11]
+% {
+% \char_set_catcode_escape:N ,
+% \char_set_catcode_group_begin:N ,
+% \char_set_catcode_group_end:N ,
+% \char_set_catcode_math_toggle:N ,
+% \char_set_catcode_alignment:N ,
+% \char_set_catcode_end_line:N ,
+% \char_set_catcode_parameter:N ,
+% \char_set_catcode_math_superscript:N ,
+% \char_set_catcode_math_subscript:N ,
+% \char_set_catcode_ignore:N ,
+% \char_set_catcode_space:N ,
+% \char_set_catcode_letter:N ,
+% \char_set_catcode_other:N ,
+% \char_set_catcode_active:N ,
+% \char_set_catcode_comment:N ,
+% \char_set_catcode_invalid:N
+% }
+% \begin{syntax}
+% \cs{char_set_catcode_letter:N} \meta{character}
+% \end{syntax}
+% Sets the category code of the \meta{character} to that indicated in
+% the function name. Depending on the current category code of the
+% \meta{token} the escape token may also be needed:
+% \begin{verbatim}
+% \char_set_catcode_other:N \%
+% \end{verbatim}
+% The assignment is local.
+% \end{function}
+%
+% \begin{function}[updated = 2015-11-11]
+% {
+% \char_set_catcode_escape:n ,
+% \char_set_catcode_group_begin:n ,
+% \char_set_catcode_group_end:n ,
+% \char_set_catcode_math_toggle:n ,
+% \char_set_catcode_alignment:n ,
+% \char_set_catcode_end_line:n ,
+% \char_set_catcode_parameter:n ,
+% \char_set_catcode_math_superscript:n ,
+% \char_set_catcode_math_subscript:n ,
+% \char_set_catcode_ignore:n ,
+% \char_set_catcode_space:n ,
+% \char_set_catcode_letter:n ,
+% \char_set_catcode_other:n ,
+% \char_set_catcode_active:n ,
+% \char_set_catcode_comment:n ,
+% \char_set_catcode_invalid:n
+% }
+% \begin{syntax}
+% \cs{char_set_catcode_letter:n} \Arg{integer expression}
+% \end{syntax}
+% Sets the category code of the \meta{character} which has character
+% code as given by the \meta{integer expression}. This version can be
+% used to set up characters which cannot otherwise be given
+% (\emph{cf.}~the \texttt{N}-type variants). The assignment is local.
+% \end{function}
+%
+% \begin{function}[updated = 2015-11-11]{\char_set_catcode:nn}
+% \begin{syntax}
+% \cs{char_set_catcode:nn} \Arg{intexpr_1} \Arg{intexpr_2}
+% \end{syntax}
+% These functions set the category code of the \meta{character} which
+% has character code as given by the \meta{integer expression}.
+% The first \meta{integer expression}
+% is the character code and the second is the category code to apply.
+% The setting applies within the current \TeX{} group. In general, the
+% symbolic functions \cs[no-index]{char_set_catcode_\meta{type}} should be preferred,
+% but there are cases where these lower-level functions may be useful.
+% \end{function}
+%
+% \begin{function}[EXP]{\char_value_catcode:n}
+% \begin{syntax}
+% \cs{char_value_catcode:n} \Arg{integer expression}
+% \end{syntax}
+% Expands to the current category code of the \meta{character} with
+% character code given by the
+% \meta{integer expression}.
+% \end{function}
+%
+% \begin{function}{\char_show_value_catcode:n}
+% \begin{syntax}
+% \cs{char_show_value_catcode:n} \Arg{integer expression}
+% \end{syntax}
+% Displays the current category code of the \meta{character} with
+% character code given by the \meta{integer expression} on the
+% terminal.
+% \end{function}
+%
+% \begin{function}[updated = 2015-08-06]{\char_set_lccode:nn}
+% \begin{syntax}
+% \cs{char_set_lccode:nn} \Arg{intexpr_1} \Arg{intexpr_2}
+% \end{syntax}
+% Sets up the behaviour of the \meta{character} when
+% found inside \cs{tl_lower_case:n}, such that \meta{character_1}
+% will be converted into \meta{character_2}. The two \meta{characters}
+% may be specified using an \meta{integer expression} for the character code
+% concerned. This may include the \TeX{} |`|\meta{character}
+% method for converting a single character into its character
+% code:
+% \begin{verbatim}
+% \char_set_lccode:nn { `\A } { `\a } % Standard behaviour
+% \char_set_lccode:nn { `\A } { `\A + 32 }
+% \char_set_lccode:nn { 50 } { 60 }
+% \end{verbatim}
+% The setting applies within the current \TeX{} group.
+% \end{function}
+%
+% \begin{function}[EXP]{\char_value_lccode:n}
+% \begin{syntax}
+% \cs{char_value_lccode:n} \Arg{integer expression}
+% \end{syntax}
+% Expands to the current lower case code of the \meta{character} with
+% character code given by the
+% \meta{integer expression}.
+% \end{function}
+%
+% \begin{function}{\char_show_value_lccode:n}
+% \begin{syntax}
+% \cs{char_show_value_lccode:n} \Arg{integer expression}
+% \end{syntax}
+% Displays the current lower case code of the \meta{character} with
+% character code given by the \meta{integer expression} on the
+% terminal.
+% \end{function}
+%
+% \begin{function}[updated = 2015-08-06]{\char_set_uccode:nn}
+% \begin{syntax}
+% \cs{char_set_uccode:nn} \Arg{intexpr_1} \Arg{intexpr_2}
+% \end{syntax}
+% Sets up the behaviour of the \meta{character} when
+% found inside \cs{tl_upper_case:n}, such that \meta{character_1}
+% will be converted into \meta{character_2}. The two \meta{characters}
+% may be specified using an \meta{integer expression} for the character code
+% concerned. This may include the \TeX{} |`|\meta{character}
+% method for converting a single character into its character
+% code:
+% \begin{verbatim}
+% \char_set_uccode:nn { `\a } { `\A } % Standard behaviour
+% \char_set_uccode:nn { `\A } { `\A - 32 }
+% \char_set_uccode:nn { 60 } { 50 }
+% \end{verbatim}
+% The setting applies within the current \TeX{} group.
+% \end{function}
+%
+% \begin{function}[EXP]{\char_value_uccode:n}
+% \begin{syntax}
+% \cs{char_value_uccode:n} \Arg{integer expression}
+% \end{syntax}
+% Expands to the current upper case code of the \meta{character} with
+% character code given by the
+% \meta{integer expression}.
+% \end{function}
+%
+% \begin{function}{\char_show_value_uccode:n}
+% \begin{syntax}
+% \cs{char_show_value_uccode:n} \Arg{integer expression}
+% \end{syntax}
+% Displays the current upper case code of the \meta{character} with
+% character code given by the \meta{integer expression} on the
+% terminal.
+% \end{function}
+%
+% \begin{function}[updated = 2015-08-06]{\char_set_mathcode:nn}
+% \begin{syntax}
+% \cs{char_set_mathcode:nn} \Arg{intexpr_1} \Arg{intexpr_2}
+% \end{syntax}
+% This function sets up the math code of \meta{character}.
+% The \meta{character} is specified as
+% an \meta{integer expression} which will be used as the character
+% code of the relevant character. The setting applies within the
+% current \TeX{} group.
+% \end{function}
+%
+% \begin{function}[EXP]{\char_value_mathcode:n}
+% \begin{syntax}
+% \cs{char_value_mathcode:n} \Arg{integer expression}
+% \end{syntax}
+% Expands to the current math code of the \meta{character} with
+% character code given by the
+% \meta{integer expression}.
+% \end{function}
+%
+% \begin{function}{\char_show_value_mathcode:n}
+% \begin{syntax}
+% \cs{char_show_value_mathcode:n} \Arg{integer expression}
+% \end{syntax}
+% Displays the current math code of the \meta{character} with
+% character code given by the \meta{integer expression} on the
+% terminal.
+% \end{function}
+%
+% \begin{function}[updated = 2015-08-06]{\char_set_sfcode:nn}
+% \begin{syntax}
+% \cs{char_set_sfcode:nn} \Arg{intexpr_1} \Arg{intexpr_2}
+% \end{syntax}
+% This function sets up the space factor for the \meta{character}.
+% The \meta{character} is specified as
+% an \meta{integer expression} which will be used as the character
+% code of the relevant character. The setting applies within the
+% current \TeX{} group.
+% \end{function}
+%
+% \begin{function}[EXP]{\char_value_sfcode:n}
+% \begin{syntax}
+% \cs{char_value_sfcode:n} \Arg{integer expression}
+% \end{syntax}
+% Expands to the current space factor for the \meta{character} with
+% character code given by the
+% \meta{integer expression}.
+% \end{function}
+%
+% \begin{function}{\char_show_value_sfcode:n}
+% \begin{syntax}
+% \cs{char_show_value_sfcode:n} \Arg{integer expression}
+% \end{syntax}
+% Displays the current space factor for the \meta{character} with
+% character code given by the \meta{integer expression} on the
+% terminal.
+% \end{function}
+%
+% \begin{variable}[added = 2012-01-23, updated = 2015-11-11]{\l_char_active_seq}
+% Used to track which tokens may require special handling at the document
+% level as they are (or have been at some point)
+% of category \meta{active} (catcode~$13$). Each entry in
+% the sequence consists of a single escaped token, for example |\~|.
+% Active tokens should be added to the sequence when they are defined for
+% general document use.
+% \end{variable}
+%
+% \begin{variable}[added = 2012-01-23, updated = 2015-11-11]{\l_char_special_seq}
+% Used to track which tokens will require special handling when working with
+% verbatim-like material at the document level as they are not of categories
+% \meta{letter} (catcode~$11$) or \meta{other} (catcode~$12$). Each entry in
+% the sequence consists of a single escaped token, for example |\\| for the
+% backslash or |\{| for an opening brace.^^A \}
+% Escaped tokens should be added to the sequence when they are defined for
+% general document use.
+% \end{variable}
+%
+% \section{Generic tokens}
+%
+% \begin{variable}
+% {
+% \c_group_begin_token,
+% \c_group_end_token,
+% \c_math_toggle_token,
+% \c_alignment_token,
+% \c_parameter_token,
+% \c_math_superscript_token,
+% \c_math_subscript_token,
+% \c_space_token
+% }
+% These are implicit tokens which have the category code described
+% by their name. They are used internally for test purposes but
+% are also available to the programmer for other uses.
+% \end{variable}
+%
+% \begin{variable}
+% {
+% \c_catcode_letter_token,
+% \c_catcode_other_token
+% }
+% These are implicit tokens which have the category code described
+% by their name. They are used internally for test purposes and should
+% not be used other than for category code tests.
+% \end{variable}
+%
+% \begin{variable}{\c_catcode_active_tl}
+% A token list containing an active token. This is used internally
+% for test purposes and should not be used other than in
+% appropriately-constructed category code tests.
+% \end{variable}
+%
+% \section{Converting tokens}
+%
+% \begin{function}[EXP]{\token_to_meaning:N, \token_to_meaning:c}
+% \begin{syntax}
+% \cs{token_to_meaning:N} \meta{token}
+% \end{syntax}
+% Inserts the current meaning of the \meta{token} into the input
+% stream as a series of characters of category code $12$ (other).
+% This is the primitive \TeX{} description of the \meta{token},
+% thus for example both functions defined by \cs{cs_set_nopar:Npn}
+% and token list variables defined using \cs{tl_new:N} are described
+% as |macro|s.
+% \begin{texnote}
+% This is the \TeX{} primitive \tn{meaning}.
+% The \meta{token} can thus be an explicit space tokens or an
+% explicit begin-group or end-group character token (|{|~or~|}| when
+% normal \TeX{} category codes apply) even though these are not
+% valid \texttt{N}-type arguments.
+% \end{texnote}
+% \end{function}
+%
+% \begin{function}[EXP]{\token_to_str:N, \token_to_str:c}
+% \begin{syntax}
+% \cs{token_to_str:N} \meta{token}
+% \end{syntax}
+% Converts the given \meta{token} into a series of characters with
+% category code $12$ (other). If the \meta{token} is a control
+% sequence, this will start with the current escape character with
+% category code $12$ (the escape character is part of the
+% \meta{token}). This function requires only a single expansion.
+% \begin{texnote}
+% \cs{token_to_str:N} is the \TeX{} primitive \tn{string} renamed.
+% The \meta{token} can thus be an explicit space tokens or an
+% explicit begin-group or end-group character token (|{|~or~|}| when
+% normal \TeX{} category codes apply) even though these are not
+% valid \texttt{N}-type arguments.
+% \end{texnote}
+% \end{function}
+%
+% \section{Token conditionals}
+%
+% \begin{function}[EXP,pTF]{\token_if_group_begin:N}
+% \begin{syntax}
+% \cs{token_if_group_begin_p:N} \meta{token} \\
+% \cs{token_if_group_begin:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of a begin group token
+% (|{| when normal \TeX{} category codes are in ^^A }
+% force).
+% Note that an explicit begin group token cannot be tested in this way,
+% as it is not a valid \texttt{N}-type argument.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_group_end:N}
+% \begin{syntax}
+% \cs{token_if_group_end_p:N} \meta{token} \\
+% \cs{token_if_group_end:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of an end group token
+% (^^A {
+% |}| when normal \TeX{} category codes are in force).
+% Note that an explicit end group token cannot be tested in this way,
+% as it is not a valid \texttt{N}-type argument.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_math_toggle:N}
+% \begin{syntax}
+% \cs{token_if_math_toggle_p:N} \meta{token} \\
+% \cs{token_if_math_toggle:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of a math shift token
+% (|$| when normal \TeX{} category codes are in force).
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_alignment:N}
+% \begin{syntax}
+% \cs{token_if_alignment_p:N} \meta{token} \\
+% \cs{token_if_alignment:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of an alignment token
+% (|&| when normal \TeX{} category codes are in force).
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_parameter:N}
+% \begin{syntax}
+% \cs{token_if_parameter_p:N} \meta{token} \\
+% \cs{token_if_alignment:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of a macro parameter token
+% (|#| when normal \TeX{} category codes are in force).
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_math_superscript:N}
+% \begin{syntax}
+% \cs{token_if_math_superscript_p:N} \meta{token} \\
+% \cs{token_if_math_superscript:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of a superscript token
+% (|^| when normal \TeX{} category codes are in force).
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_math_subscript:N}
+% \begin{syntax}
+% \cs{token_if_math_subscript_p:N} \meta{token} \\
+% \cs{token_if_math_subscript:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of a subscript token
+% (|_| when normal \TeX{} category codes are in force).
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_space:N}
+% \begin{syntax}
+% \cs{token_if_space_p:N} \meta{token} \\
+% \cs{token_if_space:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of a space token.
+% Note that an explicit space token with character code $32$ cannot
+% be tested in this way, as it is not a valid \texttt{N}-type argument.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_letter:N}
+% \begin{syntax}
+% \cs{token_if_letter_p:N} \meta{token} \\
+% \cs{token_if_letter:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of a letter token.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_other:N}
+% \begin{syntax}
+% \cs{token_if_other_p:N} \meta{token} \\
+% \cs{token_if_other:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of an \enquote{other}
+% token.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_active:N}
+% \begin{syntax}
+% \cs{token_if_active_p:N} \meta{token} \\
+% \cs{token_if_active:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of an active character.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_eq_catcode:NN}
+% \begin{syntax}
+% \cs{token_if_eq_catcode_p:NN} \meta{token_1} \meta{token_2} \\
+% \cs{token_if_eq_catcode:NNTF} \meta{token_1} \meta{token_2} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the two \meta{tokens} have the same category code.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_eq_charcode:NN}
+% \begin{syntax}
+% \cs{token_if_eq_charcode_p:NN} \meta{token_1} \meta{token_2} \\
+% \cs{token_if_eq_charcode:NNTF} \meta{token_1} \meta{token_2} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the two \meta{tokens} have the same character code.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_eq_meaning:NN}
+% \begin{syntax}
+% \cs{token_if_eq_meaning_p:NN} \meta{token_1} \meta{token_2} \\
+% \cs{token_if_eq_meaning:NNTF} \meta{token_1} \meta{token_2} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the two \meta{tokens} have the same meaning when expanded.
+% \end{function}
+%
+% \begin{function}[updated = 2011-05-23, EXP,pTF]{\token_if_macro:N}
+% \begin{syntax}
+% \cs{token_if_macro_p:N} \meta{token} \\
+% \cs{token_if_macro:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is a \TeX{} macro.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_cs:N}
+% \begin{syntax}
+% \cs{token_if_cs_p:N} \meta{token} \\
+% \cs{token_if_cs:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is a control sequence.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_expandable:N}
+% \begin{syntax}
+% \cs{token_if_expandable_p:N} \meta{token} \\
+% \cs{token_if_expandable:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is expandable. This test returns \meta{false}
+% for an undefined token.
+% \end{function}
+%
+% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_long_macro:N}
+% \begin{syntax}
+% \cs{token_if_long_macro_p:N} \meta{token} \\
+% \cs{token_if_long_macro:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is a long macro.
+% \end{function}
+%
+% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_protected_macro:N}
+% \begin{syntax}
+% \cs{token_if_protected_macro_p:N} \meta{token} \\
+% \cs{token_if_protected_macro:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is a protected macro: for a macro which
+% is both protected and long this returns \texttt{false}.
+% \end{function}
+%
+% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_protected_long_macro:N}
+% \begin{syntax}
+% \cs{token_if_protected_long_macro_p:N} \meta{token} \\
+% \cs{token_if_protected_long_macro:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is a protected long macro.
+% \end{function}
+%
+% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_chardef:N}
+% \begin{syntax}
+% \cs{token_if_chardef_p:N} \meta{token} \\
+% \cs{token_if_chardef:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is defined to be a chardef.
+% \begin{texnote}
+% Booleans, boxes and small integer constants are implemented as
+% \tn{chardef}s.
+% \end{texnote}
+% \end{function}
+%
+% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_mathchardef:N}
+% \begin{syntax}
+% \cs{token_if_mathchardef_p:N} \meta{token} \\
+% \cs{token_if_mathchardef:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is defined to be a mathchardef.
+% \end{function}
+%
+% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_dim_register:N}
+% \begin{syntax}
+% \cs{token_if_dim_register_p:N} \meta{token} \\
+% \cs{token_if_dim_register:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is defined to be a dimension register.
+% \end{function}
+%
+% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_int_register:N}
+% \begin{syntax}
+% \cs{token_if_int_register_p:N} \meta{token} \\
+% \cs{token_if_int_register:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is defined to be a integer register.
+% \begin{texnote}
+% Constant integers may be implemented as integer registers,
+% \tn{chardef}s, or \tn{mathchardef}s depending on their value.
+% \end{texnote}
+% \end{function}
+%
+% \begin{function}[EXP,pTF, added=2012-02-15]{\token_if_muskip_register:N}
+% \begin{syntax}
+% \cs{token_if_muskip_register_p:N} \meta{token} \\
+% \cs{token_if_muskip_register:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is defined to be a muskip register.
+% \end{function}
+%
+% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_skip_register:N}
+% \begin{syntax}
+% \cs{token_if_skip_register_p:N} \meta{token} \\
+% \cs{token_if_skip_register:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is defined to be a skip register.
+% \end{function}
+%
+% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_toks_register:N}
+% \begin{syntax}
+% \cs{token_if_toks_register_p:N} \meta{token} \\
+% \cs{token_if_toks_register:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is defined to be a toks register
+% (not used by \LaTeX3).
+% \end{function}
+%
+% \begin{function}[updated = 2011-05-23, EXP,pTF]{\token_if_primitive:N}
+% \begin{syntax}
+% \cs{token_if_primitive_p:N} \meta{token} \\
+% \cs{token_if_primitive:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is an engine primitive.
+% \end{function}
+%
+% \section{Peeking ahead at the next token}
+%
+% There is often a need to look ahead at the next token in the input
+% stream while leaving it in place. This is handled using the
+% \enquote{peek} functions. The generic \cs{peek_after:Nw} is
+% provided along with a family of predefined tests for common cases.
+% As peeking ahead does \emph{not} skip spaces the predefined tests
+% include both a space-respecting and space-skipping version.
+%
+% \begin{function}{\peek_after:Nw}
+% \begin{syntax}
+% \cs{peek_after:Nw} \meta{function} \meta{token}
+% \end{syntax}
+% Locally sets the test variable \cs{l_peek_token} equal to \meta{token}
+% (as an implicit token, \emph{not} as a token list), and then
+% expands the \meta{function}. The \meta{token} remains in
+% the input stream as the next item after the \meta{function}.
+% The \meta{token} here may be \verb*| |, |{| or |}| (assuming
+% normal \TeX{} category codes), \emph{i.e.}~it is not necessarily the
+% next argument which would be grabbed by a normal function.
+% \end{function}
+%
+% \begin{function}{\peek_gafter:Nw}
+% \begin{syntax}
+% \cs{peek_gafter:Nw} \meta{function} \meta{token}
+% \end{syntax}
+% Globally sets the test variable \cs{g_peek_token} equal to \meta{token}
+% (as an implicit token, \emph{not} as a token list), and then
+% expands the \meta{function}. The \meta{token} remains in
+% the input stream as the next item after the \meta{function}.
+% The \meta{token} here may be \verb*| |, |{| or |}| (assuming
+% normal \TeX{} category codes), \emph{i.e.}~it is not necessarily the
+% next argument which would be grabbed by a normal function.
+% \end{function}
+%
+% \begin{variable}{\l_peek_token}
+% Token set by \cs{peek_after:Nw} and available for testing
+% as described above.
+% \end{variable}
+%
+% \begin{variable}{\g_peek_token}
+% Token set by \cs{peek_gafter:Nw} and available for testing
+% as described above.
+% \end{variable}
+%
+% \begin{function}[updated = 2012-12-20, TF]{\peek_catcode:N}
+% \begin{syntax}
+% \cs{peek_catcode:NTF} \meta{test token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next \meta{token} in the input stream has the same
+% category code as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_catcode:NNTF}). Spaces are respected by the test
+% and the \meta{token} is left in the input stream after
+% the \meta{true code} or \meta{false code} (as appropriate to the
+% result of the test).
+% \end{function}
+%
+% \begin{function}[updated = 2012-12-20, TF]{\peek_catcode_ignore_spaces:N}
+% \begin{syntax}
+% \cs{peek_catcode_ignore_spaces:NTF} \meta{test token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next non-space \meta{token} in the input stream has the
+% same category code as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_catcode:NNTF}). Explicit and implicit space tokens
+% (with character code 32 and category code 10) are ignored and
+% removed by the test and the \meta{token} is left in the input
+% stream after the \meta{true code} or \meta{false code} (as
+% appropriate to the result of the test).
+% \end{function}
+%
+% \begin{function}[updated = 2012-12-20, TF]{\peek_catcode_remove:N}
+% \begin{syntax}
+% \cs{peek_catcode_remove:NTF} \meta{test token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next \meta{token} in the input stream has the same
+% category code as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_catcode:NNTF}). Spaces are respected by the test
+% and the \meta{token} is removed from the input stream if the
+% test is true. The function then places either the
+% \meta{true code} or \meta{false code} in the input stream (as
+% appropriate to the result of the test).
+% \end{function}
+%
+% \begin{function}[updated = 2012-12-20, TF]
+% {\peek_catcode_remove_ignore_spaces:N}
+% \begin{syntax}
+% \cs{peek_catcode_remove_ignore_spaces:NTF} \meta{test token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next non-space \meta{token} in the input stream has the
+% same category code as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_catcode:NNTF}). Explicit and implicit space tokens
+% (with character code 32 and category code 10) are ignored and
+% removed by the test and the \meta{token} is removed from the
+% input stream if the test is true. The function then places
+% either the \meta{true code} or \meta{false code} in the input stream
+% (as appropriate to the result of the test).
+% \end{function}
+%
+% \begin{function}[updated = 2012-12-20, TF]{\peek_charcode:N}
+% \begin{syntax}
+% \cs{peek_charcode:NTF} \meta{test token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next \meta{token} in the input stream has the same
+% character code as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_charcode:NNTF}). Spaces are respected by the test
+% and the \meta{token} is left in the input stream after
+% the \meta{true code} or \meta{false code} (as appropriate to the
+% result of the test).
+% \end{function}
+%
+% \begin{function}[updated = 2012-12-20, TF]{\peek_charcode_ignore_spaces:N}
+% \begin{syntax}
+% \cs{peek_charcode_ignore_spaces:NTF} \meta{test token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next non-space \meta{token} in the input stream has the
+% same character code as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_charcode:NNTF}). Explicit and implicit space tokens
+% (with character code 32 and category code 10) are ignored and removed by
+% the test and the \meta{token} is left in the input stream after
+% the \meta{true code} or \meta{false code} (as appropriate to the
+% result of the test).
+% \end{function}
+%
+% \begin{function}[updated = 2012-12-20, TF]{\peek_charcode_remove:N}
+% \begin{syntax}
+% \cs{peek_charcode_remove:NTF} \meta{test token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next \meta{token} in the input stream has the same
+% character code as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_charcode:NNTF}). Spaces are respected by the test
+% and the \meta{token} is removed from the input stream if the
+% test is true. The function then places either the
+% \meta{true code} or \meta{false code} in the input stream (as
+% appropriate to the result of the test).
+% \end{function}
+%
+% \begin{function}[updated = 2012-12-20, TF]
+% {\peek_charcode_remove_ignore_spaces:N}
+% \begin{syntax}
+% \cs{peek_charcode_remove_ignore_spaces:NTF} \meta{test token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next non-space \meta{token} in the input stream has the
+% same character code as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_charcode:NNTF}). Explicit and implicit space tokens
+% (with character code 32 and category code 10) are ignored and
+% removed by the test and the \meta{token} is removed from the
+% input stream if the test is true. The function then places
+% either the \meta{true code} or \meta{false code} in the input stream
+% (as appropriate to the result of the test).
+% \end{function}
+%
+% \begin{function}[updated = 2011-07-02, TF]{\peek_meaning:N}
+% \begin{syntax}
+% \cs{peek_meaning:NTF} \meta{test token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next \meta{token} in the input stream has the same
+% meaning as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_meaning:NNTF}). Spaces are respected by the test
+% and the \meta{token} is left in the input stream after
+% the \meta{true code} or \meta{false code} (as appropriate to the
+% result of the test).
+% \end{function}
+%
+% \begin{function}[updated = 2012-12-05, TF]{\peek_meaning_ignore_spaces:N}
+% \begin{syntax}
+% \cs{peek_meaning_ignore_spaces:NTF} \meta{test token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next non-space \meta{token} in the input stream has the
+% same meaning as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_meaning:NNTF}). Explicit and implicit space tokens
+% (with character code 32 and category code 10) are ignored and
+% removed by the test and the \meta{token} is left in the input
+% stream after the \meta{true code} or \meta{false code} (as
+% appropriate to the result of the test).
+% \end{function}
+%
+% \begin{function}[updated = 2011-07-02, TF]{\peek_meaning_remove:N}
+% \begin{syntax}
+% \cs{peek_meaning_remove:NTF} \meta{test token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next \meta{token} in the input stream has the same
+% meaning as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_meaning:NNTF}). Spaces are respected by the test
+% and the \meta{token} is removed from the input stream if the
+% test is true. The function then places either the
+% \meta{true code} or \meta{false code} in the input stream (as
+% appropriate to the result of the test).
+% \end{function}
+%
+% \begin{function}[updated = 2012-12-05, TF]
+% {\peek_meaning_remove_ignore_spaces:N}
+% \begin{syntax}
+% \cs{peek_meaning_remove_ignore_spaces:NTF} \meta{test token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next non-space \meta{token} in the input stream has the
+% same meaning as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_meaning:NNTF}). Explicit and implicit space tokens
+% (with character code 32 and category code 10) are ignored and
+% removed by the test and the \meta{token} is removed from the
+% input stream if the test is true. The function then places
+% either the \meta{true code} or \meta{false code} in the input stream
+% (as appropriate to the result of the test).
+% \end{function}
+%
+% \begin{function}[TF, updated = 2012-12-20]{\peek_N_type:}
+% \begin{syntax}
+% \cs{peek_N_type:TF} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next \meta{token} in the input stream can be safely
+% grabbed as an \texttt{N}-type argument. The test is \meta{false}
+% if the next \meta{token} is either an explicit or implicit
+% begin-group or end-group token (with any character code), or
+% an explicit or implicit space character (with character code $32$
+% and category code $10$), or an outer token (never used in \LaTeX3)
+% and \meta{true} in all other cases.
+% Note that a \meta{true} result ensures that the next \meta{token} is
+% a valid \texttt{N}-type argument. However, if the next \meta{token}
+% is for instance \cs{c_space_token}, the test takes the
+% \meta{false} branch, even though the next \meta{token} is in fact
+% a valid \texttt{N}-type argument. The \meta{token} is left
+% in the input stream after the \meta{true code} or \meta{false code}
+% (as appropriate to the result of the test).
+% \end{function}
+%
+% \section{Description of all possible tokens}
+% \label{sec:l3token:all-tokens}
+%
+% Let us end by reviewing every case that a given token can fall into.
+% This section is quite technical and some details are only meant for
+% completeness. We distinguish the meaning of the token, which controls
+% the expansion of the token and its effect on \TeX{}'s state, and its
+% shape, which is used when comparing token lists such as for delimited
+% arguments. Two tokens of the same shape must have the same meaning,
+% but the converse does not hold.
+%
+% A token has one of the following shapes.
+% \begin{itemize}
+% \item A control sequence, characterized by the sequence of
+% characters that constitute its name: for instance, \cs{use:n} is a
+% five-letter control sequence.
+% \item An active character token, characterized by its character code
+% (between $0$ and $1114111$ for \LuaTeX{} and \XeTeX{} and less for
+% other engines) and category code~$13$.
+% \item A character token, characterized by its character code and
+% category code (one of $1$, $2$, $3$, $4$, $6$, $7$, $8$, $10$,
+% $11$ or~$12$ whose meaning is described below).\footnote{In
+% \LuaTeX{}, there is also the case of \enquote{bytes}, which behave as
+% character tokens of category code $12$~(other) and character code
+% between $1114112$ and~$1114366$. They are used to output
+% individual bytes to files, rather than UTF-8.}
+% \end{itemize}
+% There are also a few internal tokens. The following list may be
+% incomplete in some engines.
+% \begin{itemize}
+% \item Expanding \tn{the}\tn{font} results in a token that looks
+% identical to the command that was used to select the current font
+% (such as \tn{tenrm}) but it differs from it in shape.
+% \item A \enquote{frozen} |\relax|, which differs from the primitive in
+% shape (but has the same meaning), is inserted when the closing \tn{fi} of a
+% conditional is encountered before the conditional is evaluated.
+% \item Expanding \tn{noexpand} \meta{token} (when the \meta{token} is
+% expandable) results in an internal token, displayed (temporarily)
+% as \cs{notexpanded: \meta{token}}, whose shape coincides with the
+% \meta{token} and whose meaning differs from \tn{relax}.
+% \item An |\outer endtemplate:| can be encountered when peeking ahead
+% at the next token; this expands to another internal token,
+% |end of alignment template|.
+% \item Tricky programming might access a frozen |\endwrite|.
+% \item Some frozen tokens can only be accessed in interactive
+% sessions: |\cr|, |\right|, |\endgroup|, |\fi|, |\inaccessible|.
+% \end{itemize}
+%
+% The meaning of a (non-active) character token is fixed by its category
+% code (and character code) and cannot be changed. We call these
+% tokens \emph{explicit} character tokens. Category codes that a
+% character token can have are listed below by giving a sample output of
+% the \TeX{} primitive \tn{meaning}, together with their \LaTeX3 names
+% and most common example:
+% \begin{itemize}
+% \item[1] begin-group character (|group_begin|, often |{|),
+% \item[2] end-group character (|group_end|, often |}|),
+% \item[3] math shift character (|math_toggle|, often |$|),
+% \item[4] alignment tab character (|alignment|, often |&|),
+% \item[6] macro parameter character (|parameter|, often |#|),
+% \item[7] superscript character (|math_superscript|, often |^|),
+% \item[8] subscript character (|math_subscript|, often |_|),
+% \item[10] blank space (|space|, often character code~$32$),
+% \item[11] the letter (|letter|, such as |A|),
+% \item[12] the character (|other|, such as |0|).
+% \end{itemize}
+% Category code~$13$ (|active|) is discussed below. Input characters
+% can also have several other category codes which do not lead to
+% character tokens for later processing: $0$~(|escape|),
+% $5$~(|end_line|), $9$~(|ignore|), $14$~(|comment|), and
+% $15$~(|invalid|).
+%
+% The meaning of a control sequence or active character can be identical
+% to that of any character token listed above (with any character code),
+% and we call such tokens \emph{implicit} character tokens. The
+% meaning is otherwise in the following list:
+% \begin{itemize}
+% \item a macro, used in \LaTeX3 for most functions and some variables
+% (|tl|, |fp|, |seq|, \ldots{}),
+% \item a primitive such as \tn{def} or \tn{topmark}, used in \LaTeX3
+% for some functions,
+% \item a register such as \tn{count}|123|, used in \LaTeX3{} for the
+% implementation of some variables (|int|, |dim|, \ldots{}),
+% \item a constant integer such as \tn{char}|"56| or \tn{mathchar}|"121|,
+% \item a font selection command,
+% \item undefined.
+% \end{itemize}
+% Macros be \tn{protected} or not, \tn{long} or not (the opposite of
+% what \LaTeX3 calls |nopar|), and \tn{outer} or not (unused in
+% \LaTeX3). Their \tn{meaning} takes the form
+% \begin{quote}
+% \meta{properties} |macro:|\meta{parameters}|->|\meta{replacement}
+% \end{quote}
+% where \meta{properties} is among \tn{protected}\tn{long}\tn{outer},
+% \meta{parameters} describes parameters that the macro expects, such as
+% |#1#2#3|, and \meta{replacement} describes how the parameters are
+% manipulated, such as~|#2/#1/#3|.
+%
+% ^^A todo Bruno: discuss here some other subtleties of space tokens? when looking for numbers, when looking for equal signs in let, in expressions, etc.
+%
+% Now is perhaps a good time to mention some subtleties relating to
+% tokens with category code $10$ (space). Any input character with this
+% category code (normally, space and tab characters) becomes a normal
+% space, with character code~$32$ and category code~$10$.
+%
+% When a macro takes an undelimited argument, explicit space characters
+% (with character code $32$ and category code $10$) are ignored. If the
+% following token is an explicit character token with category code $1$
+% (begin-group) and an arbitrary character code, then \TeX{} scans ahead
+% to obtain an equal number of explicit character tokens with category
+% code $1$ (begin-group) and $2$ (end-group), and the resulting list of
+% tokens (with outer braces removed) becomes the argument. Otherwise, a
+% single token is taken as the argument for the macro: we call such
+% single tokens \enquote{N-type}, as they are suitable to be used as an
+% argument for a function with the signature~\texttt{:N}.
+%
+% \end{documentation}
+%
+% \begin{implementation}
+%
+% \section{\pkg{l3token} implementation}
+%
+% \begin{macrocode}
+%<*initex|package>
+% \end{macrocode}
+%
+% \begin{macrocode}
+%<@@=char>
+% \end{macrocode}
+%
+% \subsection{Manipulating and interrogating character tokens}
+%
+% \begin{macro}{\char_set_catcode:nn}
+% \begin{macro}{\char_value_catcode:n}
+% \begin{macro}{\char_show_value_catcode:n}
+% Simple wrappers around the primitives.
+% \begin{macrocode}
+\cs_new_protected:Npn \char_set_catcode:nn #1#2
+ { \tex_catcode:D \int_eval:n {#1} = \int_eval:n {#2} \exp_stop_f: }
+\cs_new:Npn \char_value_catcode:n #1
+ { \tex_the:D \tex_catcode:D \int_eval:n {#1} \exp_stop_f: }
+\cs_new_protected:Npn \char_show_value_catcode:n #1
+ { \exp_args:Nf \tl_show:n { \char_value_catcode:n {#1} } }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}
+% {
+% \char_set_catcode_escape:N ,
+% \char_set_catcode_group_begin:N ,
+% \char_set_catcode_group_end:N ,
+% \char_set_catcode_math_toggle:N ,
+% \char_set_catcode_alignment:N ,
+% \char_set_catcode_end_line:N ,
+% \char_set_catcode_parameter:N ,
+% \char_set_catcode_math_superscript:N ,
+% \char_set_catcode_math_subscript:N ,
+% \char_set_catcode_ignore:N ,
+% \char_set_catcode_space:N ,
+% \char_set_catcode_letter:N ,
+% \char_set_catcode_other:N ,
+% \char_set_catcode_active:N ,
+% \char_set_catcode_comment:N ,
+% \char_set_catcode_invalid:N
+% }
+% \begin{macrocode}
+\cs_new_protected:Npn \char_set_catcode_escape:N #1
+ { \char_set_catcode:nn { `#1 } { 0 } }
+\cs_new_protected:Npn \char_set_catcode_group_begin:N #1
+ { \char_set_catcode:nn { `#1 } { 1 } }
+\cs_new_protected:Npn \char_set_catcode_group_end:N #1
+ { \char_set_catcode:nn { `#1 } { 2 } }
+\cs_new_protected:Npn \char_set_catcode_math_toggle:N #1
+ { \char_set_catcode:nn { `#1 } { 3 } }
+\cs_new_protected:Npn \char_set_catcode_alignment:N #1
+ { \char_set_catcode:nn { `#1 } { 4 } }
+\cs_new_protected:Npn \char_set_catcode_end_line:N #1
+ { \char_set_catcode:nn { `#1 } { 5 } }
+\cs_new_protected:Npn \char_set_catcode_parameter:N #1
+ { \char_set_catcode:nn { `#1 } { 6 } }
+\cs_new_protected:Npn \char_set_catcode_math_superscript:N #1
+ { \char_set_catcode:nn { `#1 } { 7 } }
+\cs_new_protected:Npn \char_set_catcode_math_subscript:N #1
+ { \char_set_catcode:nn { `#1 } { 8 } }
+\cs_new_protected:Npn \char_set_catcode_ignore:N #1
+ { \char_set_catcode:nn { `#1 } { 9 } }
+\cs_new_protected:Npn \char_set_catcode_space:N #1
+ { \char_set_catcode:nn { `#1 } { 10 } }
+\cs_new_protected:Npn \char_set_catcode_letter:N #1
+ { \char_set_catcode:nn { `#1 } { 11 } }
+\cs_new_protected:Npn \char_set_catcode_other:N #1
+ { \char_set_catcode:nn { `#1 } { 12 } }
+\cs_new_protected:Npn \char_set_catcode_active:N #1
+ { \char_set_catcode:nn { `#1 } { 13 } }
+\cs_new_protected:Npn \char_set_catcode_comment:N #1
+ { \char_set_catcode:nn { `#1 } { 14 } }
+\cs_new_protected:Npn \char_set_catcode_invalid:N #1
+ { \char_set_catcode:nn { `#1 } { 15 } }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}
+% {
+% \char_set_catcode_escape:n ,
+% \char_set_catcode_group_begin:n ,
+% \char_set_catcode_group_end:n ,
+% \char_set_catcode_math_toggle:n ,
+% \char_set_catcode_alignment:n ,
+% \char_set_catcode_end_line:n ,
+% \char_set_catcode_parameter:n ,
+% \char_set_catcode_math_superscript:n ,
+% \char_set_catcode_math_subscript:n ,
+% \char_set_catcode_ignore:n ,
+% \char_set_catcode_space:n ,
+% \char_set_catcode_letter:n ,
+% \char_set_catcode_other:n ,
+% \char_set_catcode_active:n ,
+% \char_set_catcode_comment:n ,
+% \char_set_catcode_invalid:n
+% }
+% \begin{macrocode}
+\cs_new_protected:Npn \char_set_catcode_escape:n #1
+ { \char_set_catcode:nn {#1} { 0 } }
+\cs_new_protected:Npn \char_set_catcode_group_begin:n #1
+ { \char_set_catcode:nn {#1} { 1 } }
+\cs_new_protected:Npn \char_set_catcode_group_end:n #1
+ { \char_set_catcode:nn {#1} { 2 } }
+\cs_new_protected:Npn \char_set_catcode_math_toggle:n #1
+ { \char_set_catcode:nn {#1} { 3 } }
+\cs_new_protected:Npn \char_set_catcode_alignment:n #1
+ { \char_set_catcode:nn {#1} { 4 } }
+\cs_new_protected:Npn \char_set_catcode_end_line:n #1
+ { \char_set_catcode:nn {#1} { 5 } }
+\cs_new_protected:Npn \char_set_catcode_parameter:n #1
+ { \char_set_catcode:nn {#1} { 6 } }
+\cs_new_protected:Npn \char_set_catcode_math_superscript:n #1
+ { \char_set_catcode:nn {#1} { 7 } }
+\cs_new_protected:Npn \char_set_catcode_math_subscript:n #1
+ { \char_set_catcode:nn {#1} { 8 } }
+\cs_new_protected:Npn \char_set_catcode_ignore:n #1
+ { \char_set_catcode:nn {#1} { 9 } }
+\cs_new_protected:Npn \char_set_catcode_space:n #1
+ { \char_set_catcode:nn {#1} { 10 } }
+\cs_new_protected:Npn \char_set_catcode_letter:n #1
+ { \char_set_catcode:nn {#1} { 11 } }
+\cs_new_protected:Npn \char_set_catcode_other:n #1
+ { \char_set_catcode:nn {#1} { 12 } }
+\cs_new_protected:Npn \char_set_catcode_active:n #1
+ { \char_set_catcode:nn {#1} { 13 } }
+\cs_new_protected:Npn \char_set_catcode_comment:n #1
+ { \char_set_catcode:nn {#1} { 14 } }
+\cs_new_protected:Npn \char_set_catcode_invalid:n #1
+ { \char_set_catcode:nn {#1} { 15 } }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\char_set_mathcode:nn}
+% \begin{macro}{\char_value_mathcode:n}
+% \begin{macro}{\char_show_value_mathcode:n}
+% \begin{macro}{\char_set_lccode:nn}
+% \begin{macro}{\char_value_lccode:n}
+% \begin{macro}{\char_show_value_lccode:n}
+% \begin{macro}{\char_set_uccode:nn}
+% \begin{macro}{\char_value_uccode:n}
+% \begin{macro}{\char_show_value_uccode:n}
+% \begin{macro}{\char_set_sfcode:nn}
+% \begin{macro}{\char_value_sfcode:n}
+% \begin{macro}{\char_show_value_sfcode:n}
+% Pretty repetitive, but necessary!
+% \begin{macrocode}
+\cs_new_protected:Npn \char_set_mathcode:nn #1#2
+ { \tex_mathcode:D \int_eval:n {#1} = \int_eval:n {#2} \exp_stop_f: }
+\cs_new:Npn \char_value_mathcode:n #1
+ { \tex_the:D \tex_mathcode:D \int_eval:n {#1} \exp_stop_f: }
+\cs_new_protected:Npn \char_show_value_mathcode:n #1
+ { \exp_args:Nf \tl_show:n { \char_value_mathcode:n {#1} } }
+\cs_new_protected:Npn \char_set_lccode:nn #1#2
+ { \tex_lccode:D \int_eval:n {#1} = \int_eval:n {#2} \exp_stop_f: }
+\cs_new:Npn \char_value_lccode:n #1
+ { \tex_the:D \tex_lccode:D \int_eval:n {#1} \exp_stop_f: }
+\cs_new_protected:Npn \char_show_value_lccode:n #1
+ { \exp_args:Nf \tl_show:n { \char_value_lccode:n {#1} } }
+\cs_new_protected:Npn \char_set_uccode:nn #1#2
+ { \tex_uccode:D \int_eval:n {#1} = \int_eval:n {#2} \exp_stop_f: }
+\cs_new:Npn \char_value_uccode:n #1
+ { \tex_the:D \tex_uccode:D \int_eval:n {#1} \exp_stop_f: }
+\cs_new_protected:Npn \char_show_value_uccode:n #1
+ { \exp_args:Nf \tl_show:n { \char_value_uccode:n {#1} } }
+\cs_new_protected:Npn \char_set_sfcode:nn #1#2
+ { \tex_sfcode:D \int_eval:n {#1} = \int_eval:n {#2} \exp_stop_f: }
+\cs_new:Npn \char_value_sfcode:n #1
+ { \tex_the:D \tex_sfcode:D \int_eval:n {#1} \exp_stop_f: }
+\cs_new_protected:Npn \char_show_value_sfcode:n #1
+ { \exp_args:Nf \tl_show:n { \char_value_sfcode:n {#1} } }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{variable}{\l_char_active_seq, \l_char_special_seq}
+% Two sequences for dealing with special characters. The first is characters
+% which may be active, the second longer list is for \enquote{special}
+% characters more generally. Both lists are escaped so that for example
+% bulk code assignments can be carried out. In both cases, the order is
+% by \textsc{ascii} character code (as is done in for example
+% \cs{ExplSyntaxOn}).
+% \begin{macrocode}
+\seq_new:N \l_char_special_seq
+\seq_set_split:Nnn \l_char_special_seq { }
+ { \ \" \# \$ \% \& \\ \^ \_ \{ \} \~ }
+\seq_new:N \l_char_active_seq
+\seq_set_split:Nnn \l_char_active_seq { }
+ { \" \$ \& \^ \_ \~ }
+% \end{macrocode}
+% \end{variable}
+%
+% \subsection{Creating character tokens}
+%
+% \begin{macro}
+% {
+% \char_set_active_eq:NN, \char_gset_active_eq:NN,
+% \char_set_active_eq:Nc, \char_gset_active_eq:Nc,
+% \char_set_active_eq:nN, \char_gset_active_eq:nN,
+% \char_set_active_eq:nc, \char_gset_active_eq:nc
+% }
+% Four simple functions with very similar definitions, so set up using
+% an auxiliary.
+% These are similar to \LuaTeX{}'s \tn{letcharcode} primitive.
+% \begin{macrocode}
+\group_begin:
+ \char_set_catcode_active:N \^^@
+ \cs_set_protected:Npn \@@_tmp:nN #1#2
+ {
+ \cs_new_protected:cpn { #1 :nN } ##1
+ {
+ \group_begin:
+ \char_set_lccode:nn { `\^^@ } { ##1 }
+ \tex_lowercase:D { \group_end: #2 ^^@ }
+ }
+ \cs_new_protected:cpx { #1 :NN } ##1
+ { \exp_not:c { #1 : nN } { `##1 } }
+ }
+ \@@_tmp:nN { char_set_active_eq } \cs_set_eq:NN
+ \@@_tmp:nN { char_gset_active_eq } \cs_gset_eq:NN
+\group_end:
+\cs_generate_variant:Nn \char_set_active_eq:NN { Nc }
+\cs_generate_variant:Nn \char_gset_active_eq:NN { Nc }
+\cs_generate_variant:Nn \char_set_active_eq:nN { nc }
+\cs_generate_variant:Nn \char_gset_active_eq:nN { nc }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_int_to_roman:w}
+% For efficiency in 8-bit engines, we use the faster primitive approach
+% to making roman numerals.
+% \begin{macrocode}
+\cs_new_eq:NN \@@_int_to_roman:w \tex_romannumeral:D
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[EXP]{\char_generate:nn}
+% \begin{macro}[EXP]{\@@_generate_aux:nn}
+% \begin{macro}[EXP]{\@@_generate_aux:nnw, \@@_generate_auxii:nnw}
+% \begin{variable}{\l_@@_tmp_tl}
+% \begin{macro}[EXP]{\@@_generate_invalid_catcode:}
+% The aim here is to generate characters of (broadly) arbitrary category
+% code. Where possible, that is done using engine support (\XeTeX{},
+% \LuaTeX{}). There are though various issues which are covered below. At
+% the interface layer, turn the two arguments into integers up-front so
+% this is only done once.
+% \begin{macrocode}
+\cs_new:Npn \char_generate:nn #1#2
+ {
+ \exp:w \exp_after:wN \@@_generate_aux:w
+ \int_value:w \int_eval:n {#1} \exp_after:wN ;
+ \int_value:w \int_eval:n {#2} ;
+ }
+% \end{macrocode}
+% Before doing any actual conversion, first some special case filtering.
+% Spaces are out here as \LuaTeX{} emulation only makes normal (charcode
+% $32$ spaces). However, |^^@| is filtered out separately as that can't be
+% done with macro emulation either, so is flagged up separately. That
+% done, hand off to the engine-dependent part.
+% \begin{macrocode}
+\cs_new:Npn \@@_generate_aux:w #1 ; #2 ;
+ {
+ \if_int_compare:w #2 = 10 \exp_stop_f:
+ \if_int_compare:w #1 = 0 \exp_stop_f:
+ \__kernel_msg_expandable_error:nn { kernel } { char-null-space }
+ \else:
+ \__kernel_msg_expandable_error:nn { kernel } { char-space }
+ \fi:
+ \else:
+ \if_int_odd:w 0
+ \if_int_compare:w #2 < 1 \exp_stop_f: 1 \fi:
+ \if_int_compare:w #2 = 5 \exp_stop_f: 1 \fi:
+ \if_int_compare:w #2 = 9 \exp_stop_f: 1 \fi:
+ \if_int_compare:w #2 > 13 \exp_stop_f: 1 \fi: \exp_stop_f:
+ \__kernel_msg_expandable_error:nn { kernel }
+ { char-invalid-catcode }
+ \else:
+ \if_int_odd:w 0
+ \if_int_compare:w #1 < 0 \exp_stop_f: 1 \fi:
+ \if_int_compare:w #1 > \c_max_char_int 1 \fi: \exp_stop_f:
+ \__kernel_msg_expandable_error:nn { kernel }
+ { char-out-of-range }
+ \else:
+ \@@_generate_aux:nnw {#1} {#2}
+ \fi:
+ \fi:
+ \fi:
+ \exp_end:
+ }
+\tl_new:N \l_@@_tmp_tl
+% \end{macrocode}
+% Engine-dependent definitions are now needed for the implementation. For
+% \LuaTeX{} and \XeTeX{} there is engine-level support.
+% They can do cases that macro emulation can't. All of those are filtered
+% out here using a primitive-based boolean expression to avoid fixing
+% the category code of the null character used in the false branch
+% (for 8-bit engines).
+% The final level is the basic definition at the engine level: the arguments
+% here are integers so there is no need to worry about them too much.
+% Older versions of \XeTeX{} cannot generate active characters so we filter
+% that:
+% at some future stage that may change: the slightly odd ordering of
+% auxiliaries reflects that.
+% \begin{macrocode}
+\group_begin:
+%<*package>
+ \char_set_catcode_active:N \^^L
+ \cs_set:Npn ^^L { }
+%</package>
+ \char_set_catcode_other:n { 0 }
+ \if_int_odd:w 0
+ \sys_if_engine_luatex:T { 1 }
+ \sys_if_engine_xetex:T { 1 } \exp_stop_f:
+ \sys_if_engine_luatex:TF
+ {
+ \cs_new:Npn \@@_generate_aux:nnw #1#2#3 \exp_end:
+ {
+ #3
+ \exp_after:wN \exp_after:wN \exp_after:wN \exp_end:
+ \lua_now:e { l3kernel.charcat(#1, #2) }
+ }
+ }
+ {
+ \cs_new:Npn \@@_generate_aux:nnw #1#2#3 \exp_end:
+ {
+ #3
+ \exp_after:wN \exp_end:
+ \tex_Ucharcat:D #1 \exp_stop_f: #2 \exp_stop_f:
+ }
+ \cs_if_exist:NF \tex_expanded:D
+ {
+ \cs_new_eq:NN \@@_generate_auxii:nnw \@@_generate_aux:nnw
+ \cs_gset:Npn \@@_generate_aux:nnw #1#2#3 \exp_end:
+ {
+ #3
+ \if_int_compare:w #2 = 13 \exp_stop_f:
+ \__kernel_msg_expandable_error:nn { kernel } { char-active }
+ \else:
+ \@@_generate_auxii:nnw {#1} {#2}
+ \fi:
+ \exp_end:
+ }
+ }
+ }
+ \else:
+% \end{macrocode}
+% For engines where \tn{Ucharcat} isn't available or emulated, we have
+% to work in macros, and cover only the $8$-bit range. The first stage is
+% to build up a |tl| containing |^^@| with each category code that can
+% be accessed in this way, with an error set up for the other cases. This
+% is all done such that it can be quickly accessed using a |\if_case:w|
+% low-level conditional. There are a few things to notice here.
+% As |^^L| is |\outer| we need to locally set it to avoid a problem.
+% To get open/close braces into the list, they are set up using |\if_false:|
+% pairing and are then |x|-type expanded together into the desired form.
+% \begin{macrocode}
+ \tl_set:Nn \l_@@_tmp_tl { \exp_not:N \or: }
+ \char_set_catcode_group_begin:n { 0 } % {
+ \tl_put_right:Nn \l_@@_tmp_tl { ^^@ \if_false: } }
+ \char_set_catcode_group_end:n { 0 }
+ \tl_put_right:Nn \l_@@_tmp_tl { { \fi: \exp_not:N \or: ^^@ } % }
+ \tl_set:Nx \l_@@_tmp_tl { \l_@@_tmp_tl }
+ \char_set_catcode_math_toggle:n { 0 }
+ \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+ \char_set_catcode_alignment:n { 0 }
+ \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+ \tl_put_right:Nn \l_@@_tmp_tl { \or: }
+ \char_set_catcode_parameter:n { 0 }
+ \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+ \char_set_catcode_math_superscript:n { 0 }
+ \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+ \char_set_catcode_math_subscript:n { 0 }
+ \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+ \tl_put_right:Nn \l_@@_tmp_tl { \or: }
+% \end{macrocode}
+% For making spaces, there needs to be an |o|-type expansion of a |\use:n|
+% (or some other tokenization) to avoid dropping the space. We also
+% set up active tokens although they are (currently) filtered out by the
+% interface layer (\tn{Ucharcat} cannot make active tokens).
+% \begin{macrocode}
+ \char_set_catcode_space:n { 0 }
+ \tl_put_right:No \l_@@_tmp_tl { \use:n { \or: } ^^@ }
+ \char_set_catcode_letter:n { 0 }
+ \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+ \char_set_catcode_other:n { 0 }
+ \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+ \char_set_catcode_active:n { 0 }
+ \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+% \end{macrocode}
+% Convert the above temporary list into a series of constant token
+% lists, one for each character code, using \tn{tex_lowercase:D} to
+% convert |^^@| in each case. The \texttt{x}-type expansion ensures
+% that \tn{tex_lowercase:D} receives the contents of the token list.
+% In package mode, |^^L| is awkward hence this is done in three parts.
+% Notice that at this stage |^^@| is active.
+% \begin{macrocode}
+ \cs_set_protected:Npn \@@_tmp:n #1
+ {
+ \char_set_lccode:nn { 0 } {#1}
+ \char_set_lccode:nn { 32 } {#1}
+ \exp_args:Nx \tex_lowercase:D
+ {
+ \tl_const:Nn
+ \exp_not:c { c_@@_ \@@_int_to_roman:w #1 _tl }
+ { \exp_not:o \l_@@_tmp_tl }
+ }
+ }
+%<*package>
+ \int_step_function:nnN { 0 } { 11 } \@@_tmp:n
+ \group_begin:
+ \tl_replace_once:Nnn \l_@@_tmp_tl { ^^@ } { \ERROR }
+ \@@_tmp:n { 12 }
+ \group_end:
+ \int_step_function:nnN { 13 } { 255 } \@@_tmp:n
+%</package>
+%<*initex>
+ \int_step_function:nnN { 0 } { 255 } \@@_tmp:n
+%</initex>
+% \end{macrocode}
+% As \TeX{} is very unhappy if if finds an alignment character inside
+% a primitive \tn{halign} even when skipping false branches, some
+% precautions are required. \TeX{} is happy if the token is hidden
+% between braces within \cs{if_false:} \dots{} \cs{fi:}.
+% \begin{macrocode}
+ \cs_new:Npn \@@_generate_aux:nnw #1#2#3 \exp_end:
+ {
+ #3
+ \if_false: { \fi:
+ \exp_after:wN \exp_after:wN
+ \exp_after:wN \exp_end:
+ \exp_after:wN \exp_after:wN
+ \if_case:w #2
+ \exp_last_unbraced:Nv \exp_stop_f:
+ { c_@@_ \@@_int_to_roman:w #1 _tl }
+ \or: }
+ \fi:
+ }
+ \fi:
+\group_end:
+% \end{macrocode}
+% \end{macro}
+% \end{variable}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}{\c_catcode_other_space_tl}
+% Create a space with category code $12$: an \enquote{other} space.
+% \begin{macrocode}
+\tl_const:Nx \c_catcode_other_space_tl { \char_generate:nn { `\ } { 12 } }
+% \end{macrocode}
+% \end{macro}
+%
+% \subsection{Generic tokens}
+%
+% \begin{macrocode}
+%<@@=token>
+% \end{macrocode}
+%
+% \begin{macro}{\token_to_meaning:N, \token_to_meaning:c}
+% \begin{macro}{\token_to_str:N, \token_to_str:c}
+% These are all defined in \pkg{l3basics}, as they are needed
+% \enquote{early}. This is just a reminder!
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}
+% {
+% \c_group_begin_token,
+% \c_group_end_token,
+% \c_math_toggle_token,
+% \c_alignment_token,
+% \c_parameter_token,
+% \c_math_superscript_token,
+% \c_math_subscript_token,
+% \c_space_token,
+% \c_catcode_letter_token,
+% \c_catcode_other_token
+% }
+% We define these useful tokens. For the brace and space tokens things have
+% to be done by hand: the formal argument spec.~for \cs{cs_new_eq:NN} does
+% not cover them so we do things by hand. (As currently coded it would
+% \emph{work} with \cs{cs_new_eq:NN} but that's not really a great idea to
+% show off: we want people to stick to the defined interfaces and that
+% includes us.) So that these few odd names go into the log when appropriate
+% there is a need to hand-apply the \cs{__kernel_chk_if_free_cs:N} check.
+% \begin{macrocode}
+\group_begin:
+ \__kernel_chk_if_free_cs:N \c_group_begin_token
+ \tex_global:D \tex_let:D \c_group_begin_token {
+ \__kernel_chk_if_free_cs:N \c_group_end_token
+ \tex_global:D \tex_let:D \c_group_end_token }
+ \char_set_catcode_math_toggle:N \*
+ \cs_new_eq:NN \c_math_toggle_token *
+ \char_set_catcode_alignment:N \*
+ \cs_new_eq:NN \c_alignment_token *
+ \cs_new_eq:NN \c_parameter_token #
+ \cs_new_eq:NN \c_math_superscript_token ^
+ \char_set_catcode_math_subscript:N \*
+ \cs_new_eq:NN \c_math_subscript_token *
+ \__kernel_chk_if_free_cs:N \c_space_token
+ \use:n { \tex_global:D \tex_let:D \c_space_token = ~ } ~
+ \cs_new_eq:NN \c_catcode_letter_token a
+ \cs_new_eq:NN \c_catcode_other_token 1
+\group_end:
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{variable}{\c_catcode_active_tl}
+% Not an implicit token!
+% \begin{macrocode}
+\group_begin:
+ \char_set_catcode_active:N \*
+ \tl_const:Nn \c_catcode_active_tl { \exp_not:N * }
+\group_end:
+% \end{macrocode}
+% \end{variable}
+%
+% \subsection{Token conditionals}
+%
+% \begin{macro}[pTF]{\token_if_group_begin:N}
+% Check if token is a begin group token. We use the constant
+% \cs{c_group_begin_token} for this.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_group_begin:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_group_begin_token
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_group_end:N}
+% Check if token is a end group token. We use the constant
+% \cs{c_group_end_token} for this.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_group_end:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_group_end_token
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_math_toggle:N}
+% Check if token is a math shift token. We use the constant
+% \cs{c_math_toggle_token} for this.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_math_toggle:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_math_toggle_token
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_alignment:N}
+% Check if token is an alignment tab token. We use the constant
+% \cs{c_alignment_token} for this.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_alignment:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_alignment_token
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_parameter:N}
+% Check if token is a parameter token. We use the constant
+% \cs{c_parameter_token} for this. We have to trick \TeX{} a bit to
+% avoid an error message: within a group we prevent
+% \cs{c_parameter_token} from behaving like a macro parameter character.
+% The definitions of \cs{prg_new_conditional:Npnn} are global, so they
+% remain after the group.
+% \begin{macrocode}
+\group_begin:
+\cs_set_eq:NN \c_parameter_token \scan_stop:
+\prg_new_conditional:Npnn \token_if_parameter:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_parameter_token
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+\group_end:
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_math_superscript:N}
+% Check if token is a math superscript token. We use the constant
+% \cs{c_math_superscript_token} for this.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_math_superscript:N #1
+ { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_math_superscript_token
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_math_subscript:N}
+% Check if token is a math subscript token. We use the constant
+% \cs{c_math_subscript_token} for this.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_math_subscript:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_math_subscript_token
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_space:N}
+% Check if token is a space token. We use the constant
+% \cs{c_space_token} for this.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_space:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_space_token
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_letter:N}
+% Check if token is a letter token. We use the constant
+% \cs{c_catcode_letter_token} for this.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_letter:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_catcode_letter_token
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_other:N}
+% Check if token is an other char token. We use the constant
+% \cs{c_catcode_other_token} for this.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_other:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_catcode_other_token
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_active:N}
+% Check if token is an active char token. We use the constant
+% \cs{c_catcode_active_tl} for this. A technical point is that
+% \cs{c_catcode_active_tl} is in fact a macro expanding to
+% |\exp_not:N *|, where |*| is active.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_active:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_catcode_active_tl
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_eq_meaning:NN}
+% Check if the tokens |#1| and |#2| have same meaning.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_eq_meaning:NN #1#2 { p , T , F , TF }
+ {
+ \if_meaning:w #1 #2
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_eq_catcode:NN}
+% Check if the tokens |#1| and |#2| have same category code.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_eq_catcode:NN #1#2 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \exp_not:N #2
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_eq_charcode:NN}
+% Check if the tokens |#1| and |#2| have same character code.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_eq_charcode:NN #1#2 { p , T , F , TF }
+ {
+ \if_charcode:w \exp_not:N #1 \exp_not:N #2
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_macro:N}
+% \begin{macro}{\@@_if_macro_p:w}
+% When a token is a macro, \cs{token_to_meaning:N} always outputs
+% something like |\long macro:#1->#1| so we could naively check to
+% see if the meaning contains |->|. However, this can fail the five
+% \tn[no-index]{...mark} primitives, whose meaning has the form
+% |...mark:|\meta{user material}. The problem is that the
+% \meta{user material} can contain |->|.
+%
+% However, only characters, macros, and marks can contain the colon
+% character. The idea is thus to grab until the first |:|, and analyse
+% what is left. However, macros can have any combination of |\long|,
+% |\protected| or |\outer| (not used in \LaTeX3) before the string
+% |macro:|. We thus only select the part of the meaning between
+% the first |ma| and the first following |:|. If this string is
+% |cro|, then we have a macro. If the string is |rk|, then we have
+% a mark. The string can also be |cro parameter character | for a
+% colon with a weird category code (namely the usual category code
+% of |#|). Otherwise, it is empty.
+%
+% This relies on the fact that |\long|, |\protected|, |\outer|
+% cannot contain |ma|, regardless of the escape character, even if
+% the escape character is |m|\ldots{}
+%
+% Both |ma| and |:| must be of category code $12$ (other), so are
+% detokenized.
+%
+% \begin{macrocode}
+\use:x
+ {
+ \prg_new_conditional:Npnn \exp_not:N \token_if_macro:N ##1
+ { p , T , F , TF }
+ {
+ \exp_not:N \exp_after:wN \exp_not:N \@@_if_macro_p:w
+ \exp_not:N \token_to_meaning:N ##1 \tl_to_str:n { ma : }
+ \exp_not:N \q_stop
+ }
+ \cs_new:Npn \exp_not:N \@@_if_macro_p:w
+ ##1 \tl_to_str:n { ma } ##2 \c_colon_str ##3 \exp_not:N \q_stop
+ }
+ {
+ \str_if_eq:nnTF { #2 } { cro }
+ { \prg_return_true: }
+ { \prg_return_false: }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_cs:N}
+% Check if token has same catcode as a control sequence. This
+% follows the same pattern as for \cs{token_if_letter:N} \emph{etc.}
+% We use \cs{scan_stop:} for this.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_cs:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \scan_stop:
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_expandable:N}
+% Check if token is expandable. We use the fact that \TeX{}
+% temporarily converts \cs{exp_not:N} \meta{token} into \cs{scan_stop:}
+% if \meta{token} is expandable. An \texttt{undefined} token is not
+% considered as expandable. No problem nesting the conditionals,
+% since the third |#1| is only skipped if it is non-expandable (hence
+% not part of \TeX{}'s conditional apparatus).
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_expandable:N #1 { p , T , F , TF }
+ {
+ \exp_after:wN \if_meaning:w \exp_not:N #1 #1
+ \prg_return_false:
+ \else:
+ \if_cs_exist:N #1
+ \prg_return_true:
+ \else:
+ \prg_return_false:
+ \fi:
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}
+% {
+% \@@_delimit_by_char":w,
+% \@@_delimit_by_count:w,
+% \@@_delimit_by_dimen:w,
+% \@@_delimit_by_macro:w,
+% \@@_delimit_by_muskip:w,
+% \@@_delimit_by_skip:w,
+% \@@_delimit_by_toks:w,
+% }
+% These auxiliary functions are used below to define some
+% conditionals which detect whether the \tn{meaning} of their
+% argument begins with a particular string. Each auxiliary takes an
+% argument delimited by a string, a second one delimited by
+% \cs{q_stop}, and returns the first one and its delimiter.
+% This result is eventually compared to another string.
+% \begin{macrocode}
+\group_begin:
+\cs_set_protected:Npn \@@_tmp:w #1
+ {
+ \use:x
+ {
+ \cs_new:Npn \exp_not:c { @@_delimit_by_ #1 :w }
+ ####1 \tl_to_str:n {#1} ####2 \exp_not:N \q_stop
+ { ####1 \tl_to_str:n {#1} }
+ }
+ }
+\@@_tmp:w { char" }
+\@@_tmp:w { count }
+\@@_tmp:w { dimen }
+\@@_tmp:w { macro }
+\@@_tmp:w { muskip }
+\@@_tmp:w { skip }
+\@@_tmp:w { toks }
+\group_end:
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]
+% {
+% \token_if_chardef:N, \token_if_mathchardef:N,
+% \token_if_long_macro:N,
+% \token_if_protected_macro:N, \token_if_protected_long_macro:N,
+% \token_if_dim_register:N, \token_if_int_register:N,
+% \token_if_muskip_register:N,
+% \token_if_skip_register:N, \token_if_toks_register:N,
+% }
+% Each of these conditionals tests whether its argument's
+% \tn{meaning} starts with a given string. This is essentially done
+% by having an auxiliary grab an argument delimited by the string and
+% testing whether the argument was empty. Of course, a copy of this
+% string must first be added to the end of the \tn{meaning} to avoid
+% a runaway argument in case it does not contain the string. Two
+% complications arise. First, the escape character is not fixed, and
+% cannot be included in the delimiter of the auxiliary function (this
+% function cannot be defined on the fly because tests must remain
+% expandable): instead the first argument of the auxiliary (plus the
+% delimiter to avoid complications with trailing spaces) is compared
+% using \cs{str_if_eq:eeTF} to the result of applying
+% \cs{token_to_str:N} to a control sequence. Second, the
+% \tn{meaning} of primitives such as \tn{dimen} or \tn{dimendef}
+% starts in the same way as registers such as
+% \tn{dimen}\texttt{123}, so they must be tested for.
+%
+% Characters used as delimiters must have catcode~$12$
+% and are obtained through \cs{tl_to_str:n}. This requires doing all
+% definitions within \texttt{x}-expansion. The temporary function
+% \cs{@@_tmp:w} used to define each conditional receives three
+% arguments: the name of the conditional, the auxiliary's delimiter
+% (also used to name the auxiliary), and the string to which one
+% compares the auxiliary's result. Note that the \tn{meaning} of a
+% protected long macro starts with |\protected\long macro|, with no
+% space after |\protected| but a space after |\long|, hence the
+% mixture of \cs{token_to_str:N} and \cs{tl_to_str:n}.
+%
+% For the first five conditionals, \cs{cs_if_exist:cT} turns out to
+% be \texttt{false}, and the code boils down to a string comparison
+% between the result of the auxiliary on the \tn{meaning} of the
+% conditional's argument~|####1|, and~|#3|. Both are evaluated at
+% run-time, as this is important to get the correct escape character.
+%
+% The other five conditionals have additional code that compares the
+% argument~|####1| to two \TeX{} primitives which would wrongly be
+% recognized as registers otherwise. Despite using \TeX{}'s
+% primitive conditional construction, this does not break
+% when~|####1| is itself a conditional, because branches of the
+% conditionals are only skipped if |####1|~is one of the two
+% primitives that are tested for (which are not \TeX{} conditionals).
+% \begin{macrocode}
+\group_begin:
+\cs_set_protected:Npn \@@_tmp:w #1#2#3
+ {
+ \use:x
+ {
+ \prg_new_conditional:Npnn \exp_not:c { token_if_ #1 :N } ####1
+ { p , T , F , TF }
+ {
+ \cs_if_exist:cT { tex_ #2 :D }
+ {
+ \exp_not:N \if_meaning:w ####1 \exp_not:c { tex_ #2 :D }
+ \exp_not:N \prg_return_false:
+ \exp_not:N \else:
+ \exp_not:N \if_meaning:w ####1 \exp_not:c { tex_ #2 def:D }
+ \exp_not:N \prg_return_false:
+ \exp_not:N \else:
+ }
+ \exp_not:N \str_if_eq:eeTF
+ {
+ \exp_not:N \exp_after:wN
+ \exp_not:c { @@_delimit_by_ #2 :w }
+ \exp_not:N \token_to_meaning:N ####1
+ ? \tl_to_str:n {#2} \exp_not:N \q_stop
+ }
+ { \exp_not:n {#3} }
+ { \exp_not:N \prg_return_true: }
+ { \exp_not:N \prg_return_false: }
+ \cs_if_exist:cT { tex_ #2 :D }
+ {
+ \exp_not:N \fi:
+ \exp_not:N \fi:
+ }
+ }
+ }
+ }
+\@@_tmp:w { chardef } { char" } { \token_to_str:N \char" }
+\@@_tmp:w { mathchardef } { char" } { \token_to_str:N \mathchar" }
+\@@_tmp:w { long_macro } { macro } { \tl_to_str:n { \long } macro }
+\@@_tmp:w { protected_macro } { macro }
+ { \tl_to_str:n { \protected } macro }
+\@@_tmp:w { protected_long_macro } { macro }
+ { \token_to_str:N \protected \tl_to_str:n { \long } macro }
+\@@_tmp:w { dim_register } { dimen } { \token_to_str:N \dimen }
+\@@_tmp:w { int_register } { count } { \token_to_str:N \count }
+\@@_tmp:w { muskip_register } { muskip } { \token_to_str:N \muskip }
+\@@_tmp:w { skip_register } { skip } { \token_to_str:N \skip }
+\@@_tmp:w { toks_register } { toks } { \token_to_str:N \toks }
+\group_end:
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_primitive:N}
+% \begin{macro}{\@@_if_primitive:NNw,
+% \@@_if_primitive_space:w,
+% \@@_if_primitive_nullfont:N,
+% \@@_if_primitive_loop:N,
+% \@@_if_primitive:Nw,
+% \@@_if_primitive_undefined:N}
+%^^A See http://groups.google.com/group/comp.text.tex/browse_thread/thread/0a72666873f8753d#
+%
+% We filter out macros first, because they cause endless trouble later
+% otherwise.
+%
+% Primitives are almost distinguished by the fact that the result
+% of \cs{token_to_meaning:N} is formed from letters only. Every other
+% token has either a space (e.g., |the letter A|), a digit
+% (e.g., |\count123|) or a double quote (e.g., |\char"A|).
+%
+% Ten exceptions: on the one hand, \cs{tex_undefined:D} is not a
+% primitive, but its meaning is |undefined|, only letters;
+% on the other hand, \tn{space}, \tn{italiccorr},
+% \tn{hyphen}, \tn{firstmark}, \tn{topmark},
+% \tn{botmark}, \tn{splitfirstmark}, \tn{splitbotmark},
+% and \tn{nullfont} are primitives, but have non-letters
+% in their meaning.
+%
+% We start by removing the two first (non-space) characters from
+% the meaning. This removes the escape character (which may be
+% nonexistent depending on \tn{endlinechar}), and takes care
+% of three of the exceptions: \tn{space}, \tn{italiccorr}
+% and \tn{hyphen}, whose meaning is at most two characters.
+% This leaves a string terminated by some |:|, and \cs{q_stop}.
+%
+% The meaning of each one of the five \tn[no-index]{...mark} primitives
+% has the form \meta{letters}|:|\meta{user material}. In other words,
+% the first non-letter is a colon. We remove everything after the first
+% colon.
+%
+% We are now left with a string, which we must analyze. For primitives,
+% it contains only letters. For non-primitives, it contains either
+% |"|, or a space, or a digit. Two exceptions remain: \cs{tex_undefined:D},
+% which is not a primitive, and \tn{nullfont}, which is a primitive.
+%
+% Spaces cannot be grabbed in an undelimited way, so we check them
+% separately. If there is a space, we test for \tn{nullfont}.
+% Otherwise, we go through characters one by one, and stop at the
+% first character less than |`A| (this is not quite a test for
+% \enquote{only letters}, but is close enough to work in this context).
+% If this first character is |:| then we have a primitive, or
+% \cs{tex_undefined:D}, and if it is |"| or a digit, then the token
+% is not a primitive.
+%
+% \begin{macrocode}
+\tex_chardef:D \c_@@_A_int = `A ~ %
+\use:x
+ {
+ \prg_new_conditional:Npnn \exp_not:N \token_if_primitive:N ##1
+ { p , T , F , TF }
+ {
+ \exp_not:N \token_if_macro:NTF ##1
+ \exp_not:N \prg_return_false:
+ {
+ \exp_not:N \exp_after:wN \exp_not:N \@@_if_primitive:NNw
+ \exp_not:N \token_to_meaning:N ##1
+ \tl_to_str:n { : : : } \exp_not:N \q_stop ##1
+ }
+ }
+ \cs_new:Npn \exp_not:N \@@_if_primitive:NNw
+ ##1##2 ##3 \c_colon_str ##4 \exp_not:N \q_stop
+ {
+ \exp_not:N \tl_if_empty:oTF
+ { \exp_not:N \@@_if_primitive_space:w ##3 ~ }
+ {
+ \exp_not:N \@@_if_primitive_loop:N ##3
+ \c_colon_str \exp_not:N \q_stop
+ }
+ { \exp_not:N \@@_if_primitive_nullfont:N }
+ }
+ }
+\cs_new:Npn \@@_if_primitive_space:w #1 ~ { }
+\cs_new:Npn \@@_if_primitive_nullfont:N #1
+ {
+ \if_meaning:w \tex_nullfont:D #1
+ \prg_return_true:
+ \else:
+ \prg_return_false:
+ \fi:
+ }
+\cs_new:Npn \@@_if_primitive_loop:N #1
+ {
+ \if_int_compare:w `#1 < \c_@@_A_int %
+ \exp_after:wN \@@_if_primitive:Nw
+ \exp_after:wN #1
+ \else:
+ \exp_after:wN \@@_if_primitive_loop:N
+ \fi:
+ }
+\cs_new:Npn \@@_if_primitive:Nw #1 #2 \q_stop
+ {
+ \if:w : #1
+ \exp_after:wN \@@_if_primitive_undefined:N
+ \else:
+ \prg_return_false:
+ \exp_after:wN \use_none:n
+ \fi:
+ }
+\cs_new:Npn \@@_if_primitive_undefined:N #1
+ {
+ \if_cs_exist:N #1
+ \prg_return_true:
+ \else:
+ \prg_return_false:
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \subsection{Peeking ahead at the next token}
+%
+% \begin{macrocode}
+%<@@=peek>
+% \end{macrocode}
+%
+% Peeking ahead is implemented using a two part mechanism. The
+% outer level provides a defined interface to the lower level material.
+% This allows a large amount of code to be shared. There are four
+% cases:
+% \begin{enumerate}
+% \item peek at the next token;
+% \item peek at the next non-space token;
+% \item peek at the next token and remove it;
+% \item peek at the next non-space token and remove it.
+% \end{enumerate}
+%
+% \begin{variable}{\l_peek_token}
+% \begin{variable}{\g_peek_token}
+% Storage tokens which are publicly documented: the token peeked.
+% \begin{macrocode}
+\cs_new_eq:NN \l_peek_token ?
+\cs_new_eq:NN \g_peek_token ?
+% \end{macrocode}
+% \end{variable}
+% \end{variable}
+%
+% \begin{variable}{\l_@@_search_token}
+% The token to search for as an implicit token:
+% \emph{cf.}~\cs{l_@@_search_tl}.
+% \begin{macrocode}
+\cs_new_eq:NN \l_@@_search_token ?
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_@@_search_tl}
+% The token to search for as an explicit token:
+% \emph{cf.}~\cs{l_@@_search_token}.
+% \begin{macrocode}
+\tl_new:N \l_@@_search_tl
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{macro}
+% {\@@_true:w, \@@_true_aux:w, \@@_false:w, \@@_tmp:w}
+% Functions used by the branching and space-stripping code.
+% \begin{macrocode}
+\cs_new:Npn \@@_true:w { }
+\cs_new:Npn \@@_true_aux:w { }
+\cs_new:Npn \@@_false:w { }
+\cs_new:Npn \@@_tmp:w { }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\peek_after:Nw}
+% \begin{macro}{\peek_gafter:Nw}
+% Simple wrappers for \tn{futurelet}: no arguments absorbed
+% here.
+% \begin{macrocode}
+\cs_new_protected:Npn \peek_after:Nw
+ { \tex_futurelet:D \l_peek_token }
+\cs_new_protected:Npn \peek_gafter:Nw
+ { \tex_global:D \tex_futurelet:D \g_peek_token }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}{\@@_true_remove:w}
+% A function to remove the next token and then regain control.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_true_remove:w
+ {
+ \tex_afterassignment:D \@@_true_aux:w
+ \cs_set_eq:NN \@@_tmp:w
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\peek_remove_spaces:n, \@@_remove_spaces:}
+% Repeatedly use \cs{@@_true_remove:w} to remove a space and call
+% \cs{@@_true_aux:w}.
+% \begin{macrocode}
+\cs_new_protected:Npn \peek_remove_spaces:n #1
+ {
+ \cs_set:Npx \@@_false:w { \exp_not:n {#1} }
+ \group_align_safe_begin:
+ \cs_set:Npn \@@_true_aux:w { \peek_after:Nw \@@_remove_spaces: }
+ \@@_true_aux:w
+ }
+\cs_new_protected:Npn \@@_remove_spaces:
+ {
+ \if_meaning:w \l_peek_token \c_space_token
+ \exp_after:wN \@@_true_remove:w
+ \else:
+ \group_align_safe_end:
+ \exp_after:wN \@@_false:w
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_token_generic_aux:NNNTF}
+% The generic functions store the test token in both implicit and
+% explicit modes, and the \texttt{true} and \texttt{false} code as
+% token lists, more or less. The two branches have to be absorbed here
+% as the input stream needs to be cleared for the peek function itself.
+% Here, |#1| is \cs{@@_true_remove:w} when removing the token and
+% \cs{@@_true_aux:w} otherwise.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_token_generic_aux:NNNTF #1#2#3#4#5
+ {
+ \group_align_safe_begin:
+ \cs_set_eq:NN \l_@@_search_token #3
+ \tl_set:Nn \l_@@_search_tl {#3}
+ \cs_set:Npx \@@_true_aux:w
+ {
+ \exp_not:N \group_align_safe_end:
+ \exp_not:n {#4}
+ }
+ \cs_set_eq:NN \@@_true:w #1
+ \cs_set:Npx \@@_false:w
+ {
+ \exp_not:N \group_align_safe_end:
+ \exp_not:n {#5}
+ }
+ \peek_after:Nw #2
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[TF]{\@@_token_generic:NN, \@@_token_remove_generic:NN}
+% For token removal there needs to be a call to the auxiliary
+% function which does the work.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_token_generic:NNTF
+ { \@@_token_generic_aux:NNNTF \@@_true_aux:w }
+\cs_new_protected:Npn \@@_token_generic:NNT #1#2#3
+ { \@@_token_generic:NNTF #1 #2 {#3} { } }
+\cs_new_protected:Npn \@@_token_generic:NNF #1#2#3
+ { \@@_token_generic:NNTF #1 #2 { } {#3} }
+\cs_new_protected:Npn \@@_token_remove_generic:NNTF
+ { \@@_token_generic_aux:NNNTF \@@_true_remove:w }
+\cs_new_protected:Npn \@@_token_remove_generic:NNT #1#2#3
+ { \@@_token_remove_generic:NNTF #1 #2 {#3} { } }
+\cs_new_protected:Npn \@@_token_remove_generic:NNF #1#2#3
+ { \@@_token_remove_generic:NNTF #1 #2 { } {#3} }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_execute_branches_meaning:}
+% The meaning test is straight forward.
+% \begin{macrocode}
+\cs_new:Npn \@@_execute_branches_meaning:
+ {
+ \if_meaning:w \l_peek_token \l_@@_search_token
+ \exp_after:wN \@@_true:w
+ \else:
+ \exp_after:wN \@@_false:w
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_execute_branches_catcode:, \@@_execute_branches_charcode:}
+% \begin{macro}
+% {
+% \@@_execute_branches_catcode_aux: ,
+% \@@_execute_branches_catcode_auxii:N ,
+% \@@_execute_branches_catcode_auxiii:
+% }
+% The catcode and charcode tests are very similar, and in order to use
+% the same auxiliaries we do something a little bit odd, firing
+% \cs{if_catcode:w} and \cs{if_charcode:w} before finding the operands
+% for those tests, which are only given in the |auxii:N| and
+% |auxiii:| auxiliaries. For our purposes, three kinds of tokens may
+% follow the peeking function:
+% \begin{itemize}
+% \item control sequences which are not equal to a non-active
+% character token (\emph{e.g.}, macro, primitive);
+% \item active characters which are not equal to a non-active
+% character token (\emph{e.g.}, macro, primitive);
+% \item explicit non-active character tokens, or control sequences
+% or active characters set equal to a non-active character token.
+% \end{itemize}
+% The first two cases are not distinguishable simply using \TeX{}'s
+% \tn{futurelet}, because we can only access the \tn{meaning} of
+% tokens in that way. In those cases, detected thanks to a
+% comparison with \cs{scan_stop:}, we grab the following token, and
+% compare it explicitly with the explicit search token stored in
+% \cs{l_@@_search_tl}. The \cs{exp_not:N} prevents outer macros
+% (coming from non-\LaTeX3 code) from blowing up. In the third case,
+% \cs{l_peek_token} is good enough for the test, and we compare it
+% again with the explicit search token. Just like the peek token, the
+% search token may be of any of the three types above, hence the need
+% to use the explicit token that was given to the peek function.
+% \begin{macrocode}
+\cs_new:Npn \@@_execute_branches_catcode:
+ { \if_catcode:w \@@_execute_branches_catcode_aux: }
+\cs_new:Npn \@@_execute_branches_charcode:
+ { \if_charcode:w \@@_execute_branches_catcode_aux: }
+\cs_new:Npn \@@_execute_branches_catcode_aux:
+ {
+ \if_catcode:w \exp_not:N \l_peek_token \scan_stop:
+ \exp_after:wN \exp_after:wN
+ \exp_after:wN \@@_execute_branches_catcode_auxii:N
+ \exp_after:wN \exp_not:N
+ \else:
+ \exp_after:wN \@@_execute_branches_catcode_auxiii:
+ \fi:
+ }
+\cs_new:Npn \@@_execute_branches_catcode_auxii:N #1
+ {
+ \exp_not:N #1
+ \exp_after:wN \exp_not:N \l_@@_search_tl
+ \exp_after:wN \@@_true:w
+ \else:
+ \exp_after:wN \@@_false:w
+ \fi:
+ #1
+ }
+\cs_new:Npn \@@_execute_branches_catcode_auxiii:
+ {
+ \exp_not:N \l_peek_token
+ \exp_after:wN \exp_not:N \l_@@_search_tl
+ \exp_after:wN \@@_true:w
+ \else:
+ \exp_after:wN \@@_false:w
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[TF]
+% {
+% \peek_catcode:N,
+% \peek_catcode_remove:N,
+% \peek_charcode:N,
+% \peek_charcode_remove:N,
+% \peek_meaning:N,
+% \peek_meaning_remove:N,
+% }
+% The public functions themselves cannot be defined using
+% \cs{prg_new_conditional:Npnn}. Instead, the |TF|, |T|, |F| variants
+% are defined in terms of corresponding variants of
+% \cs{@@_token_generic:NNTF} or \cs{@@_token_remove_generic:NNTF},
+% with first argument one of \cs{@@_execute_branches_catcode:},
+% \cs{@@_execute_branches_charcode:}, or
+% \cs{@@_execute_branches_meaning:}.
+% \begin{macrocode}
+\tl_map_inline:nn { { catcode } { charcode } { meaning } }
+ {
+ \tl_map_inline:nn { { } { _remove } }
+ {
+ \tl_map_inline:nn { { TF } { T } { F } }
+ {
+ \cs_new_protected:cpx { peek_ #1 ##1 :N ####1 }
+ {
+ \exp_not:c { @@_token ##1 _generic:NN ####1 }
+ \exp_not:c { @@_execute_branches_ #1 : }
+ }
+ }
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[TF]
+% {
+% \peek_catcode_ignore_spaces:N, \peek_catcode_remove_ignore_spaces:N,
+% \peek_charcode_ignore_spaces:N, \peek_charcode_remove_ignore_spaces:N,
+% \peek_meaning_ignore_spaces:N, \peek_meaning_remove_ignore_spaces:N
+% }
+% To ignore spaces, remove them using \cs{peek_remove_spaces:n}
+% before running the tests.
+% \begin{macrocode}
+\tl_map_inline:nn
+ {
+ { catcode } { catcode_remove }
+ { charcode } { charcode_remove }
+ { meaning } { meaning_remove }
+ }
+ {
+ \cs_new_protected:cpx { peek_#1_ignore_spaces:NTF } ##1##2##3
+ {
+ \peek_remove_spaces:n
+ { \exp_not:c { peek_#1:NTF } ##1 {##2} {##3} }
+ }
+ \cs_new_protected:cpx { peek_#1_ignore_spaces:NT } ##1##2
+ {
+ \peek_remove_spaces:n
+ { \exp_not:c { peek_#1:NT } ##1 {##2} }
+ }
+ \cs_new_protected:cpx { peek_#1_ignore_spaces:NF } ##1##2
+ {
+ \peek_remove_spaces:n
+ { \exp_not:c { peek_#1:NF } ##1 {##2} }
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[TF]{\peek_N_type:}
+% \begin{macro}
+% {\@@_execute_branches_N_type:, \@@_N_type:w, \@@_N_type_aux:nnw}
+% All tokens are \texttt{N}-type tokens, except in four cases:
+% begin-group tokens, end-group tokens, space tokens with character
+% code~$32$, and outer tokens. Since \cs{l_peek_token} might be
+% outer, we cannot use the convenient \cs{bool_if:nTF} function, and
+% must resort to the old trick of using \tn{ifodd} to expand a set of
+% tests. The \texttt{false} branch of this test is taken if the token
+% is one of the first three kinds of non-\texttt{N}-type tokens
+% (explicit or implicit), thus we call \cs{@@_false:w}. In the
+% \texttt{true} branch, we must detect outer tokens, without impacting
+% performance too much for non-outer tokens. The first filter is to
+% search for \texttt{outer} in the \tn{meaning} of \cs{l_peek_token}.
+% If that is absent, \cs{use_none_delimit_by_q_stop:w} cleans up, and
+% we call \cs{@@_true:w}. Otherwise, the token can be a non-outer
+% macro or a primitive mark whose parameter or replacement text
+% contains \texttt{outer}, it can be the primitive \tn{outer}, or it
+% can be an outer token. Macros and marks would have \texttt{ma} in
+% the part before the first occurrence of \texttt{outer}; the meaning
+% of \tn{outer} has nothing after \texttt{outer}, contrarily to outer
+% macros; and that covers all cases, calling \cs{@@_true:w} or
+% \cs{@@_false:w} as appropriate. Here, there is no \meta{search
+% token}, so we feed a dummy \cs{scan_stop:} to the
+% \cs{@@_token_generic:NNTF} function.
+% \begin{macrocode}
+\group_begin:
+ \cs_set_protected:Npn \@@_tmp:w #1 \q_stop
+ {
+ \cs_new_protected:Npn \@@_execute_branches_N_type:
+ {
+ \if_int_odd:w
+ \if_catcode:w \exp_not:N \l_peek_token { 0 \exp_stop_f: \fi:
+ \if_catcode:w \exp_not:N \l_peek_token } 0 \exp_stop_f: \fi:
+ \if_meaning:w \l_peek_token \c_space_token 0 \exp_stop_f: \fi:
+ 1 \exp_stop_f:
+ \exp_after:wN \@@_N_type:w
+ \token_to_meaning:N \l_peek_token
+ \q_mark \@@_N_type_aux:nnw
+ #1 \q_mark \use_none_delimit_by_q_stop:w
+ \q_stop
+ \exp_after:wN \@@_true:w
+ \else:
+ \exp_after:wN \@@_false:w
+ \fi:
+ }
+ \cs_new_protected:Npn \@@_N_type:w ##1 #1 ##2 \q_mark ##3
+ { ##3 {##1} {##2} }
+ }
+ \exp_after:wN \@@_tmp:w \tl_to_str:n { outer } \q_stop
+\group_end:
+\cs_new_protected:Npn \@@_N_type_aux:nnw #1 #2 #3 \fi:
+ {
+ \fi:
+ \tl_if_in:noTF {#1} { \tl_to_str:n {ma} }
+ { \@@_true:w }
+ { \tl_if_empty:nTF {#2} { \@@_true:w } { \@@_false:w } }
+ }
+\cs_new_protected:Npn \peek_N_type:TF
+ {
+ \@@_token_generic:NNTF
+ \@@_execute_branches_N_type: \scan_stop:
+ }
+\cs_new_protected:Npn \peek_N_type:T
+ { \@@_token_generic:NNT \@@_execute_branches_N_type: \scan_stop: }
+\cs_new_protected:Npn \peek_N_type:F
+ { \@@_token_generic:NNF \@@_execute_branches_N_type: \scan_stop: }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macrocode}
+%</initex|package>
+% \end{macrocode}
+%
+% \end{implementation}
+%
+% \PrintIndex