summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
diff options
context:
space:
mode:
authorKarl Berry <karl@freefriends.org>2011-07-31 18:27:06 +0000
committerKarl Berry <karl@freefriends.org>2011-07-31 18:27:06 +0000
commitdac945b204407dae96f70ec228206516093156d7 (patch)
tree02ca182689947e45dec95e17ba08b503c959bd17 /Master/texmf-dist/source/latex/l3kernel/l3token.dtx
parentc1840548792cfbc7c9f746da1d53d89b94406e72 (diff)
l3kernel (5jun11)
git-svn-id: svn://tug.org/texlive/trunk@23282 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/source/latex/l3kernel/l3token.dtx')
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3token.dtx2495
1 files changed, 2495 insertions, 0 deletions
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
new file mode 100644
index 00000000000..153ca92f7e0
--- /dev/null
+++ b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
@@ -0,0 +1,2495 @@
+% \iffalse meta-comment
+%
+%% File: l3token.dtx Copyright (C) 2005-2011 The LaTeX3 Project
+%%
+%% It may be distributed and/or modified under the conditions of the
+%% LaTeX Project Public License (LPPL), either version 1.3c of this
+%% license or (at your option) any later version. The latest version
+%% of this license is in the file
+%%
+%% http://www.latex-project.org/lppl.txt
+%%
+%% This file is part of the "expl3 bundle" (The Work in LPPL)
+%% and all files in that bundle must be distributed together.
+%%
+%% The released version of this bundle is available from CTAN.
+%%
+%% -----------------------------------------------------------------------
+%%
+%% The development version of the bundle can be found at
+%%
+%% http://www.latex-project.org/svnroot/experimental/trunk/
+%%
+%% for those people who are interested.
+%%
+%%%%%%%%%%%
+%% NOTE: %%
+%%%%%%%%%%%
+%%
+%% Snapshots taken from the repository represent work in progress and may
+%% not work or may contain conflicting material! We therefore ask
+%% people _not_ to put them into distributions, archives, etc. without
+%% prior consultation with the LaTeX3 Project.
+%%
+%% -----------------------------------------------------------------------
+%
+%<*driver|package>
+\RequirePackage{l3names}
+\GetIdInfo$Id: l3token.dtx 2491 2011-07-02 11:49:37Z joseph $
+ {L3 Experimental token manipulation}
+%</driver|package>
+%<*driver>
+\documentclass[full]{l3doc}
+\begin{document}
+ \DocInput{\jobname.dtx}
+\end{document}
+%</driver>
+% \fi
+%
+% \title{^^A
+% The \pkg{l3token} package\\ Token manipulation^^A
+% \thanks{This file describes v\ExplFileVersion,
+% last revised \ExplFileDate.}^^A
+% }
+%
+% \author{^^A
+% The \LaTeX3 Project\thanks
+% {^^A
+% E-mail:
+% \href{mailto:latex-team@latex-project.org}
+% {latex-team@latex-project.org}^^A
+% }^^A
+% }
+%
+% \date{Released \ExplFileDate}
+%
+% \maketitle
+%
+% \begin{documentation}
+%
+% This module deals with tokens. Now this is perhaps not the most
+% precise description so let's try with a better description: When
+% programming in \TeX{}, it is often desirable to know just what a
+% certain token is: is it a control sequence or something
+% else. Similarly one often needs to know if a control sequence is
+% expandable or not, a macro or a primitive, how many arguments it
+% takes etc. Another thing of great importance (especially when it
+% comes to document commands) is looking ahead in the token stream to
+% see if a certain character is present and maybe even remove it or
+% disregard other tokens while scanning. This module provides
+% functions for both and as such will have two primary function
+% categories: |\token| for anything that deals with tokens and
+% |\peek| for looking ahead in the token stream.
+%
+% Most of the time we will be using the term \enquote{token} but most of the
+% time the function we're describing can equally well by used on a
+% control sequence as such one is one token as well.
+%
+% We shall refer to list of tokens as |tlist|s and such lists
+% represented by a single control sequence is a \enquote{token list variable}
+% |tl var|. Functions for these two types are found in the \textsf{l3tl}
+% module.
+%
+% \section{All possible tokens}
+%
+% Let us start by reviewing every case that a given token can fall into.
+% It is very important to distinguish two aspects of a token: its meaning,
+% and what it looks like.
+%
+% For instance, \cs{if:w}, \cs{if_charcode:w}, and \cs{tex_if:D} are
+% three for the same internal operation of \TeX{}, namely the primitive
+% testing the next two characters for equality of their character code.
+% They behave identically in many situations. However, \TeX{}
+% distinguishes them when searching for a delimited argument. Namely, the
+% example function \cs{show_until_if:w} defined below will take everything
+% until \cs{if:w} as an argument, despite the presence of other copies of
+% \cs{if:w} under different names.
+% \begin{verbatim}
+% \cs_new:Npn \show_until_if:w #1 \if:w { \tl_show:n {#1} }
+% \show_until_if:w \tex_if:D \if_charcode:w \if:w
+% \end{verbatim}
+%
+% It is easier to start by considering the possibilities for what a
+% token looks like, in other words, how \TeX{} sees it from the point
+% of view of delimited arguments. Two cases: the token is either a
+% control sequence or a single character.
+% \begin{center}
+% \begin{tabular}{p{4cm}p{4cm}}
+% \toprule
+% Control sequence & Character \\
+% \midrule
+% Control word: an escape character followed by some letters. &
+% \end{tabular}
+% \end{center}
+% The token can be a control sequence,
+% in other words, an escape character followed by some letters, or by
+% a single non-letter character. Examples of this include \cs{begin},
+% \cs{;}, \cs{emph}\ldots{} The other case
+% There are two cases. either the token
+% is a single character, in which case it is associated with a category
+% code:
+%
+%
+% \section{Character tokens}
+%
+% \begin{function}
+% {
+% \char_set_catcode_escape:N ,
+% \char_set_catcode_group_begin:N ,
+% \char_set_catcode_group_end:N ,
+% \char_set_catcode_math_toggle:N ,
+% \char_set_catcode_alignment:N ,
+% \char_set_catcode_end_line:N ,
+% \char_set_catcode_parameter:N ,
+% \char_set_catcode_math_superscript:N ,
+% \char_set_catcode_math_subscript:N ,
+% \char_set_catcode_ignore:N ,
+% \char_set_catcode_space:N ,
+% \char_set_catcode_letter:N ,
+% \char_set_catcode_other:N ,
+% \char_set_catcode_active:N ,
+% \char_set_catcode_comment:N ,
+% \char_set_catcode_invalid:N
+% }
+% \begin{syntax}
+% \cs{char_make_letter:N} \meta{character}
+% \end{syntax}
+% Sets the category code of the \meta{character} to that indicated in
+% the function name. Depending on the current category code of the
+% \meta{token} the escape token may also be needed:
+% \begin{verbatim}
+% \char_set_catcode_other:N \%
+% \end{verbatim}
+% The assignment is local.
+% \end{function}
+%
+% \begin{function}
+% {
+% \char_set_catcode_escape:n ,
+% \char_set_catcode_group_begin:n ,
+% \char_set_catcode_group_end:n ,
+% \char_set_catcode_math_toggle:n ,
+% \char_set_catcode_alignment:n ,
+% \char_set_catcode_end_line:n ,
+% \char_set_catcode_parameter:n ,
+% \char_set_catcode_math_superscript:n ,
+% \char_set_catcode_math_subscript:n ,
+% \char_set_catcode_ignore:n ,
+% \char_set_catcode_space:n ,
+% \char_set_catcode_letter:n ,
+% \char_set_catcode_other:n ,
+% \char_set_catcode_active:n ,
+% \char_set_catcode_comment:n ,
+% \char_set_catcode_invalid:n
+% }
+% \begin{syntax}
+% \cs{char_make_letter:n} \Arg{integer expression}
+% \end{syntax}
+% Sets the category code of the \meta{character} which has character
+% code as given by the \meta{integer expression}. This version can be
+% used to set up characters which cannot otherwise be given
+% (\emph{cf.}~the \texttt{N}-type variants). The assignment is local.
+% \end{function}
+%
+% \begin{function}{\char_set_catcode:nn}
+% \begin{syntax}
+% \cs{char_set_catcode:nn} \Arg{intexpr1} \Arg{intexpr2}
+% \end{syntax}
+% These functions set the category code of the \meta{character} which
+% has character code as given by the \meta{integer expression}.
+% The first \meta{integer expression}
+% is the character code and the second is the category code to apply.
+% The setting applies within the current \TeX{} group. In general, the
+% symbolic functions \cs{char_make_\meta{type}} should be preferred,
+% but there are cases where these lower-level functions may be useful.
+% \end{function}
+%
+% \begin{function}[EXP]{\char_value_catcode:n}
+% \begin{syntax}
+% \cs{char_value_catcode:n} \Arg{integer expression}
+% \end{syntax}
+% Expands to the current category code of the \meta{character} with
+% character code given by the
+% \meta{integer expression}.
+% \end{function}
+%
+% \begin{function}{\char_show_value_catcode:n}
+% \begin{syntax}
+% \cs{char_show_value_catcode:n} \Arg{integer expression}
+% \end{syntax}
+% Displays the current category code of the \meta{character} with
+% character code given by the \meta{integer expression} on the
+% terminal.
+% \end{function}
+%
+% \begin{function}{\char_set_lccode:nn}
+% \begin{syntax}
+% \cs{char_set_lcode:nn} \Arg{intexpr1} \Arg{intexpr2}
+% \end{syntax}
+% This function set up the behaviour of \meta{character} when
+% found inside \cs{tl_to_lowercase:n}, such that \meta{character1}
+% will be converted into \meta{character2}. The two \meta{characters}
+% may be specified using an \meta{integer expression} for the character code
+% concerned. This may include the \TeX{} |`|\meta{character}
+% method for converting a single character into its character
+% code:
+% \begin{verbatim}
+% \char_set_lccode:nn { `\A } { `\a } % Standard behaviour
+% \char_set_lccode:nn { `\A } { `\A + 32 }
+% \char_set_lccode:nn { 50 } { 60 }
+% \end{verbatim}
+% The setting applies within the current \TeX{} group.
+% \end{function}
+%
+% \begin{function}[EXP]{\char_value_lccode:n}
+% \begin{syntax}
+% \cs{char_value_lccode:n} \Arg{integer expression}
+% \end{syntax}
+% Expands to the current lower case code of the \meta{character} with
+% character code given by the
+% \meta{integer expression}.
+% \end{function}
+%
+% \begin{function}{\char_show_value_lccode:n}
+% \begin{syntax}
+% \cs{char_show_value_lccode:n} \Arg{integer expression}
+% \end{syntax}
+% Displays the current lower case code of the \meta{character} with
+% character code given by the \meta{integer expression} on the
+% terminal.
+% \end{function}
+%
+% \begin{function}{\char_set_uccode:nn}
+% \begin{syntax}
+% \cs{char_set_uccode:nn} \Arg{intexpr1} \Arg{intexpr2}
+% \end{syntax}
+% This function set up the behaviour of \meta{character} when
+% found inside \cs{tl_to_uppeercase:n}, such that \meta{character1}
+% will be converted into \meta{character2}. The two \meta{characters}
+% may be specified using an \meta{integer expression} for the character code
+% concerned. This may include the \TeX{} |`|\meta{character}
+% method for converting a single character into its character
+% code:
+% \begin{verbatim}
+% \char_set_uccode:nn { `\a } { `\A } % Standard behaviour
+% \char_set_uccode:nn { `\A } { `\A - 32 }
+% \char_set_uccode:nn { 60 } { 50 }
+% \end{verbatim}
+% The setting applies within the current \TeX{} group.
+% \end{function}
+%
+% \begin{function}[EXP]{\char_value_uccode:n}
+% \begin{syntax}
+% \cs{char_value_uccode:n} \Arg{integer expression}
+% \end{syntax}
+% Expands to the current upper case code of the \meta{character} with
+% character code given by the
+% \meta{integer expression}.
+% \end{function}
+%
+% \begin{function}{\char_show_value_uccode:n}
+% \begin{syntax}
+% \cs{char_show_value_uccode:n} \Arg{integer expression}
+% \end{syntax}
+% Displays the current upper case code of the \meta{character} with
+% character code given by the \meta{integer expression} on the
+% terminal.
+% \end{function}
+%
+% \begin{function}{\char_set_mathcode:nn}
+% \begin{syntax}
+% \cs{char_set_mathcode:nn} \Arg{intexpr1} \Arg{intexpr2}
+% \end{syntax}
+% This function sets up the math code of \meta{character}.
+% The \meta{character} is specified as
+% an \meta{integer expression} which will be used as the character
+% code of the relevant character. The setting applies within the
+% current \TeX{} group.
+% \end{function}
+%
+% \begin{function}[EXP]{\char_value_mathcode:n}
+% \begin{syntax}
+% \cs{char_value_mathcode:n} \Arg{integer expression}
+% \end{syntax}
+% Expands to the current math code of the \meta{character} with
+% character code given by the
+% \meta{integer expression}.
+% \end{function}
+%
+% \begin{function}{\char_show_value_mathcode:n}
+% \begin{syntax}
+% \cs{char_show_value_mathcode:n} \Arg{integer expression}
+% \end{syntax}
+% Displays the current math code of the \meta{character} with
+% character code given by the \meta{integer expression} on the
+% terminal.
+% \end{function}
+%
+% \begin{function}{\char_set_sfcode:nn}
+% \begin{syntax}
+% \cs{char_set_sfcode:nn} \Arg{intexpr1} \Arg{intexpr2}
+% \end{syntax}
+% This function sets up the space factor for the \meta{character}.
+% The \meta{character} is specified as
+% an \meta{integer expression} which will be used as the character
+% code of the relevant character. The setting applies within the
+% current \TeX{} group.
+% \end{function}
+%
+% \begin{function}[EXP]{\char_value_sfcode:n}
+% \begin{syntax}
+% \cs{char_value_sfcode:n} \Arg{integer expression}
+% \end{syntax}
+% Expands to the current space factor for the \meta{character} with
+% character code given by the
+% \meta{integer expression}.
+% \end{function}
+%
+% \begin{function}{\char_show_value_sfcode:n}
+% \begin{syntax}
+% \cs{char_show_value_sfcode:n} \Arg{integer expression}
+% \end{syntax}
+% Displays the current space factor for the \meta{character} with
+% character code given by the \meta{integer expression} on the
+% terminal.
+% \end{function}
+%
+% \section{Generic tokens}
+%
+% \begin{function}{\token_new:Nn}
+% \begin{syntax}
+% \cs{token_new:Nn} \meta{token1} \Arg{token2}
+% \end{syntax}
+% Defines \meta{token1} to globally be a snapshot of \meta{token2}.
+% This will be an implicit representation of \meta{token2}.
+% \end{function}
+%
+% \begin{variable}
+% {
+% \c_group_begin_token,
+% \c_group_end_token,
+% \c_math_toggle_token,
+% \c_alignment_token,
+% \c_parameter_token,
+% \c_math_superscript_token,
+% \c_math_subscript_token,
+% \c_space_token
+% }
+% These are implicit tokens which have the category code described
+% by their name. They are used internally for test purposes but
+% are also available to the programmer for other uses.
+% \end{variable}
+%
+% \begin{variable}
+% {
+% \c_catcode_letter_token,
+% \c_catcode_other_token
+% }
+% These are implicit tokens which have the category code described
+% by their name. They are used internally for test purposes and should
+% not be used other than for category code tests.
+% \end{variable}
+%
+% \begin{variable}{\c_catcode_active_tl}
+% A token list containing an active token. This is used internally
+% for test purposes and should not be used other than in
+% appropriately-constructed category code tests.
+% \end{variable}
+%
+% \section{Converting tokens}
+%
+% \begin{function}[EXP]{\token_to_meaning:N}
+% \begin{syntax}
+% \cs{token_to_meaning:N} \meta{token}
+% \end{syntax}
+% Inserts the current meaning of the \meta{token} into the input
+% stream as a series of characters of category code $12$ (other).
+% This will be the primitive \TeX{} description of the \meta{token},
+% thus for example both functions defined by \cs{cs_set_nopar:Npn}
+% and token list variables defined using \cs{tl_new:N} will be described
+% as |macro|s.
+% \begin{texnote}
+% This is the \TeX{} primitive \tn{meaning}.
+% \end{texnote}
+% \end{function}
+%
+% \begin{function}[EXP]{\token_to_str:N, \token_to_str:c}
+% \begin{syntax}
+% \cs{token_to_str:N} \meta{token}
+% \end{syntax}
+% Converts the given \meta{token} into a series of characters with
+% category code $12$ (other). The current escape character will be
+% the first character in the sequence, although this will also have
+% category code $12$ (the escape character is part of the
+% \meta{token}). This function requires only a single expansion.
+% \begin{texnote}
+% \cs{token_to_str:N} is the \TeX{} primitive \cs{string} renamed.
+% \end{texnote}
+% \end{function}
+%
+% \section{Token conditionals}
+%
+% \begin{function}[EXP,pTF]{\token_if_group_begin:N}
+% \begin{syntax}
+% \cs{token_if_group_begin_p:N} \meta{token}
+% \cs{token_if_group_begin:NTF} \meta{token} \Arg{true code}
+% ~~\Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of a begin group token
+% (|{| when normal \TeX{} category codes are in ^^A }
+% force). The branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% Note that an explicit begin group token cannot be tested in this way,
+% as it is not a valid \texttt{N}-type argument.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_group_end:N}
+% \begin{syntax}
+% \cs{token_if_group_end_p:N} \meta{token}
+% \cs{token_if_group_end:NTF} \meta{token} \Arg{true code}
+% ~~\Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of an end group token
+% (^^A {
+% |}| when normal \TeX{} category codes are in force). The branching
+% versions then leave either \meta{true code} or \meta{false code} in the
+% input stream, as appropriate to the truth of the test and the variant
+% of the function chosen. The logical truth of the test is left in the
+% input stream by the predicate version.
+% Note that an explicit end group token cannot be tested in this way,
+% as it is not a valid \texttt{N}-type argument.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_math_toggle:N}
+% \begin{syntax}
+% \cs{token_if_math_toggle_p:N} \meta{token}
+% \cs{token_if_math_toggle:NTF} \meta{token} \Arg{true code}
+% ~~\Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of a math shift token
+% (|$| when normal \TeX{} category codes are in force). The
+% branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_alignment:N}
+% \begin{syntax}
+% \cs{token_if_alignment_p:N} \meta{token}
+% \cs{token_if_alignment:NTF} \meta{token} \Arg{true code}
+% ~~\Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of an alignment token
+% (|&| when normal \TeX{} category codes are in force). The
+% branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_parameter:N}
+% \begin{syntax}
+% \cs{token_if_parameter_p:N} \meta{token}
+% \cs{token_if_alignment:NTF} \meta{token} \Arg{true code}
+% ~~\Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of a macro parameter token
+% (|#| when normal \TeX{} category codes are in force). The
+% branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_math_superscript:N}
+% \begin{syntax}
+% \cs{token_if_math_superscript_p:N} \meta{token}
+% \cs{token_if_math_superscript:NTF} \meta{token} \Arg{true code}
+% ~~\Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of a superscript token
+% (|^| when normal \TeX{} category codes are in force). The
+% branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_math_subscript:N}
+% \begin{syntax}
+% \cs{token_if_math_subscript_p:N} \meta{token}
+% \cs{token_if_math_subscript:NTF} \meta{token} \Arg{true code}
+% ~~\Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of a subscript token
+% (|_| when normal \TeX{} category codes are in force). The
+% branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_space:N}
+% \begin{syntax}
+% \cs{token_if_space_p:N} \meta{token}
+% \cs{token_if_space:NTF} \meta{token} \Arg{true code}
+% ~~\Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of a space token. The
+% branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% Note that an explicit space token with character code $32$ cannot
+% be tested in this way, as it is not a valid \texttt{N}-type argument.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_letter:N}
+% \begin{syntax}
+% \cs{token_if_letter_p:N} \meta{token}
+% \cs{token_if_letter:NTF} \meta{token} \Arg{true code}
+% ~~\Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of a letter token. The
+% branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_other:N}
+% \begin{syntax}
+% \cs{token_if_other_p:N} \meta{token}
+% \cs{token_if_other:NTF} \meta{token} \Arg{true code}
+% ~~\Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of an \enquote{other}
+% token. The branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_active:N}
+% \begin{syntax}
+% \cs{token_if_active_p:N} \meta{token}
+% \cs{token_if_active:NTF} \meta{token} \Arg{true code}
+% ~~\Arg{false code}
+% \end{syntax}
+% Tests if \meta{token} has the category code of an active character.
+% The branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_eq_catcode:NN}
+% \begin{syntax}
+% \cs{token_if_eq_catcode_p:NN} \meta{token1} \meta{token2}
+% \cs{token_if_eq_catcode:NNTF} \meta{token1} \meta{token2}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the two \meta{tokens} have the same category code. The
+% branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_eq_charcode:NN}
+% \begin{syntax}
+% \cs{token_if_eq_charcode_p:NN} \meta{token1} \meta{token2}
+% \cs{token_if_eq_charcode:NNTF} \meta{token1} \meta{token2}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the two \meta{tokens} have the same character code. The
+% branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_eq_meaning:NN}
+% \begin{syntax}
+% \cs{token_if_eq_meaning_p:NN} \meta{token1} \meta{token2}
+% \cs{token_if_eq_meaning:NNTF} \meta{token1} \meta{token2}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the two \meta{tokens} have the same meaning when expanded.
+% The branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_macro:N}
+% \begin{syntax}
+% \cs{token_if_macro_p:N} \meta{token}
+% \cs{token_if_macro:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is a \TeX{} macro.
+% The branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_cs:N}
+% \begin{syntax}
+% \cs{token_if_cs_p:N} \meta{token}
+% \cs{token_if_cs:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is a control sequence.
+% The branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_expandable:N}
+% \begin{syntax}
+% \cs{token_if_expandable_p:N} \meta{token}
+% \cs{token_if_expandable:NTF} \meta{token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is expandable. This test returns \meta{false}
+% for an undefined token.
+% The branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_long_macro:N}
+% \begin{syntax}
+% \cs{token_if_long_macro_p:N} \meta{token}
+% \cs{token_if_long_macro:NTF} \meta{token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is a long macro.
+% The branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_protected_macro:N}
+% \begin{syntax}
+% \cs{token_if_protected_macro_p:N} \meta{token}
+% \cs{token_if_protected_macro:NTF} \meta{token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is a protected macro: a macro which
+% is both protected and long will return logical false.
+% The branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+%% \begin{function}[EXP,pTF]{\token_if_protected_long_macro:N}
+% \begin{syntax}
+% \cs{token_if_protected_long_macro_p:N} \meta{token}
+% \cs{token_if_protected_long_macro:NTF} \meta{token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is a protected long macro.
+% The branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_chardef:N}
+% \begin{syntax}
+% \cs{token_if_chardef_p:N} \meta{token}
+% \cs{token_if_chardef:NTF} \meta{token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is defined to be a chardef.
+% The branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_mathchardef:N}
+% \begin{syntax}
+% \cs{token_if_mathchardef_p:N} \meta{token}
+% \cs{token_if_mathchardef:NTF} \meta{token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is defined to be a mathchardef.
+% The branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_dim_register:N}
+% \begin{syntax}
+% \cs{token_if_dim_register_p:N} \meta{token}
+% \cs{token_if_dim_register:NTF} \meta{token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is defined to be a dimension register.
+% The branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_int_register:N}
+% \begin{syntax}
+% \cs{token_if_int_register_p:N} \meta{token}
+% \cs{token_if_int_register:NTF} \meta{token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is defined to be a integer register.
+% The branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_skip_register:N}
+% \begin{syntax}
+% \cs{token_if_skip_register_p:N} \meta{token}
+% \cs{token_if_skip_register:NTF} \meta{token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is defined to be a skip register.
+% The branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_toks_register:N}
+% \begin{syntax}
+% \cs{token_if_toks_register_p:N} \meta{token}
+% \cs{token_if_toks_register:NTF} \meta{token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is defined to be a toks register
+% (not used by\LaTeX3).
+% The branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\token_if_primitive:N}
+% \begin{syntax}
+% \cs{token_if_primitive_p:N} \meta{token}
+% \cs{token_if_primitive:NTF} \meta{token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{token} is an engine primitive.
+% The branching versions then leave either \meta{true code} or
+% \meta{false code} in the input stream, as appropriate to the truth of
+% the test and the variant of the function chosen. The logical truth of
+% the test is left in the input stream by the predicate version.
+% \end{function}
+%
+% \section{Peeking ahead at the next token}
+%
+% There is often a need to look ahead at the next token in the input
+% stream while leaving it in place. This is handled using the
+% \enquote{peek} functions. The generic \cs{peek_after:Nw} is
+% provided along with a family of predefined tests for common cases.
+% As peeking ahead does \emph{not} skip spaces the predefined tests
+% include both a space-respecting and space-skipping version.
+%
+% \begin{function}{\peek_after:Nw}
+% \begin{syntax}
+% \cs{peek_after:Nw} \meta{function} \meta{token}
+% \end{syntax}
+% Locally sets the test variable \cs{l_peek_token} equal to \meta{token}
+% (as an implicit token, \emph{not} as a token list), and then
+% expands the \meta{function}. The \meta{token} will remain in
+% the input stream as the next item after the \meta{function}.
+% The \meta{token} here may be \verb*| |, |{| or |}| (assuming
+% normal \TeX{} category codes), \emph{i.e.}~it is not necessarily the
+% next argument which would be grabbed by a normal function.
+% \end{function}
+%
+% \begin{function}{\peek_gafter:Nw}
+% \begin{syntax}
+% \cs{peek_gafter:Nw} \meta{function} \meta{token}
+% \end{syntax}
+% Globally sets the test variable \cs{g_peek_token} equal to \meta{token}
+% (as an implicit token, \emph{not} as a token list), and then
+% expands the \meta{function}. The \meta{token} will remain in
+% the input stream as the next item after the \meta{function}.
+% The \meta{token} here may be \verb*| |, |{| or |}| (assuming
+% normal \TeX{} category codes), \emph{i.e.}~it is not necessarily the
+% next argument which would be grabbed by a normal function.
+% \end{function}
+%
+% \begin{variable}{\l_peek_token}
+% Token set by \cs{peek_after:Nw} and available for testing
+% as described above.
+% \end{variable}
+%
+% \begin{variable}{\g_peek_token}
+% Token set by \cs{peek_gafter:Nw} and available for testing
+% as described above.
+% \end{variable}
+%
+% \begin{function}[TF]{\peek_catcode:N}
+% \begin{syntax}
+% \cs{peek_catcode:NTF} \meta{test token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next \meta{token} in the input stream has the same
+% category code as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_catcode:NNTF}). Spaces are respected by the test
+% and the \meta{token} will be left in the input stream after
+% the \meta{true code} or \meta{false code} (as appropriate to the
+% result of the test).
+% \end{function}
+%
+% \begin{function}[TF]{\peek_catcode_ignore_spaces:N}
+% \begin{syntax}
+% \cs{peek_catcode_ignore_spaces:NTF} \meta{test token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next \meta{token} in the input stream has the same
+% category code as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_catcode:NNTF}). Spaces are ignored by the test
+% and the \meta{token} will be left in the input stream after
+% the \meta{true code} or \meta{false code} (as appropriate to the
+% result of the test).
+% \end{function}
+%
+% \begin{function}[TF]{\peek_catcode_remove:N}
+% \begin{syntax}
+% \cs{peek_catcode_remove:NTF} \meta{test token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next \meta{token} in the input stream has the same
+% category code as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_catcode:NNTF}). Spaces are respected by the test
+% and the \meta{token} will be removed from the input stream if the
+% test is true. The function will then place either the
+% \meta{true code} or \meta{false code} in the input stream (as
+% appropriate to the result of the test).
+% \end{function}
+%
+% \begin{function}[TF]{\peek_catcode_remove_ignore_spaces:N}
+% \begin{syntax}
+% \cs{peek_catcode_remove_ignore_spaces:NTF} \meta{test token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next \meta{token} in the input stream has the same
+% category code as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_catcode:NNTF}). Spaces are ignored by the test
+% and the \meta{token} will be removed from the input stream if the
+% test is true. The function will then place either the
+% \meta{true code} or \meta{false code} in the input stream (as
+% appropriate to the result of the test).
+% \end{function}
+%
+% \begin{function}[TF]{\peek_charcode:N}
+% \begin{syntax}
+% \cs{peek_charcode:NTF} \meta{test token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next \meta{token} in the input stream has the same
+% character code as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_charcode:NNTF}). Spaces are respected by the test
+% and the \meta{token} will be left in the input stream after
+% the \meta{true code} or \meta{false code} (as appropriate to the
+% result of the test).
+% \end{function}
+%
+% \begin{function}[TF]{\peek_charcode_ignore_spaces:N}
+% \begin{syntax}
+% \cs{peek_charcode_ignore_spaces:NTF} \meta{test token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next \meta{token} in the input stream has the same
+% character code as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_charcode:NNTF}). Spaces are ignored by the test
+% and the \meta{token} will be left in the input stream after
+% the \meta{true code} or \meta{false code} (as appropriate to the
+% result of the test).
+% \end{function}
+%
+% \begin{function}[TF]{\peek_charcode_remove:N}
+% \begin{syntax}
+% \cs{peek_charcode_remove:NTF} \meta{test token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next \meta{token} in the input stream has the same
+% character code as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_charcode:NNTF}). Spaces are respected by the test
+% and the \meta{token} will be removed from the input stream if the
+% test is true. The function will then place either the
+% \meta{true code} or \meta{false code} in the input stream (as
+% appropriate to the result of the test).
+% \end{function}
+%
+% \begin{function}[TF]{\peek_charcode_remove_ignore_spaces:N}
+% \begin{syntax}
+% \cs{peek_charcode_remove_ignore_spaces:NTF} \meta{test token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next \meta{token} in the input stream has the same
+% character code as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_charcode:NNTF}). Spaces are ignored by the test
+% and the \meta{token} will be removed from the input stream if the
+% test is true. The function will then place either the
+% \meta{true code} or \meta{false code} in the input stream (as
+% appropriate to the result of the test).
+% \end{function}
+%
+% \begin{function}[TF]{\peek_meaning:N}
+% \begin{syntax}
+% \cs{peek_meaning:NTF} \meta{test token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next \meta{token} in the input stream has the same
+% meaning as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_meaning:NNTF}). Spaces are respected by the test
+% and the \meta{token} will be left in the input stream after
+% the \meta{true code} or \meta{false code} (as appropriate to the
+% result of the test).
+% \end{function}
+%
+% \begin{function}[TF]{\peek_meaning_ignore_spaces:N}
+% \begin{syntax}
+% \cs{peek_meaning_ignore_spaces:NTF} \meta{test token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next \meta{token} in the input stream has the same
+% meaning as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_meaning:NNTF}). Spaces are ignored by the test
+% and the \meta{token} will be left in the input stream after
+% the \meta{true code} or \meta{false code} (as appropriate to the
+% result of the test).
+% \end{function}
+%
+% \begin{function}[TF]{\peek_meaning_remove:N}
+% \begin{syntax}
+% \cs{peek_meaning_remove:NTF} \meta{test token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next \meta{token} in the input stream has the same
+% meaning as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_meaning:NNTF}). Spaces are respected by the test
+% and the \meta{token} will be removed from the input stream if the
+% test is true. The function will then place either the
+% \meta{true code} or \meta{false code} in the input stream (as
+% appropriate to the result of the test).
+% \end{function}
+%
+% \begin{function}[TF]{\peek_meaning_remove_ignore_spaces:N}
+% \begin{syntax}
+% \cs{peek_meaning_remove_ignore_spaces:NTF} \meta{test token}
+% ~~\Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the next \meta{token} in the input stream has the same
+% meaning as the \meta{test token} (as defined by the test
+% \cs{token_if_eq_meaning:NNTF}). Spaces are ignored by the test
+% and the \meta{token} will be removed from the input stream if the
+% test is true. The function will then place either the
+% \meta{true code} or \meta{false code} in the input stream (as
+% appropriate to the result of the test).
+% \end{function}
+%
+% \section{Decomposing a macro definition}
+%
+% These functions decompose \TeX{} macros into their constituent
+% parts: if the \meta{token} passed is not a macro then no decomposition
+% can occur. In the later case, all three functions leave \cs{scan_stop:}
+% in the input stream.
+%
+% \begin{function}[EXP]{\token_get_arg_spec:N}
+% \begin{syntax}
+% \cs{token_get_arg_spec:N} \meta{token}
+% \end{syntax}
+% If the \meta{token} is a macro, this function will leave
+% the primitive \TeX{} argument specification in input stream as
+% a string of tokens of category code $12$ (with spaces having category
+% code $10$). Thus for example for a token \cs{next} defined by
+% \begin{verbatim}
+% \cs_set:Npn \next #1#2 { x #1 y #2 }
+% \end{verbatim}
+% will leave |#1#2| in the input stream. If the \meta{token} is
+% not a macro then \cs{scan_stop:} will be left in the input stream
+% \begin{texnote}
+% If the arg~spec. contains the string |->|, then the |spec| function
+% will produce incorrect results.
+% \end{texnote}
+% \end{function}
+%
+% \begin{function}[EXP]{\token_get_replacement_text:N}
+% \begin{syntax}
+% \cs{token_get_replacement_text:N} \meta{token}
+% \end{syntax}
+% If the \meta{token} is a macro, this function will leave
+% the replacement text in input stream as
+% a string of tokens of category code $12$ (with spaces having category
+% code $10$). Thus for example for a token \cs{next} defined by
+% \begin{verbatim}
+% \cs_set:Npn \next #1#2 { x #1~y #2 }
+% \end{verbatim}
+% will leave \verb|x#1 y#2| in the input stream. If the \meta{token} is
+% not a macro then \cs{scan_stop:} will be left in the input stream
+% \end{function}
+%
+% \begin{function}[EXP]{\token_get_prefix_spec:N}
+% \begin{syntax}
+% \cs{token_get_prefix_spec:N} \meta{token}
+% \end{syntax}
+% If the \meta{token} is a macro, this function will leave
+% the \TeX{} prefixes applicable in input stream as
+% a string of tokens of category code $12$ (with spaces having category
+% code $10$). Thus for example for a token \cs{next} defined by
+% \begin{verbatim}
+% \cs_set:Npn \next #1#2 { x #1~y #2 }
+% \end{verbatim}
+% will leave |\long| in the input stream. If the \meta{token} is
+% not a macro then \cs{scan_stop:} will be left in the input stream
+% \end{function}
+%
+% \section{Experimental token functions}
+%
+% \begin{function}{\char_active_set:Npn, \char_active_set:Npx}
+% \begin{syntax}
+% \cs{char_active_set:Npn} \meta{char} \meta{parameters} \Arg{code}
+% \end{syntax}
+% Makes \meta{char} an active character to expand to \meta{code} as
+% replacement text.
+% Within the \meta{code}, the \meta{parameters} (|#1|, |#2|,
+% \emph{etc.}) will be replaced by those absorbed
+% This definition is local to the current \TeX{} group.
+% \end{function}
+%
+% \begin{function}{\char_active_gset:Npn, \char_active_gset:Npx}
+% \begin{syntax}
+% \cs{char_active_gset:Npn} \meta{char} \meta{parameters} \Arg{code}
+% \end{syntax}
+% Makes \meta{char} an active character to expand to \meta{code} as
+% replacement text.
+% Within the \meta{code}, the \meta{parameters} (|#1|, |#2|,
+% \emph{etc.}) will be replaced by those absorbed
+% This definition is global.
+% \end{function}
+%
+% \begin{function}{\char_active_set_eq:NN}
+% \begin{syntax}
+% \cs{char_active_set_eq:NN} \meta{char} \meta{function}
+% \end{syntax}
+% Makes \meta{char} an active character equivalent in meaning to the
+% \meta{fiucntion} (which may itself be an active character). This
+% definition is local to the current \TeX{} group.
+% \end{function}
+%
+% \begin{function}{\char_active_gset_eq:NN}
+% \begin{syntax}
+% \cs{char_active_gset_eq:NN} \meta{char} \meta{function}
+% \end{syntax}
+% Makes \meta{char} an active character equivalent in meaning to the
+% \meta{fiucntion} (which may itself be an active character). This
+% definition is global.
+% \end{function}
+%
+% \end{documentation}
+%
+% \begin{implementation}
+%
+% \section{\pkg{l3token} implementation}
+%
+% \begin{macrocode}
+%<*initex|package>
+% \end{macrocode}
+%
+% \begin{macrocode}
+%<*package>
+\ProvidesExplPackage
+ {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
+\package_check_loaded_expl:
+%</package>
+% \end{macrocode}
+%
+% \subsection{Character tokens}
+%
+% \begin{macro}{\char_set_catcode:nn}
+% \begin{macro}{\char_value_catcode:n}
+% \begin{macro}{\char_show_value_catcode:n}
+% Category code changes.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \char_set_catcode:nn #1#2
+ { \tex_catcode:D #1 = \int_eval:w #2 \int_eval_end: }
+\cs_new_nopar:Npn \char_value_catcode:n #1
+ { \tex_the:D \tex_catcode:D \int_eval:w #1\int_eval_end: }
+\cs_new_nopar:Npn \char_show_value_catcode:n #1
+ { \tex_showthe:D \tex_catcode:D \int_eval:w #1 \int_eval_end: }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}
+% {
+% \char_set_catcode_escape:N ,
+% \char_set_catcode_group_begin:N ,
+% \char_set_catcode_group_end:N ,
+% \char_set_catcode_math_toggle:N ,
+% \char_set_catcode_alignment:N ,
+% \char_set_catcode_end_line:N ,
+% \char_set_catcode_parameter:N ,
+% \char_set_catcode_math_superscript:N ,
+% \char_set_catcode_math_subscript:N ,
+% \char_set_catcode_ignore:N ,
+% \char_set_catcode_space:N ,
+% \char_set_catcode_letter:N ,
+% \char_set_catcode_other:N ,
+% \char_set_catcode_active:N ,
+% \char_set_catcode_comment:N ,
+% \char_set_catcode_invalid:N
+% }
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \char_set_catcode_escape:N #1
+ { \char_set_catcode:nn { `#1 } \c_zero }
+\cs_new_protected_nopar:Npn \char_set_catcode_group_begin:N #1
+ { \char_set_catcode:nn { `#1 } \c_one }
+\cs_new_protected_nopar:Npn \char_set_catcode_group_end:N #1
+ { \char_set_catcode:nn { `#1 } \c_two }
+\cs_new_protected_nopar:Npn \char_set_catcode_math_toggle:N #1
+ { \char_set_catcode:nn { `#1 } \c_three }
+\cs_new_protected_nopar:Npn \char_set_catcode_alignment:N #1
+ { \char_set_catcode:nn { `#1 } \c_four }
+\cs_new_protected_nopar:Npn \char_set_catcode_end_line:N #1
+ { \char_set_catcode:nn { `#1 } \c_five }
+\cs_new_protected_nopar:Npn \char_set_catcode_parameter:N #1
+ { \char_set_catcode:nn { `#1 } \c_six }
+\cs_new_protected_nopar:Npn \char_set_catcode_math_superscript:N #1
+ { \char_set_catcode:nn { `#1 } \c_seven }
+\cs_new_protected_nopar:Npn \char_set_catcode_math_subscript:N #1
+ { \char_set_catcode:nn { `#1 } \c_eight }
+\cs_new_protected_nopar:Npn \char_set_catcode_ignore:N #1
+ { \char_set_catcode:nn { `#1 } \c_nine }
+\cs_new_protected_nopar:Npn \char_set_catcode_space:N #1
+ { \char_set_catcode:nn { `#1 } \c_ten }
+\cs_new_protected_nopar:Npn \char_set_catcode_letter:N #1
+ { \char_set_catcode:nn { `#1 } \c_eleven }
+\cs_new_protected_nopar:Npn \char_set_catcode_other:N #1
+ { \char_set_catcode:nn { `#1 } \c_twelve }
+\cs_new_protected_nopar:Npn \char_set_catcode_active:N #1
+ { \char_set_catcode:nn { `#1 } \c_thirteen }
+\cs_new_protected_nopar:Npn \char_set_catcode_comment:N #1
+ { \char_set_catcode:nn { `#1 } \c_fourteen }
+\cs_new_protected_nopar:Npn \char_set_catcode_invalid:N #1
+ { \char_set_catcode:nn { `#1 } \c_fifteen }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}
+% {
+% \char_set_catcode_escape:n ,
+% \char_set_catcode_group_begin:n ,
+% \char_set_catcode_group_end:n ,
+% \char_set_catcode_math_toggle:n ,
+% \char_set_catcode_alignment:n ,
+% \char_set_catcode_end_line:n ,
+% \char_set_catcode_parameter:n ,
+% \char_set_catcode_math_superscript:n ,
+% \char_set_catcode_math_subscript:n ,
+% \char_set_catcode_ignore:n ,
+% \char_set_catcode_space:n ,
+% \char_set_catcode_letter:n ,
+% \char_set_catcode_other:n ,
+% \char_set_catcode_active:n ,
+% \char_set_catcode_comment:n ,
+% \char_set_catcode_invalid:n
+% }
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \char_set_catcode_escape:n #1
+ { \char_set_catcode:nn {#1} \c_zero }
+\cs_new_protected_nopar:Npn \char_set_catcode_group_begin:n #1
+ { \char_set_catcode:nn {#1} \c_one }
+\cs_new_protected_nopar:Npn \char_set_catcode_group_end:n #1
+ { \char_set_catcode:nn {#1} \c_two }
+\cs_new_protected_nopar:Npn \char_set_catcode_math_toggle:n #1
+ { \char_set_catcode:nn {#1} \c_three }
+\cs_new_protected_nopar:Npn \char_set_catcode_alignment:n #1
+ { \char_set_catcode:nn {#1} \c_four }
+\cs_new_protected_nopar:Npn \char_set_catcode_end_line:n #1
+ { \char_set_catcode:nn {#1} \c_five }
+\cs_new_protected_nopar:Npn \char_set_catcode_parameter:n #1
+ { \char_set_catcode:nn {#1} \c_six }
+\cs_new_protected_nopar:Npn \char_set_catcode_math_superscript:n #1
+ { \char_set_catcode:nn {#1} \c_seven }
+\cs_new_protected_nopar:Npn \char_set_catcode_math_subscript:n #1
+ { \char_set_catcode:nn {#1} \c_eight }
+\cs_new_protected_nopar:Npn \char_set_catcode_ignore:n #1
+ { \char_set_catcode:nn {#1} \c_nine }
+\cs_new_protected_nopar:Npn \char_set_catcode_space:n #1
+ { \char_set_catcode:nn {#1} \c_ten }
+\cs_new_protected_nopar:Npn \char_set_catcode_letter:n #1
+ { \char_set_catcode:nn {#1} \c_eleven }
+\cs_new_protected_nopar:Npn \char_set_catcode_other:n #1
+ { \char_set_catcode:nn {#1} \c_twelve }
+\cs_new_protected_nopar:Npn \char_set_catcode_active:n #1
+ { \char_set_catcode:nn {#1} \c_thirteen }
+\cs_new_protected_nopar:Npn \char_set_catcode_comment:n #1
+ { \char_set_catcode:nn {#1} \c_fourteen }
+\cs_new_protected_nopar:Npn \char_set_catcode_invalid:n #1
+ { \char_set_catcode:nn {#1} \c_fifteen }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\char_set_mathcode:nn}
+% \begin{macro}{\char_value_mathcode:n}
+% \begin{macro}{\char_show_value_mathcode:n}
+% \begin{macro}{\char_set_lccode:nn}
+% \begin{macro}{\char_value_lccode:n}
+% \begin{macro}{\char_show_value_lccode:n}
+% \begin{macro}{\char_set_uccode:nn}
+% \begin{macro}{\char_value_uccode:n}
+% \begin{macro}{\char_show_value_uccode:n}
+% \begin{macro}{\char_set_sfcode:nn}
+% \begin{macro}{\char_value_sfcode:n}
+% \begin{macro}{\char_show_value_sfcode:n}
+% Pretty repetitive, but necessary!
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \char_set_mathcode:nn #1#2
+ { \tex_mathcode:D #1 = \int_eval:w #2 \int_eval_end: }
+\cs_new_nopar:Npn \char_value_mathcode:n #1
+ { \tex_the:D \tex_mathcode:D \int_eval:w #1\int_eval_end: }
+\cs_new_nopar:Npn \char_show_value_mathcode:n #1
+ { \tex_showthe:D \tex_mathcode:D \int_eval:w #1 \int_eval_end: }
+\cs_new_protected_nopar:Npn \char_set_lccode:nn #1#2
+ { \tex_lccode:D #1 = \int_eval:w #2 \int_eval_end: }
+\cs_new_nopar:Npn \char_value_lccode:n #1
+ { \tex_the:D \tex_lccode:D \int_eval:w #1\int_eval_end: }
+\cs_new_nopar:Npn \char_show_value_lccode:n #1
+ { \tex_showthe:D \tex_lccode:D \int_eval:w #1 \int_eval_end: }
+\cs_new_protected_nopar:Npn \char_set_uccode:nn #1#2
+ { \tex_uccode:D #1 = \int_eval:w #2 \int_eval_end: }
+\cs_new_nopar:Npn \char_value_uccode:n #1
+ { \tex_the:D \tex_uccode:D \int_eval:w #1\int_eval_end: }
+\cs_new_nopar:Npn \char_show_value_uccode:n #1
+ { \tex_showthe:D \tex_uccode:D \int_eval:w #1 \int_eval_end: }
+\cs_new_protected_nopar:Npn \char_set_sfcode:nn #1#2
+ { \tex_sfcode:D #1 = \int_eval:w #2 \int_eval_end: }
+\cs_new_nopar:Npn \char_value_sfcode:n #1
+ { \tex_the:D \tex_sfcode:D \int_eval:w #1\int_eval_end: }
+\cs_new_nopar:Npn \char_show_value_sfcode:n #1
+ { \tex_showthe:D \tex_sfcode:D \int_eval:w #1 \int_eval_end: }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \subsection{Generic tokens}
+%
+% \begin{macro}{\token_new:Nn}
+% Creates a new token.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \token_new:Nn #1#2 { \cs_new_eq:NN #1 #2 }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}
+% {
+% \c_group_begin_token,
+% \c_group_end_token,
+% \c_math_toggle_token,
+% \c_alignment_token,
+% \c_parameter_token,
+% \c_math_superscript_token,
+% \c_math_subscript_token,
+% \c_space_token,
+% \c_catcode_letter_token,
+% \c_catcode_other_token
+% }
+% We define these useful tokens. We have to do it by hand with the
+% brace tokens for obvious reasons.
+% \begin{macrocode}
+\cs_new_eq:NN \c_group_begin_token {
+\cs_new_eq:NN \c_group_end_token }
+\group_begin:
+ \char_set_catcode_math_toggle:N \*
+ \token_new:Nn \c_math_toggle_token { * }
+ \char_set_catcode_alignment:N \*
+ \token_new:Nn \c_alignment_token { * }
+ \token_new:Nn \c_parameter_token { # }
+ \token_new:Nn \c_math_superscript_token { ^ }
+ \char_set_catcode_math_subscript:N \*
+ \token_new:Nn \c_math_subscript_token { * }
+ \token_new:Nn \c_space_token { ~ }
+ \token_new:Nn \c_catcode_letter_token { a }
+ \token_new:Nn \c_catcode_other_token { 1 }
+\group_end:
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{variable}{\c_catcode_active_tl}
+% Not an implicit token!
+% \begin{macrocode}
+\group_begin:
+ \char_set_catcode_active:N \*
+ \cs_new_nopar:Npn \c_catcode_active_tl { \exp_not:N * }
+\group_end:
+% \end{macrocode}
+% \end{variable}
+%
+% \subsection{Token conditionals}
+%
+% \begin{macro}[pTF]{\token_if_group_begin:N}
+% Check if token is a begin group token. We use the constant
+% |\c_group_begin_token| for this.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_group_begin:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_group_begin_token
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_group_end:N}
+% Check if token is a end group token. We use the constant
+% |\c_group_end_token| for this.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_group_end:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_group_end_token
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_math_toggle:N}
+% Check if token is a math shift token. We use the constant
+% |\c_math_toggle_token| for this.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_math_toggle:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_math_toggle_token
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_alignment:N}
+% Check if token is an alignment tab token. We use the constant
+% |\c_alignment_tab_token| for this.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_alignment:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_alignment_token
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_parameter:N}
+% Check if token is a parameter token. We use the constant
+% |\c_parameter_token| for this. We have to trick \TeX{} a bit to
+% avoid an error message: within a group we prevent
+% \cs{c_parameter_token} from behaving like a macro parameter character.
+% The definitions of \cs{prg_new_conditional:Npnn} are global, so they
+% will remain after the group.
+% \begin{macrocode}
+\group_begin:
+\cs_set_eq:NN \c_parameter_token \scan_stop:
+\prg_new_conditional:Npnn \token_if_parameter:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_parameter_token
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+\group_end:
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_math_superscript:N}
+% Check if token is a math superscript token. We use the constant
+% |\c_superscript_token| for this.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_math_superscript:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_math_superscript_token
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_math_subscript:N}
+% Check if token is a math subscript token. We use the constant
+% |\c_subscript_token| for this.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_math_subscript:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_math_subscript_token
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_space:N}
+% Check if token is a space token. We use the constant
+% |\c_space_token| for this.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_space:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_space_token
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_letter:N}
+% Check if token is a letter token. We use the constant
+% |\c_letter_token| for this.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_letter:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_catcode_letter_token
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_other:N}
+% Check if token is an other char token. We use the constant
+% |\c_other_char_token| for this.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_other:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_catcode_other_token
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_active:N}
+% Check if token is an active char token. We use the constant
+% |\c_active_char_tl| for this. A technical point is that
+% \cs{c_active_char_tl} is in fact a macro expanding to
+% |\exp_not:N *|, where |*| is active.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_active:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \c_catcode_active_tl
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_eq_meaning:NN}
+% Check if the tokens |#1| and |#2| have same meaning.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_eq_meaning:NN #1#2 { p , T , F , TF }
+ {
+ \if_meaning:w #1 #2
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_eq_catcode:NN}
+% Check if the tokens |#1| and |#2| have same category code.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_eq_catcode:NN #1#2 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \exp_not:N #2
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_eq_charcode:NN}
+% Check if the tokens |#1| and |#2| have same character code.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_eq_charcode:NN #1#2 { p , T , F , TF }
+ {
+ \if_charcode:w \exp_not:N #1 \exp_not:N #2
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_macro:N}
+% \begin{macro}[aux]{\token_if_macro_p_aux:w}
+% When a token is a macro, |\token_to_meaning:N| will always output
+% something like |\long macro:#1->#1| so we could naively check to
+% see if the meaning contains |->|. However, this can fail the five
+% \cs{tex_...mark:D} primitives, whose meaning has the form
+% |...mark:|\meta{user material}. The problem is that the
+% \meta{user material} can contain |->|.
+%
+% However, only characters, macros, and marks can contain the colon
+% character. The idea is thus to grab until the first |:|, and analyse
+% what is left. However, macros can have any combination of |\long|,
+% |\protected| or |\outer| (not used in \LaTeX3) before the string
+% |macro:|. We thus only select the part of the meaning between
+% the first |ma| and the first following |:|. If this string is
+% |cro|, then we have a macro. If the string is |rk|, then we have
+% a mark. The string can also be |cro parameter character | for a
+% colon with a weird category code (namely the usual category code
+% of |#|). Otherwise, it is empty.
+%
+% This relies on the fact that |\long|, |\protected|, |\outer|
+% cannot contain |ma|, regardless of the escape character, even if
+% the escape character is |m|\ldots{}
+%
+% Both |ma| and |:| must be of category code $12$ (other), and we
+% achieve using the standard lowercasing technique.
+%
+% \begin{macrocode}
+\group_begin:
+\char_set_catcode_other:N \M
+\char_set_catcode_other:N \A
+\char_set_lccode:nn { `\; } { `\: }
+\char_set_lccode:nn { `\T } { `\T }
+\char_set_lccode:nn { `\F } { `\F }
+\tl_to_lowercase:n
+ {
+ \group_end:
+ \prg_new_conditional:Npnn \token_if_macro:N #1 { p , T , F , TF }
+ {
+ \exp_after:wN \token_if_macro_p_aux:w
+ \token_to_meaning:N #1 MA; \q_stop
+ }
+ \cs_new_nopar:Npn \token_if_macro_p_aux:w #1 MA #2 ; #3 \q_stop
+ {
+ \if_int_compare:w \pdftex_strcmp:D { #2 } { cro } = \c_zero
+ \prg_return_true:
+ \else:
+ \prg_return_false:
+ \fi:
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_cs:N}
+% Check if token has same catcode as a control sequence. This
+% follows the same pattern as for \cs{token_if_letter:N} \emph{etc.}
+% We use |\scan_stop:| for this.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_cs:N #1 { p , T , F , TF }
+ {
+ \if_catcode:w \exp_not:N #1 \scan_stop:
+ \prg_return_true: \else: \prg_return_false: \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_expandable:N}
+% Check if token is expandable. We use the fact that \TeX{} will
+% temporarily convert |\exp_not:N| \meta{token} into |\scan_stop:| if
+% \meta{token} is expandable.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \token_if_expandable:N #1 { p , T , F , TF }
+ {
+ \cs_if_exist:NTF #1
+ {
+ \exp_after:wN \if_meaning:w \exp_not:N #1 #1
+ \prg_return_false: \else: \prg_return_true: \fi:
+ }
+ { \prg_return_false: }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_chardef:N,\token_if_mathchardef:N,
+% \token_if_long_macro:N, \token_if_protected_macro:N,
+% \token_if_protected_long_macro:N, \token_if_dim_register:N,
+% \token_if_skip_register:N, \token_if_int_register:N,
+% \token_if_toks_register:N}
+% \begin{macro}[aux]{
+% \token_if_chardef_p_aux:w,
+% \token_if_mathchardef_p_aux:w,
+% \token_if_int_register_p_aux:w,
+% \token_if_skip_register_p_aux:w,
+% \token_if_dim_register_p_aux:w,
+% \token_if_toks_register_p_aux:w,
+% \token_if_protected_macro_p_aux:w,
+% \token_if_long_macro_p_aux:w,
+% \token_if_protected_long_macro_p_aux:w}
+% Most of these functions have to check the meaning of the token in
+% question so we need to do some checkups on which characters are
+% output by |\token_to_meaning:N|. As usual, these characters have
+% catcode 12 so we must do some serious substitutions in the code
+% below\dots
+% \begin{macrocode}
+\group_begin:
+ \char_set_lccode:nn { `\T } { `\T }
+ \char_set_lccode:nn { `\F } { `\F }
+ \char_set_lccode:nn { `\X } { `\n }
+ \char_set_lccode:nn { `\Y } { `\t }
+ \char_set_lccode:nn { `\Z } { `\d }
+ \char_set_lccode:nn { `\? } { `\\ }
+ \tl_map_inline:nn { \X \Y \Z \M \C \H \A \R \O \U \S \K \I \P \L \G \P \E }
+ { \char_set_catcode:nn { `#1 } \c_twelve }
+% \end{macrocode}
+% We convert the token list to lower case and restore the catcode and
+% lowercase code changes.
+% \begin{macrocode}
+\tl_to_lowercase:n
+ {
+ \group_end:
+% \end{macrocode}
+% First up is checking if something has been defined with
+% |\tex_chardef:D| or |\tex_mathchardef:D|. This is easy since \TeX{}
+% thinks of such tokens as hexadecimal so it stores them as
+% |\char"|\meta{hex~number} or |\mathchar"|\meta{hex~number}.
+% \begin{macrocode}
+ \prg_new_conditional:Npnn \token_if_chardef:N #1 { p , T , F , TF }
+ {
+ \exp_after:wN \token_if_chardef_aux:w
+ \token_to_meaning:N #1 ?CHAR" \q_stop
+ }
+ \cs_new_nopar:Npn \token_if_chardef_aux:w #1 ?CHAR" #2 \q_stop
+ { \tl_if_empty:nTF {#1} { \prg_return_true: } { \prg_return_false: } }
+% \end{macrocode}
+%
+% \begin{macrocode}
+ \prg_new_conditional:Npnn \token_if_mathchardef:N #1 { p , T , F , TF }
+ {
+ \exp_after:wN \token_if_mathchardef_aux:w
+ \token_to_meaning:N #1 ?MAYHCHAR" \q_stop
+ }
+ \cs_new_nopar:Npn \token_if_mathchardef_aux:w #1 ?MAYHCHAR" #2 \q_stop
+ { \tl_if_empty:nTF {#1} { \prg_return_true: } { \prg_return_false: } }
+% \end{macrocode}
+% Integer registers are a little more difficult since they expand to
+% |\count|\meta{number} and there is also a primitive |\countdef|. So
+% we have to check for that primitive as well.
+% \begin{macrocode}
+ \prg_new_conditional:Npnn \token_if_int_register:N #1 { p , T , F , TF }
+ {
+ \if_meaning:w \tex_countdef:D #1
+ \prg_return_false:
+ \else:
+ \exp_after:wN \token_if_int_register_aux:w
+ \token_to_meaning:N #1 ?COUXY \q_stop
+ \fi:
+ }
+ \cs_new_nopar:Npn \token_if_int_register_aux:w #1 ?COUXY #2 \q_stop
+ { \tl_if_empty:nTF {#1} { \prg_return_true: } { \prg_return_false: } }
+% \end{macrocode}
+% Skip registers are done the same way as the integer registers.
+% \begin{macrocode}
+ \prg_new_conditional:Npnn \token_if_skip_register:N #1 { p , T , F , TF }
+ {
+ \if_meaning:w \tex_skipdef:D #1
+ \prg_return_false:
+ \else:
+ \exp_after:wN \token_if_skip_register_aux:w
+ \token_to_meaning:N #1?SKIP\q_stop
+ \fi:
+ }
+ \cs_new_nopar:Npn \token_if_skip_register_aux:w #1 ?SKIP #2 \q_stop
+ { \tl_if_empty:nTF {#1} { \prg_return_true: } { \prg_return_false: } }
+% \end{macrocode}
+% Dim registers. No news here
+% \begin{macrocode}
+ \prg_new_conditional:Npnn \token_if_dim_register:N #1 { p , T , F , TF }
+ {
+ \if_meaning:w \tex_dimendef:D #1
+ \c_false_bool
+ \else:
+ \exp_after:wN \token_if_dim_register_aux:w
+ \token_to_meaning:N #1 ?ZIMEX \q_stop
+ \fi:
+ }
+ \cs_new_nopar:Npn \token_if_dim_register_aux:w #1 ?ZIMEX #2 \q_stop
+ { \tl_if_empty:nTF {#1} { \prg_return_true: } { \prg_return_false: } }
+% \end{macrocode}
+% Toks registers.
+% \begin{macrocode}
+ \prg_new_conditional:Npnn \token_if_toks_register:N #1 { p , T , F , TF }
+ {
+ \if_meaning:w \tex_toksdef:D #1
+ \prg_return_false:
+ \else:
+ \exp_after:wN \token_if_toks_register_aux:w
+ \token_to_meaning:N #1 ?YOKS \q_stop
+ \fi:
+ }
+ \cs_new_nopar:Npn \token_if_toks_register_aux:w #1 ?YOKS #2 \q_stop
+ { \tl_if_empty:nTF {#1} { \prg_return_true: } { \prg_return_false: } }
+% \end{macrocode}
+% Protected macros.
+% \begin{macrocode}
+ \prg_new_conditional:Npnn \token_if_protected_macro:N #1
+ { p , T , F , TF }
+ {
+ \exp_after:wN \token_if_protected_macro_aux:w
+ \token_to_meaning:N #1 ?PROYECYEZ~MACRO \q_stop
+ }
+ \cs_new_nopar:Npn \token_if_protected_macro_aux:w
+ #1 ?PROYECYEZ~MACRO #2 \q_stop
+ { \tl_if_empty:nTF {#1} { \prg_return_true: } { \prg_return_false: } }
+% \end{macrocode}
+% Long macros.
+% \begin{macrocode}
+ \prg_new_conditional:Npnn \token_if_long_macro:N #1 { p , T , F , TF }
+ {
+ \exp_after:wN \token_if_long_macro_aux:w
+ \token_to_meaning:N #1 ?LOXG~MACRO \q_stop
+ }
+ \cs_new_nopar:Npn \token_if_long_macro_aux:w #1 ?LOXG~MACRO #2 \q_stop
+ { \tl_if_empty:nTF {#1} { \prg_return_true: } { \prg_return_false: } }
+% \end{macrocode}
+% Finally protected long macros where we for once don't have to add an
+% extra test since there is no primitive for the combined prefixes.
+% \begin{macrocode}
+ \prg_new_conditional:Npnn \token_if_protected_long_macro:N #1
+ { p , T , F , TF }
+ {
+ \exp_after:wN \token_if_protected_long_macro_aux:w
+ \token_to_meaning:N #1 ?PROYECYEZ?LOXG~MACRO \q_stop
+ }
+ \cs_new_nopar:Npn \token_if_protected_long_macro_aux:w
+ #1 ?PROYECYEZ?LOXG~MACRO #2 \q_stop
+ { \tl_if_empty:nTF {#1} { \prg_return_true: } { \prg_return_false: } }
+% \end{macrocode}
+% Finally the |\tl_to_lowercase:n| ends!
+% \begin{macrocode}
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_primitive:N}
+% \begin{macro}[aux]{\token_if_primitive_aux:NNw,
+% \token_if_primitive_aux_space:w,
+% \token_if_primitive_aux_nullfont:N,
+% \token_if_primitive_aux_loop:N,
+% \token_if_primitive_auxii:Nw,
+% \token_if_primitive_aux_undefined:N}
+%^^A See http://groups.google.com/group/comp.text.tex/browse_thread/thread/0a72666873f8753d#
+%
+% We filter out macros first, because they cause endless trouble later
+% otherwise.
+%
+% Primitives are almost distinguished by the fact that the result
+% of \cs{token_to_meaning:N} is formed from letters only. Every other
+% token has either a space (e.g., |the letter A|), a digit
+% (e.g., |\count123|) or a double quote (e.g., |\char"A|).
+%
+% Ten exceptions: on the one hand, \cs{c_undefined:D} is not a
+% primitive, but its meaning is |undefined|, only letters;
+% on the other hand, \cs{tex_space:D}, \cs{tex_italiccorr:D},
+% \cs{tex_hyphen:D}, \cs{tex_firstmark:D}, \cs{tex_topmark:D},
+% \cs{tex_botmark:D}, \cs{tex_splitfirstmark:D}, \cs{tex_splitbotmark:D},
+% and \cs{tex_nullfont:D} are primitives, but have non-letters
+% in their meaning.
+%
+% We start by removing the two first (non-space) characters from
+% the meaning. This removes the escape character (which may be
+% inexistent depending on \cs{tex_endlinechar:D}), and takes care
+% of three of the exceptions: \cs{tex_space:D}, \cs{tex_italiccorr:D}
+% and \cs{tex_hyphen:D}, whose meaning is at most two characters.
+% This leaves a string terminated by some |:|, and \cs{q_stop}.
+%
+% The meaning of each one of the five \cs{tex_...mark:D} primitives
+% has the form \meta{letters}|:|\meta{user material}. In other words,
+% the first non-letter is a colon. We remove everything after the first
+% colon.
+%
+% We are now left with a string, which we must analyze. For primitives,
+% it contains only letters. For non-primitives, it contains either
+% |"|, or a space, or a digit. Two exceptions remain: \cs{c_undefined:D},
+% which is not a primitive, and \cs{tex_nullfont:D}, which is a primitive.
+%
+% Spaces cannot be grabbed in an undelimited way, so we check them
+% separately. If there is a space, we test for \cs{tex_nullfont:D}.
+% Otherwise, we go through characters one by one, and stop at the
+% first character less than |`A| (this is not quite a test for
+% \enquote{only letters}, but is close enough to work in this context).
+% If this first character is |:| then we have a primitive, or
+% \cs{c_undefined:D}, and if it is |"| or a digit, then the token
+% is not a primitive.
+%
+% \begin{macrocode}
+\tex_chardef:D \c_token_A_int = `A ~ %
+\group_begin:
+\char_set_catcode_other:N \;
+\char_set_lccode:nn { `\; } { `\: }
+\char_set_lccode:nn { `\T } { `\T }
+\char_set_lccode:nn { `\F } { `\F }
+\tl_to_lowercase:n {
+ \group_end:
+ \prg_new_conditional:Npnn \token_if_primitive:N #1 { p , T , F , TF }
+ {
+ \token_if_macro:NTF #1
+ \prg_return_false:
+ {
+ \exp_after:wN \token_if_primitive_aux:NNw
+ \token_to_meaning:N #1 ; ; ; \q_stop #1
+ }
+ }
+ \cs_new_nopar:Npn \token_if_primitive_aux:NNw #1#2 #3 ; #4 \q_stop
+ {
+ \tl_if_empty:oTF { \token_if_primitive_aux_space:w #3 ~ }
+ { \token_if_primitive_aux_loop:N #3 ; \q_stop }
+ { \token_if_primitive_aux_nullfont:N }
+ }
+}
+\cs_new_nopar:Npn \token_if_primitive_aux_space:w #1 ~ { }
+\cs_new:Npn \token_if_primitive_aux_nullfont:N #1
+ {
+ \if_meaning:w \tex_nullfont:D #1
+ \prg_return_true:
+ \else:
+ \prg_return_false:
+ \fi:
+ }
+\cs_new_nopar:Npn \token_if_primitive_aux_loop:N #1
+ {
+ \if_num:w `#1 < \c_token_A_int %
+ \exp_after:wN \token_if_primitive_auxii:Nw
+ \exp_after:wN #1
+ \else:
+ \exp_after:wN \token_if_primitive_aux_loop:N
+ \fi:
+ }
+\cs_new_nopar:Npn \token_if_primitive_auxii:Nw #1 #2 \q_stop
+ {
+ \if:w : #1
+ \exp_after:wN \token_if_primitive_aux_undefined:N
+ \else:
+ \prg_return_false:
+ \exp_after:wN \use_none:n
+ \fi:
+ }
+\cs_new:Npn \token_if_primitive_aux_undefined:N #1
+ {
+ \if_cs_exist:N #1
+ \prg_return_true:
+ \else:
+ \prg_return_false:
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \subsection{Peeking ahead at the next token}
+%
+% Peeking ahead is implemented using a two part mechanism. The
+% outer level provides a defined interface to the lower level material.
+% This allows a large amount of code to be shared. There are four
+% cases:
+% \begin{enumerate}
+% \item peek at the next token;
+% \item peek at the next non-space token;
+% \item peek at the next token and remove it;
+% \item peek at the next non-space token and remove it.
+% \end{enumerate}
+%
+% \begin{variable}{\l_peek_token}
+% \begin{variable}{\g_peek_token}
+% Storage tokens which are publicly documented: the token peeked.
+% \begin{macrocode}
+\cs_new_eq:NN \l_peek_token ?
+\cs_new_eq:NN \g_peek_token ?
+% \end{macrocode}
+% \end{variable}
+% \end{variable}
+%
+% \begin{variable}{\l_peek_search_token}
+% The token to search for as an implicit token:
+% \emph{cf.}~\cs{l_peek_search_tl}.
+% \begin{macrocode}
+\cs_new_eq:NN \l_peek_search_token ?
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_peek_search_tl}
+% The token to search for as an explicit token:
+% \emph{cf.}~\cs{l_peek_search_token}.
+% \begin{macrocode}
+\cs_new_nopar:Npn \l_peek_search_tl { }
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{macro}[aux]
+% {\peek_true:w, \peek_true_aux:w, \peek_false:w, \peek_tmp:w}
+% Functions used by the branching and space-stripping code.
+% \begin{macrocode}
+\cs_new_nopar:Npn \peek_true:w { }
+\cs_new_nopar:Npn \peek_true_aux:w { }
+\cs_new_nopar:Npn \peek_false:w { }
+\cs_new:Npn \peek_tmp:w { }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\peek_after:Nw}
+% \begin{macro}{\peek_after:Nw}
+% Simple wrappers for \cs{tex_futurelet:D}: no arguments absorbed
+% here.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \peek_after:Nw
+ { \tex_futurelet:D \l_peek_token }
+\cs_new_protected_nopar:Npn \peek_gafter:Nw
+ { \pref_global:D \tex_futurelet:D \g_peek_token }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[aux]{\peek_true_remove:w}
+% A function to remove the next token and then regain control.
+% \begin{macrocode}
+\cs_new_protected:Npn \peek_true_remove:w
+ {
+ \group_align_safe_end:
+ \tex_afterassignment:D \peek_true_aux:w
+ \cs_set_eq:NN \peek_tmp:w
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[TF]{\peek_token_generic:NN}
+% The generic function stores the test token in both implicit and
+% explicit modes, and the \texttt{true} and \texttt{false} code as
+% token lists, more or less. The two branches have to be absorbed here
+% as the input stream needs to be cleared for the peek function itself.
+% \begin{macrocode}
+\cs_new_protected:Npn \peek_token_generic:NNTF #1#2#3#4
+ {
+ \cs_set_eq:NN \l_peek_search_token #2
+ \tl_set:Nn \l_peek_search_tl {#2}
+ \cs_set_nopar:Npx \peek_true:w
+ {
+ \exp_not:N \group_align_safe_end:
+ \exp_not:n {#3}
+ }
+ \cs_set_nopar:Npx \peek_false:w
+ {
+ \exp_not:N \group_align_safe_end:
+ \exp_not:n {#4}
+ }
+ \group_align_safe_begin:
+ \peek_after:Nw #1
+ }
+\cs_new_protected:Npn \peek_token_generic:NNT #1#2#3
+ { \peek_token_generic:NNTF #1 #2 {#3} { } }
+\cs_new_protected:Npn \peek_token_generic:NNF #1#2#3
+ { \peek_token_generic:NNTF #1 #2 { } {#3} }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[TF]{\peek_token_remove_generic:NN}
+% For token removal there needs to be a call to the auxiliary
+% function which does the work.
+% \begin{macrocode}
+\cs_new_protected:Npn \peek_token_remove_generic:NNTF #1#2#3#4
+ {
+ \cs_set_eq:NN \l_peek_search_token #2
+ \tl_set:Nn \l_peek_search_tl {#2}
+ \cs_set_eq:NN \peek_true:w \peek_true_remove:w
+ \cs_set_nopar:Npx \peek_true_aux:w { \exp_not:n {#3} }
+ \cs_set_nopar:Npx \peek_false:w
+ {
+ \exp_not:N \group_align_safe_end:
+ \exp_not:n {#4}
+ }
+ \group_align_safe_begin:
+ \peek_after:Nw #1
+ }
+\cs_new_protected:Npn \peek_token_remove_generic:NNT #1#2#3
+ { \peek_token_remove_generic:NNTF #1 #2 {#3} { } }
+\cs_new_protected:Npn \peek_token_remove_generic:NNF #1#2#3
+ { \peek_token_remove_generic:NNTF #1 #2 { } {#3} }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}
+% {\peek_execute_branches_catcode:, \peek_execute_branches_meaning:}
+% The category code and meaning tests are straight forward.
+% \begin{macrocode}
+\cs_new_nopar:Npn \peek_execute_branches_catcode:
+ {
+ \if_catcode:w
+ \exp_not:N \l_peek_token \exp_not:N \l_peek_search_token
+ \exp_after:wN \peek_true:w
+ \else:
+ \exp_after:wN \peek_false:w
+ \fi:
+ }
+\cs_new_nopar:Npn \peek_execute_branches_meaning:
+ {
+ \if_meaning:w \l_peek_token \l_peek_search_token
+ \exp_after:wN \peek_true:w
+ \else:
+ \exp_after:wN \peek_false:w
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\peek_execute_branches_charcode:}
+% \begin{macro}[aux]{\peek_execute_branches_charcode:NN}
+% First the character code test there is a need to worry about \TeX{}
+% grabbing brace group or skipping spaces. These are all tested for
+% using a category code check before grabbing what must be a real
+% single token and doing the comparison.
+% \begin{macrocode}
+\cs_new_nopar:Npn \peek_execute_branches_charcode:
+ {
+ \bool_if:nTF
+ {
+ \token_if_eq_catcode_p:NN \l_peek_token \c_group_begin_token
+ || \token_if_eq_catcode_p:NN \l_peek_token \c_group_end_token
+ || \token_if_eq_meaning_p:NN \l_peek_token \c_space_token
+ }
+ { \peek_false:w }
+ {
+ \exp_after:wN \peek_execute_branches_charcode_aux:NN
+ \l_peek_search_tl
+ }
+ }
+\cs_new:Npn \peek_execute_branches_charcode_aux:NN #1#2
+ {
+ \if:w \exp_not:N #1 \exp_not:N #2
+ \exp_after:wN \peek_true:w
+ \else:
+ \exp_after:wN \peek_false:w
+ \fi:
+ #2
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}{\peek_ignore_spaces_execute_branches:}
+% \begin{macro}[aux]{\peek_ignore_spaces_execute_branches_aux:}
+% This function removes one token at a time with a mechanism that can
+% be applied to things other than spaces.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \peek_ignore_spaces_execute_branches:
+ {
+ \token_if_eq_meaning:NNTF \l_peek_token \c_space_token
+ {
+ \tex_afterassignment:D \peek_ignore_spaces_execute_branches_aux:
+ \cs_set_eq:NN \peek_tmp:w
+ }
+ { \peek_execute_branches: }
+ }
+\cs_new_protected_nopar:Npn \peek_ignore_spaces_execute_branches_aux:
+ { \peek_after:Nw \peek_ignore_spaces_execute_branches: }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[aux]{\peek_def:nnnn}
+% \begin{macro}[aux]{\peek_def_aux:nnnnn}
+% The public functions themselves cannot be defined using
+% \cs{prg_set_conditional:Npnn} and so a couple of auxiliary functions
+% are used. As a result, everything is done inside a group. As a result
+% things are a bit complicated.
+% \begin{macrocode}
+\group_begin:
+ \cs_set_nopar:Npn \peek_def:nnnn #1#2#3#4
+ {
+ \peek_def_aux:nnnnn {#1} {#2} {#3} {#4} { TF }
+ \peek_def_aux:nnnnn {#1} {#2} {#3} {#4} { T }
+ \peek_def_aux:nnnnn {#1} {#2} {#3} {#4} { F }
+ }
+ \cs_set_nopar:Npn \peek_def_aux:nnnnn #1#2#3#4#5
+ {
+ \cs_gset_nopar:cpx { #1 #5 }
+ {
+ \tl_if_empty:nF {#2}
+ { \exp_not:n { \cs_set_eq:NN \peek_execute_branches: #2 } }
+ \exp_not:c { #3 #5 }
+ \exp_not:n {#4}
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \begin{macro}[TF]
+% {
+% \peek_catcode:N, \peek_catcode_ignore_spaces:N,
+% \peek_catcode_remove:N, \peek_catcode_remove_ignore_spaces:N
+% }
+% With everything in place the definitions can take place. First for
+% category codes.
+% \begin{macrocode}
+ \peek_def:nnnn { peek_catcode:N }
+ { }
+ { peek_token_generic:NN }
+ { \peek_execute_branches_catcode: }
+ \peek_def:nnnn { peek_catcode_ignore_spaces:N }
+ { \peek_execute_branches_catcode: }
+ { peek_token_generic:NN }
+ { \peek_ignore_spaces_execute_branches: }
+ \peek_def:nnnn { peek_catcode_remove:N }
+ { }
+ { peek_token_remove_generic:NN }
+ { \peek_execute_branches_catcode: }
+ \peek_def:nnnn { peek_catcode_remove_ignore_spaces:N }
+ { \peek_execute_branches_catcode: }
+ { peek_token_remove_generic:NN }
+ { \peek_ignore_spaces_execute_branches: }
+% \end{macrocode}
+% \end{macro}
+% \begin{macro}[TF]
+% {
+% \peek_charcode:N, \peek_charcode_ignore_spaces:N,
+% \peek_charcode_remove:N, \peek_charcode_remove_ignore_spaces:N
+% }
+% Then for character codes.
+% \begin{macrocode}
+ \peek_def:nnnn { peek_charcode:N }
+ { }
+ { peek_token_generic:NN }
+ { \peek_execute_branches_charcode: }
+ \peek_def:nnnn { peek_charcode_ignore_spaces:N }
+ { \peek_execute_branches_charcode: }
+ { peek_token_generic:NN }
+ { \peek_ignore_spaces_execute_branches: }
+ \peek_def:nnnn { peek_charcode_remove:N }
+ { }
+ { peek_token_remove_generic:NN }
+ { \peek_execute_branches_charcode: }
+ \peek_def:nnnn { peek_charcode_remove_ignore_spaces:N }
+ { \peek_execute_branches_charcode: }
+ { peek_token_remove_generic:NN }
+ { \peek_ignore_spaces_execute_branches: }
+% \end{macrocode}
+% \end{macro}
+% \begin{macro}[TF]
+% {
+% \peek_meaning:N, \peek_meaning_ignore_spaces:N,
+% \peek_meaning_remove:N, \peek_meaning_remove_ignore_spaces:N
+% }
+% Finally for meaning, with the group closed to remove the temporary
+% definition functions.
+% \begin{macrocode}
+ \peek_def:nnnn { peek_meaning:N }
+ { }
+ { peek_token_generic:NN }
+ { \peek_execute_branches_meaning: }
+ \peek_def:nnnn { peek_meaning_ignore_spaces:N }
+ { \peek_execute_branches_meaning: }
+ { peek_token_generic:NN }
+ { \peek_ignore_spaces_execute_branches: }
+ \peek_def:nnnn { peek_meaning_remove:N }
+ { }
+ { peek_token_remove_generic:NN }
+ { \peek_execute_branches_meaning: }
+ \peek_def:nnnn { peek_meaning_remove_ignore_spaces:N }
+ { \peek_execute_branches_meaning: }
+ { peek_token_remove_generic:NN }
+ { \peek_ignore_spaces_execute_branches: }
+\group_end:
+% \end{macrocode}
+% \end{macro}
+%
+% \subsection{Decomposing a macro definition}
+%
+% \begin{macro}{\token_get_prefix_spec:N}
+% \begin{macro}{\token_get_arg_spec:N}
+% \begin{macro}{\token_get_replacement_spec:N}
+% \begin{macro}[aux]{\token_get_prefix_arg_replacement_aux:wN}
+% We sometimes want to test if a
+% control sequence can be expanded to reveal a hidden
+% value. However, we cannot just expand the macro blindly as it may
+% have arguments and none might be present. Therefore we define
+% these functions to pick either the prefix(es), the argument
+% specification, or the replacement text from a macro. All of this
+% information is returned as characters with catcode~$12$. If the
+% token in question isn't a macro, the token |\scan_stop:| is
+% returned instead.
+% \begin{macrocode}
+\exp_args:Nno \use:nn
+ { \cs_new_nopar:Npn \token_get_prefix_arg_replacement_aux:wN #1 }
+ { \tl_to_str:n { macro : } #2 -> #3 \q_stop #4 }
+ { #4 {#1} {#2} {#3} }
+\cs_new:Npn \token_get_prefix_spec:N #1
+ {
+ \token_if_macro:NTF #1
+ {
+ \exp_after:wN \token_get_prefix_arg_replacement_aux:wN
+ \token_to_meaning:N #1 \q_stop \use_i:nnn
+ }
+ { \scan_stop: }
+ }
+\cs_new:Npn \token_get_arg_spec:N #1
+ {
+ \token_if_macro:NTF #1
+ {
+ \exp_after:wN \token_get_prefix_arg_replacement_aux:wN
+ \token_to_meaning:N #1 \q_stop \use_ii:nnn
+ }
+ { \scan_stop: }
+ }
+\cs_new:Npn \token_get_replacement_spec:N #1
+ {
+ \token_if_macro:NTF #1
+ {
+ \exp_after:wN \token_get_prefix_arg_replacement_aux:wN
+ \token_to_meaning:N #1 \q_stop \use_iii:nnn
+ }
+ { \scan_stop: }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \subsection{Experimental token functions}
+%
+% \begin{macro}{\char_active_set:Npn,\char_active_set:Npx}
+% \begin{macro}{\char_active_set:Npn,\char_active_set:Npx}
+% \begin{macro}{\char_active_set_eq:NN,\char_active_gset_eq:NN}
+% \begin{macrocode}
+\group_begin:
+ \char_set_catcode_active:N \^^@
+ \cs_set:Npn \char_tmp:NN #1#2
+ {
+ \cs_new:Npn #1 ##1
+ {
+ \char_set_catcode_active:n { `##1 }
+ \group_begin:
+ \char_set_lccode:nn { `\^^@ } { `##1 }
+ \tl_to_lowercase:n { \group_end: #2 ^^@ }
+ }
+ }
+ \char_tmp:NN \char_active_set:Npn \cs_set:Npn
+ \char_tmp:NN \char_active_set:Npx \cs_set:Npx
+ \char_tmp:NN \char_active_gset:Npn \cs_gset:Npn
+ \char_tmp:NN \char_active_gset:Npx \cs_gset:Npx
+ \char_tmp:NN \char_active_set_eq:NN \cs_set_eq:NN
+ \char_tmp:NN \char_active_gset_eq:NN \cs_gset_eq:NN
+\group_end:
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \subsection{Deprecated functions}
+%
+% Deprecated on 2011-05-27, for removal by 2011-08-31.
+%
+% \begin{macro}{\char_set_catcode:w}
+% \begin{macro}{\char_set_mathcode:w}
+% \begin{macro}{\char_set_lccode:w}
+% \begin{macro}{\char_set_uccode:w}
+% \begin{macro}{\char_set_sfcode:w}
+% Primitives renamed.
+% \begin{macrocode}
+\cs_new_eq:NN \char_set_catcode:w \tex_catcode:D
+\cs_new_eq:NN \char_set_mathcode:w \tex_mathcode:D
+\cs_new_eq:NN \char_set_lccode:w \tex_lccode:D
+\cs_new_eq:NN \char_set_uccode:w \tex_uccode:D
+\cs_new_eq:NN \char_set_sfcode:w \tex_sfcode:D
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}{\char_value_catcode:w}
+% \begin{macro}{\char_show_value_catcode:w}
+% \begin{macro}{\char_value_mathcode:w}
+% \begin{macro}{\char_show_value_mathcode:w}
+% \begin{macro}{\char_value_lccode:w}
+% \begin{macro}{\char_show_value_lccode:w}
+% \begin{macro}{\char_value_uccode:w}
+% \begin{macro}{\char_show_value_uccode:w}
+% \begin{macro}{\char_value_sfcode:w}
+% \begin{macro}{\char_show_value_sfcode:w}
+% More |w| functions we should not have.
+% \begin{macrocode}
+\cs_new_nopar:Npn \char_value_catcode:w { \tex_the:D \char_set_catcode:w }
+\cs_new_nopar:Npn \char_show_value_catcode:w
+ { \tex_showthe:D \char_set_catcode:w }
+\cs_new_nopar:Npn \char_value_mathcode:w { \tex_the:D \char_set_mathcode:w }
+\cs_new_nopar:Npn \char_show_value_mathcode:w
+ { \tex_showthe:D \char_set_mathcode:w }
+\cs_new_nopar:Npn \char_value_lccode:w { \tex_the:D \char_set_lccode:w }
+\cs_new_nopar:Npn \char_show_value_lccode:w
+ { \tex_showthe:D \char_set_lccode:w }
+\cs_new_nopar:Npn \char_value_uccode:w { \tex_the:D \char_set_uccode:w }
+\cs_new_nopar:Npn \char_show_value_uccode:w
+ { \tex_showthe:D \char_set_uccode:w }
+\cs_new_nopar:Npn \char_value_sfcode:w { \tex_the:D \char_set_sfcode:w }
+\cs_new_nopar:Npn \char_show_value_sfcode:w
+ { \tex_showthe:D \char_set_sfcode:w }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}{\peek_after:NN}
+% \begin{macro}{\peek_gafter:NN}
+% The second argument here must be |w|.
+% \begin{macrocode}
+\cs_new_eq:NN \peek_after:NN \peek_after:Nw
+\cs_new_eq:NN \peek_gafter:NN \peek_gafter:Nw
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% Functions deprecated 2011-05-28 for removal by 2011-08-31.
+%
+% \begin{macro}{\c_alignment_tab_token}
+% \begin{macro}{\c_math_shift_token}
+% \begin{macro}{\c_letter_token}
+% \begin{macro}{\c_other_char_token}
+% \begin{macrocode}
+\cs_new_eq:NN \c_alignment_tab_token \c_alignment_token
+\cs_new_eq:NN \c_math_shift_token \c_math_toggle_token
+\cs_new_eq:NN \c_letter_token \c_catcode_letter_token
+\cs_new_eq:NN \c_other_char_token \c_catcode_other_token
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}{\c_active_char_token}
+% An odd one: this was never a |token|!
+% \begin{macrocode}
+\cs_new_eq:NN \c_active_char_token \c_catcode_active_tl
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}
+% {
+% \char_make_escape:N ,
+% \char_make_group_begin:N ,
+% \char_make_group_end:N ,
+% \char_make_math_toggle:N ,
+% \char_make_alignment:N ,
+% \char_make_end_line:N ,
+% \char_make_parameter:N ,
+% \char_make_math_superscript:N ,
+% \char_make_math_subscript:N ,
+% \char_make_ignore:N ,
+% \char_make_space:N ,
+% \char_make_letter:N ,
+% \char_make_other:N ,
+% \char_make_active:N ,
+% \char_make_comment:N ,
+% \char_make_invalid:N
+% }
+% \begin{macro}
+% {
+% \char_make_escape:n ,
+% \char_make_group_begin:n ,
+% \char_make_group_end:n ,
+% \char_make_math_toggle:n ,
+% \char_make_alignment:n ,
+% \char_make_end_line:n ,
+% \char_make_parameter:n ,
+% \char_make_math_superscript:n ,
+% \char_make_math_subscript:n ,
+% \char_make_ignore:n ,
+% \char_make_space:n ,
+% \char_make_letter:n ,
+% \char_make_other:n ,
+% \char_make_active:n ,
+% \char_make_comment:n ,
+% \char_make_invalid:n
+% }
+% Two renames in one block!
+% \begin{macrocode}
+\cs_new_eq:NN \char_make_escape:N \char_set_catcode_escape:N
+\cs_new_eq:NN \char_make_begin_group:N \char_set_catcode_group_begin:N
+\cs_new_eq:NN \char_make_end_group:N \char_set_catcode_group_end:N
+\cs_new_eq:NN \char_make_math_shift:N \char_set_catcode_math_toggle:N
+\cs_new_eq:NN \char_make_alignment_tab:N \char_set_catcode_alignment:N
+\cs_new_eq:NN \char_make_end_line:N \char_set_catcode_end_line:N
+\cs_new_eq:NN \char_make_parameter:N \char_set_catcode_parameter:N
+\cs_new_eq:NN \char_make_math_superscript:N
+ \char_set_catcode_math_superscript:N
+\cs_new_eq:NN \char_make_math_subscript:N
+ \char_set_catcode_math_subscript:N
+\cs_new_eq:NN \char_make_ignore:N \char_set_catcode_ignore:N
+\cs_new_eq:NN \char_make_space:N \char_set_catcode_space:N
+\cs_new_eq:NN \char_make_letter:N \char_set_catcode_letter:N
+\cs_new_eq:NN \char_make_other:N \char_set_catcode_other:N
+\cs_new_eq:NN \char_make_active:N \char_set_catcode_active:N
+\cs_new_eq:NN \char_make_comment:N \char_set_catcode_comment:N
+\cs_new_eq:NN \char_make_invalid:N \char_set_catcode_invalid:N
+\cs_new_eq:NN \char_make_escape:n \char_set_catcode_escape:n
+\cs_new_eq:NN \char_make_begin_group:n \char_set_catcode_group_begin:n
+\cs_new_eq:NN \char_make_end_group:n \char_set_catcode_group_end:n
+\cs_new_eq:NN \char_make_math_shift:n \char_set_catcode_math_toggle:n
+\cs_new_eq:NN \char_make_alignment_tab:n \char_set_catcode_alignment:n
+\cs_new_eq:NN \char_make_end_line:n \char_set_catcode_end_line:n
+\cs_new_eq:NN \char_make_parameter:n \char_set_catcode_parameter:n
+\cs_new_eq:NN \char_make_math_superscript:n
+ \char_set_catcode_math_superscript:n
+\cs_new_eq:NN \char_make_math_subscript:n
+ \char_set_catcode_math_subscript:n
+\cs_new_eq:NN \char_make_ignore:n \char_set_catcode_ignore:n
+\cs_new_eq:NN \char_make_space:n \char_set_catcode_space:n
+\cs_new_eq:NN \char_make_letter:n \char_set_catcode_letter:n
+\cs_new_eq:NN \char_make_other:n \char_set_catcode_other:n
+\cs_new_eq:NN \char_make_active:n \char_set_catcode_active:n
+\cs_new_eq:NN \char_make_comment:n \char_set_catcode_comment:n
+\cs_new_eq:NN \char_make_invalid:n \char_set_catcode_invalid:n
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[pTF]{\token_if_alignment_tab:N}
+% \begin{macro}[pTF]{\token_if_math_shift:N}
+% \begin{macro}[pTF]{\token_if_other_char:N}
+% \begin{macro}[pTF]{\token_if_active_char:N}
+% \begin{macrocode}
+\cs_new_eq:NN \token_if_alignment_tab_p:N \token_if_alignment_p:N
+\cs_new_eq:NN \token_if_alignment_tab:NT \token_if_alignment:NT
+\cs_new_eq:NN \token_if_alignment_tab:NF \token_if_alignment:NF
+\cs_new_eq:NN \token_if_alignment_tab:NTF \token_if_alignment:NTF
+\cs_new_eq:NN \token_if_math_shift_p:N \token_if_math_toggle_p:N
+\cs_new_eq:NN \token_if_math_shift:NT \token_if_math_toggle:NT
+\cs_new_eq:NN \token_if_math_shift:NF \token_if_math_toggle:NF
+\cs_new_eq:NN \token_if_math_shift:NTF \token_if_math_toggle:NTF
+\cs_new_eq:NN \token_if_other_char_p:N \token_if_other_p:N
+\cs_new_eq:NN \token_if_other_char:NT \token_if_other:NT
+\cs_new_eq:NN \token_if_other_char:NF \token_if_other:NF
+\cs_new_eq:NN \token_if_other_char:NTF \token_if_other:NTF
+\cs_new_eq:NN \token_if_active_char_p:N \token_if_active_p:N
+\cs_new_eq:NN \token_if_active_char:NT \token_if_active:NT
+\cs_new_eq:NN \token_if_active_char:NF \token_if_active:NF
+\cs_new_eq:NN \token_if_active_char:NTF \token_if_active:NTF
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macrocode}
+%</initex|package>
+% \end{macrocode}
+%
+% \end{implementation}
+%
+% \PrintIndex