% \iffalse meta-comment % %% File: l3token.dtx % % Copyright (C) 2005-2020 The LaTeX3 Project % % It may be distributed and/or modified under the conditions of the % LaTeX Project Public License (LPPL), either version 1.3c of this % license or (at your option) any later version. The latest version % of this license is in the file % % https://www.latex-project.org/lppl.txt % % This file is part of the "l3kernel bundle" (The Work in LPPL) % and all files in that bundle must be distributed together. % % ----------------------------------------------------------------------- % % The development version of the bundle can be found at % % https://github.com/latex3/latex3 % % for those people who are interested. % %<*driver> \documentclass[full,kernel]{l3doc} \begin{document} \DocInput{\jobname.dtx} \end{document} % % \fi % % \title{^^A % The \pkg{l3token} package\\ Token manipulation^^A % } % % \author{^^A % The \LaTeX3 Project\thanks % {^^A % E-mail: % \href{mailto:latex-team@latex-project.org} % {latex-team@latex-project.org}^^A % }^^A % } % % \date{Released 2020-06-18} % % \maketitle % % \begin{documentation} % % This module deals with tokens. Now this is perhaps not the most % precise description so let's try with a better description: When % programming in \TeX{}, it is often desirable to know just what a % certain token is: is it a control sequence or something % else. Similarly one often needs to know if a control sequence is % expandable or not, a macro or a primitive, how many arguments it % takes etc. Another thing of great importance (especially when it % comes to document commands) is looking ahead in the token stream to % see if a certain character is present and maybe even remove it or % disregard other tokens while scanning. This module provides % functions for both and as such has two primary function % categories: |\token_| for anything that deals with tokens and % |\peek_| for looking ahead in the token stream. % % Most functions we describe here can be used on control sequences, % as those are tokens as well. % % It is important to distinguish two aspects of a token: its % \enquote{shape} (for lack of a better word), which affects the % matching of delimited arguments and the comparison of token lists % containing this token, and its \enquote{meaning}, which affects % whether the token expands or what operation it performs. One can have % tokens of different shapes with the same meaning, but not the % converse. % % For instance, \cs{if:w}, \cs{if_charcode:w}, and \cs{tex_if:D} are % three names for the same internal operation of \TeX{}, namely the % primitive testing the next two characters for equality of their % character code. They have the same meaning hence behave identically % in many situations. However, \TeX{} distinguishes them when searching % for a delimited argument. Namely, the example function % |\show_until_if:w| defined below takes everything until \cs{if:w} % as an argument, despite the presence of other copies of \cs{if:w} % under different names. % \begin{verbatim} % \cs_new:Npn \show_until_if:w #1 \if:w { \tl_show:n {#1} } % \show_until_if:w \tex_if:D \if_charcode:w \if:w % \end{verbatim} % A list of all possible shapes and a list of all possible meanings are % given in section~\ref{sec:l3token:all-tokens}. % % \section{Creating character tokens} % % \begin{function}[updated = 2015-11-12] % { % \char_set_active_eq:NN, \char_set_active_eq:Nc, % \char_gset_active_eq:NN, \char_gset_active_eq:Nc % } % \begin{syntax} % \cs{char_set_active_eq:NN} \meta{char} \meta{function} % \end{syntax} % Sets the behaviour of the \meta{char} in situations where it is % active (category code $13$) to be equivalent to that of the % \meta{function}. The category code of the \meta{char} is % \emph{unchanged} by this process. The \meta{function} may itself % be an active character. % \end{function} % % \begin{function}[added = 2015-11-12] % { % \char_set_active_eq:nN, \char_set_active_eq:nc, % \char_gset_active_eq:nN, \char_gset_active_eq:nc % } % \begin{syntax} % \cs{char_set_active_eq:nN} \Arg{integer expression} \meta{function} % \end{syntax} % Sets the behaviour of the \meta{char} which has character % code as given by the \meta{integer expression} in situations % where it is active (category code $13$) to be equivalent to that of the % \meta{function}. The category code of the \meta{char} is % \emph{unchanged} by this process. The \meta{function} may itself % be an active character. % \end{function} % % \begin{function}[EXP, added = 2015-09-09, updated = 2019-01-16] % {\char_generate:nn} % \begin{syntax} % \cs{char_generate:nn} \Arg{charcode} \Arg{catcode} % \end{syntax} % Generates a character token of the given \meta{charcode} and \meta{catcode} % (both of which may be integer expressions). The \meta{catcode} may be % one of % \begin{itemize} % \item $1$ (begin group) % \item $2$ (end group) % \item $3$ (math toggle) % \item $4$ (alignment) % \item $6$ (parameter) % \item $7$ (math superscript) % \item $8$ (math subscript) % \item $11$ (letter) % \item $12$ (other) % \item $13$ (active) % \end{itemize} % and other values raise an error. The \meta{charcode} may be any one valid % for the engine in use. % Active characters cannot be generated in older versions of \XeTeX{}. % \begin{texnote} % Exactly two expansions are needed to produce the character. % \end{texnote} % \end{function} % % \begin{function}[added = 2020-01-09, EXP] % { % \char_lowercase:N, \char_uppercase:N, % \char_titlecase:N, \char_foldcase:N, % \char_str_lowercase:N, \char_str_uppercase:N, % \char_str_titlecase:N, \char_str_foldcase:N % } % \begin{syntax} % \cs{char_lowercase:N} \meta{char} % \end{syntax} % Converts the \meta{char} to the equivalent case-changed character % as detailed by the function name (see \cs{str_foldcase:n} % and \cs{text_titlecase:n} for details of these terms). The case mapping % is carried out with no context-dependence (\emph{cf.}~\cs{text_uppercase:n}, % \emph{etc.}) The \texttt{str} versions always generate \enquote{other} % (category code $12$) characters, whilst the standard versions generate % characters with the category code of the \meta{char} (i.e.~only % the character code changes). % \end{function} % % \begin{variable}[added = 2011-09-05]{\c_catcode_other_space_tl} % Token list containing one character with category code $12$, % (\enquote{other}), and character code $32$ (space). % \end{variable} % % \section{Manipulating and interrogating character tokens} % % \begin{function}[updated = 2015-11-11] % { % \char_set_catcode_escape:N , % \char_set_catcode_group_begin:N , % \char_set_catcode_group_end:N , % \char_set_catcode_math_toggle:N , % \char_set_catcode_alignment:N , % \char_set_catcode_end_line:N , % \char_set_catcode_parameter:N , % \char_set_catcode_math_superscript:N , % \char_set_catcode_math_subscript:N , % \char_set_catcode_ignore:N , % \char_set_catcode_space:N , % \char_set_catcode_letter:N , % \char_set_catcode_other:N , % \char_set_catcode_active:N , % \char_set_catcode_comment:N , % \char_set_catcode_invalid:N % } % \begin{syntax} % \cs{char_set_catcode_letter:N} \meta{character} % \end{syntax} % Sets the category code of the \meta{character} to that indicated in % the function name. Depending on the current category code of the % \meta{token} the escape token may also be needed: % \begin{verbatim} % \char_set_catcode_other:N \% % \end{verbatim} % The assignment is local. % \end{function} % % \begin{function}[updated = 2015-11-11] % { % \char_set_catcode_escape:n , % \char_set_catcode_group_begin:n , % \char_set_catcode_group_end:n , % \char_set_catcode_math_toggle:n , % \char_set_catcode_alignment:n , % \char_set_catcode_end_line:n , % \char_set_catcode_parameter:n , % \char_set_catcode_math_superscript:n , % \char_set_catcode_math_subscript:n , % \char_set_catcode_ignore:n , % \char_set_catcode_space:n , % \char_set_catcode_letter:n , % \char_set_catcode_other:n , % \char_set_catcode_active:n , % \char_set_catcode_comment:n , % \char_set_catcode_invalid:n % } % \begin{syntax} % \cs{char_set_catcode_letter:n} \Arg{integer expression} % \end{syntax} % Sets the category code of the \meta{character} which has character % code as given by the \meta{integer expression}. This version can be % used to set up characters which cannot otherwise be given % (\emph{cf.}~the \texttt{N}-type variants). The assignment is local. % \end{function} % % \begin{function}[updated = 2015-11-11]{\char_set_catcode:nn} % \begin{syntax} % \cs{char_set_catcode:nn} \Arg{intexpr_1} \Arg{intexpr_2} % \end{syntax} % These functions set the category code of the \meta{character} which % has character code as given by the \meta{integer expression}. % The first \meta{integer expression} % is the character code and the second is the category code to apply. % The setting applies within the current \TeX{} group. In general, the % symbolic functions \cs[no-index]{char_set_catcode_\meta{type}} should be preferred, % but there are cases where these lower-level functions may be useful. % \end{function} % % \begin{function}[EXP]{\char_value_catcode:n} % \begin{syntax} % \cs{char_value_catcode:n} \Arg{integer expression} % \end{syntax} % Expands to the current category code of the \meta{character} with % character code given by the % \meta{integer expression}. % \end{function} % % \begin{function}{\char_show_value_catcode:n} % \begin{syntax} % \cs{char_show_value_catcode:n} \Arg{integer expression} % \end{syntax} % Displays the current category code of the \meta{character} with % character code given by the \meta{integer expression} on the % terminal. % \end{function} % % \begin{function}[updated = 2015-08-06]{\char_set_lccode:nn} % \begin{syntax} % \cs{char_set_lccode:nn} \Arg{intexpr_1} \Arg{intexpr_2} % \end{syntax} % Sets up the behaviour of the \meta{character} when % found inside \cs{text_lowercase:n}, such that \meta{character_1} % will be converted into \meta{character_2}. The two \meta{characters} % may be specified using an \meta{integer expression} for the character code % concerned. This may include the \TeX{} |`|\meta{character} % method for converting a single character into its character % code: % \begin{verbatim} % \char_set_lccode:nn { `\A } { `\a } % Standard behaviour % \char_set_lccode:nn { `\A } { `\A + 32 } % \char_set_lccode:nn { 50 } { 60 } % \end{verbatim} % The setting applies within the current \TeX{} group. % \end{function} % % \begin{function}[EXP]{\char_value_lccode:n} % \begin{syntax} % \cs{char_value_lccode:n} \Arg{integer expression} % \end{syntax} % Expands to the current lower case code of the \meta{character} with % character code given by the % \meta{integer expression}. % \end{function} % % \begin{function}{\char_show_value_lccode:n} % \begin{syntax} % \cs{char_show_value_lccode:n} \Arg{integer expression} % \end{syntax} % Displays the current lower case code of the \meta{character} with % character code given by the \meta{integer expression} on the % terminal. % \end{function} % % \begin{function}[updated = 2015-08-06]{\char_set_uccode:nn} % \begin{syntax} % \cs{char_set_uccode:nn} \Arg{intexpr_1} \Arg{intexpr_2} % \end{syntax} % Sets up the behaviour of the \meta{character} when % found inside \cs{text_uppercase:n}, such that \meta{character_1} % will be converted into \meta{character_2}. The two \meta{characters} % may be specified using an \meta{integer expression} for the character code % concerned. This may include the \TeX{} |`|\meta{character} % method for converting a single character into its character % code: % \begin{verbatim} % \char_set_uccode:nn { `\a } { `\A } % Standard behaviour % \char_set_uccode:nn { `\A } { `\A - 32 } % \char_set_uccode:nn { 60 } { 50 } % \end{verbatim} % The setting applies within the current \TeX{} group. % \end{function} % % \begin{function}[EXP]{\char_value_uccode:n} % \begin{syntax} % \cs{char_value_uccode:n} \Arg{integer expression} % \end{syntax} % Expands to the current upper case code of the \meta{character} with % character code given by the % \meta{integer expression}. % \end{function} % % \begin{function}{\char_show_value_uccode:n} % \begin{syntax} % \cs{char_show_value_uccode:n} \Arg{integer expression} % \end{syntax} % Displays the current upper case code of the \meta{character} with % character code given by the \meta{integer expression} on the % terminal. % \end{function} % % \begin{function}[updated = 2015-08-06]{\char_set_mathcode:nn} % \begin{syntax} % \cs{char_set_mathcode:nn} \Arg{intexpr_1} \Arg{intexpr_2} % \end{syntax} % This function sets up the math code of \meta{character}. % The \meta{character} is specified as % an \meta{integer expression} which will be used as the character % code of the relevant character. The setting applies within the % current \TeX{} group. % \end{function} % % \begin{function}[EXP]{\char_value_mathcode:n} % \begin{syntax} % \cs{char_value_mathcode:n} \Arg{integer expression} % \end{syntax} % Expands to the current math code of the \meta{character} with % character code given by the % \meta{integer expression}. % \end{function} % % \begin{function}{\char_show_value_mathcode:n} % \begin{syntax} % \cs{char_show_value_mathcode:n} \Arg{integer expression} % \end{syntax} % Displays the current math code of the \meta{character} with % character code given by the \meta{integer expression} on the % terminal. % \end{function} % % \begin{function}[updated = 2015-08-06]{\char_set_sfcode:nn} % \begin{syntax} % \cs{char_set_sfcode:nn} \Arg{intexpr_1} \Arg{intexpr_2} % \end{syntax} % This function sets up the space factor for the \meta{character}. % The \meta{character} is specified as % an \meta{integer expression} which will be used as the character % code of the relevant character. The setting applies within the % current \TeX{} group. % \end{function} % % \begin{function}[EXP]{\char_value_sfcode:n} % \begin{syntax} % \cs{char_value_sfcode:n} \Arg{integer expression} % \end{syntax} % Expands to the current space factor for the \meta{character} with % character code given by the % \meta{integer expression}. % \end{function} % % \begin{function}{\char_show_value_sfcode:n} % \begin{syntax} % \cs{char_show_value_sfcode:n} \Arg{integer expression} % \end{syntax} % Displays the current space factor for the \meta{character} with % character code given by the \meta{integer expression} on the % terminal. % \end{function} % % \begin{variable}[added = 2012-01-23, updated = 2015-11-11]{\l_char_active_seq} % Used to track which tokens may require special handling at the document % level as they are (or have been at some point) % of category \meta{active} (catcode~$13$). Each entry in % the sequence consists of a single escaped token, for example |\~|. % Active tokens should be added to the sequence when they are defined for % general document use. % \end{variable} % % \begin{variable}[added = 2012-01-23, updated = 2015-11-11]{\l_char_special_seq} % Used to track which tokens will require special handling when working with % verbatim-like material at the document level as they are not of categories % \meta{letter} (catcode~$11$) or \meta{other} (catcode~$12$). Each entry in % the sequence consists of a single escaped token, for example |\\| for the % backslash or |\{| for an opening brace.^^A \} % Escaped tokens should be added to the sequence when they are defined for % general document use. % \end{variable} % % \section{Generic tokens} % % \begin{variable} % { % \c_group_begin_token, % \c_group_end_token, % \c_math_toggle_token, % \c_alignment_token, % \c_parameter_token, % \c_math_superscript_token, % \c_math_subscript_token, % \c_space_token % } % These are implicit tokens which have the category code described % by their name. They are used internally for test purposes but % are also available to the programmer for other uses. % \end{variable} % % \begin{variable} % { % \c_catcode_letter_token, % \c_catcode_other_token % } % These are implicit tokens which have the category code described % by their name. They are used internally for test purposes and should % not be used other than for category code tests. % \end{variable} % % \begin{variable}{\c_catcode_active_tl} % A token list containing an active token. This is used internally % for test purposes and should not be used other than in % appropriately-constructed category code tests. % \end{variable} % % \section{Converting tokens} % % \begin{function}[EXP]{\token_to_meaning:N, \token_to_meaning:c} % \begin{syntax} % \cs{token_to_meaning:N} \meta{token} % \end{syntax} % Inserts the current meaning of the \meta{token} into the input % stream as a series of characters of category code $12$ (other). % This is the primitive \TeX{} description of the \meta{token}, % thus for example both functions defined by \cs{cs_set_nopar:Npn} % and token list variables defined using \cs{tl_new:N} are described % as |macro|s. % \begin{texnote} % This is the \TeX{} primitive \tn{meaning}. % The \meta{token} can thus be an explicit space tokens or an % explicit begin-group or end-group character token (|{|~or~|}| when % normal \TeX{} category codes apply) even though these are not % valid \texttt{N}-type arguments. % \end{texnote} % \end{function} % % \begin{function}[EXP]{\token_to_str:N, \token_to_str:c} % \begin{syntax} % \cs{token_to_str:N} \meta{token} % \end{syntax} % Converts the given \meta{token} into a series of characters with % category code $12$ (other). If the \meta{token} is a control % sequence, this will start with the current escape character with % category code $12$ (the escape character is part of the % \meta{token}). This function requires only a single expansion. % \begin{texnote} % \cs{token_to_str:N} is the \TeX{} primitive \tn{string} renamed. % The \meta{token} can thus be an explicit space tokens or an % explicit begin-group or end-group character token (|{|~or~|}| when % normal \TeX{} category codes apply) even though these are not % valid \texttt{N}-type arguments. % \end{texnote} % \end{function} % % \section{Token conditionals} % % \begin{function}[EXP,pTF]{\token_if_group_begin:N} % \begin{syntax} % \cs{token_if_group_begin_p:N} \meta{token} \\ % \cs{token_if_group_begin:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if \meta{token} has the category code of a begin group token % (|{| when normal \TeX{} category codes are in ^^A } % force). % Note that an explicit begin group token cannot be tested in this way, % as it is not a valid \texttt{N}-type argument. % \end{function} % % \begin{function}[EXP,pTF]{\token_if_group_end:N} % \begin{syntax} % \cs{token_if_group_end_p:N} \meta{token} \\ % \cs{token_if_group_end:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if \meta{token} has the category code of an end group token % (^^A { % |}| when normal \TeX{} category codes are in force). % Note that an explicit end group token cannot be tested in this way, % as it is not a valid \texttt{N}-type argument. % \end{function} % % \begin{function}[EXP,pTF]{\token_if_math_toggle:N} % \begin{syntax} % \cs{token_if_math_toggle_p:N} \meta{token} \\ % \cs{token_if_math_toggle:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if \meta{token} has the category code of a math shift token % (|$| when normal \TeX{} category codes are in force). % \end{function} % % \begin{function}[EXP,pTF]{\token_if_alignment:N} % \begin{syntax} % \cs{token_if_alignment_p:N} \meta{token} \\ % \cs{token_if_alignment:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if \meta{token} has the category code of an alignment token % (|&| when normal \TeX{} category codes are in force). % \end{function} % % \begin{function}[EXP,pTF]{\token_if_parameter:N} % \begin{syntax} % \cs{token_if_parameter_p:N} \meta{token} \\ % \cs{token_if_alignment:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if \meta{token} has the category code of a macro parameter token % (|#| when normal \TeX{} category codes are in force). % \end{function} % % \begin{function}[EXP,pTF]{\token_if_math_superscript:N} % \begin{syntax} % \cs{token_if_math_superscript_p:N} \meta{token} \\ % \cs{token_if_math_superscript:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if \meta{token} has the category code of a superscript token % (|^| when normal \TeX{} category codes are in force). % \end{function} % % \begin{function}[EXP,pTF]{\token_if_math_subscript:N} % \begin{syntax} % \cs{token_if_math_subscript_p:N} \meta{token} \\ % \cs{token_if_math_subscript:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if \meta{token} has the category code of a subscript token % (|_| when normal \TeX{} category codes are in force). % \end{function} % % \begin{function}[EXP,pTF]{\token_if_space:N} % \begin{syntax} % \cs{token_if_space_p:N} \meta{token} \\ % \cs{token_if_space:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if \meta{token} has the category code of a space token. % Note that an explicit space token with character code $32$ cannot % be tested in this way, as it is not a valid \texttt{N}-type argument. % \end{function} % % \begin{function}[EXP,pTF]{\token_if_letter:N} % \begin{syntax} % \cs{token_if_letter_p:N} \meta{token} \\ % \cs{token_if_letter:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if \meta{token} has the category code of a letter token. % \end{function} % % \begin{function}[EXP,pTF]{\token_if_other:N} % \begin{syntax} % \cs{token_if_other_p:N} \meta{token} \\ % \cs{token_if_other:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if \meta{token} has the category code of an \enquote{other} % token. % \end{function} % % \begin{function}[EXP,pTF]{\token_if_active:N} % \begin{syntax} % \cs{token_if_active_p:N} \meta{token} \\ % \cs{token_if_active:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if \meta{token} has the category code of an active character. % \end{function} % % \begin{function}[EXP,pTF]{\token_if_eq_catcode:NN} % \begin{syntax} % \cs{token_if_eq_catcode_p:NN} \meta{token_1} \meta{token_2} \\ % \cs{token_if_eq_catcode:NNTF} \meta{token_1} \meta{token_2} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the two \meta{tokens} have the same category code. % \end{function} % % \begin{function}[EXP,pTF]{\token_if_eq_charcode:NN} % \begin{syntax} % \cs{token_if_eq_charcode_p:NN} \meta{token_1} \meta{token_2} \\ % \cs{token_if_eq_charcode:NNTF} \meta{token_1} \meta{token_2} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the two \meta{tokens} have the same character code. % \end{function} % % \begin{function}[EXP,pTF]{\token_if_eq_meaning:NN} % \begin{syntax} % \cs{token_if_eq_meaning_p:NN} \meta{token_1} \meta{token_2} \\ % \cs{token_if_eq_meaning:NNTF} \meta{token_1} \meta{token_2} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the two \meta{tokens} have the same meaning when expanded. % \end{function} % % \begin{function}[updated = 2011-05-23, EXP,pTF]{\token_if_macro:N} % \begin{syntax} % \cs{token_if_macro_p:N} \meta{token} \\ % \cs{token_if_macro:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the \meta{token} is a \TeX{} macro. % \end{function} % % \begin{function}[EXP,pTF]{\token_if_cs:N} % \begin{syntax} % \cs{token_if_cs_p:N} \meta{token} \\ % \cs{token_if_cs:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the \meta{token} is a control sequence. % \end{function} % % \begin{function}[EXP,pTF]{\token_if_expandable:N} % \begin{syntax} % \cs{token_if_expandable_p:N} \meta{token} \\ % \cs{token_if_expandable:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the \meta{token} is expandable. This test returns \meta{false} % for an undefined token. % \end{function} % % \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_long_macro:N} % \begin{syntax} % \cs{token_if_long_macro_p:N} \meta{token} \\ % \cs{token_if_long_macro:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the \meta{token} is a long macro. % \end{function} % % \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_protected_macro:N} % \begin{syntax} % \cs{token_if_protected_macro_p:N} \meta{token} \\ % \cs{token_if_protected_macro:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the \meta{token} is a protected macro: for a macro which % is both protected and long this returns \texttt{false}. % \end{function} % % \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_protected_long_macro:N} % \begin{syntax} % \cs{token_if_protected_long_macro_p:N} \meta{token} \\ % \cs{token_if_protected_long_macro:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the \meta{token} is a protected long macro. % \end{function} % % \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_chardef:N} % \begin{syntax} % \cs{token_if_chardef_p:N} \meta{token} \\ % \cs{token_if_chardef:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the \meta{token} is defined to be a chardef. % \begin{texnote} % Booleans, boxes and small integer constants are implemented as % \tn{chardef}s. % \end{texnote} % \end{function} % % \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_mathchardef:N} % \begin{syntax} % \cs{token_if_mathchardef_p:N} \meta{token} \\ % \cs{token_if_mathchardef:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the \meta{token} is defined to be a mathchardef. % \end{function} % % \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_dim_register:N} % \begin{syntax} % \cs{token_if_dim_register_p:N} \meta{token} \\ % \cs{token_if_dim_register:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the \meta{token} is defined to be a dimension register. % \end{function} % % \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_int_register:N} % \begin{syntax} % \cs{token_if_int_register_p:N} \meta{token} \\ % \cs{token_if_int_register:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the \meta{token} is defined to be a integer register. % \begin{texnote} % Constant integers may be implemented as integer registers, % \tn{chardef}s, or \tn{mathchardef}s depending on their value. % \end{texnote} % \end{function} % % \begin{function}[EXP,pTF, added=2012-02-15]{\token_if_muskip_register:N} % \begin{syntax} % \cs{token_if_muskip_register_p:N} \meta{token} \\ % \cs{token_if_muskip_register:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the \meta{token} is defined to be a muskip register. % \end{function} % % \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_skip_register:N} % \begin{syntax} % \cs{token_if_skip_register_p:N} \meta{token} \\ % \cs{token_if_skip_register:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the \meta{token} is defined to be a skip register. % \end{function} % % \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_toks_register:N} % \begin{syntax} % \cs{token_if_toks_register_p:N} \meta{token} \\ % \cs{token_if_toks_register:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the \meta{token} is defined to be a toks register % (not used by \LaTeX3). % \end{function} % % \begin{function}[updated = 2011-05-23, EXP,pTF]{\token_if_primitive:N} % \begin{syntax} % \cs{token_if_primitive_p:N} \meta{token} \\ % \cs{token_if_primitive:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the \meta{token} is an engine primitive. % \end{function} % % \section{Peeking ahead at the next token} % % There is often a need to look ahead at the next token in the input % stream while leaving it in place. This is handled using the % \enquote{peek} functions. The generic \cs{peek_after:Nw} is % provided along with a family of predefined tests for common cases. % As peeking ahead does \emph{not} skip spaces the predefined tests % include both a space-respecting and space-skipping version. % % \begin{function}{\peek_after:Nw} % \begin{syntax} % \cs{peek_after:Nw} \meta{function} \meta{token} % \end{syntax} % Locally sets the test variable \cs{l_peek_token} equal to \meta{token} % (as an implicit token, \emph{not} as a token list), and then % expands the \meta{function}. The \meta{token} remains in % the input stream as the next item after the \meta{function}. % The \meta{token} here may be \verb*| |, |{| or |}| (assuming % normal \TeX{} category codes), \emph{i.e.}~it is not necessarily the % next argument which would be grabbed by a normal function. % \end{function} % % \begin{function}{\peek_gafter:Nw} % \begin{syntax} % \cs{peek_gafter:Nw} \meta{function} \meta{token} % \end{syntax} % Globally sets the test variable \cs{g_peek_token} equal to \meta{token} % (as an implicit token, \emph{not} as a token list), and then % expands the \meta{function}. The \meta{token} remains in % the input stream as the next item after the \meta{function}. % The \meta{token} here may be \verb*| |, |{| or |}| (assuming % normal \TeX{} category codes), \emph{i.e.}~it is not necessarily the % next argument which would be grabbed by a normal function. % \end{function} % % \begin{variable}{\l_peek_token} % Token set by \cs{peek_after:Nw} and available for testing % as described above. % \end{variable} % % \begin{variable}{\g_peek_token} % Token set by \cs{peek_gafter:Nw} and available for testing % as described above. % \end{variable} % % \begin{function}[updated = 2012-12-20, TF]{\peek_catcode:N} % \begin{syntax} % \cs{peek_catcode:NTF} \meta{test token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the next \meta{token} in the input stream has the same % category code as the \meta{test token} (as defined by the test % \cs{token_if_eq_catcode:NNTF}). Spaces are respected by the test % and the \meta{token} is left in the input stream after % the \meta{true code} or \meta{false code} (as appropriate to the % result of the test). % \end{function} % % \begin{function}[updated = 2012-12-20, TF]{\peek_catcode_ignore_spaces:N} % \begin{syntax} % \cs{peek_catcode_ignore_spaces:NTF} \meta{test token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the next non-space \meta{token} in the input stream has the % same category code as the \meta{test token} (as defined by the test % \cs{token_if_eq_catcode:NNTF}). Explicit and implicit space tokens % (with character code 32 and category code 10) are ignored and % removed by the test and the \meta{token} is left in the input % stream after the \meta{true code} or \meta{false code} (as % appropriate to the result of the test). % \end{function} % % \begin{function}[updated = 2012-12-20, TF]{\peek_catcode_remove:N} % \begin{syntax} % \cs{peek_catcode_remove:NTF} \meta{test token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the next \meta{token} in the input stream has the same % category code as the \meta{test token} (as defined by the test % \cs{token_if_eq_catcode:NNTF}). Spaces are respected by the test % and the \meta{token} is removed from the input stream if the % test is true. The function then places either the % \meta{true code} or \meta{false code} in the input stream (as % appropriate to the result of the test). % \end{function} % % \begin{function}[updated = 2012-12-20, TF] % {\peek_catcode_remove_ignore_spaces:N} % \begin{syntax} % \cs{peek_catcode_remove_ignore_spaces:NTF} \meta{test token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the next non-space \meta{token} in the input stream has the % same category code as the \meta{test token} (as defined by the test % \cs{token_if_eq_catcode:NNTF}). Explicit and implicit space tokens % (with character code 32 and category code 10) are ignored and % removed by the test and the \meta{token} is removed from the % input stream if the test is true. The function then places % either the \meta{true code} or \meta{false code} in the input stream % (as appropriate to the result of the test). % \end{function} % % \begin{function}[updated = 2012-12-20, TF]{\peek_charcode:N} % \begin{syntax} % \cs{peek_charcode:NTF} \meta{test token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the next \meta{token} in the input stream has the same % character code as the \meta{test token} (as defined by the test % \cs{token_if_eq_charcode:NNTF}). Spaces are respected by the test % and the \meta{token} is left in the input stream after % the \meta{true code} or \meta{false code} (as appropriate to the % result of the test). % \end{function} % % \begin{function}[updated = 2012-12-20, TF]{\peek_charcode_ignore_spaces:N} % \begin{syntax} % \cs{peek_charcode_ignore_spaces:NTF} \meta{test token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the next non-space \meta{token} in the input stream has the % same character code as the \meta{test token} (as defined by the test % \cs{token_if_eq_charcode:NNTF}). Explicit and implicit space tokens % (with character code 32 and category code 10) are ignored and removed by % the test and the \meta{token} is left in the input stream after % the \meta{true code} or \meta{false code} (as appropriate to the % result of the test). % \end{function} % % \begin{function}[updated = 2012-12-20, TF]{\peek_charcode_remove:N} % \begin{syntax} % \cs{peek_charcode_remove:NTF} \meta{test token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the next \meta{token} in the input stream has the same % character code as the \meta{test token} (as defined by the test % \cs{token_if_eq_charcode:NNTF}). Spaces are respected by the test % and the \meta{token} is removed from the input stream if the % test is true. The function then places either the % \meta{true code} or \meta{false code} in the input stream (as % appropriate to the result of the test). % \end{function} % % \begin{function}[updated = 2012-12-20, TF] % {\peek_charcode_remove_ignore_spaces:N} % \begin{syntax} % \cs{peek_charcode_remove_ignore_spaces:NTF} \meta{test token} % ~~\Arg{true code} \Arg{false code} % \end{syntax} % Tests if the next non-space \meta{token} in the input stream has the % same character code as the \meta{test token} (as defined by the test % \cs{token_if_eq_charcode:NNTF}). Explicit and implicit space tokens % (with character code 32 and category code 10) are ignored and % removed by the test and the \meta{token} is removed from the % input stream if the test is true. The function then places % either the \meta{true code} or \meta{false code} in the input stream % (as appropriate to the result of the test). % \end{function} % % \begin{function}[updated = 2011-07-02, TF]{\peek_meaning:N} % \begin{syntax} % \cs{peek_meaning:NTF} \meta{test token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the next \meta{token} in the input stream has the same % meaning as the \meta{test token} (as defined by the test % \cs{token_if_eq_meaning:NNTF}). Spaces are respected by the test % and the \meta{token} is left in the input stream after % the \meta{true code} or \meta{false code} (as appropriate to the % result of the test). % \end{function} % % \begin{function}[updated = 2012-12-05, TF]{\peek_meaning_ignore_spaces:N} % \begin{syntax} % \cs{peek_meaning_ignore_spaces:NTF} \meta{test token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the next non-space \meta{token} in the input stream has the % same meaning as the \meta{test token} (as defined by the test % \cs{token_if_eq_meaning:NNTF}). Explicit and implicit space tokens % (with character code 32 and category code 10) are ignored and % removed by the test and the \meta{token} is left in the input % stream after the \meta{true code} or \meta{false code} (as % appropriate to the result of the test). % \end{function} % % \begin{function}[updated = 2011-07-02, TF]{\peek_meaning_remove:N} % \begin{syntax} % \cs{peek_meaning_remove:NTF} \meta{test token} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the next \meta{token} in the input stream has the same % meaning as the \meta{test token} (as defined by the test % \cs{token_if_eq_meaning:NNTF}). Spaces are respected by the test % and the \meta{token} is removed from the input stream if the % test is true. The function then places either the % \meta{true code} or \meta{false code} in the input stream (as % appropriate to the result of the test). % \end{function} % % \begin{function}[updated = 2012-12-05, TF] % {\peek_meaning_remove_ignore_spaces:N} % \begin{syntax} % \cs{peek_meaning_remove_ignore_spaces:NTF} \meta{test token} % ~~\Arg{true code} \Arg{false code} % \end{syntax} % Tests if the next non-space \meta{token} in the input stream has the % same meaning as the \meta{test token} (as defined by the test % \cs{token_if_eq_meaning:NNTF}). Explicit and implicit space tokens % (with character code 32 and category code 10) are ignored and % removed by the test and the \meta{token} is removed from the % input stream if the test is true. The function then places % either the \meta{true code} or \meta{false code} in the input stream % (as appropriate to the result of the test). % \end{function} % % \begin{function}[TF, updated = 2012-12-20]{\peek_N_type:} % \begin{syntax} % \cs{peek_N_type:TF} \Arg{true code} \Arg{false code} % \end{syntax} % Tests if the next \meta{token} in the input stream can be safely % grabbed as an \texttt{N}-type argument. The test is \meta{false} % if the next \meta{token} is either an explicit or implicit % begin-group or end-group token (with any character code), or % an explicit or implicit space character (with character code $32$ % and category code $10$), or an outer token (never used in \LaTeX3) % and \meta{true} in all other cases. % Note that a \meta{true} result ensures that the next \meta{token} is % a valid \texttt{N}-type argument. However, if the next \meta{token} % is for instance \cs{c_space_token}, the test takes the % \meta{false} branch, even though the next \meta{token} is in fact % a valid \texttt{N}-type argument. The \meta{token} is left % in the input stream after the \meta{true code} or \meta{false code} % (as appropriate to the result of the test). % \end{function} % % \section{Description of all possible tokens} % \label{sec:l3token:all-tokens} % % Let us end by reviewing every case that a given token can fall into. % This section is quite technical and some details are only meant for % completeness. We distinguish the meaning of the token, which controls % the expansion of the token and its effect on \TeX{}'s state, and its % shape, which is used when comparing token lists such as for delimited % arguments. Two tokens of the same shape must have the same meaning, % but the converse does not hold. % % A token has one of the following shapes. % \begin{itemize} % \item A control sequence, characterized by the sequence of % characters that constitute its name: for instance, \cs{use:n} is a % five-letter control sequence. % \item An active character token, characterized by its character code % (between $0$ and $1114111$ for \LuaTeX{} and \XeTeX{} and less for % other engines) and category code~$13$. % \item A character token, characterized by its character code and % category code (one of $1$, $2$, $3$, $4$, $6$, $7$, $8$, $10$, % $11$ or~$12$ whose meaning is described below).\footnote{In % \LuaTeX{}, there is also the case of \enquote{bytes}, which behave as % character tokens of category code $12$~(other) and character code % between $1114112$ and~$1114366$. They are used to output % individual bytes to files, rather than UTF-8.} % \end{itemize} % There are also a few internal tokens. The following list may be % incomplete in some engines. % \begin{itemize} % \item Expanding \tn{the}\tn{font} results in a token that looks % identical to the command that was used to select the current font % (such as \tn{tenrm}) but it differs from it in shape. % \item A \enquote{frozen} |\relax|, which differs from the primitive in % shape (but has the same meaning), is inserted when the closing \tn{fi} of a % conditional is encountered before the conditional is evaluated. % \item Expanding \tn{noexpand} \meta{token} (when the \meta{token} is % expandable) results in an internal token, displayed (temporarily) % as \cs{notexpanded: \meta{token}}, whose shape coincides with the % \meta{token} and whose meaning differs from \tn{relax}. % \item An |\outer endtemplate:| can be encountered when peeking ahead % at the next token; this expands to another internal token, % |end of alignment template|. % \item Tricky programming might access a frozen |\endwrite|. % \item Some frozen tokens can only be accessed in interactive % sessions: |\cr|, |\right|, |\endgroup|, |\fi|, |\inaccessible|. % \end{itemize} % % The meaning of a (non-active) character token is fixed by its category % code (and character code) and cannot be changed. We call these % tokens \emph{explicit} character tokens. Category codes that a % character token can have are listed below by giving a sample output of % the \TeX{} primitive \tn{meaning}, together with their \LaTeX3 names % and most common example: % \begin{itemize} % \item[1] begin-group character (|group_begin|, often |{|), % \item[2] end-group character (|group_end|, often |}|), % \item[3] math shift character (|math_toggle|, often |$|), % \item[4] alignment tab character (|alignment|, often |&|), % \item[6] macro parameter character (|parameter|, often |#|), % \item[7] superscript character (|math_superscript|, often |^|), % \item[8] subscript character (|math_subscript|, often |_|), % \item[10] blank space (|space|, often character code~$32$), % \item[11] the letter (|letter|, such as |A|), % \item[12] the character (|other|, such as |0|). % \end{itemize} % Category code~$13$ (|active|) is discussed below. Input characters % can also have several other category codes which do not lead to % character tokens for later processing: $0$~(|escape|), % $5$~(|end_line|), $9$~(|ignore|), $14$~(|comment|), and % $15$~(|invalid|). % % The meaning of a control sequence or active character can be identical % to that of any character token listed above (with any character code), % and we call such tokens \emph{implicit} character tokens. The % meaning is otherwise in the following list: % \begin{itemize} % \item a macro, used in \LaTeX3 for most functions and some variables % (|tl|, |fp|, |seq|, \ldots{}), % \item a primitive such as \tn{def} or \tn{topmark}, used in \LaTeX3 % for some functions, % \item a register such as \tn{count}|123|, used in \LaTeX3{} for the % implementation of some variables (|int|, |dim|, \ldots{}), % \item a constant integer such as \tn{char}|"56| or \tn{mathchar}|"121|, % \item a font selection command, % \item undefined. % \end{itemize} % Macros be \tn{protected} or not, \tn{long} or not (the opposite of % what \LaTeX3 calls |nopar|), and \tn{outer} or not (unused in % \LaTeX3). Their \tn{meaning} takes the form % \begin{quote} % \meta{properties} |macro:|\meta{parameters}|->|\meta{replacement} % \end{quote} % where \meta{properties} is among \tn{protected}\tn{long}\tn{outer}, % \meta{parameters} describes parameters that the macro expects, such as % |#1#2#3|, and \meta{replacement} describes how the parameters are % manipulated, such as~|#2/#1/#3|. % % ^^A todo Bruno: discuss here some other subtleties of space tokens? when looking for numbers, when looking for equal signs in let, in expressions, etc. % % Now is perhaps a good time to mention some subtleties relating to % tokens with category code $10$ (space). Any input character with this % category code (normally, space and tab characters) becomes a normal % space, with character code~$32$ and category code~$10$. % % When a macro takes an undelimited argument, explicit space characters % (with character code $32$ and category code $10$) are ignored. If the % following token is an explicit character token with category code $1$ % (begin-group) and an arbitrary character code, then \TeX{} scans ahead % to obtain an equal number of explicit character tokens with category % code $1$ (begin-group) and $2$ (end-group), and the resulting list of % tokens (with outer braces removed) becomes the argument. Otherwise, a % single token is taken as the argument for the macro: we call such % single tokens \enquote{N-type}, as they are suitable to be used as an % argument for a function with the signature~\texttt{:N}. % % \end{documentation} % % \begin{implementation} % % \section{\pkg{l3token} implementation} % % \begin{macrocode} %<*initex|package> % \end{macrocode} % % \begin{macrocode} %<@@=char> % \end{macrocode} % % \subsection{Internal auxiliaries} % % \begin{variable}{\s_@@_stop} % Internal scan mark. % \begin{macrocode} \scan_new:N \s_@@_stop % \end{macrocode} % \end{variable} % % \begin{variable}{\q_@@_no_value} % Internal recursion quarks. % \begin{macrocode} \quark_new:N \q_@@_no_value % \end{macrocode} % \end{variable} % % \begin{macro}[pTF]{\@@_quark_if_no_value:N} % Functions to query recursion quarks. % \begin{macrocode} \__kernel_quark_new_conditional:Nn \@@_quark_if_no_value:N { TF } % \end{macrocode} % \end{macro} % % \subsection{Manipulating and interrogating character tokens} % % \begin{macro}{\char_set_catcode:nn} % \begin{macro}{\char_value_catcode:n} % \begin{macro}{\char_show_value_catcode:n} % Simple wrappers around the primitives. % \begin{macrocode} \cs_new_protected:Npn \char_set_catcode:nn #1#2 { \tex_catcode:D \int_eval:n {#1} = \int_eval:n {#2} \exp_stop_f: } \cs_new:Npn \char_value_catcode:n #1 { \tex_the:D \tex_catcode:D \int_eval:n {#1} \exp_stop_f: } \cs_new_protected:Npn \char_show_value_catcode:n #1 { \exp_args:Nf \tl_show:n { \char_value_catcode:n {#1} } } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro} % { % \char_set_catcode_escape:N , % \char_set_catcode_group_begin:N , % \char_set_catcode_group_end:N , % \char_set_catcode_math_toggle:N , % \char_set_catcode_alignment:N , % \char_set_catcode_end_line:N , % \char_set_catcode_parameter:N , % \char_set_catcode_math_superscript:N , % \char_set_catcode_math_subscript:N , % \char_set_catcode_ignore:N , % \char_set_catcode_space:N , % \char_set_catcode_letter:N , % \char_set_catcode_other:N , % \char_set_catcode_active:N , % \char_set_catcode_comment:N , % \char_set_catcode_invalid:N % } % \begin{macrocode} \cs_new_protected:Npn \char_set_catcode_escape:N #1 { \char_set_catcode:nn { `#1 } { 0 } } \cs_new_protected:Npn \char_set_catcode_group_begin:N #1 { \char_set_catcode:nn { `#1 } { 1 } } \cs_new_protected:Npn \char_set_catcode_group_end:N #1 { \char_set_catcode:nn { `#1 } { 2 } } \cs_new_protected:Npn \char_set_catcode_math_toggle:N #1 { \char_set_catcode:nn { `#1 } { 3 } } \cs_new_protected:Npn \char_set_catcode_alignment:N #1 { \char_set_catcode:nn { `#1 } { 4 } } \cs_new_protected:Npn \char_set_catcode_end_line:N #1 { \char_set_catcode:nn { `#1 } { 5 } } \cs_new_protected:Npn \char_set_catcode_parameter:N #1 { \char_set_catcode:nn { `#1 } { 6 } } \cs_new_protected:Npn \char_set_catcode_math_superscript:N #1 { \char_set_catcode:nn { `#1 } { 7 } } \cs_new_protected:Npn \char_set_catcode_math_subscript:N #1 { \char_set_catcode:nn { `#1 } { 8 } } \cs_new_protected:Npn \char_set_catcode_ignore:N #1 { \char_set_catcode:nn { `#1 } { 9 } } \cs_new_protected:Npn \char_set_catcode_space:N #1 { \char_set_catcode:nn { `#1 } { 10 } } \cs_new_protected:Npn \char_set_catcode_letter:N #1 { \char_set_catcode:nn { `#1 } { 11 } } \cs_new_protected:Npn \char_set_catcode_other:N #1 { \char_set_catcode:nn { `#1 } { 12 } } \cs_new_protected:Npn \char_set_catcode_active:N #1 { \char_set_catcode:nn { `#1 } { 13 } } \cs_new_protected:Npn \char_set_catcode_comment:N #1 { \char_set_catcode:nn { `#1 } { 14 } } \cs_new_protected:Npn \char_set_catcode_invalid:N #1 { \char_set_catcode:nn { `#1 } { 15 } } % \end{macrocode} % \end{macro} % % \begin{macro} % { % \char_set_catcode_escape:n , % \char_set_catcode_group_begin:n , % \char_set_catcode_group_end:n , % \char_set_catcode_math_toggle:n , % \char_set_catcode_alignment:n , % \char_set_catcode_end_line:n , % \char_set_catcode_parameter:n , % \char_set_catcode_math_superscript:n , % \char_set_catcode_math_subscript:n , % \char_set_catcode_ignore:n , % \char_set_catcode_space:n , % \char_set_catcode_letter:n , % \char_set_catcode_other:n , % \char_set_catcode_active:n , % \char_set_catcode_comment:n , % \char_set_catcode_invalid:n % } % \begin{macrocode} \cs_new_protected:Npn \char_set_catcode_escape:n #1 { \char_set_catcode:nn {#1} { 0 } } \cs_new_protected:Npn \char_set_catcode_group_begin:n #1 { \char_set_catcode:nn {#1} { 1 } } \cs_new_protected:Npn \char_set_catcode_group_end:n #1 { \char_set_catcode:nn {#1} { 2 } } \cs_new_protected:Npn \char_set_catcode_math_toggle:n #1 { \char_set_catcode:nn {#1} { 3 } } \cs_new_protected:Npn \char_set_catcode_alignment:n #1 { \char_set_catcode:nn {#1} { 4 } } \cs_new_protected:Npn \char_set_catcode_end_line:n #1 { \char_set_catcode:nn {#1} { 5 } } \cs_new_protected:Npn \char_set_catcode_parameter:n #1 { \char_set_catcode:nn {#1} { 6 } } \cs_new_protected:Npn \char_set_catcode_math_superscript:n #1 { \char_set_catcode:nn {#1} { 7 } } \cs_new_protected:Npn \char_set_catcode_math_subscript:n #1 { \char_set_catcode:nn {#1} { 8 } } \cs_new_protected:Npn \char_set_catcode_ignore:n #1 { \char_set_catcode:nn {#1} { 9 } } \cs_new_protected:Npn \char_set_catcode_space:n #1 { \char_set_catcode:nn {#1} { 10 } } \cs_new_protected:Npn \char_set_catcode_letter:n #1 { \char_set_catcode:nn {#1} { 11 } } \cs_new_protected:Npn \char_set_catcode_other:n #1 { \char_set_catcode:nn {#1} { 12 } } \cs_new_protected:Npn \char_set_catcode_active:n #1 { \char_set_catcode:nn {#1} { 13 } } \cs_new_protected:Npn \char_set_catcode_comment:n #1 { \char_set_catcode:nn {#1} { 14 } } \cs_new_protected:Npn \char_set_catcode_invalid:n #1 { \char_set_catcode:nn {#1} { 15 } } % \end{macrocode} % \end{macro} % % \begin{macro}{\char_set_mathcode:nn} % \begin{macro}{\char_value_mathcode:n} % \begin{macro}{\char_show_value_mathcode:n} % \begin{macro}{\char_set_lccode:nn} % \begin{macro}{\char_value_lccode:n} % \begin{macro}{\char_show_value_lccode:n} % \begin{macro}{\char_set_uccode:nn} % \begin{macro}{\char_value_uccode:n} % \begin{macro}{\char_show_value_uccode:n} % \begin{macro}{\char_set_sfcode:nn} % \begin{macro}{\char_value_sfcode:n} % \begin{macro}{\char_show_value_sfcode:n} % Pretty repetitive, but necessary! % \begin{macrocode} \cs_new_protected:Npn \char_set_mathcode:nn #1#2 { \tex_mathcode:D \int_eval:n {#1} = \int_eval:n {#2} \exp_stop_f: } \cs_new:Npn \char_value_mathcode:n #1 { \tex_the:D \tex_mathcode:D \int_eval:n {#1} \exp_stop_f: } \cs_new_protected:Npn \char_show_value_mathcode:n #1 { \exp_args:Nf \tl_show:n { \char_value_mathcode:n {#1} } } \cs_new_protected:Npn \char_set_lccode:nn #1#2 { \tex_lccode:D \int_eval:n {#1} = \int_eval:n {#2} \exp_stop_f: } \cs_new:Npn \char_value_lccode:n #1 { \tex_the:D \tex_lccode:D \int_eval:n {#1} \exp_stop_f: } \cs_new_protected:Npn \char_show_value_lccode:n #1 { \exp_args:Nf \tl_show:n { \char_value_lccode:n {#1} } } \cs_new_protected:Npn \char_set_uccode:nn #1#2 { \tex_uccode:D \int_eval:n {#1} = \int_eval:n {#2} \exp_stop_f: } \cs_new:Npn \char_value_uccode:n #1 { \tex_the:D \tex_uccode:D \int_eval:n {#1} \exp_stop_f: } \cs_new_protected:Npn \char_show_value_uccode:n #1 { \exp_args:Nf \tl_show:n { \char_value_uccode:n {#1} } } \cs_new_protected:Npn \char_set_sfcode:nn #1#2 { \tex_sfcode:D \int_eval:n {#1} = \int_eval:n {#2} \exp_stop_f: } \cs_new:Npn \char_value_sfcode:n #1 { \tex_the:D \tex_sfcode:D \int_eval:n {#1} \exp_stop_f: } \cs_new_protected:Npn \char_show_value_sfcode:n #1 { \exp_args:Nf \tl_show:n { \char_value_sfcode:n {#1} } } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{variable}{\l_char_active_seq, \l_char_special_seq} % Two sequences for dealing with special characters. The first is characters % which may be active, the second longer list is for \enquote{special} % characters more generally. Both lists are escaped so that for example % bulk code assignments can be carried out. In both cases, the order is % by \textsc{ascii} character code (as is done in for example % \cs{ExplSyntaxOn}). % \begin{macrocode} \seq_new:N \l_char_special_seq \seq_set_split:Nnn \l_char_special_seq { } { \ \" \# \$ \% \& \\ \^ \_ \{ \} \~ } \seq_new:N \l_char_active_seq \seq_set_split:Nnn \l_char_active_seq { } { \" \$ \& \^ \_ \~ } % \end{macrocode} % \end{variable} % % \subsection{Creating character tokens} % % \begin{macro} % { % \char_set_active_eq:NN, \char_gset_active_eq:NN, % \char_set_active_eq:Nc, \char_gset_active_eq:Nc, % \char_set_active_eq:nN, \char_gset_active_eq:nN, % \char_set_active_eq:nc, \char_gset_active_eq:nc % } % Four simple functions with very similar definitions, so set up using % an auxiliary. % These are similar to \LuaTeX{}'s \tn{letcharcode} primitive. % \begin{macrocode} \group_begin: \char_set_catcode_active:N \^^@ \cs_set_protected:Npn \@@_tmp:nN #1#2 { \cs_new_protected:cpn { #1 :nN } ##1 { \group_begin: \char_set_lccode:nn { `\^^@ } { ##1 } \tex_lowercase:D { \group_end: #2 ^^@ } } \cs_new_protected:cpx { #1 :NN } ##1 { \exp_not:c { #1 : nN } { `##1 } } } \@@_tmp:nN { char_set_active_eq } \cs_set_eq:NN \@@_tmp:nN { char_gset_active_eq } \cs_gset_eq:NN \group_end: \cs_generate_variant:Nn \char_set_active_eq:NN { Nc } \cs_generate_variant:Nn \char_gset_active_eq:NN { Nc } \cs_generate_variant:Nn \char_set_active_eq:nN { nc } \cs_generate_variant:Nn \char_gset_active_eq:nN { nc } % \end{macrocode} % \end{macro} % % \begin{macro}{\@@_int_to_roman:w} % For efficiency in 8-bit engines, we use the faster primitive approach % to making roman numerals. % \begin{macrocode} \cs_new_eq:NN \@@_int_to_roman:w \tex_romannumeral:D % \end{macrocode} % \end{macro} % % \begin{macro}[EXP]{\char_generate:nn} % \begin{macro}[EXP]{\@@_generate_aux:nn} % \begin{macro}[EXP]{\@@_generate_aux:nnw, \@@_generate_auxii:nnw} % \begin{variable}{\l_@@_tmp_tl} % \begin{macro}[EXP]{\@@_generate_invalid_catcode:} % The aim here is to generate characters of (broadly) arbitrary category % code. Where possible, that is done using engine support (\XeTeX{}, % \LuaTeX{}). There are though various issues which are covered below. At % the interface layer, turn the two arguments into integers up-front so % this is only done once. % \begin{macrocode} \cs_new:Npn \char_generate:nn #1#2 { \exp:w \exp_after:wN \@@_generate_aux:w \int_value:w \int_eval:n {#1} \exp_after:wN ; \int_value:w \int_eval:n {#2} ; } % \end{macrocode} % Before doing any actual conversion, first some special case filtering. % Spaces are out here as \LuaTeX{} emulation only makes normal (charcode % $32$ spaces). However, |^^@| is filtered out separately as that can't be % done with macro emulation either, so is flagged up separately. That % done, hand off to the engine-dependent part. % \begin{macrocode} \cs_new:Npn \@@_generate_aux:w #1 ; #2 ; { \if_int_compare:w #2 = 10 \exp_stop_f: \if_int_compare:w #1 = 0 \exp_stop_f: \__kernel_msg_expandable_error:nn { kernel } { char-null-space } \else: \__kernel_msg_expandable_error:nn { kernel } { char-space } \fi: \else: \if_int_odd:w 0 \if_int_compare:w #2 < 1 \exp_stop_f: 1 \fi: \if_int_compare:w #2 = 5 \exp_stop_f: 1 \fi: \if_int_compare:w #2 = 9 \exp_stop_f: 1 \fi: \if_int_compare:w #2 > 13 \exp_stop_f: 1 \fi: \exp_stop_f: \__kernel_msg_expandable_error:nn { kernel } { char-invalid-catcode } \else: \if_int_odd:w 0 \if_int_compare:w #1 < 0 \exp_stop_f: 1 \fi: \if_int_compare:w #1 > \c_max_char_int 1 \fi: \exp_stop_f: \__kernel_msg_expandable_error:nn { kernel } { char-out-of-range } \else: \@@_generate_aux:nnw {#1} {#2} \fi: \fi: \fi: \exp_end: } \tl_new:N \l_@@_tmp_tl % \end{macrocode} % Engine-dependent definitions are now needed for the implementation. For % \LuaTeX{} and \XeTeX{} there is engine-level support. % They can do cases that macro emulation can't. All of those are filtered % out here using a primitive-based boolean expression to avoid fixing % the category code of the null character used in the false branch % (for 8-bit engines). % The final level is the basic definition at the engine level: the arguments % here are integers so there is no need to worry about them too much. % Older versions of \XeTeX{} cannot generate active characters so we filter % that: % at some future stage that may change: the slightly odd ordering of % auxiliaries reflects that. % \begin{macrocode} \group_begin: %<*package> \char_set_catcode_active:N \^^L \cs_set:Npn ^^L { } % \char_set_catcode_other:n { 0 } \if_int_odd:w 0 \sys_if_engine_luatex:T { 1 } \sys_if_engine_xetex:T { 1 } \exp_stop_f: \sys_if_engine_luatex:TF { \cs_new:Npn \@@_generate_aux:nnw #1#2#3 \exp_end: { #3 \exp_after:wN \exp_after:wN \exp_after:wN \exp_end: \lua_now:e { l3kernel.charcat(#1, #2) } } } { \cs_new:Npn \@@_generate_aux:nnw #1#2#3 \exp_end: { #3 \exp_after:wN \exp_end: \tex_Ucharcat:D #1 \exp_stop_f: #2 \exp_stop_f: } \cs_if_exist:NF \tex_expanded:D { \cs_new_eq:NN \@@_generate_auxii:nnw \@@_generate_aux:nnw \cs_gset:Npn \@@_generate_aux:nnw #1#2#3 \exp_end: { #3 \if_int_compare:w #2 = 13 \exp_stop_f: \__kernel_msg_expandable_error:nn { kernel } { char-active } \else: \@@_generate_auxii:nnw {#1} {#2} \fi: \exp_end: } } } \else: % \end{macrocode} % For engines where \tn{Ucharcat} isn't available or emulated, we have % to work in macros, and cover only the $8$-bit range. The first stage is % to build up a |tl| containing |^^@| with each category code that can % be accessed in this way, with an error set up for the other cases. This % is all done such that it can be quickly accessed using a |\if_case:w| % low-level conditional. There are a few things to notice here. % As |^^L| is |\outer| we need to locally set it to avoid a problem. % To get open/close braces into the list, they are set up using |\if_false:| % pairing and are then |x|-type expanded together into the desired form. % \begin{macrocode} \tl_set:Nn \l_@@_tmp_tl { \exp_not:N \or: } \char_set_catcode_group_begin:n { 0 } % { \tl_put_right:Nn \l_@@_tmp_tl { ^^@ \if_false: } } \char_set_catcode_group_end:n { 0 } \tl_put_right:Nn \l_@@_tmp_tl { { \fi: \exp_not:N \or: ^^@ } % } \tl_set:Nx \l_@@_tmp_tl { \l_@@_tmp_tl } \char_set_catcode_math_toggle:n { 0 } \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } \char_set_catcode_alignment:n { 0 } \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } \tl_put_right:Nn \l_@@_tmp_tl { \or: } \char_set_catcode_parameter:n { 0 } \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } \char_set_catcode_math_superscript:n { 0 } \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } \char_set_catcode_math_subscript:n { 0 } \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } \tl_put_right:Nn \l_@@_tmp_tl { \or: } % \end{macrocode} % For making spaces, there needs to be an |o|-type expansion of a |\use:n| % (or some other tokenization) to avoid dropping the space. % \begin{macrocode} \char_set_catcode_space:n { 0 } \tl_put_right:No \l_@@_tmp_tl { \use:n { \or: } ^^@ } \char_set_catcode_letter:n { 0 } \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } \char_set_catcode_other:n { 0 } \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } \char_set_catcode_active:n { 0 } \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } % \end{macrocode} % Convert the above temporary list into a series of constant token % lists, one for each character code, using \tn{tex_lowercase:D} to % convert |^^@| in each case. The \texttt{x}-type expansion ensures % that \tn{tex_lowercase:D} receives the contents of the token list. % In package mode, |^^L| is awkward hence this is done in three parts. % Notice that at this stage |^^@| is active. % \begin{macrocode} \cs_set_protected:Npn \@@_tmp:n #1 { \char_set_lccode:nn { 0 } {#1} \char_set_lccode:nn { 32 } {#1} \exp_args:Nx \tex_lowercase:D { \tl_const:Nn \exp_not:c { c_@@_ \@@_int_to_roman:w #1 _tl } { \exp_not:o \l_@@_tmp_tl } } } %<*package> \int_step_function:nnN { 0 } { 11 } \@@_tmp:n \group_begin: \tl_replace_once:Nnn \l_@@_tmp_tl { ^^@ } { \ERROR } \@@_tmp:n { 12 } \group_end: \int_step_function:nnN { 13 } { 255 } \@@_tmp:n % %<*initex> \int_step_function:nnN { 0 } { 255 } \@@_tmp:n % % \end{macrocode} % As \TeX{} is very unhappy if it finds an alignment character inside % a primitive \tn{halign} even when skipping false branches, some % precautions are required. \TeX{} is happy if the token is hidden % between braces within \cs{if_false:} \dots{} \cs{fi:}. % \begin{macrocode} \cs_new:Npn \@@_generate_aux:nnw #1#2#3 \exp_end: { #3 \if_false: { \fi: \exp_after:wN \exp_after:wN \exp_after:wN \exp_end: \exp_after:wN \exp_after:wN \if_case:w #2 \exp_last_unbraced:Nv \exp_stop_f: { c_@@_ \@@_int_to_roman:w #1 _tl } \or: } \fi: } \fi: \group_end: % \end{macrocode} % \end{macro} % \end{variable} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}[EXP]{\char_to_utfviii_bytes:n} % \begin{macro}[EXP]{\@@_to_utfviii_bytes_auxi:n} % \begin{macro}[EXP]{\@@_to_utfviii_bytes_auxii:Nnn} % \begin{macro}[EXP]{\@@_to_utfviii_bytes_auxiii:n} % \begin{macro}[EXP] % { % \@@_to_utfviii_bytes_outputi:nw , % \@@_to_utfviii_bytes_outputii:nw , % \@@_to_utfviii_bytes_outputiii:nw , % \@@_to_utfviii_bytes_outputiv:nw % } % \begin{macro}[EXP] % {\@@_to_utfviii_bytes_output:nnn, \@@_to_utfviii_bytes_output:fnn} % \begin{macro}[EXP]{\@@_to_utfviii_bytes_end:} % This code converts a codepoint into the correct UTF-8 representation. % In terms of the algorithm itself, see % \url{https://en.wikipedia.org/wiki/UTF-8} for the octet pattern. % \begin{macrocode} \cs_new:Npn \char_to_utfviii_bytes:n #1 { \exp_args:Nf \@@_to_utfviii_bytes_auxi:n { \int_eval:n {#1} } } \cs_new:Npn \@@_to_utfviii_bytes_auxi:n #1 { \if_int_compare:w #1 > "80 \exp_stop_f: \if_int_compare:w #1 < "800 \exp_stop_f: \@@_to_utfviii_bytes_outputi:nw { \@@_to_utfviii_bytes_auxii:Nnn C {#1} { 64 } } \@@_to_utfviii_bytes_outputii:nw { \@@_to_utfviii_bytes_auxiii:n {#1} } \else: \if_int_compare:w #1 < "10000 \exp_stop_f: \@@_to_utfviii_bytes_outputi:nw { \@@_to_utfviii_bytes_auxii:Nnn E {#1} { 64 * 64 } } \@@_to_utfviii_bytes_outputii:nw { \@@_to_utfviii_bytes_auxiii:n { \int_div_truncate:nn {#1} { 64 } } } \@@_to_utfviii_bytes_outputiii:nw { \@@_to_utfviii_bytes_auxiii:n {#1} } \else: \@@_to_utfviii_bytes_outputi:nw { \@@_to_utfviii_bytes_auxii:Nnn F {#1} { 64 * 64 * 64 } } \@@_to_utfviii_bytes_outputii:nw { \@@_to_utfviii_bytes_auxiii:n { \int_div_truncate:nn {#1} { 64 * 64 } } } \@@_to_utfviii_bytes_outputiii:nw { \@@_to_utfviii_bytes_auxiii:n { \int_div_truncate:nn {#1} { 64 } } } \@@_to_utfviii_bytes_outputiv:nw { \@@_to_utfviii_bytes_auxiii:n {#1} } \fi: \fi: \else: \@@_to_utfviii_bytes_outputi:nw {#1} \fi: \@@_to_utfviii_bytes_end: { } { } { } { } } \cs_new:Npn \@@_to_utfviii_bytes_auxii:Nnn #1#2#3 { "#10 + \int_div_truncate:nn {#2} {#3} } \cs_new:Npn \@@_to_utfviii_bytes_auxiii:n #1 { \int_mod:nn {#1} { 64 } + 128 } \cs_new:Npn \@@_to_utfviii_bytes_outputi:nw #1 #2 \@@_to_utfviii_bytes_end: #3 { \@@_to_utfviii_bytes_output:fnn { \int_eval:n {#1} } { } {#2} } \cs_new:Npn \@@_to_utfviii_bytes_outputii:nw #1 #2 \@@_to_utfviii_bytes_end: #3#4 { \@@_to_utfviii_bytes_output:fnn { \int_eval:n {#1} } { {#3} } {#2} } \cs_new:Npn \@@_to_utfviii_bytes_outputiii:nw #1 #2 \@@_to_utfviii_bytes_end: #3#4#5 { \@@_to_utfviii_bytes_output:fnn { \int_eval:n {#1} } { {#3} {#4} } {#2} } \cs_new:Npn \@@_to_utfviii_bytes_outputiv:nw #1 #2 \@@_to_utfviii_bytes_end: #3#4#5#6 { \@@_to_utfviii_bytes_output:fnn { \int_eval:n {#1} } { {#3} {#4} {#5} } {#2} } \cs_new:Npn \@@_to_utfviii_bytes_output:nnn #1#2#3 { #3 \@@_to_utfviii_bytes_end: #2 {#1} } \cs_generate_variant:Nn \@@_to_utfviii_bytes_output:nnn { f } \cs_new:Npn \@@_to_utfviii_bytes_end: { } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}[rEXP]{\char_to_nfd:N} % \begin{macro}[rEXP]{\@@_to_nfd:n} % \begin{macro}[rEXP]{\@@_to_nfd:Nw} % Look up any \textsc{nfd} and recursively produce the result. % \begin{macrocode} \cs_new:Npn \char_to_nfd:N #1 { \cs_if_exist:cTF { c_@@_nfd_ \token_to_str:N #1 _ tl } { \exp_after:wN \exp_after:wN \exp_after:wN \@@_to_nfd:Nw \exp_after:wN \exp_after:wN \exp_after:wN #1 \cs:w c_@@_nfd_ \token_to_str:N #1 _ tl \cs_end: \s_@@_stop } { \exp_not:n {#1} } } \cs_set_eq:NN \@@_to_nfd:n \char_to_nfd:N \cs_new:Npn \@@_to_nfd:Nw #1#2#3 \s_@@_stop { \exp_args:Ne \@@_to_nfd:n { \char_generate:nn { `#2 } { \@@_change_case_catcode:N #1 } } \tl_if_blank:nF {#3} { \exp_args:Ne \@@_to_nfd:n { \char_generate:nn { `#3 } { \char_value_catcode:n { `#3 } } } } } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}[EXP] % { % \char_lowercase:N, \char_uppercase:N, % \char_titlecase:N, \char_foldcase:N % } % \begin{macro}[EXP]{\@@_change_case:nNN} % \begin{macro}[EXP]{\@@_change_case:nN} % \begin{macro}[EXP]{\@@_change_case_multi:nN, \@@_change_case_multi:vN} % \begin{macro}[EXP]{\@@_change_case_multi:NNNNw} % \begin{macro}[EXP]{\@@_change_case:NNN} % \begin{macro}[EXP]{\@@_change_case:NNNN} % \begin{macro}[EXP]{\@@_change_case:NN} % \begin{macro}[EXP]{\@@_change_case_catcode:N} % \begin{macro}[EXP] % { % \char_str_lowercase:N, \char_str_uppercase:N, % \char_str_titlecase:N, \char_str_foldcase:N % } % \begin{macro}[EXP]{\@@_str_change_case:nNN} % \begin{macro}[EXP]{\@@_str_change_case:nN} % To ensure that the category codes produced are predictable, every character % is re-generated even if it is otherwise unchanged. This makes life a little % interesting when we might have multiple output characters: we have to % grab each of them and case change them in reverse order to maintain % \texttt{f}-type expandability. % \begin{macrocode} \cs_new:Npn \char_lowercase:N #1 { \@@_change_case:nNN { lower } \char_value_lccode:n #1 } \cs_new:Npn \char_uppercase:N #1 { \@@_change_case:nNN { upper } \char_value_uccode:n #1 } \cs_new:Npn \char_titlecase:N #1 { \tl_if_exist:cTF { c_@@_titlecase_ \token_to_str:N #1 _tl } { \@@_change_case_multi:vN { c_@@_titlecase_ \token_to_str:N #1 _tl } #1 } { \char_uppercase:N #1 } } \cs_new:Npn \char_foldcase:N #1 { \@@_change_case:nNN { fold } \char_value_lccode:n #1 } \cs_new:Npn \@@_change_case:nNN #1#2#3 { \tl_if_exist:cTF { c_@@_ #1 case_ \token_to_str:N #3 _tl } { \@@_change_case_multi:vN { c_@@_ #1 case_ \token_to_str:N #3 _tl } #3 } { \exp_args:Nf \@@_change_case:nN { #2 { `#3 } } #3 } } \cs_new:Npn \@@_change_case:nN #1#2 { \int_compare:nNnTF {#1} = 0 { #2 } { \char_generate:nn {#1} { \@@_change_case_catcode:N #2 } } } \cs_new:Npn \@@_change_case_multi:nN #1#2 { \@@_change_case_multi:NNNNw #2 #1 \q_@@_no_value \q_@@_no_value \s_@@_stop } \cs_generate_variant:Nn \@@_change_case_multi:nN { v } \cs_new:Npn \@@_change_case_multi:NNNNw #1#2#3#4#5 \s_@@_stop { \@@_quark_if_no_value:NTF #4 { \@@_quark_if_no_value:NTF #3 { \@@_change_case:NN #1 #2 } { \@@_change_case:NNN #1 #2#3 } } { \@@_change_case:NNNN #1 #2#3#4 } } \cs_new:Npn \@@_change_case:NNN #1#2#3 { \exp_args:Nnf \use:nn { \@@_change_case:NN #1 #2 } { \@@_change_case:NN #1 #3 } } \cs_new:Npn \@@_change_case:NNNN #1#2#3#4 { \exp_args:Nnff \use:nnn { \@@_change_case:NN #1 #2 } { \@@_change_case:NN #1 #3 } { \@@_change_case:NN #1 #4 } } \cs_new:Npn \@@_change_case:NN #1#2 { \char_generate:nn { `#2 } { \@@_change_case_catcode:N #1 } } \cs_new:Npn \@@_change_case_catcode:N #1 { \if_catcode:w \exp_not:N #1 \c_math_toggle_token 3 \else: \if_catcode:w \exp_not:N #1 \c_alignment_token 4 \else: \if_catcode:w \exp_not:N #1 \c_math_superscript_token 7 \else: \if_catcode:w \exp_not:N #1 \c_math_subscript_token 8 \else: \if_catcode:w \exp_not:N #1 \c_space_token 10 \else: \if_catcode:w \exp_not:N #1 \c_catcode_letter_token 11 \else: \if_catcode:w \exp_not:N #1 \c_catcode_other_token 12 \else: 13 \fi: \fi: \fi: \fi: \fi: \fi: \fi: } % \end{macrocode} % Same story for the string version, except category code is easier % to follow. This of course makes this version significantly faster. % \begin{macrocode} \cs_new:Npn \char_str_lowercase:N #1 { \@@_str_change_case:nNN { lower } \char_value_lccode:n #1 } \cs_new:Npn \char_str_uppercase:N #1 { \@@_str_change_case:nNN { upper } \char_value_uccode:n #1 } \cs_new:Npn \char_str_titlecase:N #1 { \tl_if_exist:cTF { c_@@_titlecase_ \token_to_str:N #1 _tl } { \tl_to_str:c { c_@@_titlecase_ \token_to_str:N #1 _tl } } { \char_str_uppercase:N #1 } } \cs_new:Npn \char_str_foldcase:N #1 { \@@_str_change_case:nNN { fold } \char_value_lccode:n #1 } \cs_new:Npn \@@_str_change_case:nNN #1#2#3 { \tl_if_exist:cTF { c_@@_ #1 case_ \token_to_str:N #3 _tl } { \tl_to_str:c { c_@@_ #1 case_ \token_to_str:N #3 _tl } } { \exp_args:Nf \@@_str_change_case:nN { #2 { `#3 } } #3 } } \cs_new:Npn \@@_str_change_case:nN #1#2 { \int_compare:nNnTF {#1} = 0 { \tl_to_str:n {#2} } { \char_generate:nn {#1} { 12 } } } \bool_lazy_or:nnF { \cs_if_exist_p:N \tex_luatexversion:D } { \cs_if_exist_p:N \tex_XeTeXversion:D } { \cs_set:Npn \@@_str_change_case:nN #1#2 { \tl_to_str:n {#2} } } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\c_catcode_other_space_tl} % Create a space with category code $12$: an \enquote{other} space. % \begin{macrocode} \tl_const:Nx \c_catcode_other_space_tl { \char_generate:nn { `\ } { 12 } } % \end{macrocode} % \end{macro} % % \subsection{Generic tokens} % % \begin{macrocode} %<@@=token> % \end{macrocode} % % \begin{variable}{\s_@@_stop} % Internal scan marks. % \begin{macrocode} \scan_new:N \s_@@_stop % \end{macrocode} % \end{variable} % % \begin{macro}{\token_to_meaning:N, \token_to_meaning:c} % \begin{macro}{\token_to_str:N, \token_to_str:c} % These are all defined in \pkg{l3basics}, as they are needed % \enquote{early}. This is just a reminder! % \end{macro} % \end{macro} % % \begin{macro} % { % \c_group_begin_token, % \c_group_end_token, % \c_math_toggle_token, % \c_alignment_token, % \c_parameter_token, % \c_math_superscript_token, % \c_math_subscript_token, % \c_space_token, % \c_catcode_letter_token, % \c_catcode_other_token % } % We define these useful tokens. For the brace and space tokens things have % to be done by hand: the formal argument spec.~for \cs{cs_new_eq:NN} does % not cover them so we do things by hand. (As currently coded it would % \emph{work} with \cs{cs_new_eq:NN} but that's not really a great idea to % show off: we want people to stick to the defined interfaces and that % includes us.) So that these few odd names go into the log when appropriate % there is a need to hand-apply the \cs{__kernel_chk_if_free_cs:N} check. % \begin{macrocode} \group_begin: \__kernel_chk_if_free_cs:N \c_group_begin_token \tex_global:D \tex_let:D \c_group_begin_token { \__kernel_chk_if_free_cs:N \c_group_end_token \tex_global:D \tex_let:D \c_group_end_token } \char_set_catcode_math_toggle:N \* \cs_new_eq:NN \c_math_toggle_token * \char_set_catcode_alignment:N \* \cs_new_eq:NN \c_alignment_token * \cs_new_eq:NN \c_parameter_token # \cs_new_eq:NN \c_math_superscript_token ^ \char_set_catcode_math_subscript:N \* \cs_new_eq:NN \c_math_subscript_token * \__kernel_chk_if_free_cs:N \c_space_token \use:n { \tex_global:D \tex_let:D \c_space_token = ~ } ~ \cs_new_eq:NN \c_catcode_letter_token a \cs_new_eq:NN \c_catcode_other_token 1 \group_end: % \end{macrocode} % \end{macro} % % \begin{variable}{\c_catcode_active_tl} % Not an implicit token! % \begin{macrocode} \group_begin: \char_set_catcode_active:N \* \tl_const:Nn \c_catcode_active_tl { \exp_not:N * } \group_end: % \end{macrocode} % \end{variable} % % \subsection{Token conditionals} % % \begin{macro}[pTF]{\token_if_group_begin:N} % Check if token is a begin group token. We use the constant % \cs{c_group_begin_token} for this. % \begin{macrocode} \prg_new_conditional:Npnn \token_if_group_begin:N #1 { p , T , F , TF } { \if_catcode:w \exp_not:N #1 \c_group_begin_token \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % % \begin{macro}[pTF]{\token_if_group_end:N} % Check if token is a end group token. We use the constant % \cs{c_group_end_token} for this. % \begin{macrocode} \prg_new_conditional:Npnn \token_if_group_end:N #1 { p , T , F , TF } { \if_catcode:w \exp_not:N #1 \c_group_end_token \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % % \begin{macro}[pTF]{\token_if_math_toggle:N} % Check if token is a math shift token. We use the constant % \cs{c_math_toggle_token} for this. % \begin{macrocode} \prg_new_conditional:Npnn \token_if_math_toggle:N #1 { p , T , F , TF } { \if_catcode:w \exp_not:N #1 \c_math_toggle_token \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % % \begin{macro}[pTF]{\token_if_alignment:N} % Check if token is an alignment tab token. We use the constant % \cs{c_alignment_token} for this. % \begin{macrocode} \prg_new_conditional:Npnn \token_if_alignment:N #1 { p , T , F , TF } { \if_catcode:w \exp_not:N #1 \c_alignment_token \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % % \begin{macro}[pTF]{\token_if_parameter:N} % Check if token is a parameter token. We use the constant % \cs{c_parameter_token} for this. We have to trick \TeX{} a bit to % avoid an error message: within a group we prevent % \cs{c_parameter_token} from behaving like a macro parameter character. % The definitions of \cs{prg_new_conditional:Npnn} are global, so they % remain after the group. % \begin{macrocode} \group_begin: \cs_set_eq:NN \c_parameter_token \scan_stop: \prg_new_conditional:Npnn \token_if_parameter:N #1 { p , T , F , TF } { \if_catcode:w \exp_not:N #1 \c_parameter_token \prg_return_true: \else: \prg_return_false: \fi: } \group_end: % \end{macrocode} % \end{macro} % % \begin{macro}[pTF]{\token_if_math_superscript:N} % Check if token is a math superscript token. We use the constant % \cs{c_math_superscript_token} for this. % \begin{macrocode} \prg_new_conditional:Npnn \token_if_math_superscript:N #1 { p , T , F , TF } { \if_catcode:w \exp_not:N #1 \c_math_superscript_token \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % % \begin{macro}[pTF]{\token_if_math_subscript:N} % Check if token is a math subscript token. We use the constant % \cs{c_math_subscript_token} for this. % \begin{macrocode} \prg_new_conditional:Npnn \token_if_math_subscript:N #1 { p , T , F , TF } { \if_catcode:w \exp_not:N #1 \c_math_subscript_token \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % % \begin{macro}[pTF]{\token_if_space:N} % Check if token is a space token. We use the constant % \cs{c_space_token} for this. % \begin{macrocode} \prg_new_conditional:Npnn \token_if_space:N #1 { p , T , F , TF } { \if_catcode:w \exp_not:N #1 \c_space_token \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % % \begin{macro}[pTF]{\token_if_letter:N} % Check if token is a letter token. We use the constant % \cs{c_catcode_letter_token} for this. % \begin{macrocode} \prg_new_conditional:Npnn \token_if_letter:N #1 { p , T , F , TF } { \if_catcode:w \exp_not:N #1 \c_catcode_letter_token \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % % \begin{macro}[pTF]{\token_if_other:N} % Check if token is an other char token. We use the constant % \cs{c_catcode_other_token} for this. % \begin{macrocode} \prg_new_conditional:Npnn \token_if_other:N #1 { p , T , F , TF } { \if_catcode:w \exp_not:N #1 \c_catcode_other_token \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % % \begin{macro}[pTF]{\token_if_active:N} % Check if token is an active char token. We use the constant % \cs{c_catcode_active_tl} for this. A technical point is that % \cs{c_catcode_active_tl} is in fact a macro expanding to % |\exp_not:N *|, where |*| is active. % \begin{macrocode} \prg_new_conditional:Npnn \token_if_active:N #1 { p , T , F , TF } { \if_catcode:w \exp_not:N #1 \c_catcode_active_tl \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % % \begin{macro}[pTF]{\token_if_eq_meaning:NN} % Check if the tokens |#1| and |#2| have same meaning. % \begin{macrocode} \prg_new_conditional:Npnn \token_if_eq_meaning:NN #1#2 { p , T , F , TF } { \if_meaning:w #1 #2 \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % % \begin{macro}[pTF]{\token_if_eq_catcode:NN} % Check if the tokens |#1| and |#2| have same category code. % \begin{macrocode} \prg_new_conditional:Npnn \token_if_eq_catcode:NN #1#2 { p , T , F , TF } { \if_catcode:w \exp_not:N #1 \exp_not:N #2 \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % % \begin{macro}[pTF]{\token_if_eq_charcode:NN} % Check if the tokens |#1| and |#2| have same character code. % \begin{macrocode} \prg_new_conditional:Npnn \token_if_eq_charcode:NN #1#2 { p , T , F , TF } { \if_charcode:w \exp_not:N #1 \exp_not:N #2 \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % % \begin{macro}[pTF]{\token_if_macro:N} % \begin{macro}{\@@_if_macro_p:w} % When a token is a macro, \cs{token_to_meaning:N} always outputs % something like |\long macro:#1->#1| so we could naively check to % see if the meaning contains |->|. However, this can fail the five % \tn[no-index]{...mark} primitives, whose meaning has the form % |...mark:|\meta{user material}. The problem is that the % \meta{user material} can contain |->|. % % However, only characters, macros, and marks can contain the colon % character. The idea is thus to grab until the first |:|, and analyse % what is left. However, macros can have any combination of |\long|, % |\protected| or |\outer| (not used in \LaTeX3) before the string % |macro:|. We thus only select the part of the meaning between % the first |ma| and the first following |:|. If this string is % |cro|, then we have a macro. If the string is |rk|, then we have % a mark. The string can also be |cro parameter character | for a % colon with a weird category code (namely the usual category code % of |#|). Otherwise, it is empty. % % This relies on the fact that |\long|, |\protected|, |\outer| % cannot contain |ma|, regardless of the escape character, even if % the escape character is |m|\ldots{} % % Both |ma| and |:| must be of category code $12$ (other), so are % detokenized. % % \begin{macrocode} \use:x { \prg_new_conditional:Npnn \exp_not:N \token_if_macro:N ##1 { p , T , F , TF } { \exp_not:N \exp_after:wN \exp_not:N \@@_if_macro_p:w \exp_not:N \token_to_meaning:N ##1 \tl_to_str:n { ma : } \s_@@_stop } \cs_new:Npn \exp_not:N \@@_if_macro_p:w ##1 \tl_to_str:n { ma } ##2 \c_colon_str ##3 \s_@@_stop } { \str_if_eq:nnTF { #2 } { cro } { \prg_return_true: } { \prg_return_false: } } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}[pTF]{\token_if_cs:N} % Check if token has same catcode as a control sequence. This % follows the same pattern as for \cs{token_if_letter:N} \emph{etc.} % We use \cs{scan_stop:} for this. % \begin{macrocode} \prg_new_conditional:Npnn \token_if_cs:N #1 { p , T , F , TF } { \if_catcode:w \exp_not:N #1 \scan_stop: \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % % \begin{macro}[pTF]{\token_if_expandable:N} % Check if token is expandable. We use the fact that \TeX{} % temporarily converts \cs{exp_not:N} \meta{token} into \cs{scan_stop:} % if \meta{token} is expandable. An \texttt{undefined} token is not % considered as expandable. No problem nesting the conditionals, % since the third |#1| is only skipped if it is non-expandable (hence % not part of \TeX{}'s conditional apparatus). % \begin{macrocode} \prg_new_conditional:Npnn \token_if_expandable:N #1 { p , T , F , TF } { \exp_after:wN \if_meaning:w \exp_not:N #1 #1 \prg_return_false: \else: \if_cs_exist:N #1 \prg_return_true: \else: \prg_return_false: \fi: \fi: } % \end{macrocode} % \end{macro} % % \begin{macro} % { % \@@_delimit_by_char":w, % \@@_delimit_by_count:w, % \@@_delimit_by_dimen:w, % \@@_delimit_by_macro:w, % \@@_delimit_by_muskip:w, % \@@_delimit_by_skip:w, % \@@_delimit_by_toks:w, % } % These auxiliary functions are used below to define some % conditionals which detect whether the \tn{meaning} of their % argument begins with a particular string. Each auxiliary takes an % argument delimited by a string, a second one delimited by % \cs{s_@@_stop}, and returns the first one and its delimiter. % This result is eventually compared to another string. % \begin{macrocode} \group_begin: \cs_set_protected:Npn \@@_tmp:w #1 { \use:x { \cs_new:Npn \exp_not:c { @@_delimit_by_ #1 :w } ####1 \tl_to_str:n {#1} ####2 \s_@@_stop { ####1 \tl_to_str:n {#1} } } } \@@_tmp:w { char" } \@@_tmp:w { count } \@@_tmp:w { dimen } \@@_tmp:w { macro } \@@_tmp:w { muskip } \@@_tmp:w { skip } \@@_tmp:w { toks } \group_end: % \end{macrocode} % \end{macro} % % \begin{macro}[pTF] % { % \token_if_chardef:N, \token_if_mathchardef:N, % \token_if_long_macro:N, % \token_if_protected_macro:N, \token_if_protected_long_macro:N, % \token_if_dim_register:N, \token_if_int_register:N, % \token_if_muskip_register:N, % \token_if_skip_register:N, \token_if_toks_register:N, % } % Each of these conditionals tests whether its argument's % \tn{meaning} starts with a given string. This is essentially done % by having an auxiliary grab an argument delimited by the string and % testing whether the argument was empty. Of course, a copy of this % string must first be added to the end of the \tn{meaning} to avoid % a runaway argument in case it does not contain the string. Two % complications arise. First, the escape character is not fixed, and % cannot be included in the delimiter of the auxiliary function (this % function cannot be defined on the fly because tests must remain % expandable): instead the first argument of the auxiliary (plus the % delimiter to avoid complications with trailing spaces) is compared % using \cs{str_if_eq:eeTF} to the result of applying % \cs{token_to_str:N} to a control sequence. Second, the % \tn{meaning} of primitives such as \tn{dimen} or \tn{dimendef} % starts in the same way as registers such as % \tn{dimen}\texttt{123}, so they must be tested for. % % Characters used as delimiters must have catcode~$12$ % and are obtained through \cs{tl_to_str:n}. This requires doing all % definitions within \texttt{x}-expansion. The temporary function % \cs{@@_tmp:w} used to define each conditional receives three % arguments: the name of the conditional, the auxiliary's delimiter % (also used to name the auxiliary), and the string to which one % compares the auxiliary's result. Note that the \tn{meaning} of a % protected long macro starts with |\protected\long macro|, with no % space after |\protected| but a space after |\long|, hence the % mixture of \cs{token_to_str:N} and \cs{tl_to_str:n}. % % For the first five conditionals, \cs{cs_if_exist:cT} turns out to % be \texttt{false}, and the code boils down to a string comparison % between the result of the auxiliary on the \tn{meaning} of the % conditional's argument~|####1|, and~|#3|. Both are evaluated at % run-time, as this is important to get the correct escape character. % % The other five conditionals have additional code that compares the % argument~|####1| to two \TeX{} primitives which would wrongly be % recognized as registers otherwise. Despite using \TeX{}'s % primitive conditional construction, this does not break % when~|####1| is itself a conditional, because branches of the % conditionals are only skipped if |####1|~is one of the two % primitives that are tested for (which are not \TeX{} conditionals). % \begin{macrocode} \group_begin: \cs_set_protected:Npn \@@_tmp:w #1#2#3 { \use:x { \prg_new_conditional:Npnn \exp_not:c { token_if_ #1 :N } ####1 { p , T , F , TF } { \cs_if_exist:cT { tex_ #2 :D } { \exp_not:N \if_meaning:w ####1 \exp_not:c { tex_ #2 :D } \exp_not:N \prg_return_false: \exp_not:N \else: \exp_not:N \if_meaning:w ####1 \exp_not:c { tex_ #2 def:D } \exp_not:N \prg_return_false: \exp_not:N \else: } \exp_not:N \str_if_eq:eeTF { \exp_not:N \exp_after:wN \exp_not:c { @@_delimit_by_ #2 :w } \exp_not:N \token_to_meaning:N ####1 ? \tl_to_str:n {#2} \s_@@_stop } { \exp_not:n {#3} } { \exp_not:N \prg_return_true: } { \exp_not:N \prg_return_false: } \cs_if_exist:cT { tex_ #2 :D } { \exp_not:N \fi: \exp_not:N \fi: } } } } \@@_tmp:w { chardef } { char" } { \token_to_str:N \char" } \@@_tmp:w { mathchardef } { char" } { \token_to_str:N \mathchar" } \@@_tmp:w { long_macro } { macro } { \tl_to_str:n { \long } macro } \@@_tmp:w { protected_macro } { macro } { \tl_to_str:n { \protected } macro } \@@_tmp:w { protected_long_macro } { macro } { \token_to_str:N \protected \tl_to_str:n { \long } macro } \@@_tmp:w { dim_register } { dimen } { \token_to_str:N \dimen } \@@_tmp:w { int_register } { count } { \token_to_str:N \count } \@@_tmp:w { muskip_register } { muskip } { \token_to_str:N \muskip } \@@_tmp:w { skip_register } { skip } { \token_to_str:N \skip } \@@_tmp:w { toks_register } { toks } { \token_to_str:N \toks } \group_end: % \end{macrocode} % \end{macro} % % \begin{macro}[pTF]{\token_if_primitive:N} % \begin{macro}{\@@_if_primitive:NNw, % \@@_if_primitive_space:w, % \@@_if_primitive_nullfont:N, % \@@_if_primitive_loop:N, % \@@_if_primitive:Nw, % \@@_if_primitive_undefined:N} %^^A See http://groups.google.com/group/comp.text.tex/browse_thread/thread/0a72666873f8753d# % % We filter out macros first, because they cause endless trouble later % otherwise. % % Primitives are almost distinguished by the fact that the result % of \cs{token_to_meaning:N} is formed from letters only. Every other % token has either a space (e.g., |the letter A|), a digit % (e.g., |\count123|) or a double quote (e.g., |\char"A|). % % Ten exceptions: on the one hand, \cs{tex_undefined:D} is not a % primitive, but its meaning is |undefined|, only letters; % on the other hand, \tn{space}, \tn{italiccorr}, % \tn{hyphen}, \tn{firstmark}, \tn{topmark}, % \tn{botmark}, \tn{splitfirstmark}, \tn{splitbotmark}, % and \tn{nullfont} are primitives, but have non-letters % in their meaning. % % We start by removing the two first (non-space) characters from % the meaning. This removes the escape character (which may be % nonexistent depending on \tn{endlinechar}), and takes care % of three of the exceptions: \tn{space}, \tn{italiccorr} % and \tn{hyphen}, whose meaning is at most two characters. % This leaves a string terminated by some |:|, and \cs{s_@@_stop}. % % The meaning of each one of the five \tn[no-index]{...mark} primitives % has the form \meta{letters}|:|\meta{user material}. In other words, % the first non-letter is a colon. We remove everything after the first % colon. % % We are now left with a string, which we must analyze. For primitives, % it contains only letters. For non-primitives, it contains either % |"|, or a space, or a digit. Two exceptions remain: \cs{tex_undefined:D}, % which is not a primitive, and \tn{nullfont}, which is a primitive. % % Spaces cannot be grabbed in an undelimited way, so we check them % separately. If there is a space, we test for \tn{nullfont}. % Otherwise, we go through characters one by one, and stop at the % first character less than |`A| (this is not quite a test for % \enquote{only letters}, but is close enough to work in this context). % If this first character is |:| then we have a primitive, or % \cs{tex_undefined:D}, and if it is |"| or a digit, then the token % is not a primitive. % % \begin{macrocode} \tex_chardef:D \c_@@_A_int = `A ~ % \use:x { \prg_new_conditional:Npnn \exp_not:N \token_if_primitive:N ##1 { p , T , F , TF } { \exp_not:N \token_if_macro:NTF ##1 \exp_not:N \prg_return_false: { \exp_not:N \exp_after:wN \exp_not:N \@@_if_primitive:NNw \exp_not:N \token_to_meaning:N ##1 \tl_to_str:n { : : : } \s_@@_stop ##1 } } \cs_new:Npn \exp_not:N \@@_if_primitive:NNw ##1##2 ##3 \c_colon_str ##4 \s_@@_stop { \exp_not:N \tl_if_empty:oTF { \exp_not:N \@@_if_primitive_space:w ##3 ~ } { \exp_not:N \@@_if_primitive_loop:N ##3 \c_colon_str \s_@@_stop } { \exp_not:N \@@_if_primitive_nullfont:N } } } \cs_new:Npn \@@_if_primitive_space:w #1 ~ { } \cs_new:Npn \@@_if_primitive_nullfont:N #1 { \if_meaning:w \tex_nullfont:D #1 \prg_return_true: \else: \prg_return_false: \fi: } \cs_new:Npn \@@_if_primitive_loop:N #1 { \if_int_compare:w `#1 < \c_@@_A_int % \exp_after:wN \@@_if_primitive:Nw \exp_after:wN #1 \else: \exp_after:wN \@@_if_primitive_loop:N \fi: } \cs_new:Npn \@@_if_primitive:Nw #1 #2 \s_@@_stop { \if:w : #1 \exp_after:wN \@@_if_primitive_undefined:N \else: \prg_return_false: \exp_after:wN \use_none:n \fi: } \cs_new:Npn \@@_if_primitive_undefined:N #1 { \if_cs_exist:N #1 \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % \end{macro} % % \subsection{Peeking ahead at the next token} % % \begin{macrocode} %<@@=peek> % \end{macrocode} % % Peeking ahead is implemented using a two part mechanism. The % outer level provides a defined interface to the lower level material. % This allows a large amount of code to be shared. There are four % cases: % \begin{enumerate} % \item peek at the next token; % \item peek at the next non-space token; % \item peek at the next token and remove it; % \item peek at the next non-space token and remove it. % \end{enumerate} % % \begin{variable}{\l_peek_token} % \begin{variable}{\g_peek_token} % Storage tokens which are publicly documented: the token peeked. % \begin{macrocode} \cs_new_eq:NN \l_peek_token ? \cs_new_eq:NN \g_peek_token ? % \end{macrocode} % \end{variable} % \end{variable} % % \begin{variable}{\l_@@_search_token} % The token to search for as an implicit token: % \emph{cf.}~\cs{l_@@_search_tl}. % \begin{macrocode} \cs_new_eq:NN \l_@@_search_token ? % \end{macrocode} % \end{variable} % % \begin{variable}{\l_@@_search_tl} % The token to search for as an explicit token: % \emph{cf.}~\cs{l_@@_search_token}. % \begin{macrocode} \tl_new:N \l_@@_search_tl % \end{macrocode} % \end{variable} % % \begin{macro} % {\@@_true:w, \@@_true_aux:w, \@@_false:w, \@@_tmp:w} % Functions used by the branching and space-stripping code. % \begin{macrocode} \cs_new:Npn \@@_true:w { } \cs_new:Npn \@@_true_aux:w { } \cs_new:Npn \@@_false:w { } \cs_new:Npn \@@_tmp:w { } % \end{macrocode} % \end{macro} % % \begin{variable}{\s_@@_mark,\s_@@_stop} % Internal scan marks. % \begin{macrocode} \scan_new:N \s_@@_mark \scan_new:N \s_@@_stop % \end{macrocode} % \end{variable} % % \begin{macro}[EXP]{\@@_use_none_delimit_by_s_stop:w} % Functions to gobble up to a scan mark. % \begin{macrocode} \cs_new:Npn \@@_use_none_delimit_by_s_stop:w #1 \s_@@_stop { } % \end{macrocode} % \end{macro} % % \begin{macro}{\peek_after:Nw} % \begin{macro}{\peek_gafter:Nw} % Simple wrappers for \tn{futurelet}: no arguments absorbed % here. % \begin{macrocode} \cs_new_protected:Npn \peek_after:Nw { \tex_futurelet:D \l_peek_token } \cs_new_protected:Npn \peek_gafter:Nw { \tex_global:D \tex_futurelet:D \g_peek_token } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}{\@@_true_remove:w} % A function to remove the next token and then regain control. % \begin{macrocode} \cs_new_protected:Npn \@@_true_remove:w { \tex_afterassignment:D \@@_true_aux:w \cs_set_eq:NN \@@_tmp:w } % \end{macrocode} % \end{macro} % % \begin{macro}{\peek_remove_spaces:n, \@@_remove_spaces:} % Repeatedly use \cs{@@_true_remove:w} to remove a space and call % \cs{@@_true_aux:w}. % \begin{macrocode} \cs_new_protected:Npn \peek_remove_spaces:n #1 { \cs_set:Npx \@@_false:w { \exp_not:n {#1} } \group_align_safe_begin: \cs_set:Npn \@@_true_aux:w { \peek_after:Nw \@@_remove_spaces: } \@@_true_aux:w } \cs_new_protected:Npn \@@_remove_spaces: { \if_meaning:w \l_peek_token \c_space_token \exp_after:wN \@@_true_remove:w \else: \group_align_safe_end: \exp_after:wN \@@_false:w \fi: } % \end{macrocode} % \end{macro} % % \begin{macro}{\@@_token_generic_aux:NNNTF} % The generic functions store the test token in both implicit and % explicit modes, and the \texttt{true} and \texttt{false} code as % token lists, more or less. The two branches have to be absorbed here % as the input stream needs to be cleared for the peek function itself. % Here, |#1| is \cs{@@_true_remove:w} when removing the token and % \cs{@@_true_aux:w} otherwise. % \begin{macrocode} \cs_new_protected:Npn \@@_token_generic_aux:NNNTF #1#2#3#4#5 { \group_align_safe_begin: \cs_set_eq:NN \l_@@_search_token #3 \tl_set:Nn \l_@@_search_tl {#3} \cs_set:Npx \@@_true_aux:w { \exp_not:N \group_align_safe_end: \exp_not:n {#4} } \cs_set_eq:NN \@@_true:w #1 \cs_set:Npx \@@_false:w { \exp_not:N \group_align_safe_end: \exp_not:n {#5} } \peek_after:Nw #2 } % \end{macrocode} % \end{macro} % % \begin{macro}[TF]{\@@_token_generic:NN, \@@_token_remove_generic:NN} % For token removal there needs to be a call to the auxiliary % function which does the work. % \begin{macrocode} \cs_new_protected:Npn \@@_token_generic:NNTF { \@@_token_generic_aux:NNNTF \@@_true_aux:w } \cs_new_protected:Npn \@@_token_generic:NNT #1#2#3 { \@@_token_generic:NNTF #1 #2 {#3} { } } \cs_new_protected:Npn \@@_token_generic:NNF #1#2#3 { \@@_token_generic:NNTF #1 #2 { } {#3} } \cs_new_protected:Npn \@@_token_remove_generic:NNTF { \@@_token_generic_aux:NNNTF \@@_true_remove:w } \cs_new_protected:Npn \@@_token_remove_generic:NNT #1#2#3 { \@@_token_remove_generic:NNTF #1 #2 {#3} { } } \cs_new_protected:Npn \@@_token_remove_generic:NNF #1#2#3 { \@@_token_remove_generic:NNTF #1 #2 { } {#3} } % \end{macrocode} % \end{macro} % % \begin{macro}{\@@_execute_branches_meaning:} % The meaning test is straight forward. % \begin{macrocode} \cs_new:Npn \@@_execute_branches_meaning: { \if_meaning:w \l_peek_token \l_@@_search_token \exp_after:wN \@@_true:w \else: \exp_after:wN \@@_false:w \fi: } % \end{macrocode} % \end{macro} % % \begin{macro}{\@@_execute_branches_catcode:, \@@_execute_branches_charcode:} % \begin{macro} % { % \@@_execute_branches_catcode_aux: , % \@@_execute_branches_catcode_auxii:N , % \@@_execute_branches_catcode_auxiii: % } % The catcode and charcode tests are very similar, and in order to use % the same auxiliaries we do something a little bit odd, firing % \cs{if_catcode:w} and \cs{if_charcode:w} before finding the operands % for those tests, which are only given in the |auxii:N| and % |auxiii:| auxiliaries. For our purposes, three kinds of tokens may % follow the peeking function: % \begin{itemize} % \item control sequences which are not equal to a non-active % character token (\emph{e.g.}, macro, primitive); % \item active characters which are not equal to a non-active % character token (\emph{e.g.}, macro, primitive); % \item explicit non-active character tokens, or control sequences % or active characters set equal to a non-active character token. % \end{itemize} % The first two cases are not distinguishable simply using \TeX{}'s % \tn{futurelet}, because we can only access the \tn{meaning} of % tokens in that way. In those cases, detected thanks to a % comparison with \cs{scan_stop:}, we grab the following token, and % compare it explicitly with the explicit search token stored in % \cs{l_@@_search_tl}. The \cs{exp_not:N} prevents outer macros % (coming from non-\LaTeX3 code) from blowing up. In the third case, % \cs{l_peek_token} is good enough for the test, and we compare it % again with the explicit search token. Just like the peek token, the % search token may be of any of the three types above, hence the need % to use the explicit token that was given to the peek function. % \begin{macrocode} \cs_new:Npn \@@_execute_branches_catcode: { \if_catcode:w \@@_execute_branches_catcode_aux: } \cs_new:Npn \@@_execute_branches_charcode: { \if_charcode:w \@@_execute_branches_catcode_aux: } \cs_new:Npn \@@_execute_branches_catcode_aux: { \if_catcode:w \exp_not:N \l_peek_token \scan_stop: \exp_after:wN \exp_after:wN \exp_after:wN \@@_execute_branches_catcode_auxii:N \exp_after:wN \exp_not:N \else: \exp_after:wN \@@_execute_branches_catcode_auxiii: \fi: } \cs_new:Npn \@@_execute_branches_catcode_auxii:N #1 { \exp_not:N #1 \exp_after:wN \exp_not:N \l_@@_search_tl \exp_after:wN \@@_true:w \else: \exp_after:wN \@@_false:w \fi: #1 } \cs_new:Npn \@@_execute_branches_catcode_auxiii: { \exp_not:N \l_peek_token \exp_after:wN \exp_not:N \l_@@_search_tl \exp_after:wN \@@_true:w \else: \exp_after:wN \@@_false:w \fi: } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}[TF] % { % \peek_catcode:N, % \peek_catcode_remove:N, % \peek_charcode:N, % \peek_charcode_remove:N, % \peek_meaning:N, % \peek_meaning_remove:N, % } % The public functions themselves cannot be defined using % \cs{prg_new_conditional:Npnn}. Instead, the |TF|, |T|, |F| variants % are defined in terms of corresponding variants of % \cs{@@_token_generic:NNTF} or \cs{@@_token_remove_generic:NNTF}, % with first argument one of \cs{@@_execute_branches_catcode:}, % \cs{@@_execute_branches_charcode:}, or % \cs{@@_execute_branches_meaning:}. % \begin{macrocode} \tl_map_inline:nn { { catcode } { charcode } { meaning } } { \tl_map_inline:nn { { } { _remove } } { \tl_map_inline:nn { { TF } { T } { F } } { \cs_new_protected:cpx { peek_ #1 ##1 :N ####1 } { \exp_not:c { @@_token ##1 _generic:NN ####1 } \exp_not:c { @@_execute_branches_ #1 : } } } } } % \end{macrocode} % \end{macro} % % \begin{macro}[TF] % { % \peek_catcode_ignore_spaces:N, \peek_catcode_remove_ignore_spaces:N, % \peek_charcode_ignore_spaces:N, \peek_charcode_remove_ignore_spaces:N, % \peek_meaning_ignore_spaces:N, \peek_meaning_remove_ignore_spaces:N % } % To ignore spaces, remove them using \cs{peek_remove_spaces:n} % before running the tests. % \begin{macrocode} \tl_map_inline:nn { { catcode } { catcode_remove } { charcode } { charcode_remove } { meaning } { meaning_remove } } { \cs_new_protected:cpx { peek_#1_ignore_spaces:NTF } ##1##2##3 { \peek_remove_spaces:n { \exp_not:c { peek_#1:NTF } ##1 {##2} {##3} } } \cs_new_protected:cpx { peek_#1_ignore_spaces:NT } ##1##2 { \peek_remove_spaces:n { \exp_not:c { peek_#1:NT } ##1 {##2} } } \cs_new_protected:cpx { peek_#1_ignore_spaces:NF } ##1##2 { \peek_remove_spaces:n { \exp_not:c { peek_#1:NF } ##1 {##2} } } } % \end{macrocode} % \end{macro} % % \begin{macro}[TF]{\peek_N_type:} % \begin{macro} % {\@@_execute_branches_N_type:, \@@_N_type:w, \@@_N_type_aux:nnw} % All tokens are \texttt{N}-type tokens, except in four cases: % begin-group tokens, end-group tokens, space tokens with character % code~$32$, and outer tokens. Since \cs{l_peek_token} might be % outer, we cannot use the convenient \cs{bool_if:nTF} function, and % must resort to the old trick of using \tn{ifodd} to expand a set of % tests. The \texttt{false} branch of this test is taken if the token % is one of the first three kinds of non-\texttt{N}-type tokens % (explicit or implicit), thus we call \cs{@@_false:w}. In the % \texttt{true} branch, we must detect outer tokens, without impacting % performance too much for non-outer tokens. The first filter is to % search for \texttt{outer} in the \tn{meaning} of \cs{l_peek_token}. % If that is absent, \cs{@@_use_none_delimit_by_s_stop:w} cleans up, and % we call \cs{@@_true:w}. Otherwise, the token can be a non-outer % macro or a primitive mark whose parameter or replacement text % contains \texttt{outer}, it can be the primitive \tn{outer}, or it % can be an outer token. Macros and marks would have \texttt{ma} in % the part before the first occurrence of \texttt{outer}; the meaning % of \tn{outer} has nothing after \texttt{outer}, contrarily to outer % macros; and that covers all cases, calling \cs{@@_true:w} or % \cs{@@_false:w} as appropriate. Here, there is no \meta{search % token}, so we feed a dummy \cs{scan_stop:} to the % \cs{@@_token_generic:NNTF} function. % \begin{macrocode} \group_begin: \cs_set_protected:Npn \@@_tmp:w #1 \s_@@_stop { \cs_new_protected:Npn \@@_execute_branches_N_type: { \if_int_odd:w \if_catcode:w \exp_not:N \l_peek_token { 0 \exp_stop_f: \fi: \if_catcode:w \exp_not:N \l_peek_token } 0 \exp_stop_f: \fi: \if_meaning:w \l_peek_token \c_space_token 0 \exp_stop_f: \fi: 1 \exp_stop_f: \exp_after:wN \@@_N_type:w \token_to_meaning:N \l_peek_token \s_@@_mark \@@_N_type_aux:nnw #1 \s_@@_mark \@@_use_none_delimit_by_s_stop:w \s_@@_stop \exp_after:wN \@@_true:w \else: \exp_after:wN \@@_false:w \fi: } \cs_new_protected:Npn \@@_N_type:w ##1 #1 ##2 \s_@@_mark ##3 { ##3 {##1} {##2} } } \exp_after:wN \@@_tmp:w \tl_to_str:n { outer } \s_@@_stop \group_end: \cs_new_protected:Npn \@@_N_type_aux:nnw #1 #2 #3 \fi: { \fi: \tl_if_in:noTF {#1} { \tl_to_str:n {ma} } { \@@_true:w } { \tl_if_empty:nTF {#2} { \@@_true:w } { \@@_false:w } } } \cs_new_protected:Npn \peek_N_type:TF { \@@_token_generic:NNTF \@@_execute_branches_N_type: \scan_stop: } \cs_new_protected:Npn \peek_N_type:T { \@@_token_generic:NNT \@@_execute_branches_N_type: \scan_stop: } \cs_new_protected:Npn \peek_N_type:F { \@@_token_generic:NNF \@@_execute_branches_N_type: \scan_stop: } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macrocode} % % \end{macrocode} % % \end{implementation} % % \PrintIndex