% \iffalse %% File: l3token.dtx Copyright (C) 2005-2006 LaTeX3 project %% %% It may be distributed and/or modified under the conditions of the %% LaTeX Project Public License (LPPL), either version 1.3c of this %% license or (at your option) any later version. The latest version %% of this license is in the file %% %% http://www.latex-project.org/lppl.txt %% %% This file is part of the ``expl3 bundle'' (The Work in LPPL) %% and all files in that bundle must be distributed together. %% %% The released version of this bundle is available from CTAN. %% %% ----------------------------------------------------------------------- %% %% The development version of the bundle can be found at %% %% http://www.latex-project.org/cgi-bin/cvsweb.cgi/ %% %% for those people who are interested. %% %%%%%%%%%%% %% NOTE: %% %%%%%%%%%%% %% %% Snapshots taken from the repository represent work in progress and may %% not work or may contain conflicting material! We therefore ask %% people _not_ to put them into distributions, archives, etc. without %% prior consultation with the LaTeX Project Team. %% %% ----------------------------------------------------------------------- %<*driver|package> \RequirePackage{l3names} % %\fi \GetIdInfo$Id: l3token.dtx 701 2007-11-27 14:21:05Z morten $ {L3 Experimental token investigation and manipulation} %\iffalse %<*driver> %\fi \ProvidesFile{\filename.\filenameext} [\filedate\space v\fileversion\space\filedescription] %\iffalse \documentclass{l3doc} \begin{document} \DocInput{\filename.\filenameext} \end{document} % % \fi % % \title{The \textsf{l3token} package\thanks{This file % has version number \fileversion, last % revised \filedate.}\\ % Basic Definitions} % \author{\Team} % \date{\filedate} % \maketitle % % % \section{A token of my appreciation\dots} % % This module deals with tokens. Now this is perhaps not the most % precise description so let's try with a better description: When % programming in \TeX, it is often desirable to know just what a % certain token is: is it a control sequence or something % else. Similarly one often needs to know if a control sequence is % expandable or not, a macro or a primitive, how many arguments it % takes etc. Another thing of great importance (especially when it % comes to document commands) is looking ahead in the token stream to % see if a certain character is present and maybe even remove it or % disregard other tokens while scanning. This module provides % functions for both and as such will have two primary function % categories: |\token| for anything that deals with tokens and % |\peek| for looking ahead in the token stream. % % Most of the time we will be using the term `token' but most of the % time the function we're describing can equally well by used on a % control sequence as such one is one token as well. % % % We shall refer to list of tokens as |tlist|s and such lists % represented py a single control sequence is a `token list pointer' % |tlp|. Functions for these two types are found in the \textsf{l3tlp} % module. % % \subsection{Character tokens} % % Setting category codes of characters. % % % \begin{function}{ % \char_set_catcode:nn | % \char_set_catcode:w | % \char_value_catcode:n | % \char_value_catcode:w | % \char_show_value_catcode:n | % \char_show_value_catcode:w % } % \begin{syntax} % "\char_set_catcode:nn" "{""}" "{""}"\\ % "\char_set_catcode:w" = \\ % "\char_value_catcode:n" "{""}" \\ % "\char_show_value_catcode:n" "{""}" % \end{syntax} % "\char_set_catcode:nn" sets the category code of a character, % "\char_value_catcode:n" returns its value for use in integer tests % and "\char_show_value_catcode:n" prints the value on the terminal % and in the log file. The ":w" should be avoided unless you have to % fiddle with the catcode of "{" or "}". % \begin{texnote} % "\char_set_catcode:w" is the \TeX\ primitive \tn{catcode} renamed. % \end{texnote} % \end{function} % % \begin{function}{ % \char_set_lccode:nn | % \char_set_lccode:w | % \char_value_lccode:n | % \char_value_lccode:w | % \char_show_value_lccode:n | % \char_show_value_lccode:w % } % \begin{syntax} % "\char_set_lccode:nn" "{""}" "{""}" \\ % "\char_set_lccode:w" = \\ % "\char_value_lccode:n" "{""}"\\ % "\char_show_value_lccode:n" "{""}" % \end{syntax} % Set the lower caser representation of for when is % being converted in "\tlist_to_lowercase:n". As above, the % ":w" form is only for people who really, really know what they are % doing. % \begin{texnote} % "\char_set_lccode:w" is the \TeX\ primitive \tn{lccode} renamed. % \end{texnote} % \end{function} % % \begin{function}{ % \char_set_uccode:nn | % \char_set_uccode:w | % \char_value_uccode:n | % \char_value_uccode:w | % \char_show_value_uccode:n | % \char_show_value_uccode:w % } % \begin{syntax} % "\char_set_uccode:nn" "{""}" "{""}"\\ % "\char_set_uccode:w" = \\ % "\char_value_uccode:n" "{""}"\\ % "\char_show_value_uccode:n" "{""}" % \end{syntax} % Set the uppercase representation of for when is being % converted in "\tlist_to_uppercase:n". As above, the ":w" % form is only for people who really, really know what they are % doing. % \begin{texnote} % "\char_set_uccode:w" is the \TeX\ primitive \tn{uccode} renamed. % \end{texnote} % \end{function} % % \begin{function}{ % \char_set_sfcode:nn | % \char_set_sfcode:w | % \char_value_sfcode:n | % \char_value_sfcode:w | % \char_show_value_sfcode:n | % \char_show_value_sfcode:w % } % \begin{syntax} % "\char_set_sfcode:nn" "{""}" "{""}"\\ % "\char_set_sfcode:w" = \\ % "\char_value_sfcode:n" "{""}"\\ % "\char_show_value_sfcode:n" "{""}" % \end{syntax} % Set the space factor for . % \begin{texnote} % "\char_set_sfcode:w" is the \TeX\ primitive \tn{sfcode} renamed. % \end{texnote} % \end{function} % % \begin{function}{ % \char_set_mathcode:nn | % \char_set_mathcode:w | % \char_gset_mathcode:nn | % \char_gset_mathcode:w | % \char_value_mathcode:n | % \char_value_mathcode:w | % \char_show_value_mathcode:n | % \char_show_value_mathcode:w % } % \begin{syntax} % "\char_set_mathcode:nn" "{""}" "{""}"\\ % "\char_set_mathcode:w" = \\ % "\char_value_mathcode:n" "{""}"\\ % "\char_show_value_mathcode:n" "{""}" % \end{syntax} % Set the math code for . % \begin{texnote} % "\char_set_mathcode:w" is the \TeX\ primitive \tn{mathcode} renamed. % \end{texnote} % \end{function} % % % \subsection{Generic tokens} % % \begin{function}{% % \token_new:Nn | % } % \begin{syntax} % "\token_new:Nn" "{""}" % \end{syntax} % Defines to globally be a snapshot of . This will % be an implicit representation of . % \end{function} % % \begin{variable}{% % \c_group_begin_token | % \c_group_end_token | % \c_math_shift_token | % \c_alignment_tab_token | % \c_parameter_token | % \c_math_superscript_token | % \c_math_subscript_token | % \c_space_token | % \c_letter_token | % \c_other_char_token | % \c_active_char_token | % } % \begin{syntax} % \end{syntax} % Some useful constants. They have category codes~1, 2, 3, 4, 6, 7, % 8, 10, 11, 12, and 13 respectively. They are all implicit tokens. % \end{variable} % % \begin{function}{ % \token_if_group_begin_p:N | % \token_if_group_begin:NTF | % \token_if_group_begin:NT | % \token_if_group_begin:NF | % } % \begin{syntax} % "\token_if_group_begin:NTF" "{""}" "{""}" % \end{syntax} % Check if is a begin group token. % \end{function} % % \begin{function}{ % \token_if_group_end_p:N | % \token_if_group_end:NTF | % \token_if_group_end:NT | % \token_if_group_end:NF | % } % \begin{syntax} % "\token_if_group_end:NTF" "{""}" "{""}" % \end{syntax} % Check if is an end group token. % \end{function} % % \begin{function}{ % \token_if_math_shift_p:N | % \token_if_math_shift:NTF | % \token_if_math_shift:NT | % \token_if_math_shift:NF | % } % \begin{syntax} % "\token_if_math_shift:NTF" "{""}" "{""}" % \end{syntax} % Check if is a math shift token. % \end{function} % % % \begin{function}{ % \token_if_aligment_tab_p:N | % \token_if_aligment_tab:NTF | % \token_if_aligment_tab:NT | % \token_if_aligment_tab:NF | % } % \begin{syntax} % "\token_if_aligment_tab:NTF" "{""}" "{""}" % \end{syntax} % Check if is an aligment tab token. % \end{function} % % \begin{function}{ % \token_if_parameter_p:N | % \token_if_parameter:NTF | % \token_if_parameter:NT | % \token_if_parameter:NF | % } % \begin{syntax} % "\token_if_parameter:NTF" "{""}" "{""}" % \end{syntax} % Check if is a parameter token. % \end{function} % % \begin{function}{ % \token_if_math_superscript_p:N | % \token_if_math_superscript:NTF | % \token_if_math_superscript:NT | % \token_if_math_superscript:NF | % } % \begin{syntax} % "\token_if_math_superscript:NTF" "{""}" "{""}" % \end{syntax} % Check if is a math superscript token. % \end{function} % % \begin{function}{ % \token_if_math_subscript_p:N | % \token_if_math_subscript:NTF | % \token_if_math_subscript:NT | % \token_if_math_subscript:NF | % } % \begin{syntax} % "\token_if_math_subscript:NTF" "{""}" "{""}" % \end{syntax} % Check if is a math subscript token. % \end{function} % % \begin{function}{ % \token_if_space_p:N | % \token_if_space:NTF | % \token_if_space:NT | % \token_if_space:NF | % } % \begin{syntax} % "\token_if_space:NTF" "{""}" "{""}" % \end{syntax} % Check if is a space token. % \end{function} % % % \begin{function}{ % \token_if_letter_p:N | % \token_if_letter:NTF | % \token_if_letter:NT | % \token_if_letter:NF | % } % \begin{syntax} % "\token_if_letter:NTF" "{""}" "{""}" % \end{syntax} % Check if is a letter token. % \end{function} % % % \begin{function}{ % \token_if_other_char_p:N | % \token_if_other_char:NTF | % \token_if_other_char:NT | % \token_if_other_char:NF | % } % \begin{syntax} % "\token_if_other_char:NTF" "{""}" "{""}" % \end{syntax} % Check if is an other char token. % \end{function} % % \begin{function}{ % \token_if_active_char_p:N | % \token_if_active_char:NTF | % \token_if_active_char:NT | % \token_if_active_char:NF | % } % \begin{syntax} % "\token_if_active_char:NTF" "{""}" "{""}" % \end{syntax} % Check if is an active char token. % \end{function} % % % \begin{function}{% % \token_if_eq_meaning_p:NN | % \token_if_eq_meaning:NNTF | % \token_if_eq_meaning:NNT | % \token_if_eq_meaning:NNF % } % \begin{syntax} % "\token_if_eq_meaning:NNTF" "{""}""{""}" % \end{syntax} % Check if the meaning of two tokens are identical. % \end{function} % % \begin{function}{% % \token_if_eq_catcode_p:NN | % \token_if_eq_catcode:NNTF | % \token_if_eq_catcode:NNT | % \token_if_eq_catcode:NNF % } % \begin{syntax} % "\token_if_eq_catcode:NNTF" "{""}""{""}" % \end{syntax} % Check if the category codes of two tokens are equal. If both tokens % are control sequences the test will be true. % \end{function} % % \begin{function}{% % \token_if_eq_charcode_p:NN | % \token_if_eq_catcode:NNTF | % \token_if_eq_catcode:NNT | % \token_if_eq_catcode:NNF % } % \begin{syntax} % "\token_if_eq_catcode:NNTF" "{""}""{""}" % \end{syntax} % Check if the character codes of two tokens are equal. If both tokens % are control sequences the test will be true. % \end{function} % % \begin{function}{ % \token_if_macro_p:N | % \token_if_macro:NTF | % \token_if_macro:NT | % \token_if_macro:NF | % } % \begin{syntax} % "\token_if_macro:NTF" "{""}" "{""}" % \end{syntax} % Check if is a macro. % \end{function} % % % \begin{function}{ % \token_if_cs_p:N | % \token_if_cs:NTF | % \token_if_cs:NT | % \token_if_cs:NF | % } % \begin{syntax} % "\token_if_cs:NTF" "{""}" "{""}" % \end{syntax} % Check if is a control sequence or not. This can be useful % for situations where the next token in the input stream is being % looked at and you want to determine what should be done to it. % \end{function} % % % % % \begin{function}{ % \token_if_expandable_p:N | % \token_if_expandable:NTF | % \token_if_expandable:NT | % \token_if_expandable:NF % } % \begin{syntax} % "\token_if_expandable:NTF" "{""}" "{""}" % \end{syntax} % Check if is expandable or not. Note that can very % well be an active character. % \end{function} % % The next set of functions here are for picking apart control % sequences. Sometimes it is useful to know if a control sequence has % arguments and if so, how many. Similarly its status with respect to % "\long" or "\protected" is good to have. Finally it can be very % useful to know if a control sequence is of a certain type: Is this % \m{toks} register we're trying to to something with really a % \m{toks} register at all? % % \begin{function}{ % \token_if_long_macro_p:N | % \token_if_long_macro:NTF | % \token_if_long_macro:NT | % \token_if_long_macro:NF | % } % \begin{syntax} % "\token_if_long_macro:NTF" "{""}" "{""}" % \end{syntax} % Check if is a ``long'' macro. % \end{function} % % % \begin{function}{ % \token_if_protected_macro_p:N | % \token_if_protected_macro:NTF | % \token_if_protected_macro:NT | % \token_if_protected_macro:NF | % } % \begin{syntax} % "\token_if_long_macro:NTF" "{""}" "{""}" % \end{syntax} % Check if is a ``protected'' macro. This test does \emph{not} % return if the macro is also ``long'', see below. % \end{function} % % \begin{function}{ % \token_if_protected_long_macro_p:N | % \token_if_protected_long_macro:NTF | % \token_if_protected_long_macro:NT | % \token_if_protected_long_macro:NF | % } % \begin{syntax} % "\token_if_protected_long_macro:NTF" "{""}" "{""}" % \end{syntax} % Check if is a ``protected long'' macro. % \end{function} % % \begin{function}{ % \token_if_chardef_p:N | % \token_if_chardef:NTF | % \token_if_chardef:NT | % \token_if_chardef:NF | % } % \begin{syntax} % "\token_if_chardef:NTF" "{""}" "{""}" % \end{syntax} % Check if is defined to be a chardef. % \end{function} % % \begin{function}{ % \token_if_mathchardef_p:N | % \token_if_mathchardef:NTF | % \token_if_mathchardef:NT | % \token_if_mathchardef:NF | % } % \begin{syntax} % "\token_if_mathchardef:NTF" "{""}" "{""}" % \end{syntax} % Check if is defined to be a mathchardef. % \end{function} % % \begin{function}{ % \token_if_int_register_p:N | % \token_if_int_register:NTF | % \token_if_int_register:NT | % \token_if_int_register:NF | % } % \begin{syntax} % "\token_if_int_register:NTF" "{""}" "{""}" % \end{syntax} % Check if is defined to be an integer register. % \end{function} % % \begin{function}{ % \token_if_dim_register_p:N | % \token_if_dim_register:NTF | % \token_if_dim_register:NT | % \token_if_dim_register:NF | % } % \begin{syntax} % "\token_if_dim_register:NTF" "{""}" "{""}" % \end{syntax} % Check if is defined to be a dimension register. % \end{function} % % \begin{function}{ % \token_if_skip_register_p:N | % \token_if_skip_register:NTF | % \token_if_skip_register:NT | % \token_ifskip_register:NF | % } % \begin{syntax} % "\token_if_skip_register:NTF" "{""}" "{""}" % \end{syntax} % Check if is defined to be a skip register. % \end{function} % % \begin{function}{ % \token_if_toks_register_p:N | % \token_if_toks_register:NTF | % \token_if_toks_register:NT | % \token_if_toks_register:NF | % } % \begin{syntax} % "\token_if_toks_register:NTF" "{""}" "{""}" % \end{syntax} % Check if is defined to be a toks register. % \end{function} % % % \begin{function}{ % \token_get_prefix_spec:N | % \token_get_arg_spec:N | % \token_get_replacement_spec:N % } % \begin{syntax} % "\token_get_arg_spec:N" % \end{syntax} % If token is a macro with definition |\def_long:Npn||\next| % |#1#2{x`#1--#2'y}|, the |prefix| function will return the string % |\long|, the |arg| function returns the string |#1#2| and the % |replacement| function returns the string |x`#1--#2'y|. If % isn't a macro, these functions return the |\scan_stop:| token. % \end{function} % % \subsubsection{Useless code: because we can!} % % \begin{function}{ % \token_if_primitive_p:N | % \token_if_primitive:NTF | % \token_if_primitive:NT | % \token_if_primitive:NF | % } % \begin{syntax} % "\token_if_primitive:NTF" "{""}" "{""}" % \end{syntax} % Check if is a primitive. Probably not a very useful function. % \end{function} % % % \subsection{Peeking ahead at the next token} % % % % % % \begin{variable}{% % \l_peek_token | % \g_peek_token | % \l_peek_search_token | % } % \begin{syntax} % \end{syntax} % Some useful variables. Initially they are set to "?". % \end{variable} % % % \begin{function}{% % \peek_after:NN | % \peek_gafter:NN | % } % \begin{syntax} % "\peek_after:NN" % \end{syntax} % Assign to "\l_peek_token" and then run which % should perform some sort of test on this token. Leaves in % the input stream. "\peek_gafter:NN" does this globally to the token % "\g_peek_token". % \begin{texnote} % This is the primitive \tn{futurelet} turned into a function. % \end{texnote} % \end{function} % % \begin{function}{% % \peek_meaning:NTF | % \peek_meaning_ignore_spaces:NTF | % \peek_meaning_remove:NTF | % \peek_meaning_remove_ignore_spaces:NTF % } % \begin{syntax} % "\peek_meaning:NTF" "{""}""{""}" % \end{syntax} % "\peek_meaning:NTF" checks (by using "\if_meaning:NN") if % equals the next token in the input stream and executes either or accordingly. "\peek_meaning_remove:NTF" does % the same but additionally removes the token if found. The % "ignore_spaces" versions skips blank spaces before making the % decision. % \end{function} % % \begin{function}{% % \peek_charcode:NTF | % \peek_charcode_ignore_spaces:NTF | % \peek_charcode_remove:NTF | % \peek_charcode_remove_ignore_spaces:NTF % } % \begin{syntax} % "\peek_charcode:NTF" "{""}""{""}" % \end{syntax} % Same as for the "\peek_meaning:NTF" functions above but these use % "\if_charcode:w" to compare the tokens. % \end{function} % % \begin{function}{% % \peek_catcode:NTF | % \peek_catcode_ignore_spaces:NTF | % \peek_catcode_remove:NTF | % \peek_catcode_remove_ignore_spaces:NTF % } % \begin{syntax} % "\peek_catcode:NTF" "{""}""{""}" % \end{syntax} % Same as for the "\peek_meaning:NTF" functions above but these use % "\if_catcode:w" to compare the tokens. % \end{function} % % % \begin{function}{% % \peek_token_generic:NNTF | % \peek_token_remove_generic:NNTF % } % \begin{syntax} % "\peek_token_generic:NNTF" "{""}""{""}" % \end{syntax} % "\peek_token_generic:NNTF" looks ahead and checks if the next token % in the input stream is equal to . It uses to make % that decision. "\peek_token_remove_generic:NNTF" does the same % thing but additionally removes from the input stream if it % is found. This also works if is either % "\c_group_begin_token" or "\c_group_end_token". % \end{function} % % \begin{function}{% % \peek_execute_branches_meaning: | % \peek_execute_branches_charcode: | % \peek_execute_branches_catcode: | % } % \begin{syntax} % "\peek_execute_branches_meaning:" % \end{syntax} % These functions compare the token we are searching for with the % token found (after optional ignoring of specific tokens). They come % in the usual three versions when \TeX{} is comparing tokens: % meaning, character code, and category code. % \end{function} % % % % % \subsubsection{Internal functions} % % \begin{variable}{% % \l_peek_true_tlp | % \l_peek_false_tlp | % } % \begin{syntax} % \end{syntax} % These token list pointers are used internally when choosing either % the true or false branches of a test. % \end{variable} % % \begin{function}{% % \peek_tmp:w | % } % \begin{syntax} % \end{syntax} % Scratch function used to gobble tokens from the input stream. % \end{function} % % \begin{variable}{% % \l_peek_true_aux_tlp | % \l_peek_true_remove_next_tlp | % } % \begin{syntax} % \end{syntax} % These token list pointers are used internally when choosing either % the true or false branches of a test. % \end{variable} % % \begin{function}{% % \peek_ignore_spaces_execute_branches: | % \peek_ignore_spaces_aux: | % } % \begin{syntax} % \end{syntax} % Functions used to ignore space tokens in the input stream. % \end{function} % % % % \StopEventually{} % % % \subsection{Implementation} % First a few required packages to get this going. % \begin{macrocode} %\ProvidesExplPackage % {\filename}{\filedate}{\fileversion}{\filedescription} %\RequirePackage{l3prg} %\RequirePackage{l3int} %<*initex|package> % \end{macrocode} % % \subsubsection{Character tokens} % % \begin{macro}{\char_set_catcode:w} % \begin{macro}{\char_set_catcode:nn} % \begin{macro}{\char_value_catcode:w} % \begin{macro}{\char_value_catcode:n} % \begin{macro}{\char_show_value_catcode:w} % \begin{macro}{\char_show_value_catcode:n} % \begin{macrocode} \let_new:NN \char_set_catcode:w \tex_catcode:D \def_new:Npn \char_set_catcode:nn #1#2{ \char_set_catcode:w #1 = \int_eval:n{#2} } \def_new:Npn \char_value_catcode:w {\int_use:N\tex_catcode:D} \def_new:Npn \char_value_catcode:n #1{\char_value_catcode:w \int_eval:n{#1}} \def_new:Npn \char_show_value_catcode:w {\tex_showthe:D\tex_catcode:D} \def_new:Npn \char_show_value_catcode:n #1{ \char_show_value_catcode:w \int_eval:n{#1}} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\char_set_mathcode:w} % \begin{macro}{\char_set_mathcode:nn} % \begin{macro}{\char_gset_mathcode:w} % \begin{macro}{\char_gset_mathcode:nn} % \begin{macro}{\char_value_mathcode:w} % \begin{macro}{\char_value_mathcode:n} % \begin{macro}{\char_show_value_mathcode:w} % \begin{macro}{\char_show_value_mathcode:n} % Math codes. % \begin{macrocode} \let_new:NN \char_set_mathcode:w \tex_mathcode:D \def_new:Npn \char_set_mathcode:nn #1#2{ \char_set_mathcode:w #1 = \int_eval:n{#2} } \def_protected_new:Npn \char_gset_mathcode:w {\pref_global:D\tex_mathcode:D} \def_new:Npn \char_gset_mathcode:nn #1#2{ \char_gset_mathcode:w #1 = \int_eval:n{#2} } \def_new:Npn \char_value_mathcode:w {\int_use:N\tex_mathcode:D} \def_new:Npn \char_value_mathcode:n #1{\char_value_mathcode:w \int_eval:n{#1}} \def_new:Npn \char_show_value_mathcode:w {\tex_showthe:D\tex_mathcode:D} \def_new:Npn \char_show_value_mathcode:n #1{ \char_show_value_mathcode:w \int_eval:n{#1}} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\char_set_lccode:w} % \begin{macro}{\char_set_lccode:nn} % \begin{macro}{\char_value_lccode:w} % \begin{macro}{\char_value_lccode:n} % \begin{macro}{\char_show_value_lccode:w} % \begin{macro}{\char_show_value_lccode:n} % \begin{macrocode} \let_new:NN \char_set_lccode:w \tex_lccode:D \def_new:Npn \char_set_lccode:nn #1#2{ \char_set_lccode:w #1 = \int_eval:n{#2} } \def_new:Npn \char_value_lccode:w {\int_use:N\tex_lccode:D} \def_new:Npn \char_value_lccode:n #1{\char_value_lccode:w \int_eval:n{#1}} \def_new:Npn \char_show_value_lccode:w {\tex_showthe:D\tex_lccode:D} \def_new:Npn \char_show_value_lccode:n #1{ \char_show_value_lccode:w \int_eval:n{#1}} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\char_set_uccode:w} % \begin{macro}{\char_set_uccode:nn} % \begin{macro}{\char_value_uccode:w} % \begin{macro}{\char_value_uccode:n} % \begin{macro}{\char_show_value_uccode:w} % \begin{macro}{\char_show_value_uccode:n} % \begin{macrocode} \let_new:NN \char_set_uccode:w \tex_uccode:D \def_new:Npn \char_set_uccode:nn #1#2{ \char_set_uccode:w #1 = \int_eval:n{#2} } \def_new:Npn \char_value_uccode:w {\int_use:N\tex_uccode:D} \def_new:Npn \char_value_uccode:n #1{\char_value_uccode:w \int_eval:n{#1}} \def_new:Npn \char_show_value_uccode:w {\tex_showthe:D\tex_uccode:D} \def_new:Npn \char_show_value_uccode:n #1{ \char_show_value_uccode:w \int_eval:n{#1}} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\char_set_sfcode:w} % \begin{macro}{\char_set_sfcode:nn} % \begin{macro}{\char_value_sfcode:w} % \begin{macro}{\char_value_sfcode:n} % \begin{macro}{\char_show_value_sfcode:w} % \begin{macro}{\char_show_value_sfcode:n} % \begin{macrocode} \let_new:NN \char_set_sfcode:w \tex_sfcode:D \def_new:Npn \char_set_sfcode:nn #1#2{ \char_set_sfcode:w #1 = \int_eval:n{#2} } \def_new:Npn \char_value_sfcode:w {\int_use:N\tex_uccode:D} \def_new:Npn \char_value_sfcode:n #1{\char_value_sfcode:w \int_eval:n{#1}} \def_new:Npn \char_show_value_sfcode:w {\tex_showthe:D\tex_sfcode:D} \def_new:Npn \char_show_value_sfcode:n #1{ \char_show_value_sfcode:w \int_eval:n{#1}} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \subsubsection{Generic tokens} % % % \begin{macro}{\token_new:Nn} % Creates a new token. % \begin{macrocode} \def_new:Npn \token_new:Nn #1#2{\glet_new:NN #1#2} % \end{macrocode} % \end{macro} % \begin{macro}{\c_group_begin_token} % \begin{macro}{\c_group_end_token} % \begin{macro}{\c_math_shift_token} % \begin{macro}{\c_alignment_tab_token} % \begin{macro}{\c_parameter_token} % \begin{macro}{\c_math_superscript_token} % \begin{macro}{\c_math_subscript_token} % \begin{macro}{\c_space_token} % \begin{macro}{\c_letter_token} % \begin{macro}{\c_other_char_token} % \begin{macro}{\c_active_char_token} % We define these useful tokens. We have to do it by hand with the % brace tokens for obvious reasons. % \begin{macrocode} \let_new:NN \c_group_begin_token { \let_new:NN \c_group_end_token } \group_begin: \char_set_catcode:nn{`\*}{3} \token_new:Nn \c_math_shift_token {*} \char_set_catcode:nn{`\*}{4} \token_new:Nn \c_alignment_tab_token {*} \token_new:Nn \c_parameter_token {#} \token_new:Nn \c_math_superscript_token {^} \char_set_catcode:nn{`\*}{8} \token_new:Nn \c_math_subscript_token {*} \token_new:Nn \c_space_token {~} \token_new:Nn \c_letter_token {a} \token_new:Nn \c_other_char_token {1} \char_set_catcode:nn{`\*}{13} \token_new:Nn \c_active_char_token {*} \group_end: % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_group_begin_p:N} % \begin{macro}{\token_if_group_begin:NTF} % \begin{macro}{\token_if_group_begin:NT} % \begin{macro}{\token_if_group_begin:NF} % Check if token is a begin group token. We use the constant % "\c_group_begin_token" for this. % \begin{macrocode} \def_new:Npn \token_if_group_begin_p:N #1{ \if_catcode:w \exp_not:N #1\c_group_begin_token \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_group_begin:N} #1{ \if:w\token_if_group_begin_p:N #1} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_group_end_p:N} % \begin{macro}{\token_if_group_end:NTF} % \begin{macro}{\token_if_group_end:NT} % \begin{macro}{\token_if_group_end:NF} % Check if token is a end group token. We use the constant % "\c_group_end_token" for this. % \begin{macrocode} \def_new:Npn \token_if_group_end_p:N #1{ \if_catcode:w \exp_not:N #1\c_group_end_token \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_group_end:N} #1{ \if:w\token_if_group_end_p:N #1} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \begin{macro}{\token_if_math_shift_p:N} % \begin{macro}{\token_if_math_shift:NTF} % \begin{macro}{\token_if_math_shift:NT} % \begin{macro}{\token_if_math_shift:NF} % Check if token is a math shift token. We use the constant % "\c_math_shift_token" for this. % \begin{macrocode} \def_new:Npn \token_if_math_shift_p:N #1{ \if_catcode:w \exp_not:N #1\c_math_shift_token \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_math_shift:N} #1{ \if:w\token_if_math_shift_p:N#1} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \begin{macro}{\token_if_alignment_tab_p:N} % \begin{macro}{\token_if_alignment_tab:NTF} % \begin{macro}{\token_if_alignment_tab:NT} % \begin{macro}{\token_if_alignment_tab:NF} % Check if token is an alignment tab token. We use the constant % "\c_alignment_tab_token" for this. % \begin{macrocode} \def_new:Npn \token_if_alignment_tab_p:N #1{ \if_catcode:w \exp_not:N #1\c_alignment_tab_token \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_alignment_tab:N} #1{ \if:w\token_if_alignment_tab_p:N#1} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \begin{macro}{\token_if_parameter_p:N} % \begin{macro}{\token_if_parameter:NTF} % \begin{macro}{\token_if_parameter:NT} % \begin{macro}{\token_if_parameter:NF} % Check if token is a parameter token. We use the constant % "\c_parameter_token" for this. We have to trick \TeX{} a bit to % avoid an error message. % \begin{macrocode} \def_new:Npn \token_if_parameter_p:N #1{ \exp_after:NN\if_catcode:w \cs:w c_parameter_token\cs_end:\exp_not:N #1 \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_parameter:N} #1{ \if:w\token_if_parameter_p:N#1} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \begin{macro}{\token_if_math_superscript_p:N} % \begin{macro}{\token_if_math_superscript:NTF} % \begin{macro}{\token_if_math_superscript:NT} % \begin{macro}{\token_if_math_superscript:NF} % Check if token is a math superscript token. We use the constant % "\c_math_superscript_token" for this. % \begin{macrocode} \def_new:Npn \token_if_math_superscript_p:N #1{ \if_catcode:w \exp_not:N #1\c_math_superscript_token \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_math_superscript:N} #1{ \if:w\token_if_math_superscript_p:N #1} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \begin{macro}{\token_if_math_subscript_p:N} % \begin{macro}{\token_if_math_subscript:NTF} % \begin{macro}{\token_if_math_subscript:NT} % \begin{macro}{\token_if_math_subscript:NF} % Check if token is a math subscript token. We use the constant % "\c_math_subscript_token" for this. % \begin{macrocode} \def_new:Npn \token_if_math_subscript_p:N #1{ \if_catcode:w \exp_not:N #1\c_math_subscript_token \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_math_subscript:N} #1{ \if:w\token_if_math_subscript_p:N #1} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_space_p:N} % \begin{macro}{\token_if_space:NTF} % \begin{macro}{\token_if_space:NT} % \begin{macro}{\token_if_space:NF} % Check if token is a space token. We use the constant % "\c_space_token" for this. % \begin{macrocode} \def_new:Npn \token_if_space_p:N #1{ \if_catcode:w \exp_not:N #1\c_space_token \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_space:N} #1{ \if:w\token_if_space_p:N #1} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \begin{macro}{\token_if_letter_p:N} % \begin{macro}{\token_if_letter:NTF} % \begin{macro}{\token_if_letter:NT} % \begin{macro}{\token_if_letter:NF} % Check if token is a letter token. We use the constant % "\c_letter_token" for this. % \begin{macrocode} \def_new:Npn \token_if_letter_p:N #1{ \if_catcode:w \exp_not:N #1\c_letter_token \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_letter:N} #1{ \if:w\token_if_letter_p:N #1} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_other_char_p:N} % \begin{macro}{\token_if_other_char:NTF} % \begin{macro}{\token_if_other_char:NT} % \begin{macro}{\token_if_other_char:NF} % Check if token is an other char token. We use the constant % "\c_other_char_token" for this. % \begin{macrocode} \def_new:Npn \token_if_other_char_p:N #1{ \if_catcode:w \exp_not:N #1\c_other_char_token \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_other_char:N} #1{ \if:w\token_if_other_char_p:N #1} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_active_char_p:N} % \begin{macro}{\token_if_active_char:NTF} % \begin{macro}{\token_if_active_char:NT} % \begin{macro}{\token_if_active_char:NF} % Check if token is an active char token. We use the constant % "\c_active_char_token" for this. % \begin{macrocode} \def_new:Npn \token_if_active_char_p:N #1{ \if_catcode:w \exp_not:N #1\c_active_char_token \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_active_char:N} #1{ \if:w\token_if_active_char_p:N #1} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_eq_meaning_p:NN} % \begin{macro}{\token_if_eq_meaning:NNTF} % \begin{macro}{\token_if_eq_meaning:NNT} % \begin{macro}{\token_if_eq_meaning:NNF} % Check if the tokens |#1| and |#2| have same meaning. % \begin{macrocode} \def_new:Npn \token_if_eq_meaning_p:NN #1#2 { \if_meaning:NN #1 #2 \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_eq_meaning:NN}#1#2{ \if_meaning:NN #1 #2} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_eq_catcode_p:NN} % \begin{macro}{\token_if_eq_catcode:NNTF} % \begin{macro}{\token_if_eq_catcode:NNT} % \begin{macro}{\token_if_eq_catcode:NNF} % Check if the tokens |#1| and |#2| have same category code. % \begin{macrocode} \def_new:Npn \token_if_eq_catcode_p:NN #1#2 { \if_catcode:w \exp_not:N #1 \exp_not:N #2 \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_eq_catcode:NN}#1#2{ \if:w\token_if_eq_catcode_p:NN#1#2} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % % % \begin{macro}{\token_if_eq_charcode_p:NN} % \begin{macro}{\token_if_eq_charcode:NNTF} % \begin{macro}{\token_if_eq_charcode:NNT} % \begin{macro}{\token_if_eq_charcode:NNF} % Check if the tokens |#1| and |#2| have same character code. % \begin{macrocode} \def_new:Npn \token_if_charcode_eq_p:NN #1#2 { \if_charcode:w \exp_not:N #1 \exp_not:N #2 \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_eq_charcode:NN}#1#2{ \if:w\token_if_eq_charcode_p:NN#1#2} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_macro_p:N} % \begin{macro}{\token_if_macro_p_aux:w} % \begin{macro}{\token_if_macro:NTF} % \begin{macro}{\token_if_macro:NT} % \begin{macro}{\token_if_macro:NF} % When a token is a macro, "\token_to_meaning:N" will always output % something like "\long macro:#1->#1" so we simply check to see if % the meaning contains "->". Argument "#2" in the code below will be % empty if the string "->" isn't present, proof that the token was % not a macro (which is why we reverse the emptiness test). However % this function will fail on its own auxiliary function (and a few % other private functions as well) but that should certainly never % be a problem! % \begin{macrocode} \def_new:Npn \token_if_macro_p:N #1 { \exp_after:NN \token_if_macro_p_aux:w \token_to_meaning:N #1 -> \q_nil } \def_new:Npn \token_if_macro_p_aux:w #1 -> #2 \q_nil{ \if:w \tlist_if_empty_p:n{#2} \c_false \else: \c_true \fi: } \def_test_function_new:npn {token_if_macro:N} #1{\if:w\token_if_macro_p:N#1} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_cs_p:N} % \begin{macro}{\token_if_cs:NTF} % \begin{macro}{\token_if_cs:NT} % \begin{macro}{\token_if_cs:NF} % Check if token has same catcode as a control sequence. We use % "\scan_stop:" for this. % \begin{macrocode} \def_new:Npn \token_if_cs_p:N {\token_if_eq_catcode_p:NN \scan_stop:} \def_test_function_new:npn {token_if_cs:N} #1{ \if:w \token_if_eq_catcode_p:NN \scan_stop: #1} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % % \begin{macro}{\token_if_expandable_p:N} % \begin{macro}{\token_if_expandable:NTF} % \begin{macro}{\token_if_expandable:NT} % \begin{macro}{\token_if_expandable:NF} % Check if token is expandable. We use the fact that \TeX\ will % temporarily convert "\exp_not:N" \m{token} into "\scan_stop:" if % \m{token} is expandable. % \begin{macrocode} \def_new:Npn \token_if_expandable_p:N #1{ \exp_after:NN \if_token_eq:NN \exp_not:N #1 \scan_stop: \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_expandable:N} #1{ \if:w\token_if_expandable_p:N#1} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_chardef_p:N} % \begin{macro}{\token_if_chardef_p_aux:w} % \begin{macro}{\token_if_mathchardef_p:N} % \begin{macro}{\token_if_mathchardef_p_aux:w} % \begin{macro}{\token_if_int_register_p:N} % \begin{macro}{\token_if_int_register_p_aux:w} % \begin{macro}{\token_if_skip_register_p:N} % \begin{macro}{\token_if_skip_register_p_aux:w} % \begin{macro}{\token_if_dim_register_p:N} % \begin{macro}{\token_if_dim_register_p_aux:w} % \begin{macro}{\token_if_toks_register_p:N} % \begin{macro}{\token_if_toks_register_p_aux:w} % \begin{macro}{\token_if_protected_macro_p:N} % \begin{macro}{\token_if_protected_macro_p_aux:w} % \begin{macro}{\token_if_long_macro_p:N} % \begin{macro}{\token_if_long_macro_p_aux:w} % \begin{macro}{\token_if_protected_long_macro_p:N} % \begin{macro}{\token_if_protected_long_macro_p_aux:w} % Most of these functions have to check the meaning of the token in % question so we need to do some checkups on which characters are % output by |\token_to_meaning:N|. As usual, these characters have % catcode 12 so we must do some serious substitutions in the code % below\dots % \begin{macrocode} \group_begin: \char_set_lccode:nn {`\X}{`\n} \char_set_lccode:nn {`\Y}{`\t} \char_set_lccode:nn {`\Z}{`\d} \char_set_lccode:nn {`\?}{`\\} \tlist_map_inline:nn{\X\Y\Z\M\T\C\H\A\R\O\U\S\K\I\P\L\G\P\E} {\char_set_catcode:nn {`#1}{12}} % \end{macrocode} % We convert the token list to lowercase and restore the catcode and % lowercase code changes. % \begin{macrocode} \tlist_to_lowercase:n{ \group_end: % \end{macrocode} % First up is checking if something has been defined with % |\tex_chardef:D| or |\tex_mathchardef:D|. This is easy since \TeX\ % thinks of such tokens as hexadecimal so it stores them as % |\char"|\meta{hex~number} or |\mathchar"|\meta{hex~number}. % \begin{macrocode} \def_new:Npn \token_if_chardef_p:N #1 { \exp_after:NN \token_if_chardef_p_aux:w \token_to_meaning:N #1?CHAR"\q_nil } \def_new:Npn \token_if_chardef_p_aux:w #1?CHAR"#2\q_nil{ \tlist_if_empty_p:n{#1} } \def_new:Npn \token_if_mathchardef_p:N #1 { \exp_after:NN \token_if_mathchardef_p_aux:w \token_to_meaning:N #1?MAYHCHAR"\q_nil } \def_new:Npn \token_if_mathchardef_p_aux:w #1?MAYHCHAR"#2\q_nil{ \tlist_if_empty_p:n{#1} } % \end{macrocode} % Integer registers are a little more difficult since they expand to % |\count|\meta{number} and there is also a primitive |\countdef|. So % we have to check for that primitive as well. % \begin{macrocode} \def:Npn \token_if_int_register_p:N #1{ \if_meaning:NN \tex_countdef:D #1 \c_false \else: \exp_after:NN \token_if_int_register_p_aux:w \token_to_meaning:N #1?COUXY\q_nil \fi: } \def_new:Npn \token_if_int_register_p_aux:w #1?COUXY#2\q_nil{ \tlist_if_empty_p:n{#1} } % \end{macrocode} % Skip registers are done the same way as the integer registers. % \begin{macrocode} \def:Npn \token_if_skip_register_p:N #1{ \if_meaning:NN \tex_skipdef:D #1 \c_false \else: \exp_after:NN \token_if_skip_register_p_aux:w \token_to_meaning:N #1?SKIP\q_nil \fi: } \def_new:Npn \token_if_skip_register_p_aux:w #1?SKIP#2\q_nil{ \tlist_if_empty_p:n{#1} } % \end{macrocode} % Dim registers. No news here % \begin{macrocode} \def:Npn \token_if_dim_register_p:N #1{ \if_meaning:NN \tex_dimendef:D #1 \c_false \else: \exp_after:NN \token_if_dim_register_p_aux:w \token_to_meaning:N #1?ZIMEX\q_nil \fi: } \def_new:Npn \token_if_dim_register_p_aux:w #1?ZIMEX#2\q_nil{ \tlist_if_empty_p:n{#1} } % \end{macrocode} % Toks registers. Ho-hum. % \begin{macrocode} \def:Npn \token_if_toks_register_p:N #1{ \if_meaning:NN \tex_toksdef:D #1 \c_false \else: \exp_after:NN \token_if_toks_register_p_aux:w \token_to_meaning:N #1?YOKS\q_nil \fi: } \def_new:Npn \token_if_toks_register_p_aux:w #1?YOKS#2\q_nil{ \tlist_if_empty_p:n{#1} } % \end{macrocode} % Protected macros. % \begin{macrocode} \def_new:Npn \token_if_protected_macro_p:N #1 { \exp_after:NN \token_if_protected_macro_p_aux:w \token_to_meaning:N #1?PROYECYEZ~MACRO\q_nil } \def_new:Npn \token_if_protected_macro_p_aux:w #1?PROYECYEZ~MACRO#2\q_nil{ \tlist_if_empty_p:n{#1} } % \end{macrocode} % Long macros. % \begin{macrocode} \def_new:Npn \token_if_long_macro_p:N #1 { \exp_after:NN \token_if_long_macro_p_aux:w \token_to_meaning:N #1?LOXG~MACRO\q_nil } \def_new:Npn \token_if_long_macro_p_aux:w #1?LOXG~MACRO#2\q_nil{ \tlist_if_empty_p:n{#1} } % \end{macrocode} % Finally protected long macros where we for once don't have to add an % extra test since there is no primitive for the combined prefixes. % \begin{macrocode} \def_new:Npn \token_if_protected_long_macro_p:N #1 { \exp_after:NN \token_if_protected_long_macro_p_aux:w \token_to_meaning:N #1?PROYECYEZ~?LOXG~MACRO\q_nil } \def_new:Npn \token_if_protected_long_macro_p_aux:w #1 ?PROYECYEZ~?LOXG~MACRO#2\q_nil{ \tlist_if_empty_p:n{#1} } % \end{macrocode} % Finally the |\tlist_to_lowercase:n| ends! % \begin{macrocode} } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_chardef:NTF} % \begin{macro}{\token_if_chardef:NT} % \begin{macro}{\token_if_chardef:NF} % \begin{macrocode} \def_test_function_new:npn {token_if_chardef:N} {\if:w \token_if_chardef_p:N} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_mathchardef:NTF} % \begin{macro}{\token_if_mathchardef:NT} % \begin{macro}{\token_if_mathchardef:NF} % \begin{macrocode} \def_test_function_new:npn {token_if_mathchardef:N} { \if:w \token_if_mathchardef_p:N} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_long_macro:NTF} % \begin{macro}{\token_if_long_macro:NT} % \begin{macro}{\token_if_long_macro:NF} % \begin{macrocode} \def_test_function_new:npn {token_if_long_macro:N} { \if:w \token_if_long_macro_p:N} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_protected_macro:NTF} % \begin{macro}{\token_if_protected_macro:NT} % \begin{macro}{\token_if_protected_macro:NF} % \begin{macrocode} \def_test_function_new:npn {token_if_protected_macro:N} { \if:w \token_if_protected_macro_p:N} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_protected_long_macro:NTF} % \begin{macro}{\token_if_protected_long_macro:NT} % \begin{macro}{\token_if_protected_long_macro:NF} % \begin{macrocode} \def_test_function_new:npn {token_if_protected_long_macro:N} { \if:w \token_if_protected_long_macro_p:N} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_dim_register:NTF} % \begin{macro}{\token_if_dim_register:NT} % \begin{macro}{\token_if_dim_register:NF} % \begin{macrocode} \def_test_function_new:npn {token_if_dim_register:N} { \if:w \token_if_dim_register_p:N} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_skip_register:NTF} % \begin{macro}{\token_if_skip_register:NT} % \begin{macro}{\token_if_skip_register:NF} % \begin{macrocode} \def_test_function_new:npn {token_if_skip_register:N} { \if:w \token_if_skip_register_p:N} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_int_register:NTF} % \begin{macro}{\token_if_int_register:NT} % \begin{macro}{\token_if_int_register:NF} % \begin{macrocode} \def_test_function_new:npn {token_if_int_register:N} { \if:w \token_if_int_register_p:N} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_toks_register:NTF} % \begin{macro}{\token_if_toks_register:NT} % \begin{macro}{\token_if_toks_register:NF} % \begin{macrocode} \def_test_function_new:npn {token_if_toks_register:N} { \if:w \token_if_toks_register_p:N} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % % % % % We do not provide a function for testing if a control sequence is % ``outer'' since we don't use that in \LaTeX3. % % \begin{macro}{\token_get_prefix_arg_replacement_aux:w} % \begin{macro}{\token_get_prefix_spec:N} % \begin{macro}{\token_get_arg_spec:N} % \begin{macro}{\token_get_replacement_spec:N} % In the \textsf{xparse} package we sometimes want to test if a % control sequence can be expanded to reveal a hidden % value. However, we cannot just expand the macro blindly as it may % have arguments and none might be present. Therefore we define % these functions to pick either the prefix(es), the argument % specification, or the replacement text from a macro. All of this % information is returned as characters with catcode~12. If the % token in question isn't a macro, the token |\scan_stop:| is % returned instead. % \begin{macrocode} \group_begin: \char_set_lccode:nn {`\?}{`\:} \char_set_catcode:nn{`\M}{12} \char_set_catcode:nn{`\A}{12} \char_set_catcode:nn{`\C}{12} \char_set_catcode:nn{`\R}{12} \char_set_catcode:nn{`\O}{12} \tlist_to_lowercase:n{ \group_end: \def_new:Npn \token_get_prefix_arg_replacement_aux:w #1MACRO?#2->#3\q_nil#4{ #4{#1}{#2}{#3} } \def_new:Npn\token_get_prefix_spec:N #1{ \token_if_macro:NTF #1{ \exp_after:NN \token_get_prefix_arg_replacement_aux:w \token_to_meaning:N #1\q_nil\use_arg_i:nnn }{\scan_stop:} } \def_new:Npn\token_get_arg_spec:N #1{ \token_if_macro:NTF #1{ \exp_after:NN \token_get_prefix_arg_replacement_aux:w \token_to_meaning:N #1\q_nil\use_arg_ii:nnn }{\scan_stop:} } \def_new:Npn\token_get_replacement_spec:N #1{ \token_if_macro:NTF #1{ \exp_after:NN \token_get_prefix_arg_replacement_aux:w \token_to_meaning:N #1\q_nil\use_arg_iii:nnn }{\scan_stop:} } } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \paragraph{Useless code: because we can!} % % \begin{macro}{\token_if_primitive_p:N} % \begin{macro}{\token_if_primitive_p_aux:N} % \begin{macro}{\token_if_primitive:NTF} % \begin{macro}{\token_if_primitive:NT} % \begin{macro}{\token_if_primitive:NF} % It is rather hard to determine if a token is a primitive. First we % can check if it is a control sequence or active character. If % either, we check if it is a macro. Then we can go through a % tedious process of testing for different register types\dots{} I % don't actually think this function is useful but you never know. % \begin{macrocode} \def_new:Npn \token_if_primitive_p:N #1{ \if:w \token_if_cs_p:N #1\scan_stop: \if:w \token_if_macro_p:N #1 \c_false \else: \token_if_primitive_p_aux:N #1 \fi: \else: \if:w \token_if_active_p:N #1 \if:w \token_if_macro_p:N #1 \c_false \else: \token_if_primitive_p_aux:N #1 \fi: \else: \c_false \fi: \fi: } \def_new:Npn \token_if_primitive_p_aux:N #1{ \if:w \token_if_chardef_p:N #1 \c_false \else: \if:w \token_if_mathchardef_p:N #1 \c_false \else: \if:w \token_if_int_register_p:N #1 \c_false \else: \if:w \token_if_skip_register_p:N #1 \c_false \else: \if:w \token_if_dim_register_p:N #1 \c_false \else: \if:w \token_if_toks_register_p:N #1 \c_false \else: % \end{macrocode} % We made it! % \begin{macrocode} \c_true \fi: \fi: \fi: \fi: \fi: \fi: } \def_test_function_new:npn {token_if_primitive:N} #1{ \if:w\token_if_primitive_p:N#1} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % % % % \subsubsection{Peeking ahead at the next token} % % % % % % % \begin{macro}{\l_peek_token} % \begin{macro}{\g_peek_token} % \begin{macro}{\l_peek_search_token} % We define some other tokens which will initially be the character % |?|. % \begin{macrocode} \token_new:Nn \l_peek_token {?} \token_new:Nn \g_peek_token {?} \token_new:Nn \l_peek_search_token {?} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % % % \begin{macro}{\peek_after:NN} % \begin{macro}{\peek_gafter:NN} % |\peek_after:NN| takes two argument where the first is a function % acting on |\l_peek_token| and the second is the next token in the % input stream which |\l_peek_token| is set equal to. % |\peek_gafter:NN| does the same globally to |\g_peek_token|. % \begin{macrocode} \def_new:Npn \peek_after:NN {\tex_futurelet:D \l_peek_token } \def_new:Npn \peek_gafter:NN { \pref_global:D \tex_futurelet:D \g_peek_token } % \end{macrocode} % \end{macro} % \end{macro} % % For normal purposes there are four main cases: % \begin{enumerate} % \item peek at the next token. % \item peek at the next non-space token. % \item peek at the next token and remove it. % \item peek at the next non-space token and remove it. % \end{enumerate} % % % The generic functions will take four arguments: The token to search % for, the test function to run on it and the true/false cases. % The general algorithm is this: % \begin{enumerate} % \item % Store the token to search for in |\l_peek_search_token|. % \item % In order to avoid doubling of hash marks where it seems unnatural % we put the \meta{true} and \meta{false} cases through an |x| type % expansion but using |\exp_not:n| to avoid any expansion. This has % the same effect as putting it through a \meta{toks} register but % is faster. Also put in a special alignment safe group end. % \item % Put in an alignment safe group begin. % \item % Peek ahead and call the function which will act on the next token % in the input stream. % \end{enumerate} % % \begin{macro}{\l_peek_true_tlp} % \begin{macro}{\l_peek_false_tlp} % Two dedicated token list pointers that store the true and false % cases. % \begin{macrocode} \tlp_new:Nn \l_peek_true_tlp {} \tlp_new:Nn \l_peek_false_tlp {} % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}{\peek_tmp:w} % Scratch function used for storing the token to be removed if % found. % \begin{macrocode} \def_new:Npn \peek_tmp:w{} % \end{macrocode} % \end{macro} % % \begin{macro}{\l_peek_search_tlp} % We also use this token list pointer for storing the token we want % to compare. This turns out to be useful. % \begin{macrocode} \tlp_new:Nn \l_peek_search_tlp{} % \end{macrocode} % \end{macro} % % \begin{macro}{\peek_token_generic:NNTF} % |#1| is the function to execute (obey or ignore spaces, etc.), % |#2| is the special token we're looking for, and |#3| and |#4| % are the \meta{true} and \meta{false} branches. % \begin{macrocode} \def_long_new:Npn \peek_token_generic:NNTF #1#2#3#4{ \let:NN \l_peek_search_token #2 \tlp_set:Nn \l_peek_search_tlp {#2} \tlp_set:Nx \l_peek_true_tlp {\exp_not:n{\group_align_safe_end: #3}} \tlp_set:Nx \l_peek_false_tlp {\exp_not:n{\group_align_safe_end: #4}} \group_align_safe_begin: \peek_after:NN #1 } % \end{macrocode} % \end{macro} % \begin{macro}{\peek_token_remove_generic:NNTF} % If we want to be able to remove any character from the input % stream we might as well do it the same way for all characters so % we define this as little differently from above. % \begin{macrocode} \def_long_new:Npn \peek_token_remove_generic:NNTF #1#2#3#4{ \let:NN \l_peek_search_token #2 \tlp_set:Nn \l_peek_search_tlp {#2} \tlp_set:Nx \l_peek_true_aux_tlp { \exp_not:n{ #3 } } \tlp_set_eq:NN \l_peek_true_tlp \c_peek_true_remove_next_tlp \tlp_set:Nx \l_peek_false_tlp {\exp_not:n{\group_align_safe_end: #4}} \group_align_safe_begin: \peek_after:NN #1 } % \end{macrocode} % \end{macro} % \begin{macro}{\l_peek_true_aux_tlp} % \begin{macro}{\l_peek_true_remove_next_tlp} % Two token list pointers to help with removing the character from % the input stream. % \begin{macrocode} \tlp_new:Nn \l_peek_true_aux_tlp {} \tlp_new:Nn \c_peek_true_remove_next_tlp {\group_align_safe_end: \tex_afterassignment:D \l_peek_true_aux_tlp \let:NN \peek_tmp:w } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}{\peek_execute_braches_meaning:} % \begin{macro}{\peek_execute_braches_catcode:} % \begin{macro}{\peek_execute_braches_charcode:} % \begin{macro}{\peek_execute_branches_charcode_aux:NN} % There are three major tests between tokens in \TeX: meaning, % catcode and charcode. Hence we define three basic test functions % that set in after the ignoring phase is over and done with. % \begin{macrocode} \def_new:Npn \peek_execute_branches_meaning: { \if_meaning:NN \l_peek_token \l_peek_search_token \exp_after:NN \l_peek_true_tlp \else: \exp_after:NN \l_peek_false_tlp \fi: } \def_new:Npn \peek_execute_branches_catcode: { \if_catcode:w \exp_not:N\l_peek_token \exp_not:N\l_peek_search_token \exp_after:NN \l_peek_true_tlp \else: \exp_after:NN \l_peek_false_tlp \fi: } % \end{macrocode} % For the charcode version we do things a little differently. We want % to check the token directly but if we do this we face problems if % the next thing in the input stream is a braced group or a space % token. The braced group would be read as a complete argument and the % space would be gobbled by \TeX's argument reading routines. Hence we % test for both of these and if one of them is found we just execute % the false result directly since no one should ever try to use the % |charcode| function for searching for |\c_group_begin_token| or % |\c_space_token|. % \begin{macrocode} \def_new:Npn \peek_execute_branches_charcode: { \predicate:nTF { \token_if_eq_catcode_p:NN \l_peek_token \c_group_begin_token || \token_if_eq_meaning_p:NN \l_peek_token \c_space_token } { \l_peek_false_tlp } % \end{macrocode} % Otherwise we call a small auxiliary function that just grabs the % next token. We can do that because it really is a single token; we % just have insert it again afterwards. Also we stored the token we % were looking for in the token list pointer |\l_peek_search_tlp| so % we unpack it again for this function. % \begin{macrocode} { \exp_after:NN \peek_execute_branches_charcode_aux:NN \l_peek_search_tlp } } % \end{macrocode} % Then we just do the usual |\if_charcode:w| comparison. We also % remember to insert |#2| again after executing the true or false % branches. % \begin{macrocode} \def_long_new:Npn \peek_execute_branches_charcode_aux:NN #1#2{ \if_charcode:w \exp_not:N #1\exp_not:N#2 \exp_after:NN \l_peek_true_tlp \else: \exp_after:NN \l_peek_false_tlp \fi: #2 } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\peek_meaning:NTF} % \begin{macro}{\peek_meaning_ignore_spaces:NTF} % \begin{macro}{\peek_meaning_remove:NTF} % \begin{macro}{\peek_meaning_remove_ignore_spaces:NTF} % Here we use meaning comparison with |\if_meaning:NN|. % \begin{macrocode} \def_new:Npn \peek_meaning:NTF { \peek_token_generic:NNTF \peek_execute_branches_meaning: } \def_new:Npn \peek_meaning_ignore_spaces:NTF { \let:NN \peek_execute_branches: \peek_execute_branches_meaning: \peek_token_generic:NNTF \peek_ignore_spaces_execute_branches: } \def_new:Npn \peek_meaning_remove:NTF { \peek_token_remove_generic:NNTF \peek_execute_branches_meaning: } \def_new:Npn \peek_meaning_remove_ignore_spaces:NTF { \let:NN \peek_execute_branches: \peek_execute_branches_meaning: \peek_token_remove_generic:NNTF \peek_ignore_spaces_execute_branches: } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\peek_catcode:NTF} % \begin{macro}{\peek_catcode_ignore_spaces:NTF} % \begin{macro}{\peek_catcode_remove:NTF} % \begin{macro}{\peek_catcode_remove_ignore_spaces:NTF} % Here we use catcode comparison with |\if_catcode:w|. % \begin{macrocode} \def_new:Npn \peek_catcode:NTF { \peek_token_generic:NNTF \peek_execute_branches_catcode: } \def_new:Npn \peek_catcode_ignore_spaces:NTF { \let:NN \peek_execute_branches: \peek_execute_branches_catcode: \peek_token_generic:NNTF \peek_ignore_spaces_execute_branches: } \def_new:Npn \peek_catcode_remove:NTF { \peek_token_remove_generic:NNTF \peek_execute_branches_catcode: } \def_new:Npn \peek_catcode_remove_ignore_spaces:NTF { \let:NN \peek_execute_branches: \peek_execute_branches_catcode: \peek_token_remove_generic:NNTF \peek_ignore_spaces_execute_branches: } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\peek_charcode:NTF} % \begin{macro}{\peek_charcode_ignore_spaces:NTF} % \begin{macro}{\peek_charcode_remove:NTF} % \begin{macro}{\peek_charcode_remove_ignore_spaces:NTF} % Here we use charcode comparison with |\if_charcode:w|. % \begin{macrocode} \def_new:Npn \peek_charcode:NTF { \peek_token_generic:NNTF \peek_execute_branches_charcode: } \def_new:Npn \peek_charcode_ignore_spaces:NTF { \let:NN \peek_execute_branches: \peek_execute_branches_charcode: \peek_token_generic:NNTF \peek_ignore_spaces_execute_branches: } \def_new:Npn \peek_charcode_remove:NTF { \peek_token_remove_generic:NNTF \peek_execute_branches_charcode: } \def_new:Npn \peek_charcode_remove_ignore_spaces:NTF { \let:NN \peek_execute_branches: \peek_execute_branches_charcode: \peek_token_remove_generic:NNTF \peek_ignore_spaces_execute_branches: } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\peek_ignore_spaces_aux:} % \begin{macro}{\peek_ignore_spaces_execute_branches:} % Throw away a space token and search again. We could define this in % a more devious way where the auxiliary function gobbles the space % token but then what do we do if we decide that a certain function % should ignore more than one specific token? For example someone % might find it interesting to define a |\peek_| function that % ignores |a|'s and |b|'s! Or maybe different kinds of ``funny % spaces''\dots{} Therefore I have decided to use this version which % uses |\tex_afterassignment:D| to call the auxiliary function after % the next token has been removed by |\let:NN|. That way it is % easily extensible. % \begin{macrocode} \def_new:Npn \peek_ignore_spaces_aux: { \peek_after:NN \peek_ignore_spaces_execute_branches: } \def_new:Npn \peek_ignore_spaces_execute_branches: { \token_if_eq_meaning:NNTF \l_peek_token \c_space_token { \tex_afterassignment:D \peek_ignore_spaces_aux: \let:NN \peek_tmp:w } \peek_execute_branches: } % \end{macrocode} % \end{macro} % \end{macro} % % % % % \begin{macrocode} % %<*showmemory> \showMemUsage % % \end{macrocode} % % \endinput % % $Log$ % Revision 1.6 2006/03/20 18:26:40 braams % Updated the copyright notice (2006) and demoted all implementation % sections to subsections and so on to clean up the toc for source3.tex % % Revision 1.5 2006/01/19 22:33:53 morten % Minor changes in \char_set_XXXcode:nn functions. % % Revision 1.4 2006/01/04 01:28:06 morten % Fixed \peek_execute_branches_charcode: properly. % % Revision 1.3 2005/12/29 12:17:32 morten % Fixed \peek_execute_branches_charcode: plus minor bugs elsewhere % % Revision 1.2 2005/12/27 15:48:41 morten % Fixing initial boo-boos. Updated documentation to reflect current code % included % % Revision 1.1 2005/12/27 09:54:58 morten % New module %