% \iffalse %% File: l3token.dtx Copyright (C) 2005-2010 LaTeX3 project %% %% It may be distributed and/or modified under the conditions of the %% LaTeX Project Public License (LPPL), either version 1.3c of this %% license or (at your option) any later version. The latest version %% of this license is in the file %% %% http://www.latex-project.org/lppl.txt %% %% This file is part of the ``expl3 bundle'' (The Work in LPPL) %% and all files in that bundle must be distributed together. %% %% The released version of this bundle is available from CTAN. %% %% ----------------------------------------------------------------------- %% %% The development version of the bundle can be found at %% %% http://www.latex-project.org/svnroot/experimental/trunk/ %% %% for those people who are interested. %% %%%%%%%%%%% %% NOTE: %% %%%%%%%%%%% %% %% Snapshots taken from the repository represent work in progress and may %% not work or may contain conflicting material! We therefore ask %% people _not_ to put them into distributions, archives, etc. without %% prior consultation with the LaTeX Project Team. %% %% ----------------------------------------------------------------------- %<*driver|package> \RequirePackage{l3names} % %\fi \GetIdInfo$Id: l3token.dtx 2063 2010-10-03 08:26:49Z mittelba $ {L3 Experimental token investigation and manipulation} %\iffalse %<*driver> %\fi \ProvidesFile{\filename.\filenameext} [\filedate\space v\fileversion\space\filedescription] %\iffalse \documentclass[full]{l3doc} \begin{document} \DocInput{\filename.\filenameext} \end{document} % % \fi % % \title{The \textsf{l3token} package\thanks{This file % has version number \fileversion, last % revised \filedate.}\\ % A token of my appreciation\dots} % \author{\Team} % \date{\filedate} % \maketitle % % \begin{documentation} % % This module deals with tokens. Now this is perhaps not the most % precise description so let's try with a better description: When % programming in \TeX, it is often desirable to know just what a % certain token is: is it a control sequence or something % else. Similarly one often needs to know if a control sequence is % expandable or not, a macro or a primitive, how many arguments it % takes etc. Another thing of great importance (especially when it % comes to document commands) is looking ahead in the token stream to % see if a certain character is present and maybe even remove it or % disregard other tokens while scanning. This module provides % functions for both and as such will have two primary function % categories: |\token| for anything that deals with tokens and % |\peek| for looking ahead in the token stream. % % Most of the time we will be using the term `token' but most of the % time the function we're describing can equally well by used on a % control sequence as such one is one token as well. % % We shall refer to list of tokens as |tlist|s and such lists % represented by a single control sequence is a `token list variable' % |tl var|. Functions for these two types are found in the \textsf{l3tl} % module. % % \section{Character tokens} % % \begin{function}{ % \char_set_catcode:nn | % \char_set_catcode:w | % \char_value_catcode:n | % \char_value_catcode:w | % \char_show_value_catcode:n | % \char_show_value_catcode:w % } % \begin{syntax} % "\char_set_catcode:nn" \Arg{char number} \Arg{number}\\ % "\char_set_catcode:w" = \\ % "\char_value_catcode:n" \Arg{char number} \\ % "\char_show_value_catcode:n" \Arg{char number} % \end{syntax} % "\char_set_catcode:nn" sets the category code of a character, % "\char_value_catcode:n" returns its value for use in integer tests % and "\char_show_value_catcode:n" pausing the typesetting and prints % the value on the terminal and in the log file. % The ":w" form should be avoided. % (Will: should we then just not mention it?) % % "\char_set_catcode" is more usefully abstracted below. % \begin{texnote} % "\char_set_catcode:w" is the \TeX\ primitive \tn{catcode} renamed. % \end{texnote} % \end{function} % % \begin{function}{ % \char_make_escape:n | % \char_make_begin_group:n | % \char_make_end_group:n | % \char_make_math_shift:n | % \char_make_alignment:n | % \char_make_end_line:n | % \char_make_parameter:n | % \char_make_math_superscript:n | % \char_make_math_subscript:n | % \char_make_ignore:n | % \char_make_space:n | % \char_make_letter:n | % \char_make_other:n | % \char_make_active:n | % \char_make_comment:n | % \char_make_invalid:n } % \begin{syntax} % "\char_make_letter:n" \Arg{character number} % "\char_make_letter:n {64}" % "\char_make_letter:n {`\@}" % \end{syntax} % Sets the catcode of the character referred to by its . % \end{function} % % \begin{function}{ % \char_make_escape:N | % \char_make_begin_group:N | % \char_make_end_group:N | % \char_make_math_shift:N | % \char_make_alignment:N | % \char_make_end_line:N | % \char_make_parameter:N | % \char_make_math_superscript:N | % \char_make_math_subscript:N | % \char_make_ignore:N | % \char_make_space:N | % \char_make_letter:N | % \char_make_other:N | % \char_make_active:N | % \char_make_comment:N | % \char_make_invalid:N } % \begin{syntax} % "\char_make_letter:N" \Arg{character} % "\char_make_letter:N" "@" % "\char_make_letter:N" "\%" % \end{syntax} % Sets the catcode of the , which may have to be escaped. % \begin{texnote} % "\char_make_other:N" is \LaTeXe's "\@makeother". % \end{texnote} % \end{function} % % \begin{function}{ % \char_set_lccode:nn | % \char_set_lccode:w | % \char_value_lccode:n | % \char_value_lccode:w | % \char_show_value_lccode:n | % \char_show_value_lccode:w % } % \begin{syntax} % "\char_set_lccode:nn" \Arg{char} \Arg{number} \\ % "\char_set_lccode:w" = \\ % "\char_value_lccode:n" \Arg{char}\\ % "\char_show_value_lccode:n" \Arg{char} % \end{syntax} % Set the lower caser representation of for when is % being converted in "\tl_to_lowercase:n". As above, the % ":w" form is only for people who really, really know what they are % doing. % \begin{texnote} % "\char_set_lccode:w" is the \TeX\ primitive \tn{lccode} renamed. % \end{texnote} % \end{function} % % \begin{function}{ % \char_set_uccode:nn | % \char_set_uccode:w | % \char_value_uccode:n | % \char_value_uccode:w | % \char_show_value_uccode:n | % \char_show_value_uccode:w % } % \begin{syntax} % "\char_set_uccode:nn" \Arg{char} \Arg{number}\\ % "\char_set_uccode:w" = \\ % "\char_value_uccode:n" \Arg{char}\\ % "\char_show_value_uccode:n" \Arg{char} % \end{syntax} % Set the uppercase representation of for when is being % converted in "\tl_to_uppercase:n". As above, the ":w" % form is only for people who really, really know what they are % doing. % \begin{texnote} % "\char_set_uccode:w" is the \TeX\ primitive \tn{uccode} renamed. % \end{texnote} % \end{function} % % \begin{function}{ % \char_set_sfcode:nn | % \char_set_sfcode:w | % \char_value_sfcode:n | % \char_value_sfcode:w | % \char_show_value_sfcode:n | % \char_show_value_sfcode:w % } % \begin{syntax} % "\char_set_sfcode:nn" \Arg{char} \Arg{number}\\ % "\char_set_sfcode:w" = \\ % "\char_value_sfcode:n" \Arg{char}\\ % "\char_show_value_sfcode:n" \Arg{char} % \end{syntax} % Set the space factor for . % \begin{texnote} % "\char_set_sfcode:w" is the \TeX\ primitive \tn{sfcode} renamed. % \end{texnote} % \end{function} % % \begin{function}{ % \char_set_mathcode:nn | % \char_set_mathcode:w | % \char_gset_mathcode:nn | % \char_gset_mathcode:w | % \char_value_mathcode:n | % \char_value_mathcode:w | % \char_show_value_mathcode:n | % \char_show_value_mathcode:w % } % \begin{syntax} % "\char_set_mathcode:nn" \Arg{char} \Arg{number}\\ % "\char_set_mathcode:w" = \\ % "\char_value_mathcode:n" \Arg{char}\\ % "\char_show_value_mathcode:n" \Arg{char} % \end{syntax} % Set the math code for . % \begin{texnote} % "\char_set_mathcode:w" is the \TeX\ primitive \tn{mathcode} renamed. % \end{texnote} % \end{function} % % % \section{Generic tokens} % % \begin{function}{% % \token_new:Nn | % } % \begin{syntax} % "\token_new:Nn" \Arg{token 2} % \end{syntax} % Defines to globally be a snapshot of . This will % be an implicit representation of . % \end{function} % % \begin{variable}{% % \c_group_begin_token | % \c_group_end_token | % \c_math_shift_token | % \c_alignment_tab_token | % \c_parameter_token | % \c_math_superscript_token | % \c_math_subscript_token | % \c_space_token | % \c_letter_token | % \c_other_char_token | % \c_active_char_token | % } % \begin{syntax} % \end{syntax} % Some useful constants. They have category codes~1, 2, 3, 4, 6, 7, % 8, 10, 11, 12, and 13 respectively. They are all implicit tokens. % \end{variable} % % \begin{function}{ % \token_if_group_begin_p:N / (EXP) | % \token_if_group_begin:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_group_begin:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is a begin group token. % \end{function} % % \begin{function}{ % \token_if_group_end_p:N / (EXP) | % \token_if_group_end:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_group_end:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is an end group token. % \end{function} % % \begin{function}{ % \token_if_math_shift_p:N / (EXP) | % \token_if_math_shift:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_math_shift:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is a math shift token. % \end{function} % % % \begin{function}{ % \token_if_alignment_tab_p:N / (EXP) | % \token_if_alignment_tab:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_aligment_tab:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is an aligment tab token. % \end{function} % % \begin{function}{ % \token_if_parameter_p:N / (EXP) | % \token_if_parameter:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_parameter:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is a parameter token. % \end{function} % % \begin{function}{ % \token_if_math_superscript_p:N / (EXP) | % \token_if_math_superscript:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_math_superscript:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is a math superscript token. % \end{function} % % \begin{function}{ % \token_if_math_subscript_p:N / (EXP) | % \token_if_math_subscript:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_math_subscript:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is a math subscript token. % \end{function} % % \begin{function}{ % \token_if_space_p:N / (EXP) | % \token_if_space:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_space:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is a space token. % \end{function} % % % \begin{function}{ % \token_if_letter_p:N / (EXP) | % \token_if_letter:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_letter:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is a letter token. % \end{function} % % % \begin{function}{ % \token_if_other_char_p:N / (EXP) | % \token_if_other_char:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_other_char:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is an other char token. % \end{function} % % \begin{function}{ % \token_if_active_char_p:N / (EXP) | % \token_if_active_char:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_active_char:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is an active char token. % \end{function} % % \begin{function}{% % \token_if_eq_meaning_p:NN / (EXP) | % \token_if_eq_meaning:NN / (TF) (EXP) % } % \begin{syntax} % "\token_if_eq_meaning:NNTF" \Arg{true} \Arg{false} % \end{syntax} % Check if the meaning of two tokens are identical. % \end{function} % % \begin{function}{% % \token_if_eq_catcode_p:NN / (EXP) | % \token_if_eq_catcode:NN / (TF) (EXP) % } % \begin{syntax} % "\token_if_eq_catcode:NNTF" \Arg{true} \Arg{false} % \end{syntax} % Check if the category codes of two tokens are equal. If both tokens % are control sequences the test will be true. % \end{function} % % \begin{function}{% % \token_if_eq_charcode_p:NN / (EXP) | % \token_if_eq_charcode:NN / (TF) (EXP) % } % \begin{syntax} % "\token_if_eq_catcode:NNTF" \Arg{true} \Arg{false} % \end{syntax} % Check if the character codes of two tokens are equal. If both tokens % are control sequences the test will be true. % \end{function} % % \begin{function}{ % \token_if_macro_p:N / (EXP) | % \token_if_macro:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_macro:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is a macro. % \end{function} % % \begin{function}{ % \token_if_cs_p:N / (EXP) | % \token_if_cs:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_cs:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is a control sequence or not. This can be useful % for situations where the next token in the input stream is being % looked at and you want to determine what should be done to it. % \end{function} % % \begin{function}{ % \token_if_expandable_p:N / (EXP) | % \token_if_expandable:N / (TF) (EXP) % } % \begin{syntax} % "\token_if_expandable:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is expandable or not. Note that can very % well be an active character. % \end{function} % % The next set of functions here are for picking apart control % sequences. Sometimes it is useful to know if a control sequence has % arguments and if so, how many. Similarly its status with respect to % "\long" or "\protected" is good to have. Finally it can be very % useful to know if a control sequence is of a certain type: Is this % \m{toks} register we're trying to to something with really a % \m{toks} register at all? % % \begin{function}{ % \token_if_long_macro_p:N / (EXP) | % \token_if_long_macro:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_long_macro:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is a ``long'' macro. % \end{function} % % % \begin{function}{ % \token_if_protected_macro_p:N / (EXP) | % \token_if_protected_macro:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_long_macro:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is a ``protected'' macro. This test does \emph{not} % return if the macro is also ``long'', see below. % \end{function} % % \begin{function}{ % \token_if_protected_long_macro_p:N / (EXP) | % \token_if_protected_long_macro:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_protected_long_macro:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is a ``protected long'' macro. % \end{function} % % \begin{function}{ % \token_if_chardef_p:N / (EXP) | % \token_if_chardef:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_chardef:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is defined to be a chardef. % \end{function} % % \begin{function}{ % \token_if_mathchardef_p:N / (EXP) | % \token_if_mathchardef:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_mathchardef:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is defined to be a mathchardef. % \end{function} % % \begin{function}{ % \token_if_int_register_p:N / (EXP) | % \token_if_int_register:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_int_register:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is defined to be an integer register. % \end{function} % % \begin{function}{ % \token_if_dim_register_p:N / (EXP) | % \token_if_dim_register:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_dim_register:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is defined to be a dimension register. % \end{function} % % \begin{function}{ % \token_if_skip_register_p:N / (EXP) | % \token_if_skip_register:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_skip_register:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is defined to be a skip register. % \end{function} % % \begin{function}{ % \token_if_toks_register_p:N / (EXP) | % \token_if_toks_register:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_toks_register:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is defined to be a toks register. % \end{function} % % % \begin{function}{ % \token_get_prefix_spec:N / (EXP) | % \token_get_arg_spec:N / (EXP) | % \token_get_replacement_spec:N / (EXP) % } % \begin{syntax} % "\token_get_arg_spec:N" % \end{syntax} % If token is a macro with definition |\cs_set:Npn||\next| % |#1#2{x`#1--#2'y}|, the |prefix| function will return the string % |\long|, the |arg| function returns the string |#1#2| and the % |replacement| function returns the string |x`#1--#2'y|. If % isn't a macro, these functions return the |\scan_stop:| token. % % If the |arg_spec| contains the string |->|, then the |spec| function % will produce incorrect results. % \end{function} % % \subsection{Useless code: because we can!} % % \begin{function}{ % \token_if_primitive_p:N / (EXP) | % \token_if_primitive:N / (TF) (EXP) | % } % \begin{syntax} % "\token_if_primitive:NTF" \Arg{true} \Arg{false} % \end{syntax} % Check if is a primitive. Probably not a very useful function. % \end{function} % % % \section{Peeking ahead at the next token} % % \begin{variable}{% % \l_peek_token | % \g_peek_token | % \l_peek_search_token | % } % \begin{syntax} % \end{syntax} % Some useful variables. Initially they are set to "?". % \end{variable} % % % \begin{function}{% % \peek_after:NN | % \peek_gafter:NN | % } % \begin{syntax} % "\peek_after:NN" % \end{syntax} % Assign to "\l_peek_token" and then run which % should perform some sort of test on this token. Leaves in % the input stream. "\peek_gafter:NN" does this globally to the token % "\g_peek_token". % \begin{texnote} % This is the primitive \tn{futurelet} turned into a function. % \end{texnote} % \end{function} % % \begin{function}{% % \peek_meaning:N / (TF) | % \peek_meaning_ignore_spaces:N / (TF) | % \peek_meaning_remove:N / (TF) | % \peek_meaning_remove_ignore_spaces:N / (TF) % } % \begin{syntax} % "\peek_meaning:NTF" \Arg{true} \Arg{false} % \end{syntax} % "\peek_meaning:NTF" checks (by using "\if_meaning:w") if % equals the next token in the input stream and executes either or accordingly. "\peek_meaning_remove:NTF" does % the same but additionally removes the token if found. The % "ignore_spaces" versions skips blank spaces before making the % decision. % \begin{texnote} % This is equivalent to \LaTeXe's \tn{@ifnextchar}. % \end{texnote} % \end{function} % % \begin{function}{% % \peek_charcode:N / (TF) | % \peek_charcode_ignore_spaces:N / (TF) | % \peek_charcode_remove:N / (TF) | % \peek_charcode_remove_ignore_spaces:N / (TF) % } % \begin{syntax} % "\peek_charcode:NTF" \Arg{true} \Arg{false} % \end{syntax} % Same as for the "\peek_meaning:NTF" functions above but these use % "\if_charcode:w" to compare the tokens. % \end{function} % % \begin{function}{% % \peek_catcode:N / (TF) | % \peek_catcode_ignore_spaces:N / (TF) | % \peek_catcode_remove:N / (TF) | % \peek_catcode_remove_ignore_spaces:N / (TF) % } % \begin{syntax} % "\peek_catcode:NTF" \Arg{true} \Arg{false} % \end{syntax} % Same as for the "\peek_meaning:NTF" functions above but these use % "\if_catcode:w" to compare the tokens. % \end{function} % % % \begin{function}{% % \peek_token_generic:NN / (TF) | % \peek_token_remove_generic:NN / (TF) % } % \begin{syntax} % "\peek_token_generic:NNTF" \Arg{true} \Arg{false} % \end{syntax} % "\peek_token_generic:NNTF" looks ahead and checks if the next token % in the input stream is equal to . It uses to make % that decision. "\peek_token_remove_generic:NNTF" does the same % thing but additionally removes from the input stream if it % is found. This also works if is either % "\c_group_begin_token" or "\c_group_end_token". % \end{function} % % \begin{function}{% % \peek_execute_branches_meaning: | % \peek_execute_branches_charcode: | % \peek_execute_branches_catcode: | % } % \begin{syntax} % "\peek_execute_branches_meaning:" % \end{syntax} % These functions compare the token we are searching for with the % token found (after optional ignoring of specific tokens). They come % in the usual three versions when \TeX{} is comparing tokens: % meaning, character code, and category code. % \end{function} % % \end{documentation} % % \begin{implementation} % % \section{\pkg{l3token} implementation} % % \subsection{Documentation of internal functions} % % \begin{variable}{% % \l_peek_true_tl | % \l_peek_false_tl | % } % \begin{syntax} % \end{syntax} % These token list variables are used internally when choosing either % the true or false branches of a test. % \end{variable} % % \begin{variable}{ \l_peek_search_tl } % \begin{syntax} % \end{syntax} % Used to store "\l_peek_search_token". % \end{variable} % % \begin{function}{% % \peek_tmp:w | % } % \begin{syntax} % \end{syntax} % Scratch function used to gobble tokens from the input stream. % \end{function} % % \begin{variable}{% % \l_peek_true_aux_tl | % \c_peek_true_remove_next_tl | % } % \begin{syntax} % \end{syntax} % These token list variables are used internally when choosing either % the true or false branches of a test. % \end{variable} % % \begin{function}{% % \peek_ignore_spaces_execute_branches: | % \peek_ignore_spaces_aux: | % } % \begin{syntax} % \end{syntax} % Functions used to ignore space tokens in the input stream. % \end{function} % % \subsection{Module code} % % First a few required packages to get this going. % \begin{macrocode} %<*package> \ProvidesExplPackage {\filename}{\filedate}{\fileversion}{\filedescription} \package_check_loaded_expl: % %<*initex|package> % \end{macrocode} % % \subsection{Character tokens} % % \begin{macro}{ \char_set_catcode:w , % \char_set_catcode:nn , % \char_value_catcode:w , % \char_value_catcode:n , % \char_show_value_catcode:w , % \char_show_value_catcode:n } % \begin{macrocode} \cs_new_eq:NN \char_set_catcode:w \tex_catcode:D \cs_new_protected_nopar:Npn \char_set_catcode:nn #1#2 { \char_set_catcode:w #1 = \int_eval:w #2\int_eval_end: } \cs_new_nopar:Npn \char_value_catcode:w { \int_use:N \tex_catcode:D } \cs_new_nopar:Npn \char_value_catcode:n #1 { \char_value_catcode:w \int_eval:w #1\int_eval_end: } \cs_new_nopar:Npn \char_show_value_catcode:w { \tex_showthe:D \tex_catcode:D } \cs_new_nopar:Npn \char_show_value_catcode:n #1 { \char_show_value_catcode:w \int_eval:w #1\int_eval_end: } % \end{macrocode} % \end{macro} % % \begin{macro}{ \char_make_escape:N , \char_make_begin_group:N , % \char_make_end_group:N , \char_make_math_shift:N , % \char_make_alignment:N , \char_make_end_line:N , % \char_make_parameter:N , \char_make_math_superscript:N , % \char_make_math_subscript:N , \char_make_ignore:N , % \char_make_space:N , \char_make_letter:N , % \char_make_other:N , \char_make_active:N , % \char_make_comment:N , \char_make_invalid:N } % \begin{macrocode} \cs_new_protected_nopar:Npn \char_make_escape:N #1 { \char_set_catcode:nn {`#1} {\c_zero} } \cs_new_protected_nopar:Npn \char_make_begin_group:N #1 { \char_set_catcode:nn {`#1} {\c_one} } \cs_new_protected_nopar:Npn \char_make_end_group:N #1 { \char_set_catcode:nn {`#1} {\c_two} } \cs_new_protected_nopar:Npn \char_make_math_shift:N #1 { \char_set_catcode:nn {`#1} {\c_three} } \cs_new_protected_nopar:Npn \char_make_alignment:N #1 { \char_set_catcode:nn {`#1} {\c_four} } \cs_new_protected_nopar:Npn \char_make_end_line:N #1 { \char_set_catcode:nn {`#1} {\c_five} } \cs_new_protected_nopar:Npn \char_make_parameter:N #1 { \char_set_catcode:nn {`#1} {\c_six} } \cs_new_protected_nopar:Npn \char_make_math_superscript:N #1 { \char_set_catcode:nn {`#1} {\c_seven} } \cs_new_protected_nopar:Npn \char_make_math_subscript:N #1 { \char_set_catcode:nn {`#1} {\c_eight} } \cs_new_protected_nopar:Npn \char_make_ignore:N #1 { \char_set_catcode:nn {`#1} {\c_nine} } \cs_new_protected_nopar:Npn \char_make_space:N #1 { \char_set_catcode:nn {`#1} {\c_ten} } \cs_new_protected_nopar:Npn \char_make_letter:N #1 { \char_set_catcode:nn {`#1} {\c_eleven} } \cs_new_protected_nopar:Npn \char_make_other:N #1 { \char_set_catcode:nn {`#1} {\c_twelve} } \cs_new_protected_nopar:Npn \char_make_active:N #1 { \char_set_catcode:nn {`#1} {\c_thirteen} } \cs_new_protected_nopar:Npn \char_make_comment:N #1 { \char_set_catcode:nn {`#1} {\c_fourteen} } \cs_new_protected_nopar:Npn \char_make_invalid:N #1 { \char_set_catcode:nn {`#1} {\c_fifteen} } % \end{macrocode} % \end{macro} % % \begin{macro}{ \char_make_escape:n , \char_make_begin_group:n , % \char_make_end_group:n , \char_make_math_shift:n , % \char_make_alignment:n , \char_make_end_line:n , % \char_make_parameter:n , \char_make_math_superscript:n , % \char_make_math_subscript:n , \char_make_ignore:n , % \char_make_space:n , \char_make_letter:n , % \char_make_other:n , \char_make_active:n , % \char_make_comment:n , \char_make_invalid:n } % \begin{macrocode} \cs_new_protected_nopar:Npn \char_make_escape:n #1 { \char_set_catcode:nn {#1} {\c_zero} } \cs_new_protected_nopar:Npn \char_make_begin_group:n #1 { \char_set_catcode:nn {#1} {\c_one} } \cs_new_protected_nopar:Npn \char_make_end_group:n #1 { \char_set_catcode:nn {#1} {\c_two} } \cs_new_protected_nopar:Npn \char_make_math_shift:n #1 { \char_set_catcode:nn {#1} {\c_three} } \cs_new_protected_nopar:Npn \char_make_alignment:n #1 { \char_set_catcode:nn {#1} {\c_four} } \cs_new_protected_nopar:Npn \char_make_end_line:n #1 { \char_set_catcode:nn {#1} {\c_five} } \cs_new_protected_nopar:Npn \char_make_parameter:n #1 { \char_set_catcode:nn {#1} {\c_six} } \cs_new_protected_nopar:Npn \char_make_math_superscript:n #1 { \char_set_catcode:nn {#1} {\c_seven} } \cs_new_protected_nopar:Npn \char_make_math_subscript:n #1 { \char_set_catcode:nn {#1} {\c_eight} } \cs_new_protected_nopar:Npn \char_make_ignore:n #1 { \char_set_catcode:nn {#1} {\c_nine} } \cs_new_protected_nopar:Npn \char_make_space:n #1 { \char_set_catcode:nn {#1} {\c_ten} } \cs_new_protected_nopar:Npn \char_make_letter:n #1 { \char_set_catcode:nn {#1} {\c_eleven} } \cs_new_protected_nopar:Npn \char_make_other:n #1 { \char_set_catcode:nn {#1} {\c_twelve} } \cs_new_protected_nopar:Npn \char_make_active:n #1 { \char_set_catcode:nn {#1} {\c_thirteen} } \cs_new_protected_nopar:Npn \char_make_comment:n #1 { \char_set_catcode:nn {#1} {\c_fourteen} } \cs_new_protected_nopar:Npn \char_make_invalid:n #1 { \char_set_catcode:nn {#1} {\c_fifteen} } % \end{macrocode} % \end{macro} % % \begin{macro}{\char_set_mathcode:w, % \char_set_mathcode:nn, % \char_gset_mathcode:w, % \char_gset_mathcode:nn, % \char_value_mathcode:w, % \char_value_mathcode:n, % \char_show_value_mathcode:w, % \char_show_value_mathcode:n} % Math codes. % \begin{macrocode} \cs_new_eq:NN \char_set_mathcode:w \tex_mathcode:D \cs_new_protected_nopar:Npn \char_set_mathcode:nn #1#2 { \char_set_mathcode:w #1 = \int_eval:w #2\int_eval_end: } \cs_new_protected_nopar:Npn \char_gset_mathcode:w { \pref_global:D \tex_mathcode:D } \cs_new_protected_nopar:Npn \char_gset_mathcode:nn #1#2 { \char_gset_mathcode:w #1 = \int_eval:w #2\int_eval_end: } \cs_new_nopar:Npn \char_value_mathcode:w { \int_use:N \tex_mathcode:D } \cs_new_nopar:Npn \char_value_mathcode:n #1 { \char_value_mathcode:w \int_eval:w #1\int_eval_end: } \cs_new_nopar:Npn \char_show_value_mathcode:w { \tex_showthe:D \tex_mathcode:D } \cs_new_nopar:Npn \char_show_value_mathcode:n #1 { \char_show_value_mathcode:w \int_eval:w #1\int_eval_end: } % \end{macrocode} % \end{macro} % % \begin{macro}{\char_set_lccode:w, \char_set_lccode:nn, % \char_value_lccode:w, \char_value_lccode:n, % \char_show_value_lccode:w, \char_show_value_lccode:n} % \begin{macrocode} \cs_new_eq:NN \char_set_lccode:w \tex_lccode:D \cs_new_protected_nopar:Npn \char_set_lccode:nn #1#2{ \char_set_lccode:w #1 = \int_eval:w #2\int_eval_end: } \cs_new_nopar:Npn \char_value_lccode:w {\int_use:N\tex_lccode:D} \cs_new_nopar:Npn \char_value_lccode:n #1{\char_value_lccode:w \int_eval:w #1\int_eval_end:} \cs_new_nopar:Npn \char_show_value_lccode:w {\tex_showthe:D\tex_lccode:D} \cs_new_nopar:Npn \char_show_value_lccode:n #1{ \char_show_value_lccode:w \int_eval:w #1\int_eval_end:} % \end{macrocode} % \end{macro} % % \begin{macro}{\char_set_uccode:w, \char_set_uccode:nn, % \char_value_uccode:w, \char_value_uccode:n, % \char_show_value_uccode:w, \char_show_value_uccode:n } % \begin{macrocode} \cs_new_eq:NN \char_set_uccode:w \tex_uccode:D \cs_new_protected_nopar:Npn \char_set_uccode:nn #1#2{ \char_set_uccode:w #1 = \int_eval:w #2\int_eval_end: } \cs_new_nopar:Npn \char_value_uccode:w {\int_use:N\tex_uccode:D} \cs_new_nopar:Npn \char_value_uccode:n #1{\char_value_uccode:w \int_eval:w #1\int_eval_end:} \cs_new_nopar:Npn \char_show_value_uccode:w {\tex_showthe:D\tex_uccode:D} \cs_new_nopar:Npn \char_show_value_uccode:n #1{ \char_show_value_uccode:w \int_eval:w #1\int_eval_end:} % \end{macrocode} % \end{macro} % % \begin{macro}{\char_set_sfcode:w, \char_set_sfcode:nn, % \char_value_sfcode:w, \char_value_sfcode:n, % \char_show_value_sfcode:w, \char_show_value_sfcode:n} % \begin{macrocode} \cs_new_eq:NN \char_set_sfcode:w \tex_sfcode:D \cs_new_protected_nopar:Npn \char_set_sfcode:nn #1#2 { \char_set_sfcode:w #1 = \int_eval:w #2\int_eval_end: } \cs_new_nopar:Npn \char_value_sfcode:w { \int_use:N \tex_sfcode:D } \cs_new_nopar:Npn \char_value_sfcode:n #1 { \char_value_sfcode:w \int_eval:w #1\int_eval_end: } \cs_new_nopar:Npn \char_show_value_sfcode:w { \tex_showthe:D \tex_sfcode:D } \cs_new_nopar:Npn \char_show_value_sfcode:n #1 { \char_show_value_sfcode:w \int_eval:w #1\int_eval_end: } % \end{macrocode} % \end{macro} % % \subsection{Generic tokens} % % \begin{macro}{\token_new:Nn} % Creates a new token. % \begin{macrocode} \cs_new_protected_nopar:Npn \token_new:Nn #1#2 {\cs_new_eq:NN #1#2} % \end{macrocode} % \end{macro} % % \begin{macro}{\c_group_begin_token, % \c_group_end_token, % \c_math_shift_token, % \c_alignment_tab_token, % \c_parameter_token, % \c_math_superscript_token, % \c_math_subscript_token, % \c_space_token, % \c_letter_token, % \c_other_char_token, % \c_active_char_token } % We define these useful tokens. We have to do it by hand with the % brace tokens for obvious reasons. % \begin{macrocode} \cs_new_eq:NN \c_group_begin_token { \cs_new_eq:NN \c_group_end_token } \group_begin: \char_set_catcode:nn{`\*}{3} \token_new:Nn \c_math_shift_token {*} \char_set_catcode:nn{`\*}{4} \token_new:Nn \c_alignment_tab_token {*} \token_new:Nn \c_parameter_token {#} \token_new:Nn \c_math_superscript_token {^} \char_set_catcode:nn{`\*}{8} \token_new:Nn \c_math_subscript_token {*} \token_new:Nn \c_space_token {~} \token_new:Nn \c_letter_token {a} \token_new:Nn \c_other_char_token {1} \char_set_catcode:nn{`\*}{13} \cs_gset_nopar:Npn \c_active_char_token {\exp_not:N*} \group_end: % \end{macrocode} % \end{macro} % % \begin{macro}{\token_if_group_begin_p:N} % \begin{macro}[TF]{\token_if_group_begin:N} % Check if token is a begin group token. We use the constant % "\c_group_begin_token" for this. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_group_begin:N {p,TF,T,F} { \if_catcode:w \exp_not:N #1\c_group_begin_token \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}{ \token_if_group_end_p:N } % \begin{macro}[TF]{ \token_if_group_end:N } % Check if token is a end group token. We use the constant % "\c_group_end_token" for this. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_group_end:N {p,TF,T,F} { \if_catcode:w \exp_not:N #1\c_group_end_token \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_math_shift_p:N} % \begin{macro}[TF]{\token_if_math_shift:N} % Check if token is a math shift token. We use the constant % "\c_math_shift_token" for this. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_math_shift:N {p,TF,T,F} { \if_catcode:w \exp_not:N #1\c_math_shift_token \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_alignment_tab_p:N} % \begin{macro}[TF]{\token_if_alignment_tab:N} % Check if token is an alignment tab token. We use the constant % "\c_alignment_tab_token" for this. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_alignment_tab:N {p,TF,T,F} { \if_catcode:w \exp_not:N #1\c_alignment_tab_token \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_parameter_p:N} % \begin{macro}[TF]{\token_if_parameter:N} % Check if token is a parameter token. We use the constant % "\c_parameter_token" for this. We have to trick \TeX{} a bit to % avoid an error message. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_parameter:N {p,TF,T,F} { \exp_after:wN\if_catcode:w \cs:w c_parameter_token\cs_end:\exp_not:N #1 \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_math_superscript_p:N} % \begin{macro}[TF]{\token_if_math_superscript:N} % Check if token is a math superscript token. We use the constant % "\c_math_superscript_token" for this. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_math_superscript:N {p,TF,T,F} { \if_catcode:w \exp_not:N #1\c_math_superscript_token \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_math_subscript_p:N} % \begin{macro}[TF]{\token_if_math_subscript:N} % Check if token is a math subscript token. We use the constant % "\c_math_subscript_token" for this. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_math_subscript:N {p,TF,T,F} { \if_catcode:w \exp_not:N #1\c_math_subscript_token \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_space_p:N} % \begin{macro}[TF]{\token_if_space:N} % Check if token is a space token. We use the constant % "\c_space_token" for this. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_space:N {p,TF,T,F} { \if_catcode:w \exp_not:N #1\c_space_token \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_letter_p:N} % \begin{macro}[TF]{\token_if_letter:N} % Check if token is a letter token. We use the constant % "\c_letter_token" for this. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_letter:N {p,TF,T,F} { \if_catcode:w \exp_not:N #1\c_letter_token \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_other_char_p:N} % \begin{macro}[TF]{\token_if_other_char:N} % Check if token is an other char token. We use the constant % "\c_other_char_token" for this. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_other_char:N {p,TF,T,F} { \if_catcode:w \exp_not:N #1\c_other_char_token \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_active_char_p:N} % \begin{macro}[TF]{\token_if_active_char:N} % Check if token is an active char token. We use the constant % "\c_active_char_token" for this. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_active_char:N {p,TF,T,F} { \if_catcode:w \exp_not:N #1\c_active_char_token \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_eq_meaning_p:NN} % \begin{macro}[TF]{\token_if_eq_meaning:NN} % Check if the tokens |#1| and |#2| have same meaning. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_eq_meaning:NN {p,TF,T,F} { \if_meaning:w #1 #2 \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_eq_catcode_p:NN} % \begin{macro}[TF]{\token_if_eq_catcode:NN} % Check if the tokens |#1| and |#2| have same category code. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_eq_catcode:NN {p,TF,T,F} { \if_catcode:w \exp_not:N #1 \exp_not:N #2 \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % \end{macro} % % % % \begin{macro}{\token_if_eq_charcode_p:NN} % \begin{macro}[TF]{\token_if_eq_charcode:NN} % Check if the tokens |#1| and |#2| have same character code. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_eq_charcode:NN {p,TF,T,F} { \if_charcode:w \exp_not:N #1 \exp_not:N #2 \prg_return_true: \else: \prg_return_false: \fi: } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_macro_p:N} % \begin{macro}[TF]{\token_if_macro:N} % \begin{macro}[aux]{\token_if_macro_p_aux:w} % When a token is a macro, "\token_to_meaning:N" will always output % something like "\long macro:#1->#1" so we simply check to see if % the meaning contains "->". Argument "#2" in the code below will be % empty if the string "->" isn't present, proof that the token was % not a macro (which is why we reverse the emptiness test). However % this function will fail on its own auxiliary function (and a few % other private functions as well) but that should certainly never % be a problem! % \begin{macrocode} \prg_new_conditional:Nnn \token_if_macro:N {p,TF,T,F} { \exp_after:wN \token_if_macro_p_aux:w \token_to_meaning:N #1 -> \q_stop } \cs_new_nopar:Npn \token_if_macro_p_aux:w #1 -> #2 \q_stop{ \if_predicate:w \tl_if_empty_p:n{#2} \prg_return_false: \else: \prg_return_true: \fi: } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_cs_p:N} % \begin{macro}[TF]{\token_if_cs:N} % Check if token has same catcode as a control sequence. We use % "\scan_stop:" for this. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_cs:N {p,TF,T,F} { \if_predicate:w \token_if_eq_catcode_p:NN \scan_stop: #1 \prg_return_true: \else: \prg_return_false: \fi:} % \end{macrocode} % \end{macro} % \end{macro} % % % \begin{macro}{\token_if_expandable_p:N} % \begin{macro}[TF]{\token_if_expandable:N} % Check if token is expandable. We use the fact that \TeX\ will % temporarily convert "\exp_not:N" \m{token} into "\scan_stop:" if % \m{token} is expandable. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_expandable:N {p,TF,T,F} { \cs_if_exist:NTF #1 { \exp_after:wN \if_meaning:w \exp_not:N #1 #1 \prg_return_false: \else: \prg_return_true: \fi: } { \prg_return_false: } } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}{\token_if_chardef_p:N, % \token_if_mathchardef_p:N, % \token_if_int_register_p:N, % \token_if_skip_register_p:N, % \token_if_dim_register_p:N, % \token_if_toks_register_p:N, % \token_if_protected_macro_p:N, % \token_if_long_macro_p:N, % \token_if_protected_long_macro_p:N} % \begin{macro}[TF]{\token_if_chardef:N,\token_if_mathchardef:N, % \token_if_long_macro:N, \token_if_protected_macro:N, % \token_if_protected_long_macro:N, \token_if_dim_register:N, % \token_if_skip_register:N, \token_if_int_register:N, % \token_if_toks_register:N} % \begin{macro}[aux]{ % \token_if_chardef_p_aux:w, % \token_if_mathchardef_p_aux:w, % \token_if_int_register_p_aux:w, % \token_if_skip_register_p_aux:w, % \token_if_dim_register_p_aux:w, % \token_if_toks_register_p_aux:w, % \token_if_protected_macro_p_aux:w, % \token_if_long_macro_p_aux:w, % \token_if_protected_long_macro_p_aux:w} % Most of these functions have to check the meaning of the token in % question so we need to do some checkups on which characters are % output by |\token_to_meaning:N|. As usual, these characters have % catcode 12 so we must do some serious substitutions in the code % below\dots % \begin{macrocode} \group_begin: \char_set_lccode:nn {`\T}{`\T} \char_set_lccode:nn {`\F}{`\F} \char_set_lccode:nn {`\X}{`\n} \char_set_lccode:nn {`\Y}{`\t} \char_set_lccode:nn {`\Z}{`\d} \char_set_lccode:nn {`\?}{`\\} \tl_map_inline:nn{\X\Y\Z\M\C\H\A\R\O\U\S\K\I\P\L\G\P\E} {\char_set_catcode:nn {`#1}{12}} % \end{macrocode} % We convert the token list to lowercase and restore the catcode and % lowercase code changes. % \begin{macrocode} \tl_to_lowercase:n{ \group_end: % \end{macrocode} % First up is checking if something has been defined with % |\tex_chardef:D| or |\tex_mathchardef:D|. This is easy since \TeX\ % thinks of such tokens as hexadecimal so it stores them as % |\char"|\meta{hex~number} or |\mathchar"|\meta{hex~number}. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_chardef:N {p,TF,T,F} { \exp_after:wN \token_if_chardef_aux:w \token_to_meaning:N #1?CHAR"\q_stop } \cs_new_nopar:Npn \token_if_chardef_aux:w #1?CHAR"#2\q_stop{ \tl_if_empty:nTF {#1} {\prg_return_true:} {\prg_return_false:} } % \end{macrocode} % % \begin{macrocode} \prg_new_conditional:Nnn \token_if_mathchardef:N {p,TF,T,F} { \exp_after:wN \token_if_mathchardef_aux:w \token_to_meaning:N #1?MAYHCHAR"\q_stop } \cs_new_nopar:Npn \token_if_mathchardef_aux:w #1?MAYHCHAR"#2\q_stop{ \tl_if_empty:nTF {#1} {\prg_return_true:} {\prg_return_false:} } % \end{macrocode} % Integer registers are a little more difficult since they expand to % |\count|\meta{number} and there is also a primitive |\countdef|. So % we have to check for that primitive as well. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_int_register:N {p,TF,T,F} { \if_meaning:w \tex_countdef:D #1 \prg_return_false: \else: \exp_after:wN \token_if_int_register_aux:w \token_to_meaning:N #1?COUXY\q_stop \fi: } \cs_new_nopar:Npn \token_if_int_register_aux:w #1?COUXY#2\q_stop{ \tl_if_empty:nTF {#1} {\prg_return_true:} {\prg_return_false:} } % \end{macrocode} % Skip registers are done the same way as the integer registers. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_skip_register:N {p,TF,T,F} { \if_meaning:w \tex_skipdef:D #1 \prg_return_false: \else: \exp_after:wN \token_if_skip_register_aux:w \token_to_meaning:N #1?SKIP\q_stop \fi: } \cs_new_nopar:Npn \token_if_skip_register_aux:w #1?SKIP#2\q_stop{ \tl_if_empty:nTF {#1} {\prg_return_true:} {\prg_return_false:} } % \end{macrocode} % Dim registers. No news here % \begin{macrocode} \prg_new_conditional:Nnn \token_if_dim_register:N {p,TF,T,F} { \if_meaning:w \tex_dimendef:D #1 \c_false_bool \else: \exp_after:wN \token_if_dim_register_aux:w \token_to_meaning:N #1?ZIMEX\q_stop \fi: } \cs_new_nopar:Npn \token_if_dim_register_aux:w #1?ZIMEX#2\q_stop{ \tl_if_empty:nTF {#1} {\prg_return_true:} {\prg_return_false:} } % \end{macrocode} % Toks registers. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_toks_register:N {p,TF,T,F} { \if_meaning:w \tex_toksdef:D #1 \prg_return_false: \else: \exp_after:wN \token_if_toks_register_aux:w \token_to_meaning:N #1?YOKS\q_stop \fi: } \cs_new_nopar:Npn \token_if_toks_register_aux:w #1?YOKS#2\q_stop{ \tl_if_empty:nTF {#1} {\prg_return_true:} {\prg_return_false:} } % \end{macrocode} % Protected macros. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_protected_macro:N {p,TF,T,F} { \exp_after:wN \token_if_protected_macro_aux:w \token_to_meaning:N #1?PROYECYEZ~MACRO\q_stop } \cs_new_nopar:Npn \token_if_protected_macro_aux:w #1?PROYECYEZ~MACRO#2\q_stop{ \tl_if_empty:nTF {#1} {\prg_return_true:} {\prg_return_false:} } % \end{macrocode} % Long macros. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_long_macro:N {p,TF,T,F} { \exp_after:wN \token_if_long_macro_aux:w \token_to_meaning:N #1?LOXG~MACRO\q_stop } \cs_new_nopar:Npn \token_if_long_macro_aux:w #1?LOXG~MACRO#2\q_stop{ \tl_if_empty:nTF {#1} {\prg_return_true:} {\prg_return_false:} } % \end{macrocode} % Finally protected long macros where we for once don't have to add an % extra test since there is no primitive for the combined prefixes. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_protected_long_macro:N {p,TF,T,F} { \exp_after:wN \token_if_protected_long_macro_aux:w \token_to_meaning:N #1?PROYECYEZ?LOXG~MACRO\q_stop } \cs_new_nopar:Npn \token_if_protected_long_macro_aux:w #1 ?PROYECYEZ?LOXG~MACRO#2\q_stop{ \tl_if_empty:nTF {#1} {\prg_return_true:} {\prg_return_false:} } % \end{macrocode} % Finally the |\tl_to_lowercase:n| ends! % \begin{macrocode} } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % % We do not provide a function for testing if a control sequence is % ``outer'' since we don't use that in \LaTeX3. % % \begin{macro}[aux]{\token_get_prefix_arg_replacement_aux:w} % \begin{macro}{\token_get_prefix_spec:N} % \begin{macro}{\token_get_arg_spec:N} % \begin{macro}{\token_get_replacement_spec:N} % In the \textsf{xparse} package we sometimes want to test if a % control sequence can be expanded to reveal a hidden % value. However, we cannot just expand the macro blindly as it may % have arguments and none might be present. Therefore we define % these functions to pick either the prefix(es), the argument % specification, or the replacement text from a macro. All of this % information is returned as characters with catcode~12. If the % token in question isn't a macro, the token |\scan_stop:| is % returned instead. % \begin{macrocode} \group_begin: \char_set_lccode:nn {`\?}{`\:} \char_set_catcode:nn{`\M}{12} \char_set_catcode:nn{`\A}{12} \char_set_catcode:nn{`\C}{12} \char_set_catcode:nn{`\R}{12} \char_set_catcode:nn{`\O}{12} \tl_to_lowercase:n{ \group_end: \cs_new_nopar:Npn \token_get_prefix_arg_replacement_aux:w #1MACRO?#2->#3\q_stop#4{ #4{#1}{#2}{#3} } \cs_new_nopar:Npn\token_get_prefix_spec:N #1{ \token_if_macro:NTF #1{ \exp_after:wN \token_get_prefix_arg_replacement_aux:w \token_to_meaning:N #1\q_stop\use_i:nnn }{\scan_stop:} } \cs_new_nopar:Npn\token_get_arg_spec:N #1{ \token_if_macro:NTF #1{ \exp_after:wN \token_get_prefix_arg_replacement_aux:w \token_to_meaning:N #1\q_stop\use_ii:nnn }{\scan_stop:} } \cs_new_nopar:Npn\token_get_replacement_spec:N #1{ \token_if_macro:NTF #1{ \exp_after:wN \token_get_prefix_arg_replacement_aux:w \token_to_meaning:N #1\q_stop\use_iii:nnn }{\scan_stop:} } } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \paragraph{Useless code: because we can!} % % \begin{macro}{\token_if_primitive_p:N} % \begin{macro}[aux]{\token_if_primitive_p_aux:N} % \begin{macro}[TF]{\token_if_primitive:N} % It is rather hard to determine if a token is a primitive. First we % can check if it is a control sequence or active character. If % either, we check if it is a macro. Then we can go through a % tedious process of testing for different register types\dots{} I % don't actually think this function is useful but you never know. % \begin{macrocode} \prg_new_conditional:Nnn \token_if_primitive:N {p,TF,T,F} { \if_predicate:w \token_if_cs_p:N #1 \if_predicate:w \token_if_macro_p:N #1 \prg_return_false: \else: \token_if_primitive_p_aux:N #1 \fi: \else: \if_predicate:w \token_if_active_char_p:N #1 \if_predicate:w \token_if_macro_p:N #1 \prg_return_false: \else: \token_if_primitive_p_aux:N #1 \fi: \else: \prg_return_false: \fi: \fi: } \cs_new_nopar:Npn \token_if_primitive_p_aux:N #1{ \if_predicate:w \token_if_chardef_p:N #1 \c_false_bool \else: \if_predicate:w \token_if_mathchardef_p:N #1 \prg_return_false: \else: \if_predicate:w \token_if_int_register_p:N #1 \prg_return_false: \else: \if_predicate:w \token_if_skip_register_p:N #1 \prg_return_false: \else: \if_predicate:w \token_if_dim_register_p:N #1 \prg_return_false: \else: \if_predicate:w \token_if_toks_register_p:N #1 \prg_return_false: \else: % \end{macrocode} % We made it! % \begin{macrocode} \prg_return_true: \fi: \fi: \fi: \fi: \fi: \fi: } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % % % \subsection{Peeking ahead at the next token} % % \begin{macro}{\l_peek_token} % \begin{macro}{\g_peek_token} % \begin{macro}{\l_peek_search_token} % We define some other tokens which will initially be the character % |?|. % \begin{macrocode} \token_new:Nn \l_peek_token {?} \token_new:Nn \g_peek_token {?} \token_new:Nn \l_peek_search_token {?} % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % % % \begin{macro}{\peek_after:NN} % \begin{macro}{\peek_gafter:NN} % |\peek_after:NN| takes two argument where the first is a function % acting on |\l_peek_token| and the second is the next token in the % input stream which |\l_peek_token| is set equal to. % |\peek_gafter:NN| does the same globally to |\g_peek_token|. % \begin{macrocode} \cs_new_protected_nopar:Npn \peek_after:NN {\tex_futurelet:D \l_peek_token } \cs_new_protected_nopar:Npn \peek_gafter:NN { \pref_global:D \tex_futurelet:D \g_peek_token } % \end{macrocode} % \end{macro} % \end{macro} % % For normal purposes there are four main cases: % \begin{enumerate} % \item peek at the next token. % \item peek at the next non-space token. % \item peek at the next token and remove it. % \item peek at the next non-space token and remove it. % \end{enumerate} % % % The generic functions will take four arguments: The token to search % for, the test function to run on it and the true/false cases. % The general algorithm is this: % \begin{enumerate} % \item % Store the token to search for in |\l_peek_search_token|. % \item % In order to avoid doubling of hash marks where it seems unnatural % we put the \meta{true} and \meta{false} cases through an |x| type % expansion but using |\exp_not:n| to avoid any expansion. This has % the same effect as putting it through a \meta{toks} register but % is faster. Also put in a special alignment safe group end. % \item % Put in an alignment safe group begin. % \item % Peek ahead and call the function which will act on the next token % in the input stream. % \end{enumerate} % % \begin{macro}{\l_peek_true_tl} % \begin{macro}{\l_peek_false_tl} % Two dedicated token list variables that store the true and false % cases. % \begin{macrocode} \tl_new:N \l_peek_true_tl \tl_new:N \l_peek_false_tl % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}{\peek_tmp:w} % Scratch function used for storing the token to be removed if % found. % \begin{macrocode} \cs_new_nopar:Npn \peek_tmp:w {} % \end{macrocode} % \end{macro} % % \begin{macro}{\l_peek_search_tl} % We also use this token list variable for storing the token we want % to compare. This turns out to be useful. % \begin{macrocode} \tl_new:N \l_peek_search_tl % \end{macrocode} % \end{macro} % % \begin{macro}[TF]{\peek_token_generic:NN} % \begin{arguments} % \item the function to execute (obey or ignore spaces, etc.), % \item the special token we're looking for. % \end{arguments} % \begin{macrocode} \cs_new_protected:Npn \peek_token_generic:NNTF #1#2#3#4 { \cs_set_eq:NN \l_peek_search_token #2 \tl_set:Nn \l_peek_search_tl {#2} \tl_set:Nn \l_peek_true_tl { \group_align_safe_end: #3 } \tl_set:Nn \l_peek_false_tl { \group_align_safe_end: #4 } \group_align_safe_begin: \peek_after:NN #1 } \cs_new_protected:Npn \peek_token_generic:NNT #1#2#3 { \peek_token_generic:NNTF #1#2 {#3} {} } \cs_new_protected:Npn \peek_token_generic:NNF #1#2#3 { \peek_token_generic:NNTF #1#2 {} {#3} } % \end{macrocode} % \end{macro} % % \begin{macro}[TF]{\peek_token_remove_generic:NN} % If we want to be able to remove any character from the input % stream we might as well do it the same way for all characters so % we define this as little differently from above. % \begin{macrocode} \cs_new_protected:Npn \peek_token_remove_generic:NNTF #1#2#3#4 { \cs_set_eq:NN \l_peek_search_token #2 \tl_set:Nn \l_peek_search_tl {#2} \tl_set:Nn \l_peek_true_aux_tl {#3} \tl_set_eq:NN \l_peek_true_tl \c_peek_true_remove_next_tl \tl_set:Nn \l_peek_false_tl {\group_align_safe_end: #4} \group_align_safe_begin: \peek_after:NN #1 } \cs_new:Npn \peek_token_remove_generic:NNT #1#2#3 { \peek_token_remove_generic:NNTF #1#2 {#3} {} } \cs_new:Npn \peek_token_remove_generic:NNF #1#2#3 { \peek_token_remove_generic:NNTF #1#2 {} {#3} } % \end{macrocode} % \end{macro} % % \begin{macro}{\l_peek_true_aux_tl} % \begin{macro}{\c_peek_true_remove_next_tl} % Two token list variables to help with removing the character from % the input stream. % \begin{macrocode} \tl_new:N \l_peek_true_aux_tl \tl_const:Nn \c_peek_true_remove_next_tl {\group_align_safe_end: \tex_afterassignment:D \l_peek_true_aux_tl \cs_set_eq:NN \peek_tmp:w } % \end{macrocode} % \end{macro} % \end{macro} % % \begin{macro}{\peek_execute_branches_meaning:} % \begin{macro}{\peek_execute_branches_catcode:} % \begin{macro}{\peek_execute_branches_charcode:} % \begin{macro}[aux]{\peek_execute_branches_charcode_aux:NN} % There are three major tests between tokens in \TeX: meaning, % catcode and charcode. Hence we define three basic test functions % that set in after the ignoring phase is over and done with. % \begin{macrocode} \cs_new_nopar:Npn \peek_execute_branches_meaning: { \if_meaning:w \l_peek_token \l_peek_search_token \exp_after:wN \l_peek_true_tl \else: \exp_after:wN \l_peek_false_tl \fi: } \cs_new_nopar:Npn \peek_execute_branches_catcode: { \if_catcode:w \exp_not:N\l_peek_token \exp_not:N\l_peek_search_token \exp_after:wN \l_peek_true_tl \else: \exp_after:wN \l_peek_false_tl \fi: } % \end{macrocode} % For the charcode version we do things a little differently. We want % to check the token directly but if we do this we face problems if % the next thing in the input stream is a braced group or a space % token. The braced group would be read as a complete argument and the % space would be gobbled by \TeX's argument reading routines. Hence we % test for both of these and if one of them is found we just execute % the false result directly since no one should ever try to use the % |charcode| function for searching for |\c_group_begin_token| or % |\c_space_token|. The same is true for |\c_group_end_token|, as this % can only occur if the function is at the end of a group. % \begin{macrocode} \cs_new_nopar:Npn \peek_execute_branches_charcode: { \bool_if:nTF { \token_if_eq_catcode_p:NN \l_peek_token \c_group_begin_token || \token_if_eq_catcode_p:NN \l_peek_token \c_group_end_token || \token_if_eq_meaning_p:NN \l_peek_token \c_space_token } { \l_peek_false_tl } % \end{macrocode} % Otherwise we call a small auxiliary function that just grabs the % next token. We can do that because it really is a single token; we % just have insert it again afterwards. Also we stored the token we % were looking for in the token list variable |\l_peek_search_tl| so % we unpack it again for this function. % \begin{macrocode} { \exp_after:wN \peek_execute_branches_charcode_aux:NN \l_peek_search_tl } } % \end{macrocode} % Then we just do the usual |\if_charcode:w| comparison. We also % remember to insert |#2| again after executing the true or false % branches. % \begin{macrocode} \cs_new:Npn \peek_execute_branches_charcode_aux:NN #1#2{ \if_charcode:w \exp_not:N #1\exp_not:N#2 \exp_after:wN \l_peek_true_tl \else: \exp_after:wN \l_peek_false_tl \fi: #2 } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}[aux]{\peek_def_aux:nnnn,\peek_def_aux_ii:nnnnn} % This function aids defining conditional variants without too much % repeated code. I hope that it doesn't detract too much from the readability. % \begin{macrocode} \cs_new_nopar:Npn \peek_def_aux:nnnn #1#2#3#4 { \peek_def_aux_ii:nnnnn {#1} {#2} {#3} {#4} { TF } \peek_def_aux_ii:nnnnn {#1} {#2} {#3} {#4} { T } \peek_def_aux_ii:nnnnn {#1} {#2} {#3} {#4} { F } } \cs_new_protected_nopar:Npn \peek_def_aux_ii:nnnnn #1#2#3#4#5 { \cs_new_nopar:cpx { #1 #5 } { \tl_if_empty:nF {#2} { \exp_not:n { \cs_set_eq:NN \peek_execute_branches: #2 } } \exp_not:c { #3 #5 } \exp_not:n { #4 } } } % \end{macrocode} % \end{macro} % % \begin{macro}[TF]{\peek_meaning:N} % Here we use meaning comparison with |\if_meaning:w|. % \begin{macrocode} \peek_def_aux:nnnn { peek_meaning:N } {} { peek_token_generic:NN } { \peek_execute_branches_meaning: } % \end{macrocode} % \end{macro} % % \begin{macro}[TF]{\peek_meaning_ignore_spaces:N} % \begin{macrocode} \peek_def_aux:nnnn { peek_meaning_ignore_spaces:N } { \peek_execute_branches_meaning: } { peek_token_generic:NN } { \peek_ignore_spaces_execute_branches: } % \end{macrocode} % \end{macro} % % \begin{macro}[TF]{\peek_meaning_remove:N} % \begin{macrocode} \peek_def_aux:nnnn { peek_meaning_remove:N } {} { peek_token_remove_generic:NN } { \peek_execute_branches_meaning: } % \end{macrocode} % \end{macro} % % \begin{macro}[TF]{\peek_meaning_remove_ignore_spaces:N} % \begin{macrocode} \peek_def_aux:nnnn { peek_meaning_remove_ignore_spaces:N } { \peek_execute_branches_meaning: } { peek_token_remove_generic:NN } { \peek_ignore_spaces_execute_branches: } % \end{macrocode} % \end{macro} % % \begin{macro}[TF]{\peek_catcode:N} % Here we use catcode comparison with |\if_catcode:w|. % \begin{macrocode} \peek_def_aux:nnnn { peek_catcode:N } {} { peek_token_generic:NN } { \peek_execute_branches_catcode: } % \end{macrocode} % \end{macro} % % \begin{macro}[TF]{\peek_catcode_ignore_spaces:N} % \begin{macrocode} \peek_def_aux:nnnn { peek_catcode_ignore_spaces:N } { \peek_execute_branches_catcode: } { peek_token_generic:NN } { \peek_ignore_spaces_execute_branches: } % \end{macrocode} % \end{macro} % % \begin{macro}[TF]{\peek_catcode_remove:N} % \begin{macrocode} \peek_def_aux:nnnn { peek_catcode_remove:N } {} { peek_token_remove_generic:NN } { \peek_execute_branches_catcode: } % \end{macrocode} % \end{macro} % % \begin{macro}[TF]{\peek_catcode_remove_ignore_spaces:N} % \begin{macrocode} \peek_def_aux:nnnn { peek_catcode_remove_ignore_spaces:N } { \peek_execute_branches_catcode: } { peek_token_remove_generic:NN } { \peek_ignore_spaces_execute_branches: } % \end{macrocode} % \end{macro} % % \begin{macro}[TF]{\peek_charcode:N} % Here we use charcode comparison with |\if_charcode:w|. % \begin{macrocode} \peek_def_aux:nnnn { peek_charcode:N } {} { peek_token_generic:NN } { \peek_execute_branches_charcode: } % \end{macrocode} % \end{macro} % % \begin{macro}[TF]{\peek_charcode_ignore_spaces:N} % \begin{macrocode} \peek_def_aux:nnnn { peek_charcode_ignore_spaces:N } { \peek_execute_branches_charcode: } { peek_token_generic:NN } { \peek_ignore_spaces_execute_branches: } % \end{macrocode} % \end{macro} % % \begin{macro}[TF]{\peek_charcode_remove:N} % \begin{macrocode} \peek_def_aux:nnnn { peek_charcode_remove:N } {} { peek_token_remove_generic:NN } { \peek_execute_branches_charcode: } % \end{macrocode} % \end{macro} % % \begin{macro}[TF]{\peek_charcode_remove_ignore_spaces:N} % \begin{macrocode} \peek_def_aux:nnnn { peek_charcode_remove_ignore_spaces:N } { \peek_execute_branches_charcode: } { peek_token_remove_generic:NN } { \peek_ignore_spaces_execute_branches:} % \end{macrocode} % \end{macro} % % \begin{macro}{\peek_ignore_spaces_aux:, % \peek_ignore_spaces_execute_branches:} % Throw away a space token and search again. We could define this in % a more devious way where the auxiliary function gobbles the space % token but then what do we do if we decide that a certain function % should ignore more than one specific token? For example someone % might find it interesting to define a |\peek_| function that % ignores |a|'s and |b|'s! Or maybe different kinds of ``funny % spaces''\dots{} Therefore I have decided to use this version which % uses |\tex_afterassignment:D| to call the auxiliary function after % the next token has been removed by |\cs_set_eq:NN|. That way it is % easily extensible. % \begin{macrocode} \cs_new_nopar:Npn \peek_ignore_spaces_aux: { \peek_after:NN \peek_ignore_spaces_execute_branches: } \cs_new_protected_nopar:Npn \peek_ignore_spaces_execute_branches: { \token_if_eq_meaning:NNTF \l_peek_token \c_space_token { \tex_afterassignment:D \peek_ignore_spaces_aux: \cs_set_eq:NN \peek_tmp:w } \peek_execute_branches: } % \end{macrocode} % \end{macro} % % \begin{macrocode} % % \end{macrocode} % % \begin{macrocode} %<*showmemory> \showMemUsage % % \end{macrocode} % % \end{implementation} % \PrintIndex % % \endinput