%% %% This is file `l3token.sty', %% generated with the docstrip utility. %% %% The original source files were: %% %% l3token.dtx (with options: `package') %% %% %% EXPERIMENTAL CODE %% %% Do not distribute this file without also distributing the %% source files specified above. %% %% Do not distribute a modified version of this file. %% %% %% File: l3token.dtx Copyright (C) 2005-2006 LaTeX3 project %% %% It may be distributed and/or modified under the conditions of the %% LaTeX Project Public License (LPPL), either version 1.3c of this %% license or (at your option) any later version. The latest version %% of this license is in the file %% %% http://www.latex-project.org/lppl.txt %% %% This file is part of the ``expl3 bundle'' (The Work in LPPL) %% and all files in that bundle must be distributed together. %% %% The released version of this bundle is available from CTAN. %% %% ----------------------------------------------------------------------- %% %% The development version of the bundle can be found at %% %% http://www.latex-project.org/cgi-bin/cvsweb.cgi/ %% %% for those people who are interested. %% %%%%%%%%%%% %% NOTE: %% %%%%%%%%%%% %% %% Snapshots taken from the repository represent work in progress and may %% not work or may contain conflicting material! We therefore ask %% people _not_ to put them into distributions, archives, etc. without %% prior consultation with the LaTeX Project Team. %% %% ----------------------------------------------------------------------- \RequirePackage{l3names} \GetIdInfo$Id: l3token.dtx 701 2007-11-27 14:21:05Z morten $ {L3 Experimental token investigation and manipulation} \ProvidesExplPackage {\filename}{\filedate}{\fileversion}{\filedescription} \RequirePackage{l3prg} \RequirePackage{l3int} \let_new:NN \char_set_catcode:w \tex_catcode:D \def_new:Npn \char_set_catcode:nn #1#2{ \char_set_catcode:w #1 = \int_eval:n{#2} } \def_new:Npn \char_value_catcode:w {\int_use:N\tex_catcode:D} \def_new:Npn \char_value_catcode:n #1{\char_value_catcode:w \int_eval:n{#1}} \def_new:Npn \char_show_value_catcode:w {\tex_showthe:D\tex_catcode:D} \def_new:Npn \char_show_value_catcode:n #1{ \char_show_value_catcode:w \int_eval:n{#1}} \let_new:NN \char_set_mathcode:w \tex_mathcode:D \def_new:Npn \char_set_mathcode:nn #1#2{ \char_set_mathcode:w #1 = \int_eval:n{#2} } \def_protected_new:Npn \char_gset_mathcode:w {\pref_global:D\tex_mathcode:D} \def_new:Npn \char_gset_mathcode:nn #1#2{ \char_gset_mathcode:w #1 = \int_eval:n{#2} } \def_new:Npn \char_value_mathcode:w {\int_use:N\tex_mathcode:D} \def_new:Npn \char_value_mathcode:n #1{\char_value_mathcode:w \int_eval:n{#1}} \def_new:Npn \char_show_value_mathcode:w {\tex_showthe:D\tex_mathcode:D} \def_new:Npn \char_show_value_mathcode:n #1{ \char_show_value_mathcode:w \int_eval:n{#1}} \let_new:NN \char_set_lccode:w \tex_lccode:D \def_new:Npn \char_set_lccode:nn #1#2{ \char_set_lccode:w #1 = \int_eval:n{#2} } \def_new:Npn \char_value_lccode:w {\int_use:N\tex_lccode:D} \def_new:Npn \char_value_lccode:n #1{\char_value_lccode:w \int_eval:n{#1}} \def_new:Npn \char_show_value_lccode:w {\tex_showthe:D\tex_lccode:D} \def_new:Npn \char_show_value_lccode:n #1{ \char_show_value_lccode:w \int_eval:n{#1}} \let_new:NN \char_set_uccode:w \tex_uccode:D \def_new:Npn \char_set_uccode:nn #1#2{ \char_set_uccode:w #1 = \int_eval:n{#2} } \def_new:Npn \char_value_uccode:w {\int_use:N\tex_uccode:D} \def_new:Npn \char_value_uccode:n #1{\char_value_uccode:w \int_eval:n{#1}} \def_new:Npn \char_show_value_uccode:w {\tex_showthe:D\tex_uccode:D} \def_new:Npn \char_show_value_uccode:n #1{ \char_show_value_uccode:w \int_eval:n{#1}} \let_new:NN \char_set_sfcode:w \tex_sfcode:D \def_new:Npn \char_set_sfcode:nn #1#2{ \char_set_sfcode:w #1 = \int_eval:n{#2} } \def_new:Npn \char_value_sfcode:w {\int_use:N\tex_uccode:D} \def_new:Npn \char_value_sfcode:n #1{\char_value_sfcode:w \int_eval:n{#1}} \def_new:Npn \char_show_value_sfcode:w {\tex_showthe:D\tex_sfcode:D} \def_new:Npn \char_show_value_sfcode:n #1{ \char_show_value_sfcode:w \int_eval:n{#1}} \def_new:Npn \token_new:Nn #1#2{\glet_new:NN #1#2} \let_new:NN \c_group_begin_token { \let_new:NN \c_group_end_token } \group_begin: \char_set_catcode:nn{`\*}{3} \token_new:Nn \c_math_shift_token {*} \char_set_catcode:nn{`\*}{4} \token_new:Nn \c_alignment_tab_token {*} \token_new:Nn \c_parameter_token {#} \token_new:Nn \c_math_superscript_token {^} \char_set_catcode:nn{`\*}{8} \token_new:Nn \c_math_subscript_token {*} \token_new:Nn \c_space_token {~} \token_new:Nn \c_letter_token {a} \token_new:Nn \c_other_char_token {1} \char_set_catcode:nn{`\*}{13} \token_new:Nn \c_active_char_token {*} \group_end: \def_new:Npn \token_if_group_begin_p:N #1{ \if_catcode:w \exp_not:N #1\c_group_begin_token \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_group_begin:N} #1{ \if:w\token_if_group_begin_p:N #1} \def_new:Npn \token_if_group_end_p:N #1{ \if_catcode:w \exp_not:N #1\c_group_end_token \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_group_end:N} #1{ \if:w\token_if_group_end_p:N #1} \def_new:Npn \token_if_math_shift_p:N #1{ \if_catcode:w \exp_not:N #1\c_math_shift_token \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_math_shift:N} #1{ \if:w\token_if_math_shift_p:N#1} \def_new:Npn \token_if_alignment_tab_p:N #1{ \if_catcode:w \exp_not:N #1\c_alignment_tab_token \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_alignment_tab:N} #1{ \if:w\token_if_alignment_tab_p:N#1} \def_new:Npn \token_if_parameter_p:N #1{ \exp_after:NN\if_catcode:w \cs:w c_parameter_token\cs_end:\exp_not:N #1 \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_parameter:N} #1{ \if:w\token_if_parameter_p:N#1} \def_new:Npn \token_if_math_superscript_p:N #1{ \if_catcode:w \exp_not:N #1\c_math_superscript_token \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_math_superscript:N} #1{ \if:w\token_if_math_superscript_p:N #1} \def_new:Npn \token_if_math_subscript_p:N #1{ \if_catcode:w \exp_not:N #1\c_math_subscript_token \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_math_subscript:N} #1{ \if:w\token_if_math_subscript_p:N #1} \def_new:Npn \token_if_space_p:N #1{ \if_catcode:w \exp_not:N #1\c_space_token \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_space:N} #1{ \if:w\token_if_space_p:N #1} \def_new:Npn \token_if_letter_p:N #1{ \if_catcode:w \exp_not:N #1\c_letter_token \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_letter:N} #1{ \if:w\token_if_letter_p:N #1} \def_new:Npn \token_if_other_char_p:N #1{ \if_catcode:w \exp_not:N #1\c_other_char_token \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_other_char:N} #1{ \if:w\token_if_other_char_p:N #1} \def_new:Npn \token_if_active_char_p:N #1{ \if_catcode:w \exp_not:N #1\c_active_char_token \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_active_char:N} #1{ \if:w\token_if_active_char_p:N #1} \def_new:Npn \token_if_eq_meaning_p:NN #1#2 { \if_meaning:NN #1 #2 \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_eq_meaning:NN}#1#2{ \if_meaning:NN #1 #2} \def_new:Npn \token_if_eq_catcode_p:NN #1#2 { \if_catcode:w \exp_not:N #1 \exp_not:N #2 \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_eq_catcode:NN}#1#2{ \if:w\token_if_eq_catcode_p:NN#1#2} \def_new:Npn \token_if_charcode_eq_p:NN #1#2 { \if_charcode:w \exp_not:N #1 \exp_not:N #2 \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_eq_charcode:NN}#1#2{ \if:w\token_if_eq_charcode_p:NN#1#2} \def_new:Npn \token_if_macro_p:N #1 { \exp_after:NN \token_if_macro_p_aux:w \token_to_meaning:N #1 -> \q_nil } \def_new:Npn \token_if_macro_p_aux:w #1 -> #2 \q_nil{ \if:w \tlist_if_empty_p:n{#2} \c_false \else: \c_true \fi: } \def_test_function_new:npn {token_if_macro:N} #1{\if:w\token_if_macro_p:N#1} \def_new:Npn \token_if_cs_p:N {\token_if_eq_catcode_p:NN \scan_stop:} \def_test_function_new:npn {token_if_cs:N} #1{ \if:w \token_if_eq_catcode_p:NN \scan_stop: #1} \def_new:Npn \token_if_expandable_p:N #1{ \exp_after:NN \if_token_eq:NN \exp_not:N #1 \scan_stop: \c_true \else: \c_false \fi: } \def_test_function_new:npn {token_if_expandable:N} #1{ \if:w\token_if_expandable_p:N#1} \group_begin: \char_set_lccode:nn {`\X}{`\n} \char_set_lccode:nn {`\Y}{`\t} \char_set_lccode:nn {`\Z}{`\d} \char_set_lccode:nn {`\?}{`\\} \tlist_map_inline:nn{\X\Y\Z\M\T\C\H\A\R\O\U\S\K\I\P\L\G\P\E} {\char_set_catcode:nn {`#1}{12}} \tlist_to_lowercase:n{ \group_end: \def_new:Npn \token_if_chardef_p:N #1 { \exp_after:NN \token_if_chardef_p_aux:w \token_to_meaning:N #1?CHAR"\q_nil } \def_new:Npn \token_if_chardef_p_aux:w #1?CHAR"#2\q_nil{ \tlist_if_empty_p:n{#1} } \def_new:Npn \token_if_mathchardef_p:N #1 { \exp_after:NN \token_if_mathchardef_p_aux:w \token_to_meaning:N #1?MAYHCHAR"\q_nil } \def_new:Npn \token_if_mathchardef_p_aux:w #1?MAYHCHAR"#2\q_nil{ \tlist_if_empty_p:n{#1} } \def:Npn \token_if_int_register_p:N #1{ \if_meaning:NN \tex_countdef:D #1 \c_false \else: \exp_after:NN \token_if_int_register_p_aux:w \token_to_meaning:N #1?COUXY\q_nil \fi: } \def_new:Npn \token_if_int_register_p_aux:w #1?COUXY#2\q_nil{ \tlist_if_empty_p:n{#1} } \def:Npn \token_if_skip_register_p:N #1{ \if_meaning:NN \tex_skipdef:D #1 \c_false \else: \exp_after:NN \token_if_skip_register_p_aux:w \token_to_meaning:N #1?SKIP\q_nil \fi: } \def_new:Npn \token_if_skip_register_p_aux:w #1?SKIP#2\q_nil{ \tlist_if_empty_p:n{#1} } \def:Npn \token_if_dim_register_p:N #1{ \if_meaning:NN \tex_dimendef:D #1 \c_false \else: \exp_after:NN \token_if_dim_register_p_aux:w \token_to_meaning:N #1?ZIMEX\q_nil \fi: } \def_new:Npn \token_if_dim_register_p_aux:w #1?ZIMEX#2\q_nil{ \tlist_if_empty_p:n{#1} } \def:Npn \token_if_toks_register_p:N #1{ \if_meaning:NN \tex_toksdef:D #1 \c_false \else: \exp_after:NN \token_if_toks_register_p_aux:w \token_to_meaning:N #1?YOKS\q_nil \fi: } \def_new:Npn \token_if_toks_register_p_aux:w #1?YOKS#2\q_nil{ \tlist_if_empty_p:n{#1} } \def_new:Npn \token_if_protected_macro_p:N #1 { \exp_after:NN \token_if_protected_macro_p_aux:w \token_to_meaning:N #1?PROYECYEZ~MACRO\q_nil } \def_new:Npn \token_if_protected_macro_p_aux:w #1?PROYECYEZ~MACRO#2\q_nil{ \tlist_if_empty_p:n{#1} } \def_new:Npn \token_if_long_macro_p:N #1 { \exp_after:NN \token_if_long_macro_p_aux:w \token_to_meaning:N #1?LOXG~MACRO\q_nil } \def_new:Npn \token_if_long_macro_p_aux:w #1?LOXG~MACRO#2\q_nil{ \tlist_if_empty_p:n{#1} } \def_new:Npn \token_if_protected_long_macro_p:N #1 { \exp_after:NN \token_if_protected_long_macro_p_aux:w \token_to_meaning:N #1?PROYECYEZ~?LOXG~MACRO\q_nil } \def_new:Npn \token_if_protected_long_macro_p_aux:w #1 ?PROYECYEZ~?LOXG~MACRO#2\q_nil{ \tlist_if_empty_p:n{#1} } } \def_test_function_new:npn {token_if_chardef:N} {\if:w \token_if_chardef_p:N} \def_test_function_new:npn {token_if_mathchardef:N} { \if:w \token_if_mathchardef_p:N} \def_test_function_new:npn {token_if_long_macro:N} { \if:w \token_if_long_macro_p:N} \def_test_function_new:npn {token_if_protected_macro:N} { \if:w \token_if_protected_macro_p:N} \def_test_function_new:npn {token_if_protected_long_macro:N} { \if:w \token_if_protected_long_macro_p:N} \def_test_function_new:npn {token_if_dim_register:N} { \if:w \token_if_dim_register_p:N} \def_test_function_new:npn {token_if_skip_register:N} { \if:w \token_if_skip_register_p:N} \def_test_function_new:npn {token_if_int_register:N} { \if:w \token_if_int_register_p:N} \def_test_function_new:npn {token_if_toks_register:N} { \if:w \token_if_toks_register_p:N} \group_begin: \char_set_lccode:nn {`\?}{`\:} \char_set_catcode:nn{`\M}{12} \char_set_catcode:nn{`\A}{12} \char_set_catcode:nn{`\C}{12} \char_set_catcode:nn{`\R}{12} \char_set_catcode:nn{`\O}{12} \tlist_to_lowercase:n{ \group_end: \def_new:Npn \token_get_prefix_arg_replacement_aux:w #1MACRO?#2->#3\q_nil#4{ #4{#1}{#2}{#3} } \def_new:Npn\token_get_prefix_spec:N #1{ \token_if_macro:NTF #1{ \exp_after:NN \token_get_prefix_arg_replacement_aux:w \token_to_meaning:N #1\q_nil\use_arg_i:nnn }{\scan_stop:} } \def_new:Npn\token_get_arg_spec:N #1{ \token_if_macro:NTF #1{ \exp_after:NN \token_get_prefix_arg_replacement_aux:w \token_to_meaning:N #1\q_nil\use_arg_ii:nnn }{\scan_stop:} } \def_new:Npn\token_get_replacement_spec:N #1{ \token_if_macro:NTF #1{ \exp_after:NN \token_get_prefix_arg_replacement_aux:w \token_to_meaning:N #1\q_nil\use_arg_iii:nnn }{\scan_stop:} } } \def_new:Npn \token_if_primitive_p:N #1{ \if:w \token_if_cs_p:N #1\scan_stop: \if:w \token_if_macro_p:N #1 \c_false \else: \token_if_primitive_p_aux:N #1 \fi: \else: \if:w \token_if_active_p:N #1 \if:w \token_if_macro_p:N #1 \c_false \else: \token_if_primitive_p_aux:N #1 \fi: \else: \c_false \fi: \fi: } \def_new:Npn \token_if_primitive_p_aux:N #1{ \if:w \token_if_chardef_p:N #1 \c_false \else: \if:w \token_if_mathchardef_p:N #1 \c_false \else: \if:w \token_if_int_register_p:N #1 \c_false \else: \if:w \token_if_skip_register_p:N #1 \c_false \else: \if:w \token_if_dim_register_p:N #1 \c_false \else: \if:w \token_if_toks_register_p:N #1 \c_false \else: \c_true \fi: \fi: \fi: \fi: \fi: \fi: } \def_test_function_new:npn {token_if_primitive:N} #1{ \if:w\token_if_primitive_p:N#1} \token_new:Nn \l_peek_token {?} \token_new:Nn \g_peek_token {?} \token_new:Nn \l_peek_search_token {?} \def_new:Npn \peek_after:NN {\tex_futurelet:D \l_peek_token } \def_new:Npn \peek_gafter:NN { \pref_global:D \tex_futurelet:D \g_peek_token } \tlp_new:Nn \l_peek_true_tlp {} \tlp_new:Nn \l_peek_false_tlp {} \def_new:Npn \peek_tmp:w{} \tlp_new:Nn \l_peek_search_tlp{} \def_long_new:Npn \peek_token_generic:NNTF #1#2#3#4{ \let:NN \l_peek_search_token #2 \tlp_set:Nn \l_peek_search_tlp {#2} \tlp_set:Nx \l_peek_true_tlp {\exp_not:n{\group_align_safe_end: #3}} \tlp_set:Nx \l_peek_false_tlp {\exp_not:n{\group_align_safe_end: #4}} \group_align_safe_begin: \peek_after:NN #1 } \def_long_new:Npn \peek_token_remove_generic:NNTF #1#2#3#4{ \let:NN \l_peek_search_token #2 \tlp_set:Nn \l_peek_search_tlp {#2} \tlp_set:Nx \l_peek_true_aux_tlp { \exp_not:n{ #3 } } \tlp_set_eq:NN \l_peek_true_tlp \c_peek_true_remove_next_tlp \tlp_set:Nx \l_peek_false_tlp {\exp_not:n{\group_align_safe_end: #4}} \group_align_safe_begin: \peek_after:NN #1 } \tlp_new:Nn \l_peek_true_aux_tlp {} \tlp_new:Nn \c_peek_true_remove_next_tlp {\group_align_safe_end: \tex_afterassignment:D \l_peek_true_aux_tlp \let:NN \peek_tmp:w } \def_new:Npn \peek_execute_branches_meaning: { \if_meaning:NN \l_peek_token \l_peek_search_token \exp_after:NN \l_peek_true_tlp \else: \exp_after:NN \l_peek_false_tlp \fi: } \def_new:Npn \peek_execute_branches_catcode: { \if_catcode:w \exp_not:N\l_peek_token \exp_not:N\l_peek_search_token \exp_after:NN \l_peek_true_tlp \else: \exp_after:NN \l_peek_false_tlp \fi: } \def_new:Npn \peek_execute_branches_charcode: { \predicate:nTF { \token_if_eq_catcode_p:NN \l_peek_token \c_group_begin_token || \token_if_eq_meaning_p:NN \l_peek_token \c_space_token } { \l_peek_false_tlp } { \exp_after:NN \peek_execute_branches_charcode_aux:NN \l_peek_search_tlp } } \def_long_new:Npn \peek_execute_branches_charcode_aux:NN #1#2{ \if_charcode:w \exp_not:N #1\exp_not:N#2 \exp_after:NN \l_peek_true_tlp \else: \exp_after:NN \l_peek_false_tlp \fi: #2 } \def_new:Npn \peek_meaning:NTF { \peek_token_generic:NNTF \peek_execute_branches_meaning: } \def_new:Npn \peek_meaning_ignore_spaces:NTF { \let:NN \peek_execute_branches: \peek_execute_branches_meaning: \peek_token_generic:NNTF \peek_ignore_spaces_execute_branches: } \def_new:Npn \peek_meaning_remove:NTF { \peek_token_remove_generic:NNTF \peek_execute_branches_meaning: } \def_new:Npn \peek_meaning_remove_ignore_spaces:NTF { \let:NN \peek_execute_branches: \peek_execute_branches_meaning: \peek_token_remove_generic:NNTF \peek_ignore_spaces_execute_branches: } \def_new:Npn \peek_catcode:NTF { \peek_token_generic:NNTF \peek_execute_branches_catcode: } \def_new:Npn \peek_catcode_ignore_spaces:NTF { \let:NN \peek_execute_branches: \peek_execute_branches_catcode: \peek_token_generic:NNTF \peek_ignore_spaces_execute_branches: } \def_new:Npn \peek_catcode_remove:NTF { \peek_token_remove_generic:NNTF \peek_execute_branches_catcode: } \def_new:Npn \peek_catcode_remove_ignore_spaces:NTF { \let:NN \peek_execute_branches: \peek_execute_branches_catcode: \peek_token_remove_generic:NNTF \peek_ignore_spaces_execute_branches: } \def_new:Npn \peek_charcode:NTF { \peek_token_generic:NNTF \peek_execute_branches_charcode: } \def_new:Npn \peek_charcode_ignore_spaces:NTF { \let:NN \peek_execute_branches: \peek_execute_branches_charcode: \peek_token_generic:NNTF \peek_ignore_spaces_execute_branches: } \def_new:Npn \peek_charcode_remove:NTF { \peek_token_remove_generic:NNTF \peek_execute_branches_charcode: } \def_new:Npn \peek_charcode_remove_ignore_spaces:NTF { \let:NN \peek_execute_branches: \peek_execute_branches_charcode: \peek_token_remove_generic:NNTF \peek_ignore_spaces_execute_branches: } \def_new:Npn \peek_ignore_spaces_aux: { \peek_after:NN \peek_ignore_spaces_execute_branches: } \def_new:Npn \peek_ignore_spaces_execute_branches: { \token_if_eq_meaning:NNTF \l_peek_token \c_space_token { \tex_afterassignment:D \peek_ignore_spaces_aux: \let:NN \peek_tmp:w } \peek_execute_branches: } %% %% %% End of file `l3token.sty'.