diff options
Diffstat (limited to 'Master/texmf-dist/source/latex/l3kernel/l3token.dtx')
-rw-r--r-- | Master/texmf-dist/source/latex/l3kernel/l3token.dtx | 283 |
1 files changed, 220 insertions, 63 deletions
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx index cd310c06d3a..586097d48a8 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx @@ -1,6 +1,6 @@ % \iffalse meta-comment % -%% File: l3token.dtx Copyright (C) 2005-2014 The LaTeX3 Project +%% File: l3token.dtx Copyright (C) 2005-2015 The LaTeX3 Project %% %% It may be distributed and/or modified under the conditions of the %% LaTeX Project Public License (LPPL), either version 1.3c of this @@ -37,7 +37,7 @@ \documentclass[full]{l3doc} %</driver> %<*driver|package> -\GetIdInfo$Id: l3token.dtx 5422 2014-09-15 10:44:23Z joseph $ +\GetIdInfo$Id: l3token.dtx 5893 2015-08-26 16:16:52Z mittelba $ {L3 Experimental token manipulation} %</driver|package> %<*driver> @@ -98,11 +98,11 @@ % and what it looks like. % % For instance, \cs{if:w}, \cs{if_charcode:w}, and \cs{tex_if:D} are -% three for the same internal operation of \TeX{}, namely the primitive +% three names for the same internal operation of \TeX{}, namely the primitive % testing the next two characters for equality of their character code. % They behave identically in many situations. However, \TeX{} % distinguishes them when searching for a delimited argument. Namely, the -% example function \cs{show_until_if:w} defined below will take everything +% example function |\show_until_if:w| defined below will take everything % until \cs{if:w} as an argument, despite the presence of other copies of % \cs{if:w} under different names. % \begin{verbatim} @@ -112,7 +112,7 @@ % % \section{Character tokens} % -% \begin{function} +% \begin{function}[updated = 2015-08-09] % { % \char_set_catcode_escape:N , % \char_set_catcode_group_begin:N , @@ -140,10 +140,15 @@ % \begin{verbatim} % \char_set_catcode_other:N \% % \end{verbatim} -% The assignment is local. +% The assignment is local. These commands update +% \cs{l_char_special_seq} as appropriate by adding or removing the +% \meta{character}, and in package mode they also update +% \tn{dospecials} (plain \TeX{} and \LaTeXe{}) and \cs{@sanitize} +% (\LaTeXe{} only). The \cs{char_set_catcode_active:N} command +% also adds the active character to \cs{l_char_active_seq}. % \end{function} % -% \begin{function} +% \begin{function}[updated = 2015-08-09] % { % \char_set_catcode_escape:n , % \char_set_catcode_group_begin:n , @@ -169,9 +174,14 @@ % code as given by the \meta{integer expression}. This version can be % used to set up characters which cannot otherwise be given % (\emph{cf.}~the \texttt{N}-type variants). The assignment is local. +% These commands update \cs{l_char_special_seq} as appropriate by +% adding or removing the \meta{character}, and in package mode they +% also update \tn{dospecials} (plain \TeX{} and \LaTeXe{}) and +% \cs{@sanitize} (\LaTeXe{} only). The \cs{char_set_catcode_active:n} +% command also adds the active character to \cs{l_char_active_seq}. % \end{function} % -% \begin{function}{\char_set_catcode:nn} +% \begin{function}[updated = 2015-08-09]{\char_set_catcode:nn} % \begin{syntax} % \cs{char_set_catcode:nn} \Arg{intexpr_1} \Arg{intexpr_2} % \end{syntax} @@ -182,6 +192,12 @@ % The setting applies within the current \TeX{} group. In general, the % symbolic functions \cs{char_set_catcode_\meta{type}} should be preferred, % but there are cases where these lower-level functions may be useful. +% This command updates \cs{l_char_special_seq} as appropriate by +% adding or removing the \meta{character}, and in package mode it +% also updates \tn{dospecials} (plain \TeX{} and \LaTeXe{}) and +% \cs{@sanitize} (\LaTeXe{} only). +% If the category code is $13$ (active), the active character is +% added to \cs{l_char_active_seq}. % \end{function} % % \begin{function}[EXP]{\char_value_catcode:n} @@ -202,7 +218,7 @@ % terminal. % \end{function} % -% \begin{function}{\char_set_lccode:nn} +% \begin{function}[updated = 2015-08-06]{\char_set_lccode:nn} % \begin{syntax} % \cs{char_set_lcode:nn} \Arg{intexpr_1} \Arg{intexpr_2} % \end{syntax} @@ -239,7 +255,7 @@ % terminal. % \end{function} % -% \begin{function}{\char_set_uccode:nn} +% \begin{function}[updated = 2015-08-06]{\char_set_uccode:nn} % \begin{syntax} % \cs{char_set_uccode:nn} \Arg{intexpr_1} \Arg{intexpr_2} % \end{syntax} @@ -276,7 +292,7 @@ % terminal. % \end{function} % -% \begin{function}{\char_set_mathcode:nn} +% \begin{function}[updated = 2015-08-06]{\char_set_mathcode:nn} % \begin{syntax} % \cs{char_set_mathcode:nn} \Arg{intexpr_1} \Arg{intexpr_2} % \end{syntax} @@ -305,7 +321,7 @@ % terminal. % \end{function} % -% \begin{function}{\char_set_sfcode:nn} +% \begin{function}[updated = 2015-08-06]{\char_set_sfcode:nn} % \begin{syntax} % \cs{char_set_sfcode:nn} \Arg{intexpr_1} \Arg{intexpr_2} % \end{syntax} @@ -334,14 +350,17 @@ % terminal. % \end{function} % -% \begin{variable}[added = 2012-01-23]{\l_char_active_seq} -% Used to track which tokens will require special handling at the document -% level as they are of category \meta{active} (catcode~$13$). Each entry in +% \begin{variable}[added = 2012-01-23, updated = 2015-08-09]{\l_char_active_seq} +% Used to track which tokens may require special handling at the document +% level as they are (or have been at some point) +% of category \meta{active} (catcode~$13$). Each entry in % the sequence consists of a single active character. Active tokens should be % added to the sequence when they are defined for general document use. +% This sequence is automatically updated by \cs{char_set_catcode:nn} and +% more specific functions. % \end{variable} % -% \begin{variable}[added = 2012-01-23]{\l_char_special_seq} +% \begin{variable}[added = 2012-01-23, updated = 2015-08-09]{\l_char_special_seq} % Used to track which tokens will require special handling when working with % verbatim-like material at the document level as they are not of categories % \meta{letter} (catcode~$11$) or \meta{other} (catcode~$12$). Each entry in @@ -349,6 +368,8 @@ % backslash or |\{| for an opening brace.^^A \} % Escaped tokens should be added to the sequence when they are defined for % general document use. +% This sequence is automatically updated by \cs{char_set_catcode:nn} and +% more specific functions. % \end{variable} % % \section{Generic tokens} @@ -963,7 +984,7 @@ % \end{macrocode} % % \begin{macrocode} -%<@@=token> +%<@@=char> % \end{macrocode} % % \subsection{Character tokens} @@ -971,18 +992,137 @@ % \begin{macro}{\char_set_catcode:nn} % \begin{macro}{\char_value_catcode:n} % \begin{macro}{\char_show_value_catcode:n} -% Category code changes. +% \begin{macro}[aux] +% { +% \@@_set_catcode:Nn, +% \@@_set_catcode:NnNNN, +% \@@_set_catcode:NNN, +% } +% Category code changes. Besides setting the catcode using +% \cs{tex_catcode:D}, this function keeps up to date the sequences +% \cs{l_char_active_seq}, \cs{l_char_special_seq}, and in package +% mode \tn{dospecials} (for plain \TeX{} and \LaTeXe{}) and +% \tn{@sanitize} (\LaTeXe{} only) as well. The first list contains +% active characters, and the other three contain a single-character +% control sequence. These are constructed from the given character +% code |#1| using the \tn{lowercase} primitive. In package mode +% there are difficulties because |\+| and the active |^^L| are +% \tn{outer} macros. This pushes us to only construct the +% single-character control sequence and the active character with +% character code |#1| if they are necessary for the update. As a +% start, construct an \enquote{other} (catcode~$12$) with character +% code |#1|. The second part of the \texttt{:Nn} auxiliary takes +% care of \cs{l_char_active_seq}: if the new catcode is $13$ +% (active), then the active character is constructed through +% \tn{lowercase} and added to the sequence unless it is already in +% there. The first part is more intricate and distinguishes the +% case where the original catcode is special (different from letter +% or other, since for \eTeX{}'s \tn{numexpr}, $11/2=12/2=6$) or not. +% If it the catcode was special, the single-character control +% sequence may need to be removed from the token lists and sequence, +% otherwise it may need to be added to the lists. +% +% The addition or removal is done by the \texttt{:NnNNN} auxiliary, +% which performs the catcode assignment (note here that the arguments +% of \cs{char_set_catcode:nn} are each evaluated exactly once), then +% tests whether the new catcode is special or not. If that changed +% compared to the previous catcode, then some work is needed: pass to +% the \texttt{:NNN} auxiliary the single-character control sequence, +% a function acting on token lists, and a function acting on +% sequences. Defining this auxiliary is tricky: the odd +% \texttt{f}-expansion expands the two \cs{tl_if_exist:NT} before the +% definition is performed, and the macro parameter token |#| (we are +% not yet doing a definition) stops the \texttt{f}-expansion. +% In \LaTeXe{}, \cs{@sanitize} differs from \cs{dospecials} in that +% it should not list begin-group and end-group character tokens. +% For this, patch \cs{@@_set_catcode:Nn} by copying its code +% using its \texttt{o}-expansion with arguments |#1| and |#2| +% (braces are needed because each argument is literally two tokens, +% a macro parameter character and a digit), and appending to it some +% code that removes the single-character control sequence from +% \cs{@sanitize} if the (new) catcode is $1$ or $2$. % \begin{macrocode} +\group_begin: +\tex_catcode:D `@ = \c_eleven +\tex_catcode:D \c_zero = \c_twelve \cs_new_protected:Npn \char_set_catcode:nn #1#2 - { \tex_catcode:D #1 = \__int_eval:w #2 \__int_eval_end: } + { + \group_begin: + \char_set_lccode:nn { 0 } {#1} + \tl_to_lowercase:n + { \group_end: \@@_set_catcode:Nn ^^@ } {#2} + } +\tex_catcode:D \c_zero = \c_thirteen +\cs_new_protected:Npn \@@_set_catcode:Nn #1#2 + { + \int_compare:nNnTF { \tex_catcode:D `#1 / \c_two } = \c_six + { + \@@_set_catcode:NnNNN #1 {#2} \int_compare:nNnF + \tl_put_right:Nn \seq_put_right:Nn + } + { + \@@_set_catcode:NnNNN #1 {#2} \int_compare:nNnT + \tl_remove_all:Nn \seq_remove_all:Nn + } + \int_compare:nNnT { \tex_catcode:D `#1 } = \c_thirteen + { + \group_begin: + \char_set_lccode:nn { 0 } { `#1 } + \tl_to_lowercase:n + { + \group_end: + \seq_if_in:NnF \l_char_active_seq { ^^@ } + { \seq_put_right:Nn \l_char_active_seq { ^^@ } } + } + } + } +\cs_new_protected:Npn \@@_set_catcode:NnNNN #1#2#3#4#5 + { + \tex_catcode:D `#1 = \__int_eval:w #2 \__int_eval_end: + #3 { \tex_catcode:D `#1 / \c_two } = \c_six + { + \group_begin: \exp_args:NNc \group_end: + \@@_set_catcode:NNN {#1} #4 #5 + } + } +\exp_args:Nnf \use:n + { \cs_new_protected:Npn \@@_set_catcode:NNN #1#2#3 } + { +%<*package> + \tl_if_exist:NT \dospecials + { + \tl_if_exist:NT \@sanitize { #2 \@sanitize { \@makeother #1 } } + #2 \dospecials { \do #1 } + } +%</package> + #3 \l_char_special_seq {#1} + } +%<*package> +\tl_if_exist:NT \@sanitize + { + \exp_args:Nno \use:n + { \cs_gset_protected:Npn \@@_set_catcode:Nn #1#2 } + { + \@@_set_catcode:Nn {#1} {#2} + \int_compare:nNnT { \tex_catcode:D `#1 / \c_two } = \c_one + { + \group_begin: \exp_args:NNNx \group_end: + \tl_remove_all:Nn \@sanitize + { \exp_not:N \@makeother \exp_not:c {#1} } + } + } + } +%</package> \cs_new:Npn \char_value_catcode:n #1 { \tex_the:D \tex_catcode:D \__int_eval:w #1\__int_eval_end: } \cs_new_protected:Npn \char_show_value_catcode:n #1 - { \int_show:n { \char_value_catcode:n {#1} } } + { \__msg_show_wrap:n { > ~ \char_value_catcode:n {#1} } } +\group_end: % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} +% \end{macro} % % \begin{macro} % { @@ -1109,29 +1249,41 @@ % Pretty repetitive, but necessary! % \begin{macrocode} \cs_new_protected:Npn \char_set_mathcode:nn #1#2 - { \tex_mathcode:D #1 = \__int_eval:w #2 \__int_eval_end: } + { + \tex_mathcode:D \__int_eval:w #1 \__int_eval_end: + = \__int_eval:w #2 \__int_eval_end: + } \cs_new:Npn \char_value_mathcode:n #1 { \tex_the:D \tex_mathcode:D \__int_eval:w #1\__int_eval_end: } \cs_new_protected:Npn \char_show_value_mathcode:n #1 - { \int_show:n { \char_value_mathcode:n {#1} } } + { \__msg_show_wrap:n { > ~ \char_value_mathcode:n {#1} } } \cs_new_protected:Npn \char_set_lccode:nn #1#2 - { \tex_lccode:D #1 = \__int_eval:w #2 \__int_eval_end: } + { + \tex_lccode:D \__int_eval:w #1 \__int_eval_end: + = \__int_eval:w #2 \__int_eval_end: + } \cs_new:Npn \char_value_lccode:n #1 { \tex_the:D \tex_lccode:D \__int_eval:w #1\__int_eval_end: } \cs_new_protected:Npn \char_show_value_lccode:n #1 - { \int_show:n { \char_value_lccode:n {#1} } } + { \__msg_show_wrap:n { > ~ \char_value_lccode:n {#1} } } \cs_new_protected:Npn \char_set_uccode:nn #1#2 - { \tex_uccode:D #1 = \__int_eval:w #2 \__int_eval_end: } + { + \tex_uccode:D \__int_eval:w #1 \__int_eval_end: + = \__int_eval:w #2 \__int_eval_end: + } \cs_new:Npn \char_value_uccode:n #1 { \tex_the:D \tex_uccode:D \__int_eval:w #1\__int_eval_end: } \cs_new_protected:Npn \char_show_value_uccode:n #1 - { \int_show:n { \char_value_uccode:n {#1} } } + { \__msg_show_wrap:n { > ~ \char_value_uccode:n {#1} } } \cs_new_protected:Npn \char_set_sfcode:nn #1#2 - { \tex_sfcode:D #1 = \__int_eval:w #2 \__int_eval_end: } + { + \tex_sfcode:D \__int_eval:w #1 \__int_eval_end: + = \__int_eval:w #2 \__int_eval_end: + } \cs_new:Npn \char_value_sfcode:n #1 { \tex_the:D \tex_sfcode:D \__int_eval:w #1\__int_eval_end: } \cs_new_protected:Npn \char_show_value_sfcode:n #1 - { \int_show:n { \char_value_sfcode:n {#1} } } + { \__msg_show_wrap:n { > ~ \char_value_sfcode:n {#1} } } % \end{macrocode} % \end{macro} % \end{macro} @@ -1146,8 +1298,46 @@ % \end{macro} % \end{macro} % +% \begin{variable}{\l_char_active_seq, \l_char_special_seq} +% Two sequences for dealing with special characters. They must be defined +% before \cs{char_set_catcode:nn} can be used. The first is characters +% which may be active, and contains the active characters themselves to +% allow easy redefinition. The second longer list is for \enquote{special} +% characters more generally, and these are escaped so that for example +% bulk code assignments can be carried out. In both cases, the order is +% by \textsc{ascii} character code (as is done in for example +% \cs{ExplSyntaxOn}). The only complication is dealing with |_|, which +% requires the use of \cs{use:n} \emph{and} \cs{use:nn}. +% \begin{macrocode} +\seq_new:N \l_char_special_seq +\seq_set_split:Nnn \l_char_special_seq { } + { \ \" \# \$ \% \& \\ \^ \_ \{ \} \~ } +\seq_new:N \l_char_active_seq +\use:n + { + \group_begin: + \char_set_catcode_active:N \" + \char_set_catcode_active:N \$ + \char_set_catcode_active:N \& + \char_set_catcode_active:N \^ + \char_set_catcode_active:N \_ + \char_set_catcode_active:N \~ + \use:nn + { + \group_end: + \seq_set_split:Nnn \l_char_active_seq { } + } + } + { { " $ & ^ _ ~ } } %$ +% \end{macrocode} +% \end{variable} +% % \subsection{Generic tokens} % +% \begin{macrocode} +%<@@=token> +% \end{macrocode} +% % \begin{macro}{\token_to_meaning:N, \token_to_meaning:c} % \begin{macro}{\token_to_str:N, \token_to_str:c} % These are all defined in \pkg{l3basics}, as they are needed @@ -1214,39 +1404,6 @@ % \end{macrocode} % \end{variable} % -% \begin{variable}{\l_char_active_seq, \l_char_special_seq} -% Two sequences for dealing with special characters. The first is characters -% which may be active, and contains the active characters themselves to -% allow easy redefinition. The second longer list is for \enquote{special} -% characters more generally, and these are escaped so that for example -% bulk code assignments can be carried out. In both cases, the order is -% by \textsc{ascii} character code (as is done in for example -% \cs{ExplSyntaxOn}). The only complication is dealing with |_|, which -% requires the use of \cs{use:n} \emph{and} \cs{use:nn}. -% \begin{macrocode} -\seq_new:N \l_char_active_seq -\use:n - { - \group_begin: - \char_set_catcode_active:N \" - \char_set_catcode_active:N \$ - \char_set_catcode_active:N \& - \char_set_catcode_active:N \^ - \char_set_catcode_active:N \_ - \char_set_catcode_active:N \~ - \use:nn - { - \group_end: - \seq_set_split:Nnn \l_char_active_seq { } - } - } - { { " $ & ^ _ ~ } } %$ -\seq_new:N \l_char_special_seq -\seq_set_split:Nnn \l_char_special_seq { } - { \ \" \# \$ \% \& \\ \^ \_ \{ \} \~ } -% \end{macrocode} -% \end{variable} -% % \subsection{Token conditionals} % % \begin{macro}[pTF]{\token_if_group_begin:N} @@ -2090,14 +2247,14 @@ % \cs{token_if_eq_meaning:NNTF} because \cs{l_peek_token} may be an % outer macro (coming from non-\LaTeX3 packages). Spaces are removed % using a side-effect of \texttt{f}-expansion: -% |\tex_romannumeral:D -`0| removes one space. +% |\exp:w \exp_end_continue_f:w| removes one space. % \begin{macrocode} \cs_new_protected_nopar:Npn \@@_ignore_spaces_execute_branches: { \if_meaning:w \l_peek_token \c_space_token \exp_after:wN \peek_after:Nw \exp_after:wN \@@_ignore_spaces_execute_branches: - \tex_romannumeral:D -`0 + \exp:w \exp_end_continue_f:w \else: \exp_after:wN \@@_execute_branches: \fi: |