summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/source/latex/l3kernel/l3token.dtx')
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3token.dtx283
1 files changed, 220 insertions, 63 deletions
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
index cd310c06d3a..586097d48a8 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
@@ -1,6 +1,6 @@
% \iffalse meta-comment
%
-%% File: l3token.dtx Copyright (C) 2005-2014 The LaTeX3 Project
+%% File: l3token.dtx Copyright (C) 2005-2015 The LaTeX3 Project
%%
%% It may be distributed and/or modified under the conditions of the
%% LaTeX Project Public License (LPPL), either version 1.3c of this
@@ -37,7 +37,7 @@
\documentclass[full]{l3doc}
%</driver>
%<*driver|package>
-\GetIdInfo$Id: l3token.dtx 5422 2014-09-15 10:44:23Z joseph $
+\GetIdInfo$Id: l3token.dtx 5893 2015-08-26 16:16:52Z mittelba $
{L3 Experimental token manipulation}
%</driver|package>
%<*driver>
@@ -98,11 +98,11 @@
% and what it looks like.
%
% For instance, \cs{if:w}, \cs{if_charcode:w}, and \cs{tex_if:D} are
-% three for the same internal operation of \TeX{}, namely the primitive
+% three names for the same internal operation of \TeX{}, namely the primitive
% testing the next two characters for equality of their character code.
% They behave identically in many situations. However, \TeX{}
% distinguishes them when searching for a delimited argument. Namely, the
-% example function \cs{show_until_if:w} defined below will take everything
+% example function |\show_until_if:w| defined below will take everything
% until \cs{if:w} as an argument, despite the presence of other copies of
% \cs{if:w} under different names.
% \begin{verbatim}
@@ -112,7 +112,7 @@
%
% \section{Character tokens}
%
-% \begin{function}
+% \begin{function}[updated = 2015-08-09]
% {
% \char_set_catcode_escape:N ,
% \char_set_catcode_group_begin:N ,
@@ -140,10 +140,15 @@
% \begin{verbatim}
% \char_set_catcode_other:N \%
% \end{verbatim}
-% The assignment is local.
+% The assignment is local. These commands update
+% \cs{l_char_special_seq} as appropriate by adding or removing the
+% \meta{character}, and in package mode they also update
+% \tn{dospecials} (plain \TeX{} and \LaTeXe{}) and \cs{@sanitize}
+% (\LaTeXe{} only). The \cs{char_set_catcode_active:N} command
+% also adds the active character to \cs{l_char_active_seq}.
% \end{function}
%
-% \begin{function}
+% \begin{function}[updated = 2015-08-09]
% {
% \char_set_catcode_escape:n ,
% \char_set_catcode_group_begin:n ,
@@ -169,9 +174,14 @@
% code as given by the \meta{integer expression}. This version can be
% used to set up characters which cannot otherwise be given
% (\emph{cf.}~the \texttt{N}-type variants). The assignment is local.
+% These commands update \cs{l_char_special_seq} as appropriate by
+% adding or removing the \meta{character}, and in package mode they
+% also update \tn{dospecials} (plain \TeX{} and \LaTeXe{}) and
+% \cs{@sanitize} (\LaTeXe{} only). The \cs{char_set_catcode_active:n}
+% command also adds the active character to \cs{l_char_active_seq}.
% \end{function}
%
-% \begin{function}{\char_set_catcode:nn}
+% \begin{function}[updated = 2015-08-09]{\char_set_catcode:nn}
% \begin{syntax}
% \cs{char_set_catcode:nn} \Arg{intexpr_1} \Arg{intexpr_2}
% \end{syntax}
@@ -182,6 +192,12 @@
% The setting applies within the current \TeX{} group. In general, the
% symbolic functions \cs{char_set_catcode_\meta{type}} should be preferred,
% but there are cases where these lower-level functions may be useful.
+% This command updates \cs{l_char_special_seq} as appropriate by
+% adding or removing the \meta{character}, and in package mode it
+% also updates \tn{dospecials} (plain \TeX{} and \LaTeXe{}) and
+% \cs{@sanitize} (\LaTeXe{} only).
+% If the category code is $13$ (active), the active character is
+% added to \cs{l_char_active_seq}.
% \end{function}
%
% \begin{function}[EXP]{\char_value_catcode:n}
@@ -202,7 +218,7 @@
% terminal.
% \end{function}
%
-% \begin{function}{\char_set_lccode:nn}
+% \begin{function}[updated = 2015-08-06]{\char_set_lccode:nn}
% \begin{syntax}
% \cs{char_set_lcode:nn} \Arg{intexpr_1} \Arg{intexpr_2}
% \end{syntax}
@@ -239,7 +255,7 @@
% terminal.
% \end{function}
%
-% \begin{function}{\char_set_uccode:nn}
+% \begin{function}[updated = 2015-08-06]{\char_set_uccode:nn}
% \begin{syntax}
% \cs{char_set_uccode:nn} \Arg{intexpr_1} \Arg{intexpr_2}
% \end{syntax}
@@ -276,7 +292,7 @@
% terminal.
% \end{function}
%
-% \begin{function}{\char_set_mathcode:nn}
+% \begin{function}[updated = 2015-08-06]{\char_set_mathcode:nn}
% \begin{syntax}
% \cs{char_set_mathcode:nn} \Arg{intexpr_1} \Arg{intexpr_2}
% \end{syntax}
@@ -305,7 +321,7 @@
% terminal.
% \end{function}
%
-% \begin{function}{\char_set_sfcode:nn}
+% \begin{function}[updated = 2015-08-06]{\char_set_sfcode:nn}
% \begin{syntax}
% \cs{char_set_sfcode:nn} \Arg{intexpr_1} \Arg{intexpr_2}
% \end{syntax}
@@ -334,14 +350,17 @@
% terminal.
% \end{function}
%
-% \begin{variable}[added = 2012-01-23]{\l_char_active_seq}
-% Used to track which tokens will require special handling at the document
-% level as they are of category \meta{active} (catcode~$13$). Each entry in
+% \begin{variable}[added = 2012-01-23, updated = 2015-08-09]{\l_char_active_seq}
+% Used to track which tokens may require special handling at the document
+% level as they are (or have been at some point)
+% of category \meta{active} (catcode~$13$). Each entry in
% the sequence consists of a single active character. Active tokens should be
% added to the sequence when they are defined for general document use.
+% This sequence is automatically updated by \cs{char_set_catcode:nn} and
+% more specific functions.
% \end{variable}
%
-% \begin{variable}[added = 2012-01-23]{\l_char_special_seq}
+% \begin{variable}[added = 2012-01-23, updated = 2015-08-09]{\l_char_special_seq}
% Used to track which tokens will require special handling when working with
% verbatim-like material at the document level as they are not of categories
% \meta{letter} (catcode~$11$) or \meta{other} (catcode~$12$). Each entry in
@@ -349,6 +368,8 @@
% backslash or |\{| for an opening brace.^^A \}
% Escaped tokens should be added to the sequence when they are defined for
% general document use.
+% This sequence is automatically updated by \cs{char_set_catcode:nn} and
+% more specific functions.
% \end{variable}
%
% \section{Generic tokens}
@@ -963,7 +984,7 @@
% \end{macrocode}
%
% \begin{macrocode}
-%<@@=token>
+%<@@=char>
% \end{macrocode}
%
% \subsection{Character tokens}
@@ -971,18 +992,137 @@
% \begin{macro}{\char_set_catcode:nn}
% \begin{macro}{\char_value_catcode:n}
% \begin{macro}{\char_show_value_catcode:n}
-% Category code changes.
+% \begin{macro}[aux]
+% {
+% \@@_set_catcode:Nn,
+% \@@_set_catcode:NnNNN,
+% \@@_set_catcode:NNN,
+% }
+% Category code changes. Besides setting the catcode using
+% \cs{tex_catcode:D}, this function keeps up to date the sequences
+% \cs{l_char_active_seq}, \cs{l_char_special_seq}, and in package
+% mode \tn{dospecials} (for plain \TeX{} and \LaTeXe{}) and
+% \tn{@sanitize} (\LaTeXe{} only) as well. The first list contains
+% active characters, and the other three contain a single-character
+% control sequence. These are constructed from the given character
+% code |#1| using the \tn{lowercase} primitive. In package mode
+% there are difficulties because |\+| and the active |^^L| are
+% \tn{outer} macros. This pushes us to only construct the
+% single-character control sequence and the active character with
+% character code |#1| if they are necessary for the update. As a
+% start, construct an \enquote{other} (catcode~$12$) with character
+% code |#1|. The second part of the \texttt{:Nn} auxiliary takes
+% care of \cs{l_char_active_seq}: if the new catcode is $13$
+% (active), then the active character is constructed through
+% \tn{lowercase} and added to the sequence unless it is already in
+% there. The first part is more intricate and distinguishes the
+% case where the original catcode is special (different from letter
+% or other, since for \eTeX{}'s \tn{numexpr}, $11/2=12/2=6$) or not.
+% If it the catcode was special, the single-character control
+% sequence may need to be removed from the token lists and sequence,
+% otherwise it may need to be added to the lists.
+%
+% The addition or removal is done by the \texttt{:NnNNN} auxiliary,
+% which performs the catcode assignment (note here that the arguments
+% of \cs{char_set_catcode:nn} are each evaluated exactly once), then
+% tests whether the new catcode is special or not. If that changed
+% compared to the previous catcode, then some work is needed: pass to
+% the \texttt{:NNN} auxiliary the single-character control sequence,
+% a function acting on token lists, and a function acting on
+% sequences. Defining this auxiliary is tricky: the odd
+% \texttt{f}-expansion expands the two \cs{tl_if_exist:NT} before the
+% definition is performed, and the macro parameter token |#| (we are
+% not yet doing a definition) stops the \texttt{f}-expansion.
+% In \LaTeXe{}, \cs{@sanitize} differs from \cs{dospecials} in that
+% it should not list begin-group and end-group character tokens.
+% For this, patch \cs{@@_set_catcode:Nn} by copying its code
+% using its \texttt{o}-expansion with arguments |#1| and |#2|
+% (braces are needed because each argument is literally two tokens,
+% a macro parameter character and a digit), and appending to it some
+% code that removes the single-character control sequence from
+% \cs{@sanitize} if the (new) catcode is $1$ or $2$.
% \begin{macrocode}
+\group_begin:
+\tex_catcode:D `@ = \c_eleven
+\tex_catcode:D \c_zero = \c_twelve
\cs_new_protected:Npn \char_set_catcode:nn #1#2
- { \tex_catcode:D #1 = \__int_eval:w #2 \__int_eval_end: }
+ {
+ \group_begin:
+ \char_set_lccode:nn { 0 } {#1}
+ \tl_to_lowercase:n
+ { \group_end: \@@_set_catcode:Nn ^^@ } {#2}
+ }
+\tex_catcode:D \c_zero = \c_thirteen
+\cs_new_protected:Npn \@@_set_catcode:Nn #1#2
+ {
+ \int_compare:nNnTF { \tex_catcode:D `#1 / \c_two } = \c_six
+ {
+ \@@_set_catcode:NnNNN #1 {#2} \int_compare:nNnF
+ \tl_put_right:Nn \seq_put_right:Nn
+ }
+ {
+ \@@_set_catcode:NnNNN #1 {#2} \int_compare:nNnT
+ \tl_remove_all:Nn \seq_remove_all:Nn
+ }
+ \int_compare:nNnT { \tex_catcode:D `#1 } = \c_thirteen
+ {
+ \group_begin:
+ \char_set_lccode:nn { 0 } { `#1 }
+ \tl_to_lowercase:n
+ {
+ \group_end:
+ \seq_if_in:NnF \l_char_active_seq { ^^@ }
+ { \seq_put_right:Nn \l_char_active_seq { ^^@ } }
+ }
+ }
+ }
+\cs_new_protected:Npn \@@_set_catcode:NnNNN #1#2#3#4#5
+ {
+ \tex_catcode:D `#1 = \__int_eval:w #2 \__int_eval_end:
+ #3 { \tex_catcode:D `#1 / \c_two } = \c_six
+ {
+ \group_begin: \exp_args:NNc \group_end:
+ \@@_set_catcode:NNN {#1} #4 #5
+ }
+ }
+\exp_args:Nnf \use:n
+ { \cs_new_protected:Npn \@@_set_catcode:NNN #1#2#3 }
+ {
+%<*package>
+ \tl_if_exist:NT \dospecials
+ {
+ \tl_if_exist:NT \@sanitize { #2 \@sanitize { \@makeother #1 } }
+ #2 \dospecials { \do #1 }
+ }
+%</package>
+ #3 \l_char_special_seq {#1}
+ }
+%<*package>
+\tl_if_exist:NT \@sanitize
+ {
+ \exp_args:Nno \use:n
+ { \cs_gset_protected:Npn \@@_set_catcode:Nn #1#2 }
+ {
+ \@@_set_catcode:Nn {#1} {#2}
+ \int_compare:nNnT { \tex_catcode:D `#1 / \c_two } = \c_one
+ {
+ \group_begin: \exp_args:NNNx \group_end:
+ \tl_remove_all:Nn \@sanitize
+ { \exp_not:N \@makeother \exp_not:c {#1} }
+ }
+ }
+ }
+%</package>
\cs_new:Npn \char_value_catcode:n #1
{ \tex_the:D \tex_catcode:D \__int_eval:w #1\__int_eval_end: }
\cs_new_protected:Npn \char_show_value_catcode:n #1
- { \int_show:n { \char_value_catcode:n {#1} } }
+ { \__msg_show_wrap:n { > ~ \char_value_catcode:n {#1} } }
+\group_end:
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
+% \end{macro}
%
% \begin{macro}
% {
@@ -1109,29 +1249,41 @@
% Pretty repetitive, but necessary!
% \begin{macrocode}
\cs_new_protected:Npn \char_set_mathcode:nn #1#2
- { \tex_mathcode:D #1 = \__int_eval:w #2 \__int_eval_end: }
+ {
+ \tex_mathcode:D \__int_eval:w #1 \__int_eval_end:
+ = \__int_eval:w #2 \__int_eval_end:
+ }
\cs_new:Npn \char_value_mathcode:n #1
{ \tex_the:D \tex_mathcode:D \__int_eval:w #1\__int_eval_end: }
\cs_new_protected:Npn \char_show_value_mathcode:n #1
- { \int_show:n { \char_value_mathcode:n {#1} } }
+ { \__msg_show_wrap:n { > ~ \char_value_mathcode:n {#1} } }
\cs_new_protected:Npn \char_set_lccode:nn #1#2
- { \tex_lccode:D #1 = \__int_eval:w #2 \__int_eval_end: }
+ {
+ \tex_lccode:D \__int_eval:w #1 \__int_eval_end:
+ = \__int_eval:w #2 \__int_eval_end:
+ }
\cs_new:Npn \char_value_lccode:n #1
{ \tex_the:D \tex_lccode:D \__int_eval:w #1\__int_eval_end: }
\cs_new_protected:Npn \char_show_value_lccode:n #1
- { \int_show:n { \char_value_lccode:n {#1} } }
+ { \__msg_show_wrap:n { > ~ \char_value_lccode:n {#1} } }
\cs_new_protected:Npn \char_set_uccode:nn #1#2
- { \tex_uccode:D #1 = \__int_eval:w #2 \__int_eval_end: }
+ {
+ \tex_uccode:D \__int_eval:w #1 \__int_eval_end:
+ = \__int_eval:w #2 \__int_eval_end:
+ }
\cs_new:Npn \char_value_uccode:n #1
{ \tex_the:D \tex_uccode:D \__int_eval:w #1\__int_eval_end: }
\cs_new_protected:Npn \char_show_value_uccode:n #1
- { \int_show:n { \char_value_uccode:n {#1} } }
+ { \__msg_show_wrap:n { > ~ \char_value_uccode:n {#1} } }
\cs_new_protected:Npn \char_set_sfcode:nn #1#2
- { \tex_sfcode:D #1 = \__int_eval:w #2 \__int_eval_end: }
+ {
+ \tex_sfcode:D \__int_eval:w #1 \__int_eval_end:
+ = \__int_eval:w #2 \__int_eval_end:
+ }
\cs_new:Npn \char_value_sfcode:n #1
{ \tex_the:D \tex_sfcode:D \__int_eval:w #1\__int_eval_end: }
\cs_new_protected:Npn \char_show_value_sfcode:n #1
- { \int_show:n { \char_value_sfcode:n {#1} } }
+ { \__msg_show_wrap:n { > ~ \char_value_sfcode:n {#1} } }
% \end{macrocode}
% \end{macro}
% \end{macro}
@@ -1146,8 +1298,46 @@
% \end{macro}
% \end{macro}
%
+% \begin{variable}{\l_char_active_seq, \l_char_special_seq}
+% Two sequences for dealing with special characters. They must be defined
+% before \cs{char_set_catcode:nn} can be used. The first is characters
+% which may be active, and contains the active characters themselves to
+% allow easy redefinition. The second longer list is for \enquote{special}
+% characters more generally, and these are escaped so that for example
+% bulk code assignments can be carried out. In both cases, the order is
+% by \textsc{ascii} character code (as is done in for example
+% \cs{ExplSyntaxOn}). The only complication is dealing with |_|, which
+% requires the use of \cs{use:n} \emph{and} \cs{use:nn}.
+% \begin{macrocode}
+\seq_new:N \l_char_special_seq
+\seq_set_split:Nnn \l_char_special_seq { }
+ { \ \" \# \$ \% \& \\ \^ \_ \{ \} \~ }
+\seq_new:N \l_char_active_seq
+\use:n
+ {
+ \group_begin:
+ \char_set_catcode_active:N \"
+ \char_set_catcode_active:N \$
+ \char_set_catcode_active:N \&
+ \char_set_catcode_active:N \^
+ \char_set_catcode_active:N \_
+ \char_set_catcode_active:N \~
+ \use:nn
+ {
+ \group_end:
+ \seq_set_split:Nnn \l_char_active_seq { }
+ }
+ }
+ { { " $ & ^ _ ~ } } %$
+% \end{macrocode}
+% \end{variable}
+%
% \subsection{Generic tokens}
%
+% \begin{macrocode}
+%<@@=token>
+% \end{macrocode}
+%
% \begin{macro}{\token_to_meaning:N, \token_to_meaning:c}
% \begin{macro}{\token_to_str:N, \token_to_str:c}
% These are all defined in \pkg{l3basics}, as they are needed
@@ -1214,39 +1404,6 @@
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\l_char_active_seq, \l_char_special_seq}
-% Two sequences for dealing with special characters. The first is characters
-% which may be active, and contains the active characters themselves to
-% allow easy redefinition. The second longer list is for \enquote{special}
-% characters more generally, and these are escaped so that for example
-% bulk code assignments can be carried out. In both cases, the order is
-% by \textsc{ascii} character code (as is done in for example
-% \cs{ExplSyntaxOn}). The only complication is dealing with |_|, which
-% requires the use of \cs{use:n} \emph{and} \cs{use:nn}.
-% \begin{macrocode}
-\seq_new:N \l_char_active_seq
-\use:n
- {
- \group_begin:
- \char_set_catcode_active:N \"
- \char_set_catcode_active:N \$
- \char_set_catcode_active:N \&
- \char_set_catcode_active:N \^
- \char_set_catcode_active:N \_
- \char_set_catcode_active:N \~
- \use:nn
- {
- \group_end:
- \seq_set_split:Nnn \l_char_active_seq { }
- }
- }
- { { " $ & ^ _ ~ } } %$
-\seq_new:N \l_char_special_seq
-\seq_set_split:Nnn \l_char_special_seq { }
- { \ \" \# \$ \% \& \\ \^ \_ \{ \} \~ }
-% \end{macrocode}
-% \end{variable}
-%
% \subsection{Token conditionals}
%
% \begin{macro}[pTF]{\token_if_group_begin:N}
@@ -2090,14 +2247,14 @@
% \cs{token_if_eq_meaning:NNTF} because \cs{l_peek_token} may be an
% outer macro (coming from non-\LaTeX3 packages). Spaces are removed
% using a side-effect of \texttt{f}-expansion:
-% |\tex_romannumeral:D -`0| removes one space.
+% |\exp:w \exp_end_continue_f:w| removes one space.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \@@_ignore_spaces_execute_branches:
{
\if_meaning:w \l_peek_token \c_space_token
\exp_after:wN \peek_after:Nw
\exp_after:wN \@@_ignore_spaces_execute_branches:
- \tex_romannumeral:D -`0
+ \exp:w \exp_end_continue_f:w
\else:
\exp_after:wN \@@_execute_branches:
\fi: