From 88ad42f78998b5210fa58a05e2e7750f897b2550 Mon Sep 17 00:00:00 2001
From: Karl Berry <karl@freefriends.org>
Date: Mon, 6 Feb 2012 01:22:10 +0000
Subject: l3kernel 3287

git-svn-id: svn://tug.org/texlive/trunk@25306 c570f23f-e606-0410-a88d-b1316a301751
---
 .../texmf-dist/source/latex/l3kernel/l3token.dtx   | 295 ++++++++++++++-------
 1 file changed, 201 insertions(+), 94 deletions(-)

(limited to 'Master/texmf-dist/source/latex/l3kernel/l3token.dtx')

diff --git a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
index 0c145dd47ea..107abc826ee 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
@@ -1,6 +1,6 @@
 % \iffalse meta-comment
 %
-%% File: l3token.dtx Copyright (C) 2005-2011 The LaTeX3 Project
+%% File: l3token.dtx Copyright (C) 2005-2012 The LaTeX3 Project
 %%
 %% It may be distributed and/or modified under the conditions of the
 %% LaTeX Project Public License (LPPL), either version 1.3c of this
@@ -35,7 +35,7 @@
 %
 %<*driver|package>
 \RequirePackage{l3names}
-\GetIdInfo$Id: l3token.dtx 3123 2011-12-31 21:51:51Z joseph $
+\GetIdInfo$Id: l3token.dtx 3234 2012-01-25 12:55:06Z joseph $
   {L3 Experimental token manipulation}
 %</driver|package>
 %<*driver>
@@ -332,6 +332,23 @@
 %   character code given by the  \meta{integer expression} on the
 %   terminal.
 % \end{function}
+% 
+% \begin{variable}[added = 2012-01-23]{\l_char_active_seq}
+%   Used to track which tokens will require special handling at the document
+%   level as they are of category \meta{active} (catcode~$13$). Each entry in
+%   the sequence consists of a single active character. Active tokens should be
+%   added to the sequence when they are defined for general document use.
+% \end{variable}
+% 
+% \begin{variable}[added = 2012-01-23]{\l_char_special_seq}
+%   Used to track which tokens will require special handling when working with
+%   verbatim-like material at the document level as they are not of categories
+%   \meta{letter} (catcode~$11$) or \meta{other} (catcode~$12$). Each entry in
+%   the sequence consists of a single escaped token, for example |\\| for the
+%   backslash or |\{| for an opening brace.^^A \}
+%   Escaped tokens should be added to the sequence when they are defined for
+%   general document use.
+% \end{variable}
 %
 % \section{Generic tokens}
 %
@@ -545,7 +562,7 @@
 %   Tests if the two \meta{tokens} have the same meaning when expanded.
 % \end{function}
 %
-% \begin{function}[updated = 2001-05-23, EXP,pTF]{\token_if_macro:N}
+% \begin{function}[updated = 2011-05-23, EXP,pTF]{\token_if_macro:N}
 %   \begin{syntax}
 %     \cs{token_if_macro_p:N} \meta{token} \\
 %     \cs{token_if_macro:NTF} \meta{token} \Arg{true code} \Arg{false code}
@@ -570,7 +587,7 @@
 %   for an undefined token.
 % \end{function}
 %
-% \begin{function}[EXP,pTF]{\token_if_long_macro:N}
+% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_long_macro:N}
 %   \begin{syntax}
 %     \cs{token_if_long_macro_p:N} \meta{token} \\
 %     \cs{token_if_long_macro:NTF} \meta{token} \Arg{true code} \Arg{false code}
@@ -578,7 +595,7 @@
 %   Tests if the \meta{token} is a long macro.
 % \end{function}
 %
-% \begin{function}[EXP,pTF]{\token_if_protected_macro:N}
+% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_protected_macro:N}
 %   \begin{syntax}
 %     \cs{token_if_protected_macro_p:N} \meta{token} \\
 %     \cs{token_if_protected_macro:NTF} \meta{token} \Arg{true code} \Arg{false code}
@@ -587,7 +604,7 @@
 %   is both protected and long will return logical \texttt{false}.
 % \end{function}
 %
-%% \begin{function}[EXP,pTF]{\token_if_protected_long_macro:N}
+% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_protected_long_macro:N}
 %   \begin{syntax}
 %     \cs{token_if_protected_long_macro_p:N} \meta{token} \\
 %     \cs{token_if_protected_long_macro:NTF} \meta{token} \Arg{true code} \Arg{false code}
@@ -595,15 +612,19 @@
 %   Tests if the \meta{token} is a protected long macro.
 % \end{function}
 %
-% \begin{function}[EXP,pTF]{\token_if_chardef:N}
+% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_chardef:N}
 %   \begin{syntax}
 %     \cs{token_if_chardef_p:N} \meta{token} \\
 %     \cs{token_if_chardef:NTF} \meta{token} \Arg{true code} \Arg{false code}
 %   \end{syntax}
 %   Tests if the \meta{token} is defined to be a chardef.
+%   \begin{texnote}
+%     Booleans, boxes and small integer constants are implemented as
+%     chardefs.
+%   \end{texnote}
 % \end{function}
 %
-% \begin{function}[EXP,pTF]{\token_if_mathchardef:N}
+% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_mathchardef:N}
 %   \begin{syntax}
 %     \cs{token_if_mathchardef_p:N} \meta{token} \\
 %     \cs{token_if_mathchardef:NTF} \meta{token} \Arg{true code} \Arg{false code}
@@ -611,7 +632,7 @@
 %   Tests if the \meta{token} is defined to be a mathchardef.
 % \end{function}
 %
-% \begin{function}[EXP,pTF]{\token_if_dim_register:N}
+% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_dim_register:N}
 %   \begin{syntax}
 %     \cs{token_if_dim_register_p:N} \meta{token} \\
 %     \cs{token_if_dim_register:NTF} \meta{token} \Arg{true code} \Arg{false code}
@@ -619,15 +640,19 @@
 %   Tests if the \meta{token} is defined to be a dimension register.
 % \end{function}
 %
-% \begin{function}[EXP,pTF]{\token_if_int_register:N}
+% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_int_register:N}
 %   \begin{syntax}
 %     \cs{token_if_int_register_p:N} \meta{token} \\
 %     \cs{token_if_int_register:NTF} \meta{token} \Arg{true code} \Arg{false code}
 %   \end{syntax}
 %   Tests if the \meta{token} is defined to be a integer register.
+%   \begin{texnote}
+%     Constant integers may be implemented as integer registers,
+%     chardefs, or mathchardefs depending on their value.
+%   \end{texnote}
 % \end{function}
 %
-% \begin{function}[EXP,pTF]{\token_if_skip_register:N}
+% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_skip_register:N}
 %   \begin{syntax}
 %     \cs{token_if_skip_register_p:N} \meta{token} \\
 %     \cs{token_if_skip_register:NTF} \meta{token} \Arg{true code} \Arg{false code}
@@ -635,7 +660,7 @@
 %   Tests if the \meta{token} is defined to be a skip register.
 % \end{function}
 %
-% \begin{function}[EXP,pTF]{\token_if_toks_register:N}
+% \begin{function}[EXP,pTF, updated=2012-01-20]{\token_if_toks_register:N}
 %   \begin{syntax}
 %     \cs{token_if_toks_register_p:N} \meta{token} \\
 %     \cs{token_if_toks_register:NTF} \meta{token} \Arg{true code} \Arg{false code}
@@ -644,7 +669,7 @@
 %   (not used by\LaTeX3).
 % \end{function}
 %
-% \begin{function}[updated = 2001-05-23, EXP,pTF]{\token_if_primitive:N}
+% \begin{function}[updated = 2011-05-23, EXP,pTF]{\token_if_primitive:N}
 %   \begin{syntax}
 %     \cs{token_if_primitive_p:N} \meta{token} \\
 %     \cs{token_if_primitive:NTF} \meta{token} \Arg{true code} \Arg{false code}
@@ -1231,6 +1256,39 @@
 \group_end:
 %    \end{macrocode}
 % \end{variable}
+% 
+% \begin{variable}{\l_char_active_seq, \l_char_special_seq}
+%   Two sequences for dealing with special characters. The first is characters
+%   which may be active, and contains the active characters themselves  to
+%   allow easy redefinition. The second longer list is for \enquote{special}
+%   characters more generally, and these are escaped so that for example
+%   bulk code assignments can be carried out. In both cases, the order is
+%   by \textsc{ascii} character code (as is done in for example
+%   \cs{ExplSyntaxOn}). The only complication is dealing with |_|, which has to
+%   be done using \cs{tl_to_lowercase:n}
+%    \begin{macrocode}
+\seq_new:N \l_char_active_seq
+\use:n
+  {
+    \group_begin:
+    \char_set_catcode_active:N \"
+    \char_set_catcode_active:N \$
+    \char_set_catcode_active:N \&
+    \char_set_catcode_active:N \^
+    \char_set_catcode_active:N \_
+    \char_set_catcode_active:N \~
+    \use:nn
+      {
+        \group_end:
+        \seq_set_from_clist:Nn \l_char_active_seq
+      }
+  }
+    { { " , $ , & , ^ , _ , ~ } }
+\seq_new:N \l_char_special_seq
+\seq_set_from_clist:Nn \l_char_special_seq
+  { \  , \" , \# , \$ , \% , \& , \\ , \^ , \_ , \{ , \} , \~ }
+%    \end{macrocode}
+% \end{variable}
 %
 % \subsection{Token conditionals}
 %
@@ -1266,7 +1324,7 @@
   {
     \if_catcode:w \exp_not:N #1 \c_math_toggle_token
       \prg_return_true: \else: \prg_return_false: \fi:
-  }
+  } 
 %    \end{macrocode}
 % \end{macro}
 %
@@ -1493,21 +1551,24 @@
 %    \end{macrocode}
 % \end{macro}
 %
-% \begin{macro}[pTF]{\token_if_chardef:N,\token_if_mathchardef:N,
-%   \token_if_long_macro:N,             \token_if_protected_macro:N,
-%   \token_if_protected_long_macro:N,   \token_if_dim_register:N,
-%   \token_if_skip_register:N,          \token_if_int_register:N,
-%   \token_if_toks_register:N}
-% \begin{macro}[aux]{
-%               \token_if_chardef_p_aux:w,
-%               \token_if_mathchardef_p_aux:w,
-%               \token_if_int_register_p_aux:w,
-%               \token_if_skip_register_p_aux:w,
-%               \token_if_dim_register_p_aux:w,
-%               \token_if_toks_register_p_aux:w,
-%               \token_if_protected_macro_p_aux:w,
-%               \token_if_long_macro_p_aux:w,
-%               \token_if_protected_long_macro_p_aux:w}
+% \begin{macro}[pTF]
+%   {
+%     \token_if_chardef:N,               \token_if_mathchardef:N,
+%     \token_if_dim_register:N,          \token_if_int_register:N,
+%     \token_if_skip_register:N,         \token_if_toks_register:N,
+%                                        \token_if_long_macro:N,
+%     \token_if_protected_macro:N,       \token_if_protected_long_macro:N,
+%   }
+% \begin{macro}[aux]
+%   {
+%     \token_if_chardef_aux:w,
+%     \token_if_dim_register_aux:w,
+%     \token_if_int_register_aux:w,
+%     \token_if_skip_register_aux:w,
+%     \token_if_toks_register_aux:w,
+%     \token_if_protected_macro_aux:w,
+%     \token_if_long_macro_aux:w,
+%   }
 %   Most of these functions have to check the meaning of the token in
 %   question so we need to do some checkups on which characters are
 %   output by |\token_to_meaning:N|. As usual, these characters have
@@ -1515,13 +1576,12 @@
 %   below\dots
 %    \begin{macrocode}
 \group_begin:
-  \char_set_lccode:nn { `\T } { `\T }
-  \char_set_lccode:nn { `\F } { `\F }
-  \char_set_lccode:nn { `\X } { `\n }
-  \char_set_lccode:nn { `\Y } { `\t }
-  \char_set_lccode:nn { `\Z } { `\d }
-  \char_set_lccode:nn { `\? } { `\\ }
-  \tl_map_inline:nn { \X \Y \Z \M \C \H \A \R \O \U \S \K \I \P \L \G \P \E }
+  \char_set_lccode:nn { `T } { `T }
+  \char_set_lccode:nn { `F } { `F }
+  \char_set_lccode:nn { `X } { `n }
+  \char_set_lccode:nn { `Y } { `t }
+  \char_set_lccode:nn { `Z } { `d }
+  \tl_map_inline:nn { A C E G H I K L M O P R S U X Y Z R " }
     { \char_set_catcode:nn { `#1 } \c_twelve }
 %    \end{macrocode}
 %   We convert the token list to lower case and restore the catcode and
@@ -1535,117 +1595,164 @@
 %   \tn{chardef} or \tn{mathchardef}. This is easy since \TeX{}
 %   thinks of such tokens as hexadecimal so it stores them as
 %   |\char"|\meta{hex~number} or |\mathchar"|\meta{hex~number}.
+%   Grab until the first occurrence of |char"|, and compare what
+%   preceeds with |\| or |\math|. In fact, the escape character
+%   may not be a backslash, so we compare with the result of
+%   converting some other control sequence to a string, namely
+%   |\char| or |\mathchar| (the auxiliary adds the |char| back).
 %    \begin{macrocode}
     \prg_new_conditional:Npnn \token_if_chardef:N #1 { p , T ,  F , TF }
       {
-        \exp_after:wN \token_if_chardef_aux:w
-          \token_to_meaning:N #1 ?CHAR" \q_stop
+        \str_if_eq_return:xx
+          {
+            \exp_after:wN \token_if_chardef_aux:w
+              \token_to_meaning:N #1 CHAR" \q_stop
+          }
+          { \token_to_str:N \char }
       }
-    \cs_new:Npn \token_if_chardef_aux:w #1 ?CHAR" #2 \q_stop
-      { \tl_if_empty:nTF {#1} { \prg_return_true: } { \prg_return_false: } }
-%    \end{macrocode}
-%
-%    \begin{macrocode}
     \prg_new_conditional:Npnn \token_if_mathchardef:N #1 { p , T ,  F , TF }
       {
-        \exp_after:wN \token_if_mathchardef_aux:w
-          \token_to_meaning:N #1 ?MAYHCHAR" \q_stop
+        \str_if_eq_return:xx
+          {
+            \exp_after:wN \token_if_chardef_aux:w
+              \token_to_meaning:N #1 CHAR" \q_stop
+          }
+          { \token_to_str:N \mathchar }
       }
-    \cs_new:Npn \token_if_mathchardef_aux:w #1 ?MAYHCHAR" #2 \q_stop
-      { \tl_if_empty:nTF {#1} { \prg_return_true: } { \prg_return_false: } }
+    \cs_new:Npn \token_if_chardef_aux:w #1 CHAR" #2 \q_stop { #1 CHAR }
 %    \end{macrocode}
-%   Integer registers are a little more difficult since they expand to
-%   |\count|\meta{number} and there is also a primitive |\countdef|. So
-%   we have to check for that primitive as well.
+%
+%   Dim registers are a little more difficult since their \tn{meaning}
+%   has the form |\dimen|\meta{number}, and we must take care of the
+%   two primitives \tn{dimen} and \tn{dimendef}.
 %    \begin{macrocode}
-    \prg_new_conditional:Npnn \token_if_int_register:N #1 { p , T ,  F , TF }
+    \prg_new_conditional:Npnn \token_if_dim_register:N #1 { p , T ,  F , TF }
       {
-        \if_meaning:w \tex_countdef:D #1
+        \if_meaning:w \tex_dimen:D #1
           \prg_return_false:
         \else:
-          \exp_after:wN \token_if_int_register_aux:w
-            \token_to_meaning:N #1 ?COUXY \q_stop
+          \if_meaning:w \tex_dimendef:D #1
+            \prg_return_false:
+          \else:
+            \str_if_eq_return:xx
+              {
+                \exp_after:wN \token_if_dim_register_aux:w
+                  \token_to_meaning:N #1 ZIMEX \q_stop
+              }
+              { \token_to_str:N \  }
+          \fi:
         \fi:
       }
-    \cs_new:Npn \token_if_int_register_aux:w #1 ?COUXY #2 \q_stop
-      { \tl_if_empty:nTF {#1} { \prg_return_true: } { \prg_return_false: } }
+    \cs_new:Npn \token_if_dim_register_aux:w #1 ZIMEX #2 \q_stop { #1 ~ }
 %    \end{macrocode}
-%   Skip registers are done the same way as the integer registers.
+% Integer registers are one step harder since constants are implemented
+% differently from variables, and we also have to take care of the
+% primitives \tn{count} and \tn{countdef}.
 %    \begin{macrocode}
-    \prg_new_conditional:Npnn \token_if_skip_register:N #1 { p , T ,  F , TF }
+    \prg_new_conditional:Npnn \token_if_int_register:N #1 { p , T ,  F , TF }
       {
-        \if_meaning:w \tex_skipdef:D #1
+        % \token_if_chardef:NTF #1 { \prg_return_true: }
+        %   {
+        %     \token_if_mathchardef:NTF #1 { \prg_return_true: }
+        %       {
+        \if_meaning:w \tex_count:D #1
           \prg_return_false:
         \else:
-          \exp_after:wN \token_if_skip_register_aux:w
-            \token_to_meaning:N #1?SKIP\q_stop
+          \if_meaning:w \tex_countdef:D #1
+            \prg_return_false:
+          \else:
+            \str_if_eq_return:xx
+              {
+                \exp_after:wN \token_if_int_register_aux:w
+                  \token_to_meaning:N #1 COUXY \q_stop
+              }
+              { \token_to_str:N \  }
+          \fi:
         \fi:
+        %       }
+        %   }
       }
-    \cs_new:Npn \token_if_skip_register_aux:w #1 ?SKIP #2 \q_stop
-      { \tl_if_empty:nTF {#1} { \prg_return_true: } { \prg_return_false: } }
+    \cs_new:Npn \token_if_int_register_aux:w #1 COUXY #2 \q_stop { #1 ~ }
 %    \end{macrocode}
-%   Dim registers. No news here
+%   Skip registers are done the same way as the dimension registers.
 %    \begin{macrocode}
-    \prg_new_conditional:Npnn \token_if_dim_register:N #1 { p , T ,  F , TF }
+    \prg_new_conditional:Npnn \token_if_skip_register:N #1 { p , T ,  F , TF }
       {
-        \if_meaning:w \tex_dimendef:D #1
-          \c_false_bool
+        \if_meaning:w \tex_skip:D #1
+          \prg_return_false:
         \else:
-          \exp_after:wN \token_if_dim_register_aux:w
-            \token_to_meaning:N #1 ?ZIMEX \q_stop
+          \if_meaning:w \tex_skipdef:D #1
+            \prg_return_false:
+          \else:
+            \str_if_eq_return:xx
+              {
+                \exp_after:wN \token_if_skip_register_aux:w
+                  \token_to_meaning:N #1 SKIP \q_stop
+              }
+              { \token_to_str:N \  }
+          \fi:
         \fi:
       }
-    \cs_new:Npn \token_if_dim_register_aux:w #1 ?ZIMEX #2 \q_stop
-      { \tl_if_empty:nTF {#1} { \prg_return_true: } { \prg_return_false: } }
+    \cs_new:Npn \token_if_skip_register_aux:w #1 SKIP #2 \q_stop { #1 ~ }
 %    \end{macrocode}
 %   Toks registers.
 %    \begin{macrocode}
     \prg_new_conditional:Npnn \token_if_toks_register:N #1 { p , T ,  F , TF }
       {
-        \if_meaning:w \tex_toksdef:D #1
+        \if_meaning:w \tex_toks:D #1
           \prg_return_false:
         \else:
-          \exp_after:wN \token_if_toks_register_aux:w
-            \token_to_meaning:N #1 ?YOKS \q_stop
+          \if_meaning:w \tex_toksdef:D #1
+            \prg_return_false:
+          \else:
+            \str_if_eq_return:xx
+              {
+                \exp_after:wN \token_if_toks_register_aux:w
+                  \token_to_meaning:N #1 YOKS \q_stop
+              }
+              { \token_to_str:N \  }
+          \fi:
         \fi:
       }
-     \cs_new:Npn \token_if_toks_register_aux:w #1 ?YOKS #2 \q_stop
-       { \tl_if_empty:nTF {#1} { \prg_return_true: } { \prg_return_false: } }
+     \cs_new:Npn \token_if_toks_register_aux:w #1 YOKS #2 \q_stop { #1 ~ }
 %    \end{macrocode}
 %   Protected macros.
 %    \begin{macrocode}
     \prg_new_conditional:Npnn \token_if_protected_macro:N #1
       { p , T ,  F , TF }
       {
-        \exp_after:wN \token_if_protected_macro_aux:w
-          \token_to_meaning:N #1 ?PROYECYEZ~MACRO \q_stop
+        \str_if_eq_return:xx
+          {
+            \exp_after:wN \token_if_protected_macro_aux:w
+              \token_to_meaning:N #1 PROYECYEZ~MACRO \q_stop
+          }
+          { \token_to_str:N \  }
       }
     \cs_new:Npn \token_if_protected_macro_aux:w
-      #1 ?PROYECYEZ~MACRO #2 \q_stop
-      { \tl_if_empty:nTF {#1} { \prg_return_true: } { \prg_return_false: } }
+      #1 PROYECYEZ~MACRO #2 \q_stop { #1 ~ }
 %    \end{macrocode}
-%   Long macros.
+%   Long macros and protected long macros share an auxiliary.
 %    \begin{macrocode}
     \prg_new_conditional:Npnn \token_if_long_macro:N #1 { p , T ,  F , TF }
       {
-        \exp_after:wN \token_if_long_macro_aux:w
-          \token_to_meaning:N #1 ?LOXG~MACRO \q_stop
+        \str_if_eq_return:xx
+          {
+            \exp_after:wN \token_if_long_macro_aux:w
+              \token_to_meaning:N #1 LOXG~MACRO \q_stop
+          }
+          { \token_to_str:N \  }
       }
-    \cs_new:Npn \token_if_long_macro_aux:w #1 ?LOXG~MACRO #2 \q_stop
-      { \tl_if_empty:nTF {#1} { \prg_return_true: } { \prg_return_false: } }
-%    \end{macrocode}
-%   Finally protected long macros where we for once don't have to add an
-%   extra test since there is no primitive for the combined prefixes.
-%    \begin{macrocode}
     \prg_new_conditional:Npnn \token_if_protected_long_macro:N #1
       { p , T ,  F , TF }
       {
-        \exp_after:wN \token_if_protected_long_macro_aux:w
-          \token_to_meaning:N #1 ?PROYECYEZ?LOXG~MACRO \q_stop
+        \str_if_eq_return:xx
+          {
+            \exp_after:wN \token_if_long_macro_aux:w
+              \token_to_meaning:N #1 LOXG~MACRO \q_stop
+          }
+          { \token_to_str:N \protected \token_to_str:N \  }
       }
-    \cs_new:Npn \token_if_protected_long_macro_aux:w
-      #1 ?PROYECYEZ?LOXG~MACRO #2 \q_stop
-      { \tl_if_empty:nTF {#1} { \prg_return_true: } { \prg_return_false: } }
+    \cs_new:Npn \token_if_long_macro_aux:w #1 LOXG~MACRO #2 \q_stop { #1 ~ }
 %    \end{macrocode}
 % Finally the |\tl_to_lowercase:n| ends!
 %    \begin{macrocode}
-- 
cgit v1.2.3

