From d501b4397bbc0f4389add919be80ccfca5b3863e Mon Sep 17 00:00:00 2001
From: Karl Berry <karl@freefriends.org>
Date: Sat, 12 Sep 2015 21:24:44 +0000
Subject: l3 (12sep15)

git-svn-id: svn://tug.org/texlive/trunk@38360 c570f23f-e606-0410-a88d-b1316a301751
---
 .../source/latex/l3kernel/l3candidates.dtx         | 267 +++++++++++++++++++--
 1 file changed, 247 insertions(+), 20 deletions(-)

(limited to 'Master/texmf-dist/source/latex/l3kernel/l3candidates.dtx')

diff --git a/Master/texmf-dist/source/latex/l3kernel/l3candidates.dtx b/Master/texmf-dist/source/latex/l3kernel/l3candidates.dtx
index 3b8cf3368c5..f5ad9fc0a00 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3candidates.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3candidates.dtx
@@ -38,7 +38,7 @@
 \documentclass[full]{l3doc}
 %</driver>
 %<*driver|package>
-\GetIdInfo$Id: l3candidates.dtx 5911 2015-09-03 08:24:00Z joseph $
+\GetIdInfo$Id: l3candidates.dtx 6002 2015-09-11 22:13:30Z joseph $
   {L3 Experimental additions to l3kernel}
 %</driver|package>
 %<*driver>
@@ -822,7 +822,7 @@
 %   letter-like characters which can also be case-changed).  Begin-group and
 %   end-group characters in the \meta{tokens} are normalized and become |{|
 %   and |}|, respectively.
-%   
+%
 %   Importantly, notice that these functions are intended for working with
 %   user text for typesetting. For case changing programmatic data see the
 %   \pkg{l3str} module and discussion there of \cs{str_lower_case:n},
@@ -833,7 +833,7 @@
 % input in the form of token lists or expandable functions will be expanded
 % \emph{unless} it falls within one of the special handling classes described
 % below. This expansion approach means that in general the result of case
-% changing will match the \enquote{natural} outcome expected from a 
+% changing will match the \enquote{natural} outcome expected from a
 % \enquote{functional} approach to case modification. For example
 % \begin{verbatim}
 %   \tl_set:Nn \l_tmpa_tl { hello }
@@ -859,7 +859,7 @@
 %   \end{verbatim}
 %   Material inside math mode is left entirely unchanged: in particular, no
 %   expansion is undertaken.
-%   
+%
 %   Detection of math mode is controlled by the list of tokens in
 %   \cs{l_tl_case_change_math_tl}, which should be in open--close pairs. In
 %   package mode the standard settings is
@@ -875,7 +875,7 @@
 %   the case changing functions are intended to apply to this should not be
 %   an issue.
 % \end{variable}
-%   
+%
 % \begin{variable}{\l_tl_case_change_exclude_tl}
 %   Case changing can be prevented by using any command on the list
 %   \cs{l_tl_case_change_exclude_tl}. Each entry should be a function
@@ -902,7 +902,7 @@
 %   and |\ref| are automatically included in the list for exclusion from
 %   case changing.
 % \end{variable}
-% 
+%
 % In package mode, the case change system will also
 % convert text stored using the \LaTeXe{} \enquote{LICR} approach. This
 % will upper/lower case tokens as implemented for the font encodings
@@ -926,12 +926,12 @@
 % \end{verbatim}
 % When finding the first \enquote{letter} for this process, any content in
 % math mode or covered by \cs{l_tl_case_change_exclude_tl} is ignored.
-% 
+%
 % (Note that the Unicode Consortium describe this as \enquote{title case}, but
 % that in English title case applies on a word-by-word basis. The
 % \enquote{mixed} case implemented here is a lower level concept needed for
 % both \enquote{title} and \enquote{sentence} casing of text.)
-% 
+%
 % \begin{variable}{\l_tl_mixed_case_ignore_tl}
 %   The list of characters to ignore when searching for the first
 %   \enquote{letter} in mixed-casing is determined by
@@ -1040,7 +1040,7 @@
 %      \cs{char_set_active_eq:NN} \meta{char} \meta{function}
 %   \end{syntax}
 %   Sets the behaviour of the \meta{char} in situations where it is
-%   active (category code $13$) to be equivalent to that of the 
+%   active (category code $13$) to be equivalent to that of the
 %   \meta{function}. The category code of the \meta{char} is
 %   \emph{unchanged} by this process. The \meta{function} may itself
 %   be an active character.
@@ -1059,6 +1059,31 @@
 %   be an active character.
 % \end{function}
 %
+% \begin{function}[EXP, added = 2015-09-09]{\char_generate:nn}
+%   \begin{syntax}
+%      \cs{char_generate:nn} \Arg{charcode} \Arg{catcode}
+%   \end{syntax}
+%   Generates a character token of the given \meta{charcode} and \meta{catcode}
+%   (both of which may be integer expressions). The \meta{catcode} may be
+%   one of
+%   \begin{itemize}
+%     \item $1$ (begin group)
+%     \item $2$ (end group)
+%     \item $3$ (math toggle)
+%     \item $4$ (alignment)
+%     \item $6$ (parameter)
+%     \item $7$ (math superscript)
+%     \item $8$ (math subscript)
+%     \item $11$ (letter)
+%     \item $12$ (other)
+%   \end{itemize}
+%   and other values will raise an error.
+%
+%   The \meta{catcode} may be any one valid for the engine in use. Note
+%   however that for \XeTeX{} releases prior to 0.99992 only the 8-bit
+%   range ($0$ to $255$) is accepted due to engine limitations.
+% \end{function}
+%
 % \begin{function}[TF, updated = 2012-12-20]{\peek_N_type:}
 %   \begin{syntax}
 %     \cs{peek_N_type:TF} \Arg{true code} \Arg{false code}
@@ -2969,7 +2994,7 @@
 \cs_generate_variant:Nn \tl_if_head_eq_catcode:nNTF { o }
 %    \end{macrocode}
 % \end{macro}
-% 
+%
 % \begin{macro}[EXP]{\tl_lower_case:n, \tl_upper_case:n, \tl_mixed_case:n}
 % \begin{macro}[EXP]{\tl_lower_case:nn, \tl_upper_case:nn, \tl_mixed_case:nn}
 %   The user level functions here are all wrappers around the internal
@@ -3269,7 +3294,7 @@
 %   a string then examining the first four characters. For Cyrillic, the
 %   fourth character can be used for a second split based on the character
 %   code.
-%   
+%
 %   Note that as this is dependent on \LaTeXe{}, in format mode the code
 %   goes straight to the final phase of handling control sequences.
 %    \begin{macrocode}
@@ -3308,8 +3333,8 @@
 \cs_new:Npn \@@_change_case_cs_cyr:NnNNNNw #1#2#3#4#5#6#7 \q_stop
   {
     \@@_change_case_cs_type:Nnnnn #1
-      { cyrillic } 
-      { 
+      { cyrillic }
+      {
         #2 _
         \int_to_roman:n
           {
@@ -3437,7 +3462,7 @@
 % \end{macro}
 % \end{macro}
 % \end{macro}
-%    
+%
 % \begin{macro}[aux, EXP]{\@@_change_case_lower_sigma:Nnw}
 % \begin{macro}[aux, EXP]{\@@_change_case_lower_sigma:w}
 % \begin{macro}[aux, EXP]{\@@_change_case_lower_sigma:Nw}
@@ -3742,7 +3767,7 @@
 %   to generic routines, but at the cost of making the process rather opaque.
 %   Instead, the approach taken here is to use a dedicated set of functions
 %   which keep the different loop requirements clearly separate.
-%   
+%
 %   The main loop looks for the first \enquote{real} char in the input
 %   (skipping any pre-letter chars). Once one is found, it is case changed to
 %   upper case but first checking that there is not an entry in the exceptions
@@ -3837,7 +3862,7 @@
         \@@_change_case_cs:Nnnn #1 { upper }
           {
             \@@_change_case_loop:wnn
-              #2 \q_recursion_stop { lower } {#3} 
+              #2 \q_recursion_stop { lower } {#3}
           }
           {
             \@@_change_case_cs:N #1
@@ -3991,7 +4016,7 @@
 %</package>
 %    \end{macrocode}
 % \end{variable}
-% 
+%
 % \begin{variable}{\l_tl_mixed_case_ignore_tl}
 %   Characters to skip over when finding the first letter in a word to be
 %   mixed cased.
@@ -4132,7 +4157,7 @@
       \cyrishrt     \CYRISHRT
       \cyrishrtdsc  \CYRISHRTDSC
       \cyrizh       \CYRIZH
-      \cyrje        \CYRJE 
+      \cyrje        \CYRJE
       \cyrk         \CYRK
       \cyrkbeak     \CYRKBEAK
       \cyrkdsc      \CYRKDSC
@@ -4200,7 +4225,7 @@
       \cyrz         \CYRZ
       \cyrzdsc      \CYRZDSC
       \cyrzh        \CYRZH
-      \cyrzhdsc     \CYRZHDSC 
+      \cyrzhdsc     \CYRZHDSC
     }
     { \q_recursion_tail }
   \@@_change_case_setup:nnnn
@@ -4255,7 +4280,7 @@
       \accdialytikatonos       { \exp_stop_f: \LGR@accDialytika }
       \accdialytikavaria       { \exp_stop_f: \LGR@accDialytika }
       \accdialytikaperispomeni { \exp_stop_f: \LGR@accDialytika }
-      \accperispomeni          { \exp_stop_f: \LGR@accdropped }   
+      \accperispomeni          { \exp_stop_f: \LGR@accdropped }
     }
   \tl_const:Nn \c_@@_change_case_acc_lower_tl { }
   \tl_const:Nn \c_@@_change_case_misc_upper_tl
@@ -4340,6 +4365,208 @@
 %    \end{macrocode}
 % \end{macro}
 %
+% \begin{macro}[EXP]{\char_generate:nn}
+% \begin{macro}[EXP, aux]{\@@_generate_aux:nn}
+% \begin{macro}[EXP, aux]{\@@_generate_aux:nnw}
+% \begin{variable}{\l_@@_tmp_tl}
+% \begin{variable}{\c_@@_max_int}
+% \begin{macro}[EXP, aux]{\@@_generate_invalid_catcode:}
+%   The aim here is to generate characters of (broadly) arbitrary category
+%   code. Where possible, that is done using engine support (\XeTeX{},
+%   \LuaTeX{}). There are though various issues which are covered below. At
+%   the interface layer, turn the two arguments into integers up-front so
+%   this is only done once.
+%    \begin{macrocode}
+\cs_new:Npn \char_generate:nn #1#2
+  {
+    \exp:w \exp_after:wN \@@_generate_aux:w
+      \int_use:N \__int_eval:w #1 \exp_after:wN ;
+      \int_use:N \__int_eval:w #2 ;
+  }
+%    \end{macrocode}
+%   Before doing any actual conversion, first some special case filtering.
+%   The \tn{Ucharcat} primitive cannot make active chars, so that is turned
+%   off here: if the primitive gets altered then the code is already in
+%   place for 8-bit engines and will kick in for \LuaTeX{} too. Spaces
+%   are also banned here as \LuaTeX{} emulation only makes normal (charcode
+%   $32$ spaces. However, |^^@| is filtered out separately as that can't be
+%   done with macro emulation either, so is flagged up separately. That
+%   done, hand off to the engine-dependent part.
+%    \begin{macrocode}
+\cs_new:Npn \@@_generate_aux:w #1 ; #2 ;
+  {
+    \if_int_compare:w #2 = \c_thirteen
+      \__msg_kernel_expandable_error:nn { kernel } { char-active }
+    \else:
+      \if_int_compare:w #2 = \c_ten
+        \if_int_compare:w #1 =  \c_zero
+          \__msg_kernel_expandable_error:nn { kernel } { char-null-space }
+        \else:
+          \__msg_kernel_expandable_error:nn { kernel } { char-space }
+        \fi:
+      \else:
+        \if_int_odd:w 0
+            \if_int_compare:w #2 < \c_one      1 \fi:
+            \if_int_compare:w #2 = \c_five     1 \fi:
+            \if_int_compare:w #2 = \c_nine     1 \fi:
+            \if_int_compare:w #2 > \c_thirteen 1 \fi: \exp_stop_f:
+          \__msg_kernel_expandable_error:nn { kernel }
+            { char-invalid-catcode }
+        \else:
+          \if_int_odd:w 0
+            \if_int_compare:w #1 < \c_zero       1 \fi:
+            \if_int_compare:w #1 > \c_@@_max_int 1 \fi: \exp_stop_f:
+            \__msg_kernel_expandable_error:nn { kernel }
+            { char-out-of-range }
+          \else:
+            \@@_generate_aux:nnw {#1} {#2}
+          \fi:
+        \fi:
+      \fi:
+    \fi:
+    \exp_end:
+  }
+\tl_new:N \l_@@_tmp_tl
+%    \end{macrocode}
+%   Engine-dependent definitions are now needed for the implementation. For
+%   \LuaTeX{} and recent \XeTeX{} releases there is engine-level support.
+%   They can do cases that macro emulation can't. All of those are filtered
+%   out here using a primitive-based boolean expression for speed.
+%   The final level is the basic definition at the engine level: the arguments
+%   here are integers so there is no need to worry about them too much.
+%    \begin{macrocode}
+\group_begin:
+%<*package>
+  \cs_set_nopar:Npn ^^L { }
+%</package>
+  \char_set_catcode_other:n { 0 }
+  \if_int_odd:w 0
+      \cs_if_exist:NT \luatex_directlua:D { 1 }
+      \cs_if_exist:NT \utex_charcat:D     { 1 } \exp_stop_f:
+    \int_const:Nn \c_@@_max_int { 1114111 }
+    \cs_if_exist:NTF \luatex_directlua:D
+      {
+        \cs_new:Npn \@@_generate_aux:nnw #1#2#3 \exp_end:
+          {
+            #3
+            \exp_after:wN \exp_end:
+            \luatex_directlua:D { l3kernel.charcat(#1, #2) }
+          }
+      }
+      {
+        \cs_new:Npn \@@_generate_aux:nnw #1#2#3 \exp_end:
+          {
+            #3
+            \exp_after:wN \exp_end:
+            \utex_charcat:D  #1 ~ #2 ~
+          }
+      }
+  \else:
+%    \end{macrocode}
+%   For engines where \tn{Ucharcat} isn't available (or emulated) then we have
+%   to work in macros, and cover only the $8$-bit range. The first stage is
+%   to build up a |tl| containing |^^@| with each category code that can
+%   be accessed in this way, with an error set up for the other cases. This
+%   is all done such that it can be quickly accessed using a |\if_case:w|
+%   low-level conditional. There are a few things to notice here.
+%   As |^^L| is |\outer| we need to locally set it to avoid a problem.
+%   To get open/close braces into the list, they are set up using |\if_false:|
+%   pairing here and will later be |x|-type expanded into the desired form.
+%   For making spaces, there needs to be an |o|-type expansion of a |\use:n|
+%   (or some other tokenization) to avoid dropping the space. We also
+%   set up active tokens although they are (currently) filtered out by the
+%   interface layer (\tn{Ucharcat} cannot make active tokens).
+%    \begin{macrocode}
+      \int_const:Nn \c_@@_max_int { 255 }
+      \tl_set:Nn \l_@@_tmp_tl { \exp_not:N \or: }
+      \char_set_catcode_group_begin:n { 0 } % {
+      \tl_put_right:Nn \l_@@_tmp_tl { ^^@ \if_false: } }
+      \char_set_catcode_group_end:n { 0 }
+      \tl_put_right:Nn \l_@@_tmp_tl { { \fi: \exp_not:N \or: ^^@ } % }
+      \tl_set:Nx \l_@@_tmp_tl { \l_@@_tmp_tl }
+      \char_set_catcode_math_toggle:n { 0 }
+      \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+      \char_set_catcode_alignment:n { 0 }
+      \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+      \tl_put_right:Nn \l_@@_tmp_tl { \or: }
+      \char_set_catcode_parameter:n { 0 }
+      \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+      \char_set_catcode_math_superscript:n { 0 }
+      \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+      \char_set_catcode_math_subscript:n { 0 }
+      \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+      \tl_put_right:Nn \l_@@_tmp_tl { \or: }
+      \char_set_catcode_space:n { 0 }
+      \tl_put_right:No \l_@@_tmp_tl { \use:n { \or: } ^^@ }
+      \char_set_catcode_letter:n { 0 }
+      \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+      \char_set_catcode_other:n { 0 }
+      \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+      \char_set_catcode_active:n { 0 }
+      \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+%    \end{macrocode}
+%   Convert the above temporary list into a series of constant token
+%   lists, one for each character code, using \tn{tex_lowercase:D} to
+%   convert |^^@| in each case. The \texttt{x}-type expansion deals with
+%   the |\if_false:| stuff introduced earlier. This is done in three parts
+%   as |^^L| is awkward. Notice that at this stage |^^@| is active. In
+%   format mode this is not needed.
+%    \begin{macrocode}
+      \cs_set_protected:Npn \@@_tmp:n #1
+        {
+          \char_set_lccode:nn { 0 } {#1}
+          \char_set_lccode:nn { 32 } {#1}
+          \exp_args:Nx \tex_lowercase:D
+            {
+              \tl_const:cn { c_@@_ \__int_to_roman:w #1 _tl }
+                { \exp_not:o \l_@@_tmp_tl }
+            }
+        }
+%<*package>
+      \int_step_function:nnnN { 0 }  { 1 } { 11 }  \@@_tmp:n
+      \group_begin:
+        \tl_replace_once:Nnn \l_@@_tmp_tl { ^^@ } { \ERROR }
+        \@@_tmp:n { 12 }
+      \group_end:
+      \int_step_function:nnnN { 13 } { 1 } { 255 } \@@_tmp:n
+%</package>
+%<*initex>
+      \int_step_function:nnnN { 0 }  { 1 } { 255 }  \@@_tmp:n
+%</initex>
+      \cs_new:Npn \@@_generate_aux:nnw #1#2#3 \exp_end:
+        {
+          #3
+          \exp_after:wN \exp_after:wN
+          \exp_after:wN \exp_end:
+          \exp_after:wN \exp_after:wN
+          \if_case:w #2
+            \exp_last_unbraced:Nv \exp_stop_f:
+              { c_@@_ \__int_to_roman:w #1 _tl }
+          \fi:
+        }
+  \fi:
+\group_end:
+%    \end{macrocode}
+%   Job done, set up a few messages.
+%    \begin{macrocode}
+\__msg_kernel_new:nnn { kernel } { char-active }
+  { Cannot~generate~active~chars. }
+\__msg_kernel_new:nnn { kernel } { char-invalid-catcode }
+  { Invalid~catcode~for~char~generation. }
+\__msg_kernel_new:nnn { kernel } { char-null-space }
+  { Cannot~generate~null~char~as~a~space. }
+\__msg_kernel_new:nnn { kernel } { char-out-of-range }
+  { Charcode~requested~out~of~engine~range. }
+\__msg_kernel_new:nnn { kernel } { char-space }
+  { Cannot~generate~space~chars. }
+%    \end{macrocode}
+% \end{macro}
+% \end{variable}
+% \end{variable}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
 %    \begin{macrocode}
 %<@@=peek>
 %    \end{macrocode}
-- 
cgit v1.2.3

