summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/source/latex/l3kernel/l3token.dtx')
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3token.dtx321
1 files changed, 311 insertions, 10 deletions
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
index cc2cf5d3f4d..3485c03c978 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
@@ -1,6 +1,6 @@
% \iffalse meta-comment
%
-%% File: l3token.dtx Copyright (C) 2005-2015 The LaTeX3 Project
+%% File: l3token.dtx Copyright (C) 2005-2016 The LaTeX3 Project
%%
%% It may be distributed and/or modified under the conditions of the
%% LaTeX Project Public License (LPPL), either version 1.3c of this
@@ -37,7 +37,7 @@
\documentclass[full]{l3doc}
%</driver>
%<*driver|package>
-\GetIdInfo$Id: l3token.dtx 6249 2015-11-11 08:40:42Z joseph $
+\GetIdInfo$Id: l3token.dtx 6465 2016-03-26 16:15:09Z joseph $
{L3 Experimental token manipulation}
%</driver|package>
%<*driver>
@@ -86,11 +86,6 @@
% time the function we're describing can equally well by used on a
% control sequence as such one is one token as well.
%
-% We shall refer to list of tokens as |tlist|s and such lists
-% represented by a single control sequence is a \enquote{token list variable}
-% |tl var|. Functions for these two types are found in the \textsf{l3tl}
-% module.
-%
% \section{All possible tokens}
%
% Let us start by reviewing every case that a given token can fall into.
@@ -110,7 +105,65 @@
% \show_until_if:w \tex_if:D \if_charcode:w \if:w
% \end{verbatim}
%
-% \section{Character tokens}
+% \section{Creating character tokens}
+%
+% \begin{function}[updated = 2015-11-12]
+% {
+% \char_set_active_eq:NN, \char_set_active_eq:Nc,
+% \char_gset_active_eq:NN, \char_gset_active_eq:Nc
+% }
+% \begin{syntax}
+% \cs{char_set_active_eq:NN} \meta{char} \meta{function}
+% \end{syntax}
+% Sets the behaviour of the \meta{char} in situations where it is
+% active (category code $13$) to be equivalent to that of the
+% \meta{function}. The category code of the \meta{char} is
+% \emph{unchanged} by this process. The \meta{function} may itself
+% be an active character.
+% \end{function}
+%
+% \begin{function}[added = 2015-11-12]
+% {
+% \char_set_active_eq:nN, \char_set_active_eq:nc
+% \char_gset_active_eq:nN, \char_gset_active_eq:nc
+% }
+% \begin{syntax}
+% \cs{char_set_active_eq:nN} \Arg{integer expression} \meta{function}
+% \end{syntax}
+% Sets the behaviour of the \meta{char} which has character
+% code as given by the \meta{integer expression} in situations
+% where it is active (category code $13$) to be equivalent to that of the
+% \meta{function}. The category code of the \meta{char} is
+% \emph{unchanged} by this process. The \meta{function} may itself
+% be an active character.
+% \end{function}
+%
+% \begin{function}[EXP, added = 2015-09-09]{\char_generate:nn}
+% \begin{syntax}
+% \cs{char_generate:nn} \Arg{charcode} \Arg{catcode}
+% \end{syntax}
+% Generates a character token of the given \meta{charcode} and \meta{catcode}
+% (both of which may be integer expressions). The \meta{catcode} may be
+% one of
+% \begin{itemize}
+% \item $1$ (begin group)
+% \item $2$ (end group)
+% \item $3$ (math toggle)
+% \item $4$ (alignment)
+% \item $6$ (parameter)
+% \item $7$ (math superscript)
+% \item $8$ (math subscript)
+% \item $11$ (letter)
+% \item $12$ (other)
+% \end{itemize}
+% and other values will raise an error.
+%
+% The \meta{charcode} may be any one valid for the engine in use. Note
+% however that for \XeTeX{} releases prior to 0.99992 only the 8-bit
+% range ($0$ to $255$) is accepted due to engine limitations.
+% \end{function}
+%
+% \section{Manipulating and interrogating character tokens}
%
% \begin{function}[updated = 2015-11-11]
% {
@@ -204,7 +257,7 @@
%
% \begin{function}[updated = 2015-08-06]{\char_set_lccode:nn}
% \begin{syntax}
-% \cs{char_set_lcode:nn} \Arg{intexpr_1} \Arg{intexpr_2}
+% \cs{char_set_lccode:nn} \Arg{intexpr_1} \Arg{intexpr_2}
% \end{syntax}
% Sets up the behaviour of the \meta{character} when
% found inside \cs{tl_to_lowercase:n}, such that \meta{character_1}
@@ -954,6 +1007,18 @@
% not a macro then \cs{scan_stop:} will be left in the input stream
% \end{function}
%
+% \section{Internal functions}
+%
+% \begin{function}[EXP, added = 2016-03-25]{\__char_generate:nn}
+% \begin{syntax}
+% \cs{__char_generate:nn} \Arg{charcode} \Arg{catcode}
+% \end{syntax}
+% This function is identical in operation to the public
+% \cs{char_generate:nn} but omits various sanity tests. In particular, this
+% means it is used in certain places where engine variations need to be
+% accounted for by the kernel.
+% \end{function}
+%
% \end{documentation}
%
% \begin{implementation}
@@ -968,7 +1033,7 @@
%<@@=char>
% \end{macrocode}
%
-% \subsection{Character tokens}
+% \section{Manipulating and interrogating character tokens}
%
% \begin{macro}{\char_set_catcode:nn}
% \begin{macro}{\char_value_catcode:n}
@@ -1180,6 +1245,242 @@
% \end{macrocode}
% \end{variable}
%
+% \section{Creating character tokens}
+%
+% \begin{macro}
+% {^^A
+% \char_set_active_eq:NN, \char_gset_active_eq:NN,
+% \char_set_active_eq:Nc, \char_gset_active_eq:Nc,
+% \char_set_active_eq:nN, \char_gset_active_eq:nN,
+% \char_set_active_eq:nc, \char_gset_active_eq:nc
+% }
+% Four simple functions with very similar definitions, so set up using
+% an auxiliary.
+% \begin{macrocode}
+\group_begin:
+ \char_set_catcode_active:N \^^@
+ \cs_set_protected:Npn \@@_tmp:nN #1#2
+ {
+ \cs_new_protected:cpn { #1 :nN } ##1
+ {
+ \group_begin:
+ \char_set_catcode_active:n { ##1 }
+ \char_set_lccode:nn { `\^^@ } { ##1 }
+ \tex_lowercase:D { \group_end: #2 ^^@ }
+ }
+ \cs_new_protected:cpx { #1 :NN } ##1
+ { \exp_not:c { #1 : nN } { `##1 } }
+ }
+ \@@_tmp:nN { char_set_active_eq } \cs_set_eq:NN
+ \@@_tmp:nN { char_gset_active_eq } \cs_gset_eq:NN
+\group_end:
+\cs_generate_variant:Nn \char_set_active_eq:NN { Nc }
+\cs_generate_variant:Nn \char_gset_active_eq:NN { Nc }
+\cs_generate_variant:Nn \char_set_active_eq:nN { nc }
+\cs_generate_variant:Nn \char_gset_active_eq:nN { nc }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[EXP]{\char_generate:nn}
+% \begin{macro}[EXP, int]{\__char_generate:nn}
+% \begin{macro}[EXP, aux]{\@@_generate_aux:nn}
+% \begin{macro}[EXP, aux]{\@@_generate_aux:nnw}
+% \begin{variable}{\l_@@_tmp_tl}
+% \begin{variable}{\c_@@_max_int}
+% \begin{macro}[EXP, aux]{\@@_generate_invalid_catcode:}
+% The aim here is to generate characters of (broadly) arbitrary category
+% code. Where possible, that is done using engine support (\XeTeX{},
+% \LuaTeX{}). There are though various issues which are covered below. At
+% the interface layer, turn the two arguments into integers up-front so
+% this is only done once.
+% \begin{macrocode}
+\cs_new:Npn \char_generate:nn #1#2
+ {
+ \exp:w \exp_after:wN \@@_generate_aux:w
+ \__int_value:w \__int_eval:w #1 \exp_after:wN ;
+ \__int_value:w \__int_eval:w #2 ;
+ }
+\cs_new:Npn \__char_generate:nn #1#2
+ {
+ \exp:w \exp_after:wN
+ \@@_generate_aux:nnw \exp_after:wN
+ { \__int_value:w \__int_eval:w #1 \exp_after:wN }
+ {#2} \exp_end:
+ }
+% \end{macrocode}
+% Before doing any actual conversion, first some special case filtering.
+% The \tn{Ucharcat} primitive cannot make active chars, so that is turned
+% off here: if the primitive gets altered then the code is already in
+% place for 8-bit engines and will kick in for \LuaTeX{} too. Spaces
+% are also banned here as \LuaTeX{} emulation only makes normal (charcode
+% $32$ spaces. However, |^^@| is filtered out separately as that can't be
+% done with macro emulation either, so is flagged up separately. That
+% done, hand off to the engine-dependent part.
+% \begin{macrocode}
+\cs_new:Npn \@@_generate_aux:w #1 ; #2 ;
+ {
+ \if_int_compare:w #2 = \c_thirteen
+ \__msg_kernel_expandable_error:nn { kernel } { char-active }
+ \else:
+ \if_int_compare:w #2 = \c_ten
+ \if_int_compare:w #1 = \c_zero
+ \__msg_kernel_expandable_error:nn { kernel } { char-null-space }
+ \else:
+ \__msg_kernel_expandable_error:nn { kernel } { char-space }
+ \fi:
+ \else:
+ \if_int_odd:w 0
+ \if_int_compare:w #2 < \c_one 1 \fi:
+ \if_int_compare:w #2 = \c_five 1 \fi:
+ \if_int_compare:w #2 = \c_nine 1 \fi:
+ \if_int_compare:w #2 > \c_thirteen 1 \fi: \exp_stop_f:
+ \__msg_kernel_expandable_error:nn { kernel }
+ { char-invalid-catcode }
+ \else:
+ \if_int_odd:w 0
+ \if_int_compare:w #1 < \c_zero 1 \fi:
+ \if_int_compare:w #1 > \c_@@_max_int 1 \fi: \exp_stop_f:
+ \__msg_kernel_expandable_error:nn { kernel }
+ { char-out-of-range }
+ \else:
+ \@@_generate_aux:nnw {#1} {#2}
+ \fi:
+ \fi:
+ \fi:
+ \fi:
+ \exp_end:
+ }
+\tl_new:N \l_@@_tmp_tl
+% \end{macrocode}
+% Engine-dependent definitions are now needed for the implementation. For
+% \LuaTeX{} and recent \XeTeX{} releases there is engine-level support.
+% They can do cases that macro emulation can't. All of those are filtered
+% out here using a primitive-based boolean expression for speed.
+% The final level is the basic definition at the engine level: the arguments
+% here are integers so there is no need to worry about them too much.
+% \begin{macrocode}
+\group_begin:
+%<*package>
+ \char_set_catcode_active:N \^^L
+ \cs_set_nopar:Npn ^^L { }
+%</package>
+ \char_set_catcode_other:n { 0 }
+ \if_int_odd:w 0
+ \cs_if_exist:NT \luatex_directlua:D { 1 }
+ \cs_if_exist:NT \utex_charcat:D { 1 } \exp_stop_f:
+ \int_const:Nn \c_@@_max_int { 1114111 }
+ \cs_if_exist:NTF \luatex_directlua:D
+ {
+ \cs_new:Npn \@@_generate_aux:nnw #1#2#3 \exp_end:
+ {
+ #3
+ \exp_after:wN \exp_end:
+ \luatex_directlua:D { l3kernel.charcat(#1, #2) }
+ }
+ }
+ {
+ \cs_new:Npn \@@_generate_aux:nnw #1#2#3 \exp_end:
+ {
+ #3
+ \exp_after:wN \exp_end:
+ \utex_charcat:D #1 ~ #2 ~
+ }
+ }
+ \else:
+% \end{macrocode}
+% For engines where \tn{Ucharcat} isn't available (or emulated) then we have
+% to work in macros, and cover only the $8$-bit range. The first stage is
+% to build up a |tl| containing |^^@| with each category code that can
+% be accessed in this way, with an error set up for the other cases. This
+% is all done such that it can be quickly accessed using a |\if_case:w|
+% low-level conditional. There are a few things to notice here.
+% As |^^L| is |\outer| we need to locally set it to avoid a problem.
+% To get open/close braces into the list, they are set up using |\if_false:|
+% pairing here and will later be |x|-type expanded into the desired form.
+% For making spaces, there needs to be an |o|-type expansion of a |\use:n|
+% (or some other tokenization) to avoid dropping the space. We also
+% set up active tokens although they are (currently) filtered out by the
+% interface layer (\tn{Ucharcat} cannot make active tokens).
+% \begin{macrocode}
+ \int_const:Nn \c_@@_max_int { 255 }
+ \tl_set:Nn \l_@@_tmp_tl { \exp_not:N \or: }
+ \char_set_catcode_group_begin:n { 0 } % {
+ \tl_put_right:Nn \l_@@_tmp_tl { ^^@ \if_false: } }
+ \char_set_catcode_group_end:n { 0 }
+ \tl_put_right:Nn \l_@@_tmp_tl { { \fi: \exp_not:N \or: ^^@ } % }
+ \tl_set:Nx \l_@@_tmp_tl { \l_@@_tmp_tl }
+ \char_set_catcode_math_toggle:n { 0 }
+ \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+ \char_set_catcode_alignment:n { 0 }
+ \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+ \tl_put_right:Nn \l_@@_tmp_tl { \or: }
+ \char_set_catcode_parameter:n { 0 }
+ \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+ \char_set_catcode_math_superscript:n { 0 }
+ \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+ \char_set_catcode_math_subscript:n { 0 }
+ \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+ \tl_put_right:Nn \l_@@_tmp_tl { \or: }
+ \char_set_catcode_space:n { 0 }
+ \tl_put_right:No \l_@@_tmp_tl { \use:n { \or: } ^^@ }
+ \char_set_catcode_letter:n { 0 }
+ \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+ \char_set_catcode_other:n { 0 }
+ \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+ \char_set_catcode_active:n { 0 }
+ \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ }
+% \end{macrocode}
+% Convert the above temporary list into a series of constant token
+% lists, one for each character code, using \tn{tex_lowercase:D} to
+% convert |^^@| in each case. The \texttt{x}-type expansion deals with
+% the |\if_false:| stuff introduced earlier. This is done in three parts
+% as |^^L| is awkward. Notice that at this stage |^^@| is active. In
+% format mode this is not needed.
+% \begin{macrocode}
+ \cs_set_protected:Npn \@@_tmp:n #1
+ {
+ \char_set_lccode:nn { 0 } {#1}
+ \char_set_lccode:nn { 32 } {#1}
+ \exp_args:Nx \tex_lowercase:D
+ {
+ \tl_const:Nn
+ \exp_not:c { c_@@_ \__int_to_roman:w #1 _tl }
+ { \exp_not:o \l_@@_tmp_tl }
+ }
+ }
+%<*package>
+ \int_step_function:nnnN { 0 } { 1 } { 11 } \@@_tmp:n
+ \group_begin:
+ \tl_replace_once:Nnn \l_@@_tmp_tl { ^^@ } { \ERROR }
+ \@@_tmp:n { 12 }
+ \group_end:
+ \int_step_function:nnnN { 13 } { 1 } { 255 } \@@_tmp:n
+%</package>
+%<*initex>
+ \int_step_function:nnnN { 0 } { 1 } { 255 } \@@_tmp:n
+%</initex>
+ \cs_new:Npn \@@_generate_aux:nnw #1#2#3 \exp_end:
+ {
+ #3
+ \exp_after:wN \exp_after:wN
+ \exp_after:wN \exp_end:
+ \exp_after:wN \exp_after:wN
+ \if_case:w #2
+ \exp_last_unbraced:Nv \exp_stop_f:
+ { c_@@_ \__int_to_roman:w #1 _tl }
+ \fi:
+ }
+ \fi:
+\group_end:
+% \end{macrocode}
+% \end{macro}
+% \end{variable}
+% \end{variable}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
% \subsection{Generic tokens}
%
% \begin{macrocode}