diff options
author | Karl Berry <karl@freefriends.org> | 2016-03-26 22:56:01 +0000 |
---|---|---|
committer | Karl Berry <karl@freefriends.org> | 2016-03-26 22:56:01 +0000 |
commit | acbc4b3c1d3629ae290305ff7c6056eb83d1151c (patch) | |
tree | 59c4ad13167b9bad138b2c231cbd6bc9f2c9c311 /Master/texmf-dist/source/latex/l3kernel/l3token.dtx | |
parent | 473c539aefe98fc932ea0ed651464b9f91482938 (diff) |
l3 (26mar16)
git-svn-id: svn://tug.org/texlive/trunk@40146 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/source/latex/l3kernel/l3token.dtx')
-rw-r--r-- | Master/texmf-dist/source/latex/l3kernel/l3token.dtx | 321 |
1 files changed, 311 insertions, 10 deletions
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx index cc2cf5d3f4d..3485c03c978 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx @@ -1,6 +1,6 @@ % \iffalse meta-comment % -%% File: l3token.dtx Copyright (C) 2005-2015 The LaTeX3 Project +%% File: l3token.dtx Copyright (C) 2005-2016 The LaTeX3 Project %% %% It may be distributed and/or modified under the conditions of the %% LaTeX Project Public License (LPPL), either version 1.3c of this @@ -37,7 +37,7 @@ \documentclass[full]{l3doc} %</driver> %<*driver|package> -\GetIdInfo$Id: l3token.dtx 6249 2015-11-11 08:40:42Z joseph $ +\GetIdInfo$Id: l3token.dtx 6465 2016-03-26 16:15:09Z joseph $ {L3 Experimental token manipulation} %</driver|package> %<*driver> @@ -86,11 +86,6 @@ % time the function we're describing can equally well by used on a % control sequence as such one is one token as well. % -% We shall refer to list of tokens as |tlist|s and such lists -% represented by a single control sequence is a \enquote{token list variable} -% |tl var|. Functions for these two types are found in the \textsf{l3tl} -% module. -% % \section{All possible tokens} % % Let us start by reviewing every case that a given token can fall into. @@ -110,7 +105,65 @@ % \show_until_if:w \tex_if:D \if_charcode:w \if:w % \end{verbatim} % -% \section{Character tokens} +% \section{Creating character tokens} +% +% \begin{function}[updated = 2015-11-12] +% { +% \char_set_active_eq:NN, \char_set_active_eq:Nc, +% \char_gset_active_eq:NN, \char_gset_active_eq:Nc +% } +% \begin{syntax} +% \cs{char_set_active_eq:NN} \meta{char} \meta{function} +% \end{syntax} +% Sets the behaviour of the \meta{char} in situations where it is +% active (category code $13$) to be equivalent to that of the +% \meta{function}. The category code of the \meta{char} is +% \emph{unchanged} by this process. The \meta{function} may itself +% be an active character. +% \end{function} +% +% \begin{function}[added = 2015-11-12] +% { +% \char_set_active_eq:nN, \char_set_active_eq:nc +% \char_gset_active_eq:nN, \char_gset_active_eq:nc +% } +% \begin{syntax} +% \cs{char_set_active_eq:nN} \Arg{integer expression} \meta{function} +% \end{syntax} +% Sets the behaviour of the \meta{char} which has character +% code as given by the \meta{integer expression} in situations +% where it is active (category code $13$) to be equivalent to that of the +% \meta{function}. The category code of the \meta{char} is +% \emph{unchanged} by this process. The \meta{function} may itself +% be an active character. +% \end{function} +% +% \begin{function}[EXP, added = 2015-09-09]{\char_generate:nn} +% \begin{syntax} +% \cs{char_generate:nn} \Arg{charcode} \Arg{catcode} +% \end{syntax} +% Generates a character token of the given \meta{charcode} and \meta{catcode} +% (both of which may be integer expressions). The \meta{catcode} may be +% one of +% \begin{itemize} +% \item $1$ (begin group) +% \item $2$ (end group) +% \item $3$ (math toggle) +% \item $4$ (alignment) +% \item $6$ (parameter) +% \item $7$ (math superscript) +% \item $8$ (math subscript) +% \item $11$ (letter) +% \item $12$ (other) +% \end{itemize} +% and other values will raise an error. +% +% The \meta{charcode} may be any one valid for the engine in use. Note +% however that for \XeTeX{} releases prior to 0.99992 only the 8-bit +% range ($0$ to $255$) is accepted due to engine limitations. +% \end{function} +% +% \section{Manipulating and interrogating character tokens} % % \begin{function}[updated = 2015-11-11] % { @@ -204,7 +257,7 @@ % % \begin{function}[updated = 2015-08-06]{\char_set_lccode:nn} % \begin{syntax} -% \cs{char_set_lcode:nn} \Arg{intexpr_1} \Arg{intexpr_2} +% \cs{char_set_lccode:nn} \Arg{intexpr_1} \Arg{intexpr_2} % \end{syntax} % Sets up the behaviour of the \meta{character} when % found inside \cs{tl_to_lowercase:n}, such that \meta{character_1} @@ -954,6 +1007,18 @@ % not a macro then \cs{scan_stop:} will be left in the input stream % \end{function} % +% \section{Internal functions} +% +% \begin{function}[EXP, added = 2016-03-25]{\__char_generate:nn} +% \begin{syntax} +% \cs{__char_generate:nn} \Arg{charcode} \Arg{catcode} +% \end{syntax} +% This function is identical in operation to the public +% \cs{char_generate:nn} but omits various sanity tests. In particular, this +% means it is used in certain places where engine variations need to be +% accounted for by the kernel. +% \end{function} +% % \end{documentation} % % \begin{implementation} @@ -968,7 +1033,7 @@ %<@@=char> % \end{macrocode} % -% \subsection{Character tokens} +% \section{Manipulating and interrogating character tokens} % % \begin{macro}{\char_set_catcode:nn} % \begin{macro}{\char_value_catcode:n} @@ -1180,6 +1245,242 @@ % \end{macrocode} % \end{variable} % +% \section{Creating character tokens} +% +% \begin{macro} +% {^^A +% \char_set_active_eq:NN, \char_gset_active_eq:NN, +% \char_set_active_eq:Nc, \char_gset_active_eq:Nc, +% \char_set_active_eq:nN, \char_gset_active_eq:nN, +% \char_set_active_eq:nc, \char_gset_active_eq:nc +% } +% Four simple functions with very similar definitions, so set up using +% an auxiliary. +% \begin{macrocode} +\group_begin: + \char_set_catcode_active:N \^^@ + \cs_set_protected:Npn \@@_tmp:nN #1#2 + { + \cs_new_protected:cpn { #1 :nN } ##1 + { + \group_begin: + \char_set_catcode_active:n { ##1 } + \char_set_lccode:nn { `\^^@ } { ##1 } + \tex_lowercase:D { \group_end: #2 ^^@ } + } + \cs_new_protected:cpx { #1 :NN } ##1 + { \exp_not:c { #1 : nN } { `##1 } } + } + \@@_tmp:nN { char_set_active_eq } \cs_set_eq:NN + \@@_tmp:nN { char_gset_active_eq } \cs_gset_eq:NN +\group_end: +\cs_generate_variant:Nn \char_set_active_eq:NN { Nc } +\cs_generate_variant:Nn \char_gset_active_eq:NN { Nc } +\cs_generate_variant:Nn \char_set_active_eq:nN { nc } +\cs_generate_variant:Nn \char_gset_active_eq:nN { nc } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[EXP]{\char_generate:nn} +% \begin{macro}[EXP, int]{\__char_generate:nn} +% \begin{macro}[EXP, aux]{\@@_generate_aux:nn} +% \begin{macro}[EXP, aux]{\@@_generate_aux:nnw} +% \begin{variable}{\l_@@_tmp_tl} +% \begin{variable}{\c_@@_max_int} +% \begin{macro}[EXP, aux]{\@@_generate_invalid_catcode:} +% The aim here is to generate characters of (broadly) arbitrary category +% code. Where possible, that is done using engine support (\XeTeX{}, +% \LuaTeX{}). There are though various issues which are covered below. At +% the interface layer, turn the two arguments into integers up-front so +% this is only done once. +% \begin{macrocode} +\cs_new:Npn \char_generate:nn #1#2 + { + \exp:w \exp_after:wN \@@_generate_aux:w + \__int_value:w \__int_eval:w #1 \exp_after:wN ; + \__int_value:w \__int_eval:w #2 ; + } +\cs_new:Npn \__char_generate:nn #1#2 + { + \exp:w \exp_after:wN + \@@_generate_aux:nnw \exp_after:wN + { \__int_value:w \__int_eval:w #1 \exp_after:wN } + {#2} \exp_end: + } +% \end{macrocode} +% Before doing any actual conversion, first some special case filtering. +% The \tn{Ucharcat} primitive cannot make active chars, so that is turned +% off here: if the primitive gets altered then the code is already in +% place for 8-bit engines and will kick in for \LuaTeX{} too. Spaces +% are also banned here as \LuaTeX{} emulation only makes normal (charcode +% $32$ spaces. However, |^^@| is filtered out separately as that can't be +% done with macro emulation either, so is flagged up separately. That +% done, hand off to the engine-dependent part. +% \begin{macrocode} +\cs_new:Npn \@@_generate_aux:w #1 ; #2 ; + { + \if_int_compare:w #2 = \c_thirteen + \__msg_kernel_expandable_error:nn { kernel } { char-active } + \else: + \if_int_compare:w #2 = \c_ten + \if_int_compare:w #1 = \c_zero + \__msg_kernel_expandable_error:nn { kernel } { char-null-space } + \else: + \__msg_kernel_expandable_error:nn { kernel } { char-space } + \fi: + \else: + \if_int_odd:w 0 + \if_int_compare:w #2 < \c_one 1 \fi: + \if_int_compare:w #2 = \c_five 1 \fi: + \if_int_compare:w #2 = \c_nine 1 \fi: + \if_int_compare:w #2 > \c_thirteen 1 \fi: \exp_stop_f: + \__msg_kernel_expandable_error:nn { kernel } + { char-invalid-catcode } + \else: + \if_int_odd:w 0 + \if_int_compare:w #1 < \c_zero 1 \fi: + \if_int_compare:w #1 > \c_@@_max_int 1 \fi: \exp_stop_f: + \__msg_kernel_expandable_error:nn { kernel } + { char-out-of-range } + \else: + \@@_generate_aux:nnw {#1} {#2} + \fi: + \fi: + \fi: + \fi: + \exp_end: + } +\tl_new:N \l_@@_tmp_tl +% \end{macrocode} +% Engine-dependent definitions are now needed for the implementation. For +% \LuaTeX{} and recent \XeTeX{} releases there is engine-level support. +% They can do cases that macro emulation can't. All of those are filtered +% out here using a primitive-based boolean expression for speed. +% The final level is the basic definition at the engine level: the arguments +% here are integers so there is no need to worry about them too much. +% \begin{macrocode} +\group_begin: +%<*package> + \char_set_catcode_active:N \^^L + \cs_set_nopar:Npn ^^L { } +%</package> + \char_set_catcode_other:n { 0 } + \if_int_odd:w 0 + \cs_if_exist:NT \luatex_directlua:D { 1 } + \cs_if_exist:NT \utex_charcat:D { 1 } \exp_stop_f: + \int_const:Nn \c_@@_max_int { 1114111 } + \cs_if_exist:NTF \luatex_directlua:D + { + \cs_new:Npn \@@_generate_aux:nnw #1#2#3 \exp_end: + { + #3 + \exp_after:wN \exp_end: + \luatex_directlua:D { l3kernel.charcat(#1, #2) } + } + } + { + \cs_new:Npn \@@_generate_aux:nnw #1#2#3 \exp_end: + { + #3 + \exp_after:wN \exp_end: + \utex_charcat:D #1 ~ #2 ~ + } + } + \else: +% \end{macrocode} +% For engines where \tn{Ucharcat} isn't available (or emulated) then we have +% to work in macros, and cover only the $8$-bit range. The first stage is +% to build up a |tl| containing |^^@| with each category code that can +% be accessed in this way, with an error set up for the other cases. This +% is all done such that it can be quickly accessed using a |\if_case:w| +% low-level conditional. There are a few things to notice here. +% As |^^L| is |\outer| we need to locally set it to avoid a problem. +% To get open/close braces into the list, they are set up using |\if_false:| +% pairing here and will later be |x|-type expanded into the desired form. +% For making spaces, there needs to be an |o|-type expansion of a |\use:n| +% (or some other tokenization) to avoid dropping the space. We also +% set up active tokens although they are (currently) filtered out by the +% interface layer (\tn{Ucharcat} cannot make active tokens). +% \begin{macrocode} + \int_const:Nn \c_@@_max_int { 255 } + \tl_set:Nn \l_@@_tmp_tl { \exp_not:N \or: } + \char_set_catcode_group_begin:n { 0 } % { + \tl_put_right:Nn \l_@@_tmp_tl { ^^@ \if_false: } } + \char_set_catcode_group_end:n { 0 } + \tl_put_right:Nn \l_@@_tmp_tl { { \fi: \exp_not:N \or: ^^@ } % } + \tl_set:Nx \l_@@_tmp_tl { \l_@@_tmp_tl } + \char_set_catcode_math_toggle:n { 0 } + \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } + \char_set_catcode_alignment:n { 0 } + \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } + \tl_put_right:Nn \l_@@_tmp_tl { \or: } + \char_set_catcode_parameter:n { 0 } + \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } + \char_set_catcode_math_superscript:n { 0 } + \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } + \char_set_catcode_math_subscript:n { 0 } + \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } + \tl_put_right:Nn \l_@@_tmp_tl { \or: } + \char_set_catcode_space:n { 0 } + \tl_put_right:No \l_@@_tmp_tl { \use:n { \or: } ^^@ } + \char_set_catcode_letter:n { 0 } + \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } + \char_set_catcode_other:n { 0 } + \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } + \char_set_catcode_active:n { 0 } + \tl_put_right:Nn \l_@@_tmp_tl { \or: ^^@ } +% \end{macrocode} +% Convert the above temporary list into a series of constant token +% lists, one for each character code, using \tn{tex_lowercase:D} to +% convert |^^@| in each case. The \texttt{x}-type expansion deals with +% the |\if_false:| stuff introduced earlier. This is done in three parts +% as |^^L| is awkward. Notice that at this stage |^^@| is active. In +% format mode this is not needed. +% \begin{macrocode} + \cs_set_protected:Npn \@@_tmp:n #1 + { + \char_set_lccode:nn { 0 } {#1} + \char_set_lccode:nn { 32 } {#1} + \exp_args:Nx \tex_lowercase:D + { + \tl_const:Nn + \exp_not:c { c_@@_ \__int_to_roman:w #1 _tl } + { \exp_not:o \l_@@_tmp_tl } + } + } +%<*package> + \int_step_function:nnnN { 0 } { 1 } { 11 } \@@_tmp:n + \group_begin: + \tl_replace_once:Nnn \l_@@_tmp_tl { ^^@ } { \ERROR } + \@@_tmp:n { 12 } + \group_end: + \int_step_function:nnnN { 13 } { 1 } { 255 } \@@_tmp:n +%</package> +%<*initex> + \int_step_function:nnnN { 0 } { 1 } { 255 } \@@_tmp:n +%</initex> + \cs_new:Npn \@@_generate_aux:nnw #1#2#3 \exp_end: + { + #3 + \exp_after:wN \exp_after:wN + \exp_after:wN \exp_end: + \exp_after:wN \exp_after:wN + \if_case:w #2 + \exp_last_unbraced:Nv \exp_stop_f: + { c_@@_ \__int_to_roman:w #1 _tl } + \fi: + } + \fi: +\group_end: +% \end{macrocode} +% \end{macro} +% \end{variable} +% \end{variable} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% % \subsection{Generic tokens} % % \begin{macrocode} |