summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/source/latex/l3kernel/l3token.dtx')
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3token.dtx332
1 files changed, 325 insertions, 7 deletions
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
index 36ca7200714..93b1e6ffc55 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
@@ -2,7 +2,7 @@
%
%% File: l3token.dtx
%
-% Copyright (C) 2005-2019 The LaTeX3 Project
+% Copyright (C) 2005-2020 The LaTeX3 Project
%
% It may be distributed and/or modified under the conditions of the
% LaTeX Project Public License (LPPL), either version 1.3c of this
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2019-11-07}
+% \date{Released 2020-01-12}
%
% \maketitle
%
@@ -151,6 +151,26 @@
% \end{texnote}
% \end{function}
%
+% \begin{function}[added = 2020-01-09, EXP]
+% {
+% \char_lowercase:N, \char_uppercase:N,
+% \char_titlecase:N, \char_foldcase:N,
+% \char_str_lowercase:N, \char_str_uppercase:N,
+% \char_str_titlecase:N, \char_str_foldcase:N
+% }
+% \begin{syntax}
+% \cs{char_lowercase:N} \meta{char}
+% \end{syntax}
+% Converts the \meta{char} to the equivalent case-changed character
+% as detailed by the function name (see \cs{str_foldcase:n}
+% and \cs{text_titlecase:n} for details of these terms). The case mapping
+% is carried out with no context-dependence (\emph{cf.}~\cs{text_uppercase:n},
+% \emph{etc.}) The \texttt{str} versions always generate \enquote{other}
+% (category code $12$) characters, whilst the standard versions generate
+% characters with the category code of the \meta{char} (i.e.~only
+% the character code changes).
+% \end{function}
+%
% \begin{variable}[added = 2011-09-05]{\c_catcode_other_space_tl}
% Token list containing one character with category code $12$,
% (\enquote{other}), and character code $32$ (space).
@@ -253,7 +273,7 @@
% \cs{char_set_lccode:nn} \Arg{intexpr_1} \Arg{intexpr_2}
% \end{syntax}
% Sets up the behaviour of the \meta{character} when
-% found inside \cs{tl_lower_case:n}, such that \meta{character_1}
+% found inside \cs{text_lowercase:n}, such that \meta{character_1}
% will be converted into \meta{character_2}. The two \meta{characters}
% may be specified using an \meta{integer expression} for the character code
% concerned. This may include the \TeX{} |`|\meta{character}
@@ -290,7 +310,7 @@
% \cs{char_set_uccode:nn} \Arg{intexpr_1} \Arg{intexpr_2}
% \end{syntax}
% Sets up the behaviour of the \meta{character} when
-% found inside \cs{tl_upper_case:n}, such that \meta{character_1}
+% found inside \cs{text_uppercase:n}, such that \meta{character_1}
% will be converted into \meta{character_2}. The two \meta{characters}
% may be specified using an \meta{integer expression} for the character code
% concerned. This may include the \TeX{} |`|\meta{character}
@@ -1462,9 +1482,7 @@
\tl_put_right:Nn \l_@@_tmp_tl { \or: }
% \end{macrocode}
% For making spaces, there needs to be an |o|-type expansion of a |\use:n|
-% (or some other tokenization) to avoid dropping the space. We also
-% set up active tokens although they are (currently) filtered out by the
-% interface layer (\tn{Ucharcat} cannot make active tokens).
+% (or some other tokenization) to avoid dropping the space.
% \begin{macrocode}
\char_set_catcode_space:n { 0 }
\tl_put_right:No \l_@@_tmp_tl { \use:n { \or: } ^^@ }
@@ -1532,6 +1550,306 @@
% \end{macro}
% \end{macro}
%
+% \begin{macro}[EXP]{\char_to_utfviii_bytes:n}
+% \begin{macro}[EXP]{\@@_to_utfviii_bytes_auxi:n}
+% \begin{macro}[EXP]{\@@_to_utfviii_bytes_auxii:Nnn}
+% \begin{macro}[EXP]{\@@_to_utfviii_bytes_auxiii:n}
+% \begin{macro}[EXP]
+% {
+% \@@_to_utfviii_bytes_outputi:nw ,
+% \@@_to_utfviii_bytes_outputii:nw ,
+% \@@_to_utfviii_bytes_outputiii:nw ,
+% \@@_to_utfviii_bytes_outputiv:nw
+% }
+% \begin{macro}[EXP]
+% {\@@_to_utfviii_bytes_output:nnn, \@@_to_utfviii_bytes_output:fnn}
+% \begin{macro}[EXP]{\@@_to_utfviii_bytes_end:}
+% This code converts a codepoint into the correct UTF-8 representation.
+% In terms of the algorithm itself, see
+% \url{https://en.wikipedia.org/wiki/UTF-8} for the octet pattern.
+% \begin{macrocode}
+\cs_new:Npn \char_to_utfviii_bytes:n #1
+ {
+ \exp_args:Nf \@@_to_utfviii_bytes_auxi:n
+ { \int_eval:n {#1} }
+ }
+\cs_new:Npn \@@_to_utfviii_bytes_auxi:n #1
+ {
+ \if_int_compare:w #1 > "80 \exp_stop_f:
+ \if_int_compare:w #1 < "800 \exp_stop_f:
+ \@@_to_utfviii_bytes_outputi:nw
+ { \@@_to_utfviii_bytes_auxii:Nnn C {#1} { 64 } }
+ \@@_to_utfviii_bytes_outputii:nw
+ { \@@_to_utfviii_bytes_auxiii:n {#1} }
+ \else:
+ \if_int_compare:w #1 < "10000 \exp_stop_f:
+ \@@_to_utfviii_bytes_outputi:nw
+ { \@@_to_utfviii_bytes_auxii:Nnn E {#1} { 64 * 64 } }
+ \@@_to_utfviii_bytes_outputii:nw
+ {
+ \@@_to_utfviii_bytes_auxiii:n
+ { \int_div_truncate:nn {#1} { 64 } }
+ }
+ \@@_to_utfviii_bytes_outputiii:nw
+ { \@@_to_utfviii_bytes_auxiii:n {#1} }
+ \else:
+ \@@_to_utfviii_bytes_outputi:nw
+ {
+ \@@_to_utfviii_bytes_auxii:Nnn F
+ {#1} { 64 * 64 * 64 }
+ }
+ \@@_to_utfviii_bytes_outputii:nw
+ {
+ \@@_to_utfviii_bytes_auxiii:n
+ { \int_div_truncate:nn {#1} { 64 * 64 } }
+ }
+ \@@_to_utfviii_bytes_outputiii:nw
+ {
+ \@@_to_utfviii_bytes_auxiii:n
+ { \int_div_truncate:nn {#1} { 64 } }
+ }
+ \@@_to_utfviii_bytes_outputiv:nw
+ { \@@_to_utfviii_bytes_auxiii:n {#1} }
+ \fi:
+ \fi:
+ \else:
+ \@@_to_utfviii_bytes_outputi:nw {#1}
+ \fi:
+ \@@_to_utfviii_bytes_end: { } { } { } { }
+ }
+\cs_new:Npn \@@_to_utfviii_bytes_auxii:Nnn #1#2#3
+ { "#10 + \int_div_truncate:nn {#2} {#3} }
+\cs_new:Npn \@@_to_utfviii_bytes_auxiii:n #1
+ { \int_mod:nn {#1} { 64 } + 128 }
+\cs_new:Npn \@@_to_utfviii_bytes_outputi:nw
+ #1 #2 \@@_to_utfviii_bytes_end: #3
+ { \@@_to_utfviii_bytes_output:fnn { \int_eval:n {#1} } { } {#2} }
+\cs_new:Npn \@@_to_utfviii_bytes_outputii:nw
+ #1 #2 \@@_to_utfviii_bytes_end: #3#4
+ { \@@_to_utfviii_bytes_output:fnn { \int_eval:n {#1} } { {#3} } {#2} }
+\cs_new:Npn \@@_to_utfviii_bytes_outputiii:nw
+ #1 #2 \@@_to_utfviii_bytes_end: #3#4#5
+ {
+ \@@_to_utfviii_bytes_output:fnn
+ { \int_eval:n {#1} } { {#3} {#4} } {#2}
+ }
+\cs_new:Npn \@@_to_utfviii_bytes_outputiv:nw
+ #1 #2 \@@_to_utfviii_bytes_end: #3#4#5#6
+ {
+ \@@_to_utfviii_bytes_output:fnn
+ { \int_eval:n {#1} } { {#3} {#4} {#5} } {#2}
+ }
+\cs_new:Npn \@@_to_utfviii_bytes_output:nnn #1#2#3
+ {
+ #3
+ \@@_to_utfviii_bytes_end: #2 {#1}
+ }
+\cs_generate_variant:Nn \@@_to_utfviii_bytes_output:nnn { f }
+\cs_new:Npn \@@_to_utfviii_bytes_end: { }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[rEXP]{\char_to_nfd:N}
+% \begin{macro}[rEXP]{\@@_to_nfd:n}
+% \begin{macro}[rEXP]{\@@_to_nfd:Nw}
+% Look up any \textsc{nfd} and recursively produce the result.
+% \begin{macrocode}
+\cs_new:Npn \char_to_nfd:N #1
+ {
+ \cs_if_exist:cTF { c_@@_nfd_ \token_to_str:N #1 _ tl }
+ {
+ \exp_after:wN \exp_after:wN \exp_after:wN \@@_to_nfd:Nw
+ \exp_after:wN \exp_after:wN \exp_after:wN #1
+ \cs:w c_@@_nfd_ \token_to_str:N #1 _ tl \cs_end:
+ \q_stop
+ }
+ { \exp_not:n {#1} }
+ }
+\cs_set_eq:NN \@@_to_nfd:n \char_to_nfd:N
+\cs_new:Npn \@@_to_nfd:Nw #1#2#3 \q_stop
+ {
+ \exp_args:Ne \@@_to_nfd:n
+ { \char_generate:nn { `#2 } { \@@_change_case_catcode:N #1 } }
+ \tl_if_blank:nF {#3}
+ {
+ \exp_args:Ne \@@_to_nfd:n
+ { \char_generate:nn { `#3 } { \char_value_catcode:n { `#3 } } }
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[EXP]
+% {
+% \char_lowercase:N, \char_uppercase:N,
+% \char_titlecase:N, \char_foldcase:N
+% }
+% \begin{macro}[EXP]{\@@_change_case:nNN}
+% \begin{macro}[EXP]{\@@_change_case:nN}
+% \begin{macro}[EXP]{\@@_change_case_multi:nN, \@@_change_case_multi:vN}
+% \begin{macro}[EXP]{\@@_change_case_multi:NNNNw}
+% \begin{macro}[EXP]{\@@_change_case:NNN}
+% \begin{macro}[EXP]{\@@_change_case:NNNN}
+% \begin{macro}[EXP]{\@@_change_case:NN}
+% \begin{macro}[EXP]{\@@_change_case_catcode:N}
+% \begin{macro}[EXP]
+% {
+% \char_str_lowercase:N, \char_str_uppercase:N,
+% \char_str_titlecase:N, \char_str_foldcase:N
+% }
+% \begin{macro}[EXP]{\@@_str_change_case:nNN}
+% \begin{macro}[EXP]{\@@_str_change_case:nN}
+% To ensure that the category codes produced are predictable, every character
+% is re-generated even if it is otherwise unchanged. This makes life a little
+% interesting when we might have multiple output characters: we have to
+% grab each of them and case change them in reverse order to maintain
+% \texttt{f}-type expandability.
+% \begin{macrocode}
+\cs_new:Npn \char_lowercase:N #1
+ { \@@_change_case:nNN { lower } \char_value_lccode:n #1 }
+\cs_new:Npn \char_uppercase:N #1
+ { \@@_change_case:nNN { upper } \char_value_uccode:n #1 }
+\cs_new:Npn \char_titlecase:N #1
+ {
+ \tl_if_exist:cTF { c_@@_titlecase_ \token_to_str:N #1 _tl }
+ {
+ \@@_change_case_multi:vN
+ { c_@@_titlecase_ \token_to_str:N #1 _tl } #1
+ }
+ { \char_uppercase:N #1 }
+ }
+\cs_new:Npn \char_foldcase:N #1
+ { \@@_change_case:nNN { fold } \char_value_lccode:n #1 }
+\cs_new:Npn \@@_change_case:nNN #1#2#3
+ {
+ \tl_if_exist:cTF { c_@@_ #1 case_ \token_to_str:N #3 _tl }
+ {
+ \@@_change_case_multi:vN
+ { c_@@_ #1 case_ \token_to_str:N #3 _tl } #3
+ }
+ { \exp_args:Nf \@@_change_case:nN { #2 { `#3 } } #3 }
+ }
+\cs_new:Npn \@@_change_case:nN #1#2
+ {
+ \int_compare:nNnTF {#1} = 0
+ { #2 }
+ { \char_generate:nn {#1} { \@@_change_case_catcode:N #2 } }
+ }
+\cs_new:Npn \@@_change_case_multi:nN #1#2
+ { \@@_change_case_multi:NNNNw #2 #1 \q_no_value \q_no_value \q_stop }
+\cs_generate_variant:Nn \@@_change_case_multi:nN { v }
+\cs_new:Npn \@@_change_case_multi:NNNNw #1#2#3#4#5 \q_stop
+ {
+ \quark_if_no_value:NTF #4
+ {
+ \quark_if_no_value:NTF #3
+ { \@@_change_case:NN #1 #2 }
+ { \@@_change_case:NNN #1 #2#3 }
+ }
+ { \@@_change_case:NNNN #1 #2#3#4 }
+ }
+\cs_new:Npn \@@_change_case:NNN #1#2#3
+ {
+ \exp_args:Nnf \use:nn
+ { \@@_change_case:NN #1 #2 }
+ { \@@_change_case:NN #1 #3 }
+ }
+\cs_new:Npn \@@_change_case:NNNN #1#2#3#4
+ {
+ \exp_args:Nnff \use:nnn
+ { \@@_change_case:NN #1 #2 }
+ { \@@_change_case:NN #1 #3 }
+ { \@@_change_case:NN #1 #4 }
+ }
+\cs_new:Npn \@@_change_case:NN #1#2
+ { \char_generate:nn { `#2 } { \@@_change_case_catcode:N #1 } }
+\cs_new:Npn \@@_change_case_catcode:N #1
+ {
+ \if_catcode:w \exp_not:N #1 \c_math_toggle_token
+ 3
+ \else:
+ \if_catcode:w \exp_not:N #1 \c_alignment_token
+ 4
+ \else:
+ \if_catcode:w \exp_not:N #1 \c_math_superscript_token
+ 7
+ \else:
+ \if_catcode:w \exp_not:N #1 \c_math_subscript_token
+ 8
+ \else:
+ \if_catcode:w \exp_not:N #1 \c_space_token
+ 10
+ \else:
+ \if_catcode:w \exp_not:N #1 \c_catcode_letter_token
+ 11
+ \else:
+ \if_catcode:w \exp_not:N #1 \c_catcode_other_token
+ 12
+ \else:
+ 13
+ \fi:
+ \fi:
+ \fi:
+ \fi:
+ \fi:
+ \fi:
+ \fi:
+ }
+% \end{macrocode}
+% Same story for the string version, except category code is easier
+% to follow. This of course makes this version significantly faster.
+% \begin{macrocode}
+\cs_new:Npn \char_str_lowercase:N #1
+ { \@@_str_change_case:nNN { lower } \char_value_lccode:n #1 }
+\cs_new:Npn \char_str_uppercase:N #1
+ { \@@_str_change_case:nNN { upper } \char_value_uccode:n #1 }
+\cs_new:Npn \char_str_titlecase:N #1
+ {
+ \tl_if_exist:cTF { c_@@_titlecase_ \token_to_str:N #1 _tl }
+ { \tl_to_str:c { c_@@_titlecase_ \token_to_str:N #1 _tl } }
+ { \char_str_uppercase:N #1 }
+ }
+\cs_new:Npn \char_str_foldcase:N #1
+ { \@@_str_change_case:nNN { fold } \char_value_lccode:n #1 }
+\cs_new:Npn \@@_str_change_case:nNN #1#2#3
+ {
+ \tl_if_exist:cTF { c_@@_ #1 case_ \token_to_str:N #3 _tl }
+ { \tl_to_str:c { c_@@_ #1 case_ \token_to_str:N #3 _tl } }
+ { \exp_args:Nf \@@_str_change_case:nN { #2 { `#3 } } #3 }
+ }
+\cs_new:Npn \@@_str_change_case:nN #1#2
+ {
+ \int_compare:nNnTF {#1} = 0
+ { \tl_to_str:n {#2} }
+ { \char_generate:nn {#1} { 12 } }
+ }
+\cs_if_exist:NF \tex_Uchar:D
+ {
+ \cs_set:Npn \@@_str_change_case:nN #1#2
+ { \tl_to_str:n {#2} }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
% \begin{macro}{\c_catcode_other_space_tl}
% Create a space with category code $12$: an \enquote{other} space.
% \begin{macrocode}
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-06 19:10:00 +0000