summaryrefslogtreecommitdiff
path: root/macros/latex/contrib/l3kernel/l3token.dtx
diff options
context:
space:
mode:
Diffstat (limited to 'macros/latex/contrib/l3kernel/l3token.dtx')
-rw-r--r--macros/latex/contrib/l3kernel/l3token.dtx423
1 files changed, 96 insertions, 327 deletions
diff --git a/macros/latex/contrib/l3kernel/l3token.dtx b/macros/latex/contrib/l3kernel/l3token.dtx
index e8af4d063f..6fa3508c45 100644
--- a/macros/latex/contrib/l3kernel/l3token.dtx
+++ b/macros/latex/contrib/l3kernel/l3token.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2022-11-02}
+% \date{Released 2022-12-17}
%
% \maketitle
%
@@ -155,26 +155,6 @@
% \end{texnote}
% \end{function}
%
-% \begin{function}[added = 2020-01-09, EXP]
-% {
-% \char_lowercase:N, \char_uppercase:N,
-% \char_titlecase:N, \char_foldcase:N,
-% \char_str_lowercase:N, \char_str_uppercase:N,
-% \char_str_titlecase:N, \char_str_foldcase:N
-% }
-% \begin{syntax}
-% \cs{char_lowercase:N} \meta{char}
-% \end{syntax}
-% Converts the \meta{char} to the equivalent case-changed character
-% as detailed by the function name (see \cs{str_foldcase:n}
-% and \cs{text_titlecase:n} for details of these terms). The case mapping
-% is carried out with no context-dependence (\emph{cf.}~\cs{text_uppercase:n},
-% \emph{etc.}) The \texttt{str} versions always generate \enquote{other}
-% (category code $12$) characters, whilst the standard versions generate
-% characters with the category code of the \meta{char} (i.e.~only
-% the character code changes).
-% \end{function}
-%
% \begin{variable}[added = 2011-09-05]{\c_catcode_other_space_tl}
% Token list containing one character with category code $12$,
% (\enquote{other}), and character code $32$ (space).
@@ -1222,6 +1202,28 @@
% characters at the start of the argument are \emph{not} ignored in this
% case (and they prevent brace-stripping).
%
+% \section{Deprecated functions}
+%
+% \begin{function}[added = 2020-01-09, EXP]
+% {
+% \char_lowercase:N, \char_uppercase:N,
+% \char_titlecase:N, \char_foldcase:N,
+% \char_str_lowercase:N, \char_str_uppercase:N,
+% \char_str_titlecase:N, \char_str_foldcase:N
+% }
+% \begin{syntax}
+% \cs{char_lowercase:N} \meta{char}
+% \end{syntax}
+% Converts the \meta{char} to the equivalent case-changed character
+% as detailed by the function name (see \cs{str_foldcase:n}
+% and \cs{text_titlecase:n} for details of these terms). The case mapping
+% is carried out with no context-dependence (\emph{cf.}~\cs{text_uppercase:n},
+% \emph{etc.}) The \texttt{str} versions always generate \enquote{other}
+% (category code $12$) characters, whilst the standard versions generate
+% characters with the category code of the \meta{char} (i.e.~only
+% the character code changes).
+% \end{function}
+%
% \end{documentation}
%
% \begin{implementation}
@@ -1690,304 +1692,78 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}[EXP]{\char_to_utfviii_bytes:n}
-% \begin{macro}[EXP]{\@@_to_utfviii_bytes_auxi:n}
-% \begin{macro}[EXP]{\@@_to_utfviii_bytes_auxii:Nnn}
-% \begin{macro}[EXP]{\@@_to_utfviii_bytes_auxiii:n}
-% \begin{macro}[EXP]
-% {
-% \@@_to_utfviii_bytes_outputi:nw ,
-% \@@_to_utfviii_bytes_outputii:nw ,
-% \@@_to_utfviii_bytes_outputiii:nw ,
-% \@@_to_utfviii_bytes_outputiv:nw
-% }
-% \begin{macro}[EXP]
-% {\@@_to_utfviii_bytes_output:nnn, \@@_to_utfviii_bytes_output:fnn}
-% \begin{macro}[EXP]{\@@_to_utfviii_bytes_end:}
-% This code converts a codepoint into the correct UTF-8 representation.
-% In terms of the algorithm itself, see
-% \url{https://en.wikipedia.org/wiki/UTF-8} for the octet pattern.
-% \begin{macrocode}
-\cs_new:Npn \char_to_utfviii_bytes:n #1
- {
- \exp_args:Nf \@@_to_utfviii_bytes_auxi:n
- { \int_eval:n {#1} }
- }
-\cs_new:Npn \@@_to_utfviii_bytes_auxi:n #1
- {
- \if_int_compare:w #1 > "80 \exp_stop_f:
- \if_int_compare:w #1 < "800 \exp_stop_f:
- \@@_to_utfviii_bytes_outputi:nw
- { \@@_to_utfviii_bytes_auxii:Nnn C {#1} { 64 } }
- \@@_to_utfviii_bytes_outputii:nw
- { \@@_to_utfviii_bytes_auxiii:n {#1} }
- \else:
- \if_int_compare:w #1 < "10000 \exp_stop_f:
- \@@_to_utfviii_bytes_outputi:nw
- { \@@_to_utfviii_bytes_auxii:Nnn E {#1} { 64 * 64 } }
- \@@_to_utfviii_bytes_outputii:nw
- {
- \@@_to_utfviii_bytes_auxiii:n
- { \int_div_truncate:nn {#1} { 64 } }
- }
- \@@_to_utfviii_bytes_outputiii:nw
- { \@@_to_utfviii_bytes_auxiii:n {#1} }
- \else:
- \@@_to_utfviii_bytes_outputi:nw
- {
- \@@_to_utfviii_bytes_auxii:Nnn F
- {#1} { 64 * 64 * 64 }
- }
- \@@_to_utfviii_bytes_outputii:nw
- {
- \@@_to_utfviii_bytes_auxiii:n
- { \int_div_truncate:nn {#1} { 64 * 64 } }
- }
- \@@_to_utfviii_bytes_outputiii:nw
- {
- \@@_to_utfviii_bytes_auxiii:n
- { \int_div_truncate:nn {#1} { 64 } }
- }
- \@@_to_utfviii_bytes_outputiv:nw
- { \@@_to_utfviii_bytes_auxiii:n {#1} }
- \fi:
- \fi:
- \else:
- \@@_to_utfviii_bytes_outputi:nw {#1}
- \fi:
- \@@_to_utfviii_bytes_end: { } { } { } { }
- }
-\cs_new:Npn \@@_to_utfviii_bytes_auxii:Nnn #1#2#3
- { "#10 + \int_div_truncate:nn {#2} {#3} }
-\cs_new:Npn \@@_to_utfviii_bytes_auxiii:n #1
- { \int_mod:nn {#1} { 64 } + 128 }
-\cs_new:Npn \@@_to_utfviii_bytes_outputi:nw
- #1 #2 \@@_to_utfviii_bytes_end: #3
- { \@@_to_utfviii_bytes_output:fnn { \int_eval:n {#1} } { } {#2} }
-\cs_new:Npn \@@_to_utfviii_bytes_outputii:nw
- #1 #2 \@@_to_utfviii_bytes_end: #3#4
- { \@@_to_utfviii_bytes_output:fnn { \int_eval:n {#1} } { {#3} } {#2} }
-\cs_new:Npn \@@_to_utfviii_bytes_outputiii:nw
- #1 #2 \@@_to_utfviii_bytes_end: #3#4#5
- {
- \@@_to_utfviii_bytes_output:fnn
- { \int_eval:n {#1} } { {#3} {#4} } {#2}
- }
-\cs_new:Npn \@@_to_utfviii_bytes_outputiv:nw
- #1 #2 \@@_to_utfviii_bytes_end: #3#4#5#6
- {
- \@@_to_utfviii_bytes_output:fnn
- { \int_eval:n {#1} } { {#3} {#4} {#5} } {#2}
- }
-\cs_new:Npn \@@_to_utfviii_bytes_output:nnn #1#2#3
- {
- #3
- \@@_to_utfviii_bytes_end: #2 {#1}
- }
-\cs_generate_variant:Nn \@@_to_utfviii_bytes_output:nnn { f }
-\cs_new:Npn \@@_to_utfviii_bytes_end: { }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-%
-% \begin{macro}[rEXP]{\char_to_nfd:N}
-% \begin{macro}[rEXP]{\@@_to_nfd:n, \@@_to_nfd:e}
-% \begin{macro}[rEXP]{\@@_to_nfd:Nw}
-% \begin{macro}[rEXP]{\char_to_nfd:n}
-% \begin{macro}[rEXP]{\char_to_nfd:w}
-% \begin{macro}[rEXP]{\char_nfd_generate:n}
-% \begin{macro}[rEXP]{\char_nfd_generate:nnnn}
-% Look up any \textsc{nfd} and recursively produce the result. Having shared
-% code between Unicode and $8$-bit engines would be ideal, but this would be
-% awkward as we have completely different treatment of catcodes, numbers
-% of tokens, etc. The apparent saving becomes more of a headache than it's
-% worth \dots
-% \begin{macrocode}
-\bool_lazy_or:nnTF
- { \sys_if_engine_luatex_p: }
- { \sys_if_engine_xetex_p: }
- {
- \cs_new:Npn \char_to_nfd:N #1
- {
- \cs_if_exist:cTF { c_@@_nfd_ \token_to_str:N #1 _ tl }
- {
- \exp_after:wN \exp_after:wN \exp_after:wN \@@_to_nfd:Nw
- \exp_after:wN \exp_after:wN \exp_after:wN #1
- \cs:w c_@@_nfd_ \token_to_str:N #1 _tl \cs_end:
- \s_@@_stop
- }
- { \exp_not:n {#1} }
- }
- \cs_new_eq:NN \@@_to_nfd:n \char_to_nfd:N
- \cs_generate_variant:Nn \@@_to_nfd:n { e }
- \cs_new:Npn \@@_to_nfd:Nw #1#2#3 \s_@@_stop
- {
- \@@_to_nfd:e
- { \char_generate:nn { `#2 } { \@@_change_case_catcode:N #1 } }
- \tl_if_blank:nF {#3}
- {
- \@@_to_nfd:e
- { \char_generate:nn { `#3 } { \char_value_catcode:n { `#3 } } }
- }
- }
- \cs_new:Npn \char_to_nfd:n #1
- {
- \@@_to_nfd:e { \char_generate:nn {#1} { \char_value_catcode:n {#1} } }
- }
- }
- {
- \cs_new:Npn \char_to_nfd:N #1 { \exp_not:n {#1} }
- \cs_new:Npn \char_to_nfd:n #1
- {
- \int_compare:nNnTF {#1} > { "80 }
- { \exp_args:Ne \@@_to_nfd:n { \@@_nfd_generate:n {#1} } }
- { \@@_nfd_generate:n {#1} }
- }
- \cs_new:Npn \@@_to_nfd:n #1
- {
- \cs_if_exist:cTF { c_@@_nfd_ \tl_to_str:n {#1} _ tl }
- {
- \exp_after:wN \exp_after:wN \exp_after:wN \@@_to_nfd:w
- \cs:w c_@@_nfd_ \tl_to_str:n {#1} _tl \cs_end:
- \s_@@_stop
- }
- { \exp_not:n {#1} }
- }
- \cs_new:Npn \@@_to_nfd:w #1#2 \s_@@_stop
- {
- \@@_to_nfd:n {#1}
- \tl_if_blank:nF {#2}
- { \@@_to_nfd:n {#2} }
- }
- \cs_new:Npn \@@_nfd_generate:n #1
- {
- \use:e
- {
- \exp_not:N \@@_nfd_generate:nnnn
- \char_to_utfviii_bytes:n {#1}
- }
- }
- \cs_new:Npn \@@_nfd_generate:nnnn #1#2#3#4
- {
- \tl_if_blank:nTF {#2}
- { \char_generate:nn {#1} { \char_value_catcode:n {#1} } }
- {
- \exp_after:wN \exp_after:wN \exp_after:wN
- \exp_not:N \char_generate:nn {#1} { 13 }
- \exp_after:wN \exp_after:wN \exp_after:wN
- \exp_not:N \char_generate:nn {#2} { 13 }
- \tl_if_blank:nF {#3}
- {
- \exp_after:wN \exp_after:wN \exp_after:wN
- \exp_not:N \char_generate:nn {#3} { 13 }
- \tl_if_blank:nF {#4}
- {
- \exp_after:wN \exp_after:wN \exp_after:wN
- \exp_not:N \char_generate:nn {#4} { 13 }
- }
- }
- }
-
- }
- }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-%
% \begin{macro}[EXP]
% {
% \char_lowercase:N, \char_uppercase:N,
% \char_titlecase:N, \char_foldcase:N
% }
-% \begin{macro}[EXP]{\@@_change_case:nNN}
% \begin{macro}[EXP]{\@@_change_case:nN}
-% \begin{macro}[EXP]{\@@_change_case_multi:nN, \@@_change_case_multi:vN}
-% \begin{macro}[EXP]{\@@_change_case_multi:NNNNw}
-% \begin{macro}[EXP]{\@@_change_case:NNN}
-% \begin{macro}[EXP]{\@@_change_case:NNNN}
-% \begin{macro}[EXP]{\@@_change_case:NN}
+% \begin{macro}[EXP]{\@@_change_case:nnnN}
+% \begin{macro}[EXP]{\@@_change_case_auxi:nN, \@@_change_case_auxii:nN}
% \begin{macro}[EXP]{\@@_change_case_catcode:N}
% \begin{macro}[EXP]
% {
% \char_str_lowercase:N, \char_str_uppercase:N,
% \char_str_titlecase:N, \char_str_foldcase:N
% }
-% \begin{macro}[EXP]{\@@_str_change_case:nNN}
-% \begin{macro}[EXP]{\@@_str_change_case:nN}
+% \begin{macro}[EXP]{\@@_str_change_case:nN, \@@_str_change_case_aux:nN}
+% \begin{macro}[EXP]{\@@_str_change_case:nnnN}
+% \begin{macro}[EXP]{\@@_str_change_case:n}
% To ensure that the category codes produced are predictable, every character
-% is re-generated even if it is otherwise unchanged. This makes life a little
-% interesting when we might have multiple output characters: we have to
-% grab each of them and case change them in reverse order to maintain
-% \texttt{f}-type expandability.
-% \begin{macrocode}
-\cs_new:Npn \char_lowercase:N #1
- { \@@_change_case:nNN { lower } \char_value_lccode:n #1 }
-\cs_new:Npn \char_uppercase:N #1
- { \@@_change_case:nNN { upper } \char_value_uccode:n #1 }
-\cs_new:Npn \char_titlecase:N #1
+% is re-generated even if it is otherwise unchanged.
+% \begin{macrocode}
+\cs_new:Npn \char_lowercase:N
+ { \@@_change_case:nN { lowercase } }
+\cs_new:Npn \char_uppercase:N
+ { \@@_change_case:nN { uppercase } }
+\cs_new:Npn \char_titlecase:N
+ { \@@_change_case:nN { titlecase } }
+\cs_new:Npn \char_foldcase:N
+ { \@@_change_case:nN { casefold } }
+\cs_new:Npn \@@_change_case:nN #1#2
{
- \tl_if_exist:cTF { c_@@_titlecase_ \token_to_str:N #1 _tl }
+ \int_compare:nNnTF {`#2} = { `\ }
+ { ~ }
{
- \@@_change_case_multi:vN
- { c_@@_titlecase_ \token_to_str:N #1 _tl } #1
+ \exp_args:Ne \@@_change_case_auxi:nN
+ { \__kernel_codepoint_case:nn {#1} {`#2} } #2
}
- { \char_uppercase:N #1 }
}
-\cs_new:Npn \char_foldcase:N #1
- { \@@_change_case:nNN { fold } \char_value_lccode:n #1 }
-\cs_new:Npn \@@_change_case:nNN #1#2#3
+\cs_new:Npn \@@_change_case_auxi:nN #1#2
+ { \use:e { \@@_change_case:nnnN #1 #2 } }
+\cs_new:Npn \@@_change_case:nnnN #1#2#3#4
{
- \tl_if_exist:cTF { c_@@_ #1 case_ \token_to_str:N #3 _tl }
+ \int_compare:nNnTF {#1} = {`#4}
+ { \exp_not:n {#4} }
{
- \@@_change_case_multi:vN
- { c_@@_ #1 case_ \token_to_str:N #3 _tl } #3
+ \@@_change_case_auxii:nN {#1} {#4}
+ \tl_if_blank:nF {#2}
+ {
+ \@@_change_case_auxii:nN {#2} {#4}
+ \tl_if_blank:nF {#3}
+ { \@@_change_case_auxii:nN {#3} {#4} }
+ }
}
- { \exp_args:Nf \@@_change_case:nN { #2 { `#3 } } #3 }
}
-\cs_new:Npn \@@_change_case:nN #1#2
+\cs_new:Npn \@@_change_case_auxii:nN #1#2
{
- \int_compare:nNnTF {#1} = 0
- { #2 }
- { \char_generate:nn {#1} { \@@_change_case_catcode:N #2 } }
+ \char_generate:nn {#1}
+ { \@@_change_case_catcode:N #2 }
}
-\cs_new:Npn \@@_change_case_multi:nN #1#2
- { \@@_change_case_multi:NNNNw #2 #1 \q_@@_no_value \q_@@_no_value \s_@@_stop }
-\cs_generate_variant:Nn \@@_change_case_multi:nN { v }
-\cs_new:Npn \@@_change_case_multi:NNNNw #1#2#3#4#5 \s_@@_stop
+\bool_lazy_or:nnF
+ { \sys_if_engine_luatex_p: }
+ { \sys_if_engine_xetex_p: }
{
- \@@_quark_if_no_value:NTF #4
+ \cs_gset:Npn \@@_change_case_auxii:nN #1#2
{
- \@@_quark_if_no_value:NTF #3
- { \@@_change_case:NN #1 #2 }
- { \@@_change_case:NNN #1 #2#3 }
+ \int_compare:nNnTF {#1} < { "80 }
+ {
+ \char_generate:nn {#1}
+ { \@@_change_case_catcode:N #2 }
+ }
+ { \exp_not:n {#2} }
}
- { \@@_change_case:NNNN #1 #2#3#4 }
}
-\cs_new:Npn \@@_change_case:NNN #1#2#3
- {
- \exp_args:Nnf \use:nn
- { \@@_change_case:NN #1 #2 }
- { \@@_change_case:NN #1 #3 }
- }
-\cs_new:Npn \@@_change_case:NNNN #1#2#3#4
- {
- \exp_args:Nnff \use:nnn
- { \@@_change_case:NN #1 #2 }
- { \@@_change_case:NN #1 #3 }
- { \@@_change_case:NN #1 #4 }
- }
-\cs_new:Npn \@@_change_case:NN #1#2
- { \char_generate:nn { `#2 } { \@@_change_case_catcode:N #1 } }
\cs_new:Npn \@@_change_case_catcode:N #1
{
\if_catcode:w \exp_not:N #1 \c_math_toggle_token
@@ -2022,47 +1798,43 @@
}
% \end{macrocode}
% Same story for the string version, except category code is easier
-% to follow. This of course makes this version significantly faster.
-% \begin{macrocode}
-\cs_new:Npn \char_str_lowercase:N #1
- { \@@_str_change_case:nNN { lower } \char_value_lccode:n #1 }
-\cs_new:Npn \char_str_uppercase:N #1
- { \@@_str_change_case:nNN { upper } \char_value_uccode:n #1 }
-\cs_new:Npn \char_str_titlecase:N #1
- {
- \tl_if_exist:cTF { c_@@_titlecase_ \token_to_str:N #1 _tl }
- { \tl_to_str:c { c_@@_titlecase_ \token_to_str:N #1 _tl } }
- { \char_str_uppercase:N #1 }
- }
-\cs_new:Npn \char_str_foldcase:N #1
- { \@@_str_change_case:nNN { fold } \char_value_lccode:n #1 }
-\cs_new:Npn \@@_str_change_case:nNN #1#2#3
- {
- \tl_if_exist:cTF { c_@@_ #1 case_ \token_to_str:N #3 _tl }
- { \tl_to_str:c { c_@@_ #1 case_ \token_to_str:N #3 _tl } }
- { \exp_args:Nf \@@_str_change_case:nN { #2 { `#3 } } #3 }
- }
+% to follow. This of course makes this version faster.
+% \begin{macrocode}
+\cs_new:Npn \char_str_lowercase:N
+ { \@@_str_change_case:nN { lowercase } }
+\cs_new:Npn \char_str_uppercase:N
+ { \@@_str_change_case:nN { uppercase } }
+\cs_new:Npn \char_str_titlecase:N
+ { \@@_str_change_case:nN { titlecase } }
+\cs_new:Npn \char_str_foldcase:N
+ { \@@_str_change_case:nN { casefold } }
\cs_new:Npn \@@_str_change_case:nN #1#2
{
- \int_compare:nNnTF {#1} = 0
- { \tl_to_str:n {#2} }
- { \char_generate:nn {#1} { 12 } }
+ \int_compare:nNnTF {`#2} = { `\ }
+ { ~ }
+ {
+ \exp_args:Ne \@@_str_change_case_aux:nN
+ { \__kernel_codepoint_case:nn {#1} {`#2} } #2
+ }
}
-\bool_lazy_or:nnF
- { \cs_if_exist_p:N \tex_luatexversion:D }
- { \cs_if_exist_p:N \tex_XeTeXversion:D }
+\cs_new:Npn \@@_str_change_case_aux:nN #1#2
+ { \use:e { \@@_str_change_case:nnnN #1 #2 } }
+\cs_new:Npn \@@_str_change_case:nnnN #1#2#3#4
{
- \cs_set:Npn \@@_str_change_case:nN #1#2
+ \int_compare:nNnTF {#1} = {`#4}
+ { \tl_to_str:n {#4} }
{
- \int_compare:nNnTF {#1} = 0
- { \tl_to_str:n {#2} }
+ \@@_str_change_case:n {#1}
+ \tl_if_blank:nF {#2}
{
- \int_compare:nTF { `A <= #1 <= `z }
- { \char_generate:nn {#1} { 12 } }
- { \tl_to_str:n {#2} }
+ \@@_str_change_case:n {#2}
+ \tl_if_blank:nF {#3}
+ { \@@_str_change_case:n {#3} }
}
}
}
+\cs_new:Npn \@@_str_change_case:n #1
+ { \char_generate:nn {#1} { 12 } }
% \end{macrocode}
% \end{macro}
% \end{macro}
@@ -2073,9 +1845,6 @@
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
%
% \begin{macro}{\c_catcode_other_space_tl}
% Create a space with category code $12$: an \enquote{other} space.