diff options
author | Karl Berry <karl@freefriends.org> | 2022-12-17 20:55:34 +0000 |
---|---|---|
committer | Karl Berry <karl@freefriends.org> | 2022-12-17 20:55:34 +0000 |
commit | fa3635a5e8d3d4cc58dc54018698b29cca4b7574 (patch) | |
tree | 19cef36a90c509bedbf75aea308d0e3c0fc86d5b /Master/texmf-dist/source/latex/l3kernel/l3token.dtx | |
parent | a6cd99e3fcbf111c9e512a932f418f98186d4dc1 (diff) |
l3kernel (17dec22)
git-svn-id: svn://tug.org/texlive/trunk@65299 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/source/latex/l3kernel/l3token.dtx')
-rw-r--r-- | Master/texmf-dist/source/latex/l3kernel/l3token.dtx | 423 |
1 files changed, 96 insertions, 327 deletions
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx index e8af4d063f7..6fa3508c45f 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2022-11-02} +% \date{Released 2022-12-17} % % \maketitle % @@ -155,26 +155,6 @@ % \end{texnote} % \end{function} % -% \begin{function}[added = 2020-01-09, EXP] -% { -% \char_lowercase:N, \char_uppercase:N, -% \char_titlecase:N, \char_foldcase:N, -% \char_str_lowercase:N, \char_str_uppercase:N, -% \char_str_titlecase:N, \char_str_foldcase:N -% } -% \begin{syntax} -% \cs{char_lowercase:N} \meta{char} -% \end{syntax} -% Converts the \meta{char} to the equivalent case-changed character -% as detailed by the function name (see \cs{str_foldcase:n} -% and \cs{text_titlecase:n} for details of these terms). The case mapping -% is carried out with no context-dependence (\emph{cf.}~\cs{text_uppercase:n}, -% \emph{etc.}) The \texttt{str} versions always generate \enquote{other} -% (category code $12$) characters, whilst the standard versions generate -% characters with the category code of the \meta{char} (i.e.~only -% the character code changes). -% \end{function} -% % \begin{variable}[added = 2011-09-05]{\c_catcode_other_space_tl} % Token list containing one character with category code $12$, % (\enquote{other}), and character code $32$ (space). @@ -1222,6 +1202,28 @@ % characters at the start of the argument are \emph{not} ignored in this % case (and they prevent brace-stripping). % +% \section{Deprecated functions} +% +% \begin{function}[added = 2020-01-09, EXP] +% { +% \char_lowercase:N, \char_uppercase:N, +% \char_titlecase:N, \char_foldcase:N, +% \char_str_lowercase:N, \char_str_uppercase:N, +% \char_str_titlecase:N, \char_str_foldcase:N +% } +% \begin{syntax} +% \cs{char_lowercase:N} \meta{char} +% \end{syntax} +% Converts the \meta{char} to the equivalent case-changed character +% as detailed by the function name (see \cs{str_foldcase:n} +% and \cs{text_titlecase:n} for details of these terms). The case mapping +% is carried out with no context-dependence (\emph{cf.}~\cs{text_uppercase:n}, +% \emph{etc.}) The \texttt{str} versions always generate \enquote{other} +% (category code $12$) characters, whilst the standard versions generate +% characters with the category code of the \meta{char} (i.e.~only +% the character code changes). +% \end{function} +% % \end{documentation} % % \begin{implementation} @@ -1690,304 +1692,78 @@ % \end{macro} % \end{macro} % -% \begin{macro}[EXP]{\char_to_utfviii_bytes:n} -% \begin{macro}[EXP]{\@@_to_utfviii_bytes_auxi:n} -% \begin{macro}[EXP]{\@@_to_utfviii_bytes_auxii:Nnn} -% \begin{macro}[EXP]{\@@_to_utfviii_bytes_auxiii:n} -% \begin{macro}[EXP] -% { -% \@@_to_utfviii_bytes_outputi:nw , -% \@@_to_utfviii_bytes_outputii:nw , -% \@@_to_utfviii_bytes_outputiii:nw , -% \@@_to_utfviii_bytes_outputiv:nw -% } -% \begin{macro}[EXP] -% {\@@_to_utfviii_bytes_output:nnn, \@@_to_utfviii_bytes_output:fnn} -% \begin{macro}[EXP]{\@@_to_utfviii_bytes_end:} -% This code converts a codepoint into the correct UTF-8 representation. -% In terms of the algorithm itself, see -% \url{https://en.wikipedia.org/wiki/UTF-8} for the octet pattern. -% \begin{macrocode} -\cs_new:Npn \char_to_utfviii_bytes:n #1 - { - \exp_args:Nf \@@_to_utfviii_bytes_auxi:n - { \int_eval:n {#1} } - } -\cs_new:Npn \@@_to_utfviii_bytes_auxi:n #1 - { - \if_int_compare:w #1 > "80 \exp_stop_f: - \if_int_compare:w #1 < "800 \exp_stop_f: - \@@_to_utfviii_bytes_outputi:nw - { \@@_to_utfviii_bytes_auxii:Nnn C {#1} { 64 } } - \@@_to_utfviii_bytes_outputii:nw - { \@@_to_utfviii_bytes_auxiii:n {#1} } - \else: - \if_int_compare:w #1 < "10000 \exp_stop_f: - \@@_to_utfviii_bytes_outputi:nw - { \@@_to_utfviii_bytes_auxii:Nnn E {#1} { 64 * 64 } } - \@@_to_utfviii_bytes_outputii:nw - { - \@@_to_utfviii_bytes_auxiii:n - { \int_div_truncate:nn {#1} { 64 } } - } - \@@_to_utfviii_bytes_outputiii:nw - { \@@_to_utfviii_bytes_auxiii:n {#1} } - \else: - \@@_to_utfviii_bytes_outputi:nw - { - \@@_to_utfviii_bytes_auxii:Nnn F - {#1} { 64 * 64 * 64 } - } - \@@_to_utfviii_bytes_outputii:nw - { - \@@_to_utfviii_bytes_auxiii:n - { \int_div_truncate:nn {#1} { 64 * 64 } } - } - \@@_to_utfviii_bytes_outputiii:nw - { - \@@_to_utfviii_bytes_auxiii:n - { \int_div_truncate:nn {#1} { 64 } } - } - \@@_to_utfviii_bytes_outputiv:nw - { \@@_to_utfviii_bytes_auxiii:n {#1} } - \fi: - \fi: - \else: - \@@_to_utfviii_bytes_outputi:nw {#1} - \fi: - \@@_to_utfviii_bytes_end: { } { } { } { } - } -\cs_new:Npn \@@_to_utfviii_bytes_auxii:Nnn #1#2#3 - { "#10 + \int_div_truncate:nn {#2} {#3} } -\cs_new:Npn \@@_to_utfviii_bytes_auxiii:n #1 - { \int_mod:nn {#1} { 64 } + 128 } -\cs_new:Npn \@@_to_utfviii_bytes_outputi:nw - #1 #2 \@@_to_utfviii_bytes_end: #3 - { \@@_to_utfviii_bytes_output:fnn { \int_eval:n {#1} } { } {#2} } -\cs_new:Npn \@@_to_utfviii_bytes_outputii:nw - #1 #2 \@@_to_utfviii_bytes_end: #3#4 - { \@@_to_utfviii_bytes_output:fnn { \int_eval:n {#1} } { {#3} } {#2} } -\cs_new:Npn \@@_to_utfviii_bytes_outputiii:nw - #1 #2 \@@_to_utfviii_bytes_end: #3#4#5 - { - \@@_to_utfviii_bytes_output:fnn - { \int_eval:n {#1} } { {#3} {#4} } {#2} - } -\cs_new:Npn \@@_to_utfviii_bytes_outputiv:nw - #1 #2 \@@_to_utfviii_bytes_end: #3#4#5#6 - { - \@@_to_utfviii_bytes_output:fnn - { \int_eval:n {#1} } { {#3} {#4} {#5} } {#2} - } -\cs_new:Npn \@@_to_utfviii_bytes_output:nnn #1#2#3 - { - #3 - \@@_to_utfviii_bytes_end: #2 {#1} - } -\cs_generate_variant:Nn \@@_to_utfviii_bytes_output:nnn { f } -\cs_new:Npn \@@_to_utfviii_bytes_end: { } -% \end{macrocode} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% -% \begin{macro}[rEXP]{\char_to_nfd:N} -% \begin{macro}[rEXP]{\@@_to_nfd:n, \@@_to_nfd:e} -% \begin{macro}[rEXP]{\@@_to_nfd:Nw} -% \begin{macro}[rEXP]{\char_to_nfd:n} -% \begin{macro}[rEXP]{\char_to_nfd:w} -% \begin{macro}[rEXP]{\char_nfd_generate:n} -% \begin{macro}[rEXP]{\char_nfd_generate:nnnn} -% Look up any \textsc{nfd} and recursively produce the result. Having shared -% code between Unicode and $8$-bit engines would be ideal, but this would be -% awkward as we have completely different treatment of catcodes, numbers -% of tokens, etc. The apparent saving becomes more of a headache than it's -% worth \dots -% \begin{macrocode} -\bool_lazy_or:nnTF - { \sys_if_engine_luatex_p: } - { \sys_if_engine_xetex_p: } - { - \cs_new:Npn \char_to_nfd:N #1 - { - \cs_if_exist:cTF { c_@@_nfd_ \token_to_str:N #1 _ tl } - { - \exp_after:wN \exp_after:wN \exp_after:wN \@@_to_nfd:Nw - \exp_after:wN \exp_after:wN \exp_after:wN #1 - \cs:w c_@@_nfd_ \token_to_str:N #1 _tl \cs_end: - \s_@@_stop - } - { \exp_not:n {#1} } - } - \cs_new_eq:NN \@@_to_nfd:n \char_to_nfd:N - \cs_generate_variant:Nn \@@_to_nfd:n { e } - \cs_new:Npn \@@_to_nfd:Nw #1#2#3 \s_@@_stop - { - \@@_to_nfd:e - { \char_generate:nn { `#2 } { \@@_change_case_catcode:N #1 } } - \tl_if_blank:nF {#3} - { - \@@_to_nfd:e - { \char_generate:nn { `#3 } { \char_value_catcode:n { `#3 } } } - } - } - \cs_new:Npn \char_to_nfd:n #1 - { - \@@_to_nfd:e { \char_generate:nn {#1} { \char_value_catcode:n {#1} } } - } - } - { - \cs_new:Npn \char_to_nfd:N #1 { \exp_not:n {#1} } - \cs_new:Npn \char_to_nfd:n #1 - { - \int_compare:nNnTF {#1} > { "80 } - { \exp_args:Ne \@@_to_nfd:n { \@@_nfd_generate:n {#1} } } - { \@@_nfd_generate:n {#1} } - } - \cs_new:Npn \@@_to_nfd:n #1 - { - \cs_if_exist:cTF { c_@@_nfd_ \tl_to_str:n {#1} _ tl } - { - \exp_after:wN \exp_after:wN \exp_after:wN \@@_to_nfd:w - \cs:w c_@@_nfd_ \tl_to_str:n {#1} _tl \cs_end: - \s_@@_stop - } - { \exp_not:n {#1} } - } - \cs_new:Npn \@@_to_nfd:w #1#2 \s_@@_stop - { - \@@_to_nfd:n {#1} - \tl_if_blank:nF {#2} - { \@@_to_nfd:n {#2} } - } - \cs_new:Npn \@@_nfd_generate:n #1 - { - \use:e - { - \exp_not:N \@@_nfd_generate:nnnn - \char_to_utfviii_bytes:n {#1} - } - } - \cs_new:Npn \@@_nfd_generate:nnnn #1#2#3#4 - { - \tl_if_blank:nTF {#2} - { \char_generate:nn {#1} { \char_value_catcode:n {#1} } } - { - \exp_after:wN \exp_after:wN \exp_after:wN - \exp_not:N \char_generate:nn {#1} { 13 } - \exp_after:wN \exp_after:wN \exp_after:wN - \exp_not:N \char_generate:nn {#2} { 13 } - \tl_if_blank:nF {#3} - { - \exp_after:wN \exp_after:wN \exp_after:wN - \exp_not:N \char_generate:nn {#3} { 13 } - \tl_if_blank:nF {#4} - { - \exp_after:wN \exp_after:wN \exp_after:wN - \exp_not:N \char_generate:nn {#4} { 13 } - } - } - } - - } - } -% \end{macrocode} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% % \begin{macro}[EXP] % { % \char_lowercase:N, \char_uppercase:N, % \char_titlecase:N, \char_foldcase:N % } -% \begin{macro}[EXP]{\@@_change_case:nNN} % \begin{macro}[EXP]{\@@_change_case:nN} -% \begin{macro}[EXP]{\@@_change_case_multi:nN, \@@_change_case_multi:vN} -% \begin{macro}[EXP]{\@@_change_case_multi:NNNNw} -% \begin{macro}[EXP]{\@@_change_case:NNN} -% \begin{macro}[EXP]{\@@_change_case:NNNN} -% \begin{macro}[EXP]{\@@_change_case:NN} +% \begin{macro}[EXP]{\@@_change_case:nnnN} +% \begin{macro}[EXP]{\@@_change_case_auxi:nN, \@@_change_case_auxii:nN} % \begin{macro}[EXP]{\@@_change_case_catcode:N} % \begin{macro}[EXP] % { % \char_str_lowercase:N, \char_str_uppercase:N, % \char_str_titlecase:N, \char_str_foldcase:N % } -% \begin{macro}[EXP]{\@@_str_change_case:nNN} -% \begin{macro}[EXP]{\@@_str_change_case:nN} +% \begin{macro}[EXP]{\@@_str_change_case:nN, \@@_str_change_case_aux:nN} +% \begin{macro}[EXP]{\@@_str_change_case:nnnN} +% \begin{macro}[EXP]{\@@_str_change_case:n} % To ensure that the category codes produced are predictable, every character -% is re-generated even if it is otherwise unchanged. This makes life a little -% interesting when we might have multiple output characters: we have to -% grab each of them and case change them in reverse order to maintain -% \texttt{f}-type expandability. -% \begin{macrocode} -\cs_new:Npn \char_lowercase:N #1 - { \@@_change_case:nNN { lower } \char_value_lccode:n #1 } -\cs_new:Npn \char_uppercase:N #1 - { \@@_change_case:nNN { upper } \char_value_uccode:n #1 } -\cs_new:Npn \char_titlecase:N #1 +% is re-generated even if it is otherwise unchanged. +% \begin{macrocode} +\cs_new:Npn \char_lowercase:N + { \@@_change_case:nN { lowercase } } +\cs_new:Npn \char_uppercase:N + { \@@_change_case:nN { uppercase } } +\cs_new:Npn \char_titlecase:N + { \@@_change_case:nN { titlecase } } +\cs_new:Npn \char_foldcase:N + { \@@_change_case:nN { casefold } } +\cs_new:Npn \@@_change_case:nN #1#2 { - \tl_if_exist:cTF { c_@@_titlecase_ \token_to_str:N #1 _tl } + \int_compare:nNnTF {`#2} = { `\ } + { ~ } { - \@@_change_case_multi:vN - { c_@@_titlecase_ \token_to_str:N #1 _tl } #1 + \exp_args:Ne \@@_change_case_auxi:nN + { \__kernel_codepoint_case:nn {#1} {`#2} } #2 } - { \char_uppercase:N #1 } } -\cs_new:Npn \char_foldcase:N #1 - { \@@_change_case:nNN { fold } \char_value_lccode:n #1 } -\cs_new:Npn \@@_change_case:nNN #1#2#3 +\cs_new:Npn \@@_change_case_auxi:nN #1#2 + { \use:e { \@@_change_case:nnnN #1 #2 } } +\cs_new:Npn \@@_change_case:nnnN #1#2#3#4 { - \tl_if_exist:cTF { c_@@_ #1 case_ \token_to_str:N #3 _tl } + \int_compare:nNnTF {#1} = {`#4} + { \exp_not:n {#4} } { - \@@_change_case_multi:vN - { c_@@_ #1 case_ \token_to_str:N #3 _tl } #3 + \@@_change_case_auxii:nN {#1} {#4} + \tl_if_blank:nF {#2} + { + \@@_change_case_auxii:nN {#2} {#4} + \tl_if_blank:nF {#3} + { \@@_change_case_auxii:nN {#3} {#4} } + } } - { \exp_args:Nf \@@_change_case:nN { #2 { `#3 } } #3 } } -\cs_new:Npn \@@_change_case:nN #1#2 +\cs_new:Npn \@@_change_case_auxii:nN #1#2 { - \int_compare:nNnTF {#1} = 0 - { #2 } - { \char_generate:nn {#1} { \@@_change_case_catcode:N #2 } } + \char_generate:nn {#1} + { \@@_change_case_catcode:N #2 } } -\cs_new:Npn \@@_change_case_multi:nN #1#2 - { \@@_change_case_multi:NNNNw #2 #1 \q_@@_no_value \q_@@_no_value \s_@@_stop } -\cs_generate_variant:Nn \@@_change_case_multi:nN { v } -\cs_new:Npn \@@_change_case_multi:NNNNw #1#2#3#4#5 \s_@@_stop +\bool_lazy_or:nnF + { \sys_if_engine_luatex_p: } + { \sys_if_engine_xetex_p: } { - \@@_quark_if_no_value:NTF #4 + \cs_gset:Npn \@@_change_case_auxii:nN #1#2 { - \@@_quark_if_no_value:NTF #3 - { \@@_change_case:NN #1 #2 } - { \@@_change_case:NNN #1 #2#3 } + \int_compare:nNnTF {#1} < { "80 } + { + \char_generate:nn {#1} + { \@@_change_case_catcode:N #2 } + } + { \exp_not:n {#2} } } - { \@@_change_case:NNNN #1 #2#3#4 } } -\cs_new:Npn \@@_change_case:NNN #1#2#3 - { - \exp_args:Nnf \use:nn - { \@@_change_case:NN #1 #2 } - { \@@_change_case:NN #1 #3 } - } -\cs_new:Npn \@@_change_case:NNNN #1#2#3#4 - { - \exp_args:Nnff \use:nnn - { \@@_change_case:NN #1 #2 } - { \@@_change_case:NN #1 #3 } - { \@@_change_case:NN #1 #4 } - } -\cs_new:Npn \@@_change_case:NN #1#2 - { \char_generate:nn { `#2 } { \@@_change_case_catcode:N #1 } } \cs_new:Npn \@@_change_case_catcode:N #1 { \if_catcode:w \exp_not:N #1 \c_math_toggle_token @@ -2022,47 +1798,43 @@ } % \end{macrocode} % Same story for the string version, except category code is easier -% to follow. This of course makes this version significantly faster. -% \begin{macrocode} -\cs_new:Npn \char_str_lowercase:N #1 - { \@@_str_change_case:nNN { lower } \char_value_lccode:n #1 } -\cs_new:Npn \char_str_uppercase:N #1 - { \@@_str_change_case:nNN { upper } \char_value_uccode:n #1 } -\cs_new:Npn \char_str_titlecase:N #1 - { - \tl_if_exist:cTF { c_@@_titlecase_ \token_to_str:N #1 _tl } - { \tl_to_str:c { c_@@_titlecase_ \token_to_str:N #1 _tl } } - { \char_str_uppercase:N #1 } - } -\cs_new:Npn \char_str_foldcase:N #1 - { \@@_str_change_case:nNN { fold } \char_value_lccode:n #1 } -\cs_new:Npn \@@_str_change_case:nNN #1#2#3 - { - \tl_if_exist:cTF { c_@@_ #1 case_ \token_to_str:N #3 _tl } - { \tl_to_str:c { c_@@_ #1 case_ \token_to_str:N #3 _tl } } - { \exp_args:Nf \@@_str_change_case:nN { #2 { `#3 } } #3 } - } +% to follow. This of course makes this version faster. +% \begin{macrocode} +\cs_new:Npn \char_str_lowercase:N + { \@@_str_change_case:nN { lowercase } } +\cs_new:Npn \char_str_uppercase:N + { \@@_str_change_case:nN { uppercase } } +\cs_new:Npn \char_str_titlecase:N + { \@@_str_change_case:nN { titlecase } } +\cs_new:Npn \char_str_foldcase:N + { \@@_str_change_case:nN { casefold } } \cs_new:Npn \@@_str_change_case:nN #1#2 { - \int_compare:nNnTF {#1} = 0 - { \tl_to_str:n {#2} } - { \char_generate:nn {#1} { 12 } } + \int_compare:nNnTF {`#2} = { `\ } + { ~ } + { + \exp_args:Ne \@@_str_change_case_aux:nN + { \__kernel_codepoint_case:nn {#1} {`#2} } #2 + } } -\bool_lazy_or:nnF - { \cs_if_exist_p:N \tex_luatexversion:D } - { \cs_if_exist_p:N \tex_XeTeXversion:D } +\cs_new:Npn \@@_str_change_case_aux:nN #1#2 + { \use:e { \@@_str_change_case:nnnN #1 #2 } } +\cs_new:Npn \@@_str_change_case:nnnN #1#2#3#4 { - \cs_set:Npn \@@_str_change_case:nN #1#2 + \int_compare:nNnTF {#1} = {`#4} + { \tl_to_str:n {#4} } { - \int_compare:nNnTF {#1} = 0 - { \tl_to_str:n {#2} } + \@@_str_change_case:n {#1} + \tl_if_blank:nF {#2} { - \int_compare:nTF { `A <= #1 <= `z } - { \char_generate:nn {#1} { 12 } } - { \tl_to_str:n {#2} } + \@@_str_change_case:n {#2} + \tl_if_blank:nF {#3} + { \@@_str_change_case:n {#3} } } } } +\cs_new:Npn \@@_str_change_case:n #1 + { \char_generate:nn {#1} { 12 } } % \end{macrocode} % \end{macro} % \end{macro} @@ -2073,9 +1845,6 @@ % \end{macro} % \end{macro} % \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} % % \begin{macro}{\c_catcode_other_space_tl} % Create a space with category code $12$: an \enquote{other} space. |