diff options
Diffstat (limited to 'Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx')
-rw-r--r-- | Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx | 346 |
1 files changed, 215 insertions, 131 deletions
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx b/Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx index 8ae0f44fe87..737f9704843 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2022-08-23} +% \date{Released 2022-08-30} % % \maketitle % @@ -67,10 +67,9 @@ %<@@=char> % \end{macrocode} % -% Case changing both for strings and \enquote{text} requires data from -% the Unicode Consortium. Some of this is build in to the format (as -% \tn{lccode} and \tn{uccode} values) but this covers only the simple -% one-to-one situations and does not fully handle for example case folding. +% Text operations requires data from the Unicode Consortium. Data read into +% Unicode engine formats is at best a small part of what we need, so there +% is a loader here to set up the appropriate data structures. % % As only the data needs to remain at the end of this process, everything % is set up inside a group. The only thing that is outside is creating a @@ -78,30 +77,67 @@ % all engines. For performance reasons, some of the code here is very % low-level: the material is read during loading \pkg{expl3} in package % mode. +% +% Parsing the data files can be the same way for all engines, but the detail +% of data storage varies depending on whether they are Unicode or $8$-bit +% internally.Parsing is therefore done by common functions, with the data +% storage usign engine-specific auxiliaries. +% +% Conversion of a codepoint to a character (Unicode engines) or to one +% or more bytes ($8$-bit engines) is required. We might need those to be +% detokenized: if not, for Unicode engines they have the current category +% code. % \begin{macrocode} \ior_new:N \g_@@_data_ior -\bool_lazy_or:nnTF { \sys_if_engine_luatex_p: } { \sys_if_engine_xetex_p: } - { - \group_begin: -% \end{macrocode} -% Access the primitive but suppress further expansion: active chars are -% otherwise an issue. -% \begin{macrocode} - \cs_set:Npn \@@_generate_char:n #1 - { \tex_detokenize:D \tex_expandafter:D { \tex_Uchar:D " #1 } } -% \end{macrocode} -% A fast local implementation for generating characters; the chars may -% be active, so we prevent further expansion. -% \begin{macrocode} - \cs_set:Npx \@@_generate:n #1 +\group_begin: + \bool_lazy_or:nnTF + { \sys_if_engine_luatex_p: } + { \sys_if_engine_xetex_p: } + { + \cs_set:Npn \@@_generate_other:n #1 + { \tex_detokenize:D \tex_expandafter:D { \tex_Uchar:D #1 } } + \cs_set:Npn \@@_generate:n #1 + { + \tex_unexpanded:D \exp_after:wN + { \tex_Ucharcat:D #1 ~ \tex_catcode:D #1 ~ } + } + } + { + \cs_set:Npn \@@_generate_other:n #1 + { + \tex_detokenize:D \tex_expandafter:D + { \tex_expanded:D { \@@_generate:n {#1} } } + } + \cs_set:Npn \@@_generate:n #1 { - \exp_not:N \tex_unexpanded:D \exp_not:N \exp_after:wN + \use:e { - \exp_not:N \tex_Ucharcat:D - #1 ~ - \tex_catcode:D #1 ~ + \exp_not:N \@@_generate:nnnn + \char_to_utfviii_bytes:n {#1} } - } + } + \cs_set:Npn \@@_generate:nnnn #1#2#3#4 + { + \tex_unexpanded:D \exp_after:wN \exp_after:wN \exp_after:wN + { \char_generate:nn {#1} { 13 } } + \tl_if_blank:nF {#2} + { + \tex_unexpanded:D \exp_after:wN \exp_after:wN \exp_after:wN + { \char_generate:nn {#2} { 13 } } + \tl_if_blank:nF {#3} + { + \tex_unexpanded:D \exp_after:wN \exp_after:wN \exp_after:wN + { \char_generate:nn {#3} { 13 } } + \tl_if_blank:nF {#4} + { + \tex_unexpanded:D + \exp_after:wN \exp_after:wN \exp_after:wN + { \char_generate:nn {#4} { 13 } } + } + } + } + } + } % \end{macrocode} % Parse the main Unicode data file for two things. First, we want the titlecase % exceptions: the one-to-one lower- and uppercase mappings it contains are all @@ -109,51 +145,52 @@ % just the canonical \textsc{nfd} mappings. Those all yield either one or two % codepoints, so the split is relatively easy. % \begin{macrocode} - \ior_open:Nn \g_@@_data_ior { UnicodeData.txt } - \cs_set_protected:Npn \@@_data_auxi:w - #1 ; #2 ; #3 ; #4 ; #5 ; #6 ; #7 ; #8 ; #9 ; + \cs_set_protected:Npn \@@_data_auxi:w + #1 ; #2 ; #3 ; #4 ; #5 ; #6 ; #7 ; #8 ; #9 ; + { + \tl_if_blank:nF {#6} { - \tl_if_blank:nF {#6} - { - \tl_if_head_eq_charcode:nNF {#6} < % > - { \@@_data_auxii:w #1 ; #6 ~ \q_stop } - } - \@@_data_auxiii:w #1 ; + \tl_if_head_eq_charcode:nNF {#6} < % > + { \@@_data_auxii:w #1 ; #6 ~ \q_stop } } - \cs_set_protected:Npn \@@_data_auxii:w #1 ; #2 ~ #3 \q_stop + \@@_data_auxiii:w #1 ; + } + \cs_set_protected:Npn \@@_data_auxii:w #1 ; #2 ~ #3 \q_stop + { + \tl_const:cx + { c_@@_nfd_ \@@_generate_other:n { "#1 } _tl } { - \tl_const:cx - { c_@@_nfd_ \@@_generate_char:n {#1} _tl } - { - \@@_generate:n { "#2 } - \tl_if_blank:nF {#3} - { \@@_generate:n { "#3 } } + { \@@_generate:n { "#2 } } + { + \tl_if_blank:nF {#3} + { \@@_generate:n { "#3 } } } } - \cs_set_protected:Npn \@@_data_auxiii:w - #1 ; #2 ; #3 ; #4 ; #5 ; #6 ; #7 ~ \q_stop - { - \cs_set_nopar:Npn \l_@@_tmpa_tl {#7} - \reverse_if:N \if_meaning:w \l_@@_tmpa_tl \c_empty_tl - \cs_set_nopar:Npn \l_@@_tmpb_tl {#5} - \reverse_if:N \if_meaning:w \l_@@_tmpa_tl \l_@@_tmpb_tl - \tl_const:cx - { c_@@_titlecase_ \@@_generate_char:n {#1} _tl } - { \@@_generate:n { "#7 } } - \fi: - \fi: - } - \group_begin: - \char_set_catcode_space:n { `\ }% - \ior_map_variable:NNn \g_@@_data_ior \l_@@_tmpa_tl - {% - \if_meaning:w \l_@@_tmpa_tl \c_space_tl - \exp_after:wN \ior_map_break: - \fi: - \exp_after:wN \@@_data_auxi:w \l_@@_tmpa_tl \q_stop - }% - \group_end: - \ior_close:N \g_@@_data_ior + } + \cs_set_protected:Npn \@@_data_auxiii:w + #1 ; #2 ; #3 ; #4 ; #5 ; #6 ; #7 ~ \q_stop + { + \cs_set_nopar:Npn \l_@@_tmpa_tl {#7} + \reverse_if:N \if_meaning:w \l_@@_tmpa_tl \c_empty_tl + \cs_set_nopar:Npn \l_@@_tmpb_tl {#5} + \reverse_if:N \if_meaning:w \l_@@_tmpa_tl \l_@@_tmpb_tl + \tl_const:cx + { c_@@_titlecase_ \@@_generate_other:n { "#1 } _tl } + { \@@_generate:n { "#7 } } + \fi: + \fi: + } + \ior_open:Nn \g_@@_data_ior { UnicodeData.txt } + \group_begin: + \char_set_catcode_space:n { `\ }% + \ior_map_variable:NNn \g_@@_data_ior \l_@@_tmpa_tl + {% + \if_meaning:w \l_@@_tmpa_tl \c_space_tl + \exp_after:wN \ior_map_break: + \fi: + \exp_after:wN \@@_data_auxi:w \l_@@_tmpa_tl \q_stop + }% + \group_end: % \end{macrocode} % The other data files all use C-style comments so we have to worry about % |#| tokens (and reading as strings). The set up for case folding is in two @@ -162,93 +199,140 @@ % always store the result, splitting up the two or three code points in the input % as required. % \begin{macrocode} - \ior_open:Nn \g_@@_data_ior { CaseFolding.txt } + \ior_open:Nn \g_@@_data_ior { CaseFolding.txt } + \cs_set_protected:Npn \@@_data_auxi:w #1 ;~ #2 ;~ #3 ; #4 \q_stop + { + \if:w \tl_head:n { #2 ? } C + \reverse_if:N \if_int_compare:w + \char_value_lccode:n {"#1} = "#3 ~ + \tl_const:cx + { c_@@_foldcase_ \@@_generate_other:n { "#1 } _tl } + { \@@_generate:n { "#3 } } + \fi: + \else: + \if:w \tl_head:n { #2 ? } F + \@@_data_auxii:w #1 ~ #3 ~ \q_stop + \fi: + \fi: + } + \bool_lazy_or:nnF + { \sys_if_engine_luatex_p: } + { \sys_if_engine_xetex_p: } + { \cs_set_protected:Npn \@@_data_auxi:w #1 ;~ #2 ;~ #3 ; #4 \q_stop { - \if:w \tl_head:n { #2 ? } C - \reverse_if:N \if_int_compare:w - \char_value_lccode:n {"#1} = "#3 ~ - \tl_const:cx - { c_@@_foldcase_ \@@_generate_char:n {#1} _tl } - { \@@_generate:n { "#3 } } - \fi: - \else: - \if:w \tl_head:n { #2 ? } F - \@@_data_auxii:w #1 ~ #3 ~ \q_stop - \fi: + \if:w \tl_head:n { #2 ? } F + \@@_data_auxii:w #1 ~ #3 ~ \q_stop \fi: } - \cs_set_protected:Npn \@@_data_auxii:w #1 ~ #2 ~ #3 ~ #4 \q_stop + } + \cs_set_protected:Npn \@@_data_auxii:w #1 ~ #2 ~ #3 ~ #4 \q_stop + { + \tl_const:cx { c_@@_foldcase_ \@@_generate_other:n { "#1 } _tl } { - \tl_const:cx { c_@@_foldcase_ \@@_generate_char:n {#1} _tl } - { - \@@_generate:n { "#2 } - \@@_generate:n { "#3 } - \tl_if_blank:nF {#4} - { \@@_generate:n { \int_value:w "#4 } } - } - } - \ior_str_map_inline:Nn \g_@@_data_ior - { - \reverse_if:N \if:w \c_hash_str \tl_head:w #1 \c_hash_str \q_stop - \@@_data_auxi:w #1 \q_stop - \fi: + \@@_generate:n { "#2 } + \@@_generate:n { "#3 } + \tl_if_blank:nF {#4} + { \@@_generate:n { \int_value:w "#4 } } } - \ior_close:N \g_@@_data_ior + } + \ior_str_map_inline:Nn \g_@@_data_ior + { + \reverse_if:N \if:w \c_hash_str \tl_head:w #1 \c_hash_str \q_stop + \@@_data_auxi:w #1 \q_stop + \fi: + } + \ior_close:N \g_@@_data_ior % \end{macrocode} % For upper- and lowercasing special situations, there is a bit more to % do as we also have title casing to consider, plus we need to stop part-way % through the file. % \begin{macrocode} - \ior_open:Nn \g_@@_data_ior { SpecialCasing.txt } - \cs_set_protected:Npn \@@_data_auxi:w - #1 ;~ #2 ;~ #3 ;~ #4 ; #5 \q_stop + \ior_open:Nn \g_@@_data_ior { SpecialCasing.txt } + \cs_set_protected:Npn \@@_data_auxi:w + #1 ;~ #2 ;~ #3 ;~ #4 ; #5 \q_stop + { + \use:n { \@@_data_auxii:w #1 ~ lower ~ #2 ~ } ~ \q_stop + \use:n { \@@_data_auxii:w #1 ~ upper ~ #4 ~ } ~ \q_stop + \str_if_eq:nnF {#3} {#4} + { \use:n { \@@_data_auxii:w #1 ~ title ~ #3 ~ } ~ \q_stop } + } + \cs_set_protected:Npn \@@_data_auxii:w + #1 ~ #2 ~ #3 ~ #4 ~ #5 \q_stop + { + \tl_if_empty:nF {#4} { - \use:n { \@@_data_auxii:w #1 ~ lower ~ #2 ~ } ~ \q_stop - \use:n { \@@_data_auxii:w #1 ~ upper ~ #4 ~ } ~ \q_stop - \str_if_eq:nnF {#3} {#4} - { \use:n { \@@_data_auxii:w #1 ~ title ~ #3 ~ } ~ \q_stop } - } - \cs_set_protected:Npn \@@_data_auxii:w - #1 ~ #2 ~ #3 ~ #4 ~ #5 \q_stop - { - \tl_if_empty:nF {#4} + \tl_const:cx { c_@@_ #2 case_ \@@_generate_other:n { "#1 } _tl } { - \tl_const:cx { c_@@_ #2 case_ \@@_generate_char:n {#1} _tl } - { - \@@_generate:n { "#3 } - \@@_generate:n { "#4 } - \tl_if_blank:nF {#5} - { \@@_generate:n { "#5 } } - } + \@@_generate:n { "#3 } + \@@_generate:n { "#4 } + \tl_if_blank:nF {#5} + { \@@_generate:n { "#5 } } } } - \ior_str_map_inline:Nn \g_@@_data_ior + } + \ior_str_map_inline:Nn \g_@@_data_ior + { + \str_if_eq:eeTF { \tl_head:w #1 \c_hash_str \q_stop } { \c_hash_str } { - \str_if_eq:eeTF - { \tl_head:w #1 \c_hash_str \q_stop } - { \c_hash_str } + \str_if_eq:eeT + {#1} + { \c_hash_str \c_space_tl Conditional~Mappings } + { \ior_map_break: } + } + { \@@_data_auxi:w #1 \q_stop } + } + \ior_close:N \g_@@_data_ior +\group_end: + +% \end{macrocode} +% +% \begin{macrocode} +%<@@=text> +% \end{macrocode} +% +% Read the Unicode grapheme data. This is quite easy to handle and we only need +% codepoints, not characters, so there is no need to worry about the engine in use. +% As reading as a string is most convenient, we have to do some work to remove +% spaces: the hardest part of the entire process! +% \begin{macrocode} +\ior_new:N \g_@@_data_ior +\group_begin: + \ior_open:Nn \g_@@_data_ior { GraphemeBreakProperty.txt } + \cs_set_nopar:Npn \l_@@_tmpa_str { } + \cs_set_nopar:Npn \l_@@_tmpb_str { } + \cs_set_protected:Npn \@@_data_auxi:w #1 ;~ #2 ~ #3 \q_stop + { + \str_if_eq:VnF \l_@@_tmpb_str {#2} + { + \str_if_empty:NF \l_@@_tmpb_str { - \str_if_eq:eeT - {#1} - { \c_hash_str \c_space_tl Conditional~Mappings } - { \ior_map_break: } + \clist_const:cx { c_@@_grapheme_ \l_@@_tmpb_str _clist } + { \exp_after:wN \use_none:n \l_@@_tmpa_str } + \cs_set_nopar:Npn \l_@@_tmpa_str { } } + \cs_set_nopar:Npn \l_@@_tmpb_str {#2} + } + \@@_data_auxii:w #1 .. #1 .. #1 \q_stop + } + \cs_set_protected:Npn \@@_data_auxii:w #1 .. #2 .. #3 \q_stop + { + \cs_set_nopar:Npx \l_@@_tmpa_str + { + \l_@@_tmpa_str , + \tl_trim_spaces:n {#1} .. \tl_trim_spaces:n {#2} + } + } + \ior_str_map_inline:Nn \g_@@_data_ior + { + \str_if_eq:eeF { \tl_head:w #1 \c_hash_str \q_stop } { \c_hash_str } + { + \tl_if_blank:nF {#1} { \@@_data_auxi:w #1 \q_stop } } - \ior_close:N \g_@@_data_ior - \group_end: - } -% \end{macrocode} -% For the $8$-bit engines, open the stream and close again: this keeps -% file records the same. -% \begin{macrocode} - { - \group_begin: - \ior_open:Nn \g_@@_data_ior { UnicodeData.txt } - \ior_close:N \g_@@_data_ior - \group_end: - } + } + \ior_close:N \g_@@_data_ior +\group_end: % \end{macrocode} % % \begin{macrocode} |