% \iffalse meta-comment % %% File: l3unicode.dtx % % Copyright (C) 2018-2021 The LaTeX Project % % It may be distributed and/or modified under the conditions of the % LaTeX Project Public License (LPPL), either version 1.3c of this % license or (at your option) any later version. The latest version % of this license is in the file % % https://www.latex-project.org/lppl.txt % % This file is part of the "l3kernel bundle" (The Work in LPPL) % and all files in that bundle must be distributed together. % % ----------------------------------------------------------------------- % % The development version of the bundle can be found at % % https://github.com/latex3/latex3 % % for those people who are interested. % %<*driver> \documentclass[full,kernel]{l3doc} \begin{document} \DocInput{\jobname.dtx} \end{document} % % \fi % % \title{^^A % The \pkg{l3unicode} package: Unicode support functions^^A % } % % \author{^^A % The \LaTeX{} Project\thanks % {^^A % E-mail: % \href{mailto:latex-team@latex-project.org} % {latex-team@latex-project.org}^^A % }^^A % } % % \date{Released 2021-05-07} % % \maketitle % % \begin{documentation} % % This module provides Unicode-specific functions along with loading data % from a range of Unicode Consortium files. At present, it provides no % public functions. % % \end{documentation} % % \begin{implementation} % % \section{\pkg{l3unicode} implementation} % % \begin{macrocode} %<*package> % \end{macrocode} % % \begin{macrocode} %<@@=char> % \end{macrocode} % % Case changing both for strings and \enquote{text} requires data from % the Unicode Consortium. Some of this is build in to the format (as % \tn{lccode} and \tn{uccode} values) but this covers only the simple % one-to-one situations and does not fully handle for example case folding. % % As only the data needs to remain at the end of this process, everything % is set up inside a group. The only thing that is outside is creating a % stream: they are global anyway and it is best to force a stream for % all engines. For performance reasons, some of the code here is very % low-level: the material is read during loading \pkg{expl3} in package % mode. % \begin{macrocode} \ior_new:N \g_@@_data_ior \bool_lazy_or:nnTF { \sys_if_engine_luatex_p: } { \sys_if_engine_xetex_p: } { \group_begin: % \end{macrocode} % Access the primitive but suppress further expansion: active chars are % otherwise an issue. % \begin{macrocode} \cs_set:Npn \@@_generate_char:n #1 { \tex_detokenize:D \tex_expandafter:D { \tex_Uchar:D " #1 } } % \end{macrocode} % A fast local implementation for generating characters; the chars may % be active, so we prevent further expansion. % \begin{macrocode} \cs_set:Npx \@@_generate:n #1 { \exp_not:N \tex_unexpanded:D \exp_not:N \exp_after:wN { \exp_not:N \tex_Ucharcat:D #1 ~ \tex_catcode:D #1 ~ } } % \end{macrocode} % Parse the main Unicode data file for two things. First, we want the titlecase % exceptions: the one-to-one lower- and uppercase mappings it contains are all % be covered by the \TeX{} data. Second, we need normalization data: at present, % just the canonical \textsc{nfd} mappings. Those all yield either one or two % codepoints, so the split is relatively easy. % \begin{macrocode} \ior_open:Nn \g_@@_data_ior { UnicodeData.txt } \cs_set_protected:Npn \@@_data_auxi:w #1 ; #2 ; #3 ; #4 ; #5 ; #6 ; #7 ; #8 ; #9 ; { \tl_if_blank:nF {#6} { \tl_if_head_eq_charcode:nNF {#6} < % > { \@@_data_auxii:w #1 ; #6 ~ \q_stop } } \@@_data_auxiii:w #1 ; } \cs_set_protected:Npn \@@_data_auxii:w #1 ; #2 ~ #3 \q_stop { \tl_const:cx { c_@@_nfd_ \@@_generate_char:n {#1} _tl } { \@@_generate:n { "#2 } \tl_if_blank:nF {#3} { \@@_generate:n { "#3 } } } } \cs_set_protected:Npn \@@_data_auxiii:w #1 ; #2 ; #3 ; #4 ; #5 ; #6 ; #7 ~ \q_stop { \cs_set_nopar:Npn \l_@@_tmpa_tl {#7} \reverse_if:N \if_meaning:w \l_@@_tmpa_tl \c_empty_tl \cs_set_nopar:Npn \l_@@_tmpb_tl {#5} \reverse_if:N \if_meaning:w \l_@@_tmpa_tl \l_@@_tmpb_tl \tl_const:cx { c_@@_titlecase_ \@@_generate_char:n {#1} _tl } { \@@_generate:n { "#7 } } \fi: \fi: } \group_begin: \char_set_catcode_space:n { `\ }% \ior_map_variable:NNn \g_@@_data_ior \l_@@_tmpa_tl {% \if_meaning:w \l_@@_tmpa_tl \c_space_tl \exp_after:wN \ior_map_break: \fi: \exp_after:wN \@@_data_auxi:w \l_@@_tmpa_tl \q_stop }% \group_end: \ior_close:N \g_@@_data_ior % \end{macrocode} % The other data files all use C-style comments so we have to worry about % |#| tokens (and reading as strings). The set up for case folding is in two % parts. For the basic (core) mappings, folding is the same as lower casing in % most positions so only store the differences. For the more complex foldings, % always store the result, splitting up the two or three code points in the input % as required. % \begin{macrocode} \ior_open:Nn \g_@@_data_ior { CaseFolding.txt } \cs_set_protected:Npn \@@_data_auxi:w #1 ;~ #2 ;~ #3 ; #4 \q_stop { \if:w \tl_head:n { #2 ? } C \reverse_if:N \if_int_compare:w \char_value_lccode:n {"#1} = "#3 ~ \tl_const:cx { c_@@_foldcase_ \@@_generate_char:n {#1} _tl } { \@@_generate:n { "#3 } } \fi: \else: \if:w \tl_head:n { #2 ? } F \@@_data_auxii:w #1 ~ #3 ~ \q_stop \fi: \fi: } \cs_set_protected:Npn \@@_data_auxii:w #1 ~ #2 ~ #3 ~ #4 \q_stop { \tl_const:cx { c_@@_foldcase_ \@@_generate_char:n {#1} _tl } { \@@_generate:n { "#2 } \@@_generate:n { "#3 } \tl_if_blank:nF {#4} { \@@_generate:n { \int_value:w "#4 } } } } \ior_str_map_inline:Nn \g_@@_data_ior { \reverse_if:N \if:w \c_hash_str \tl_head:w #1 \c_hash_str \q_stop \@@_data_auxi:w #1 \q_stop \fi: } \ior_close:N \g_@@_data_ior % \end{macrocode} % For upper- and lowercasing special situations, there is a bit more to % do as we also have title casing to consider, plus we need to stop part-way % through the file. % \begin{macrocode} \ior_open:Nn \g_@@_data_ior { SpecialCasing.txt } \cs_set_protected:Npn \@@_data_auxi:w #1 ;~ #2 ;~ #3 ;~ #4 ; #5 \q_stop { \use:n { \@@_data_auxii:w #1 ~ lower ~ #2 ~ } ~ \q_stop \use:n { \@@_data_auxii:w #1 ~ upper ~ #4 ~ } ~ \q_stop \str_if_eq:nnF {#3} {#4} { \use:n { \@@_data_auxii:w #1 ~ title ~ #3 ~ } ~ \q_stop } } \cs_set_protected:Npn \@@_data_auxii:w #1 ~ #2 ~ #3 ~ #4 ~ #5 \q_stop { \tl_if_empty:nF {#4} { \tl_const:cx { c_@@_ #2 case_ \@@_generate_char:n {#1} _tl } { \@@_generate:n { "#3 } \@@_generate:n { "#4 } \tl_if_blank:nF {#5} { \@@_generate:n { "#5 } } } } } \ior_str_map_inline:Nn \g_@@_data_ior { \str_if_eq:eeTF { \tl_head:w #1 \c_hash_str \q_stop } { \c_hash_str } { \str_if_eq:eeT {#1} { \c_hash_str \c_space_tl Conditional~Mappings } { \ior_map_break: } } { \@@_data_auxi:w #1 \q_stop } } \ior_close:N \g_@@_data_ior \group_end: } % \end{macrocode} % For the $8$-bit engines, the above is skipped but there is still some % set up required. As case changing can only be applied to bytes, and % they have to be in the ASCII range, we define a series of data stores % to represent them, and the data are used such that only these are % ever case-changed. We do open and close one file to force allocation of % a read: this keeps all engines in line. % \begin{macrocode} { \group_begin: \cs_set_protected:Npn \@@_tmp:NN #1#2 { \quark_if_recursion_tail_stop:N #2 \tl_const:cn { c_@@_uppercase_ #2 _tl } {#1} \tl_const:cn { c_@@_lowercase_ #1 _tl } {#2} \tl_const:cn { c_@@_foldcase_ #1 _tl } {#2} \@@_tmp:NN } \@@_tmp:NN AaBbCcDdEeFfGgHhIiJjKkLlMmNnOoPpQqRrSsTtUuVvWwXxYyZz ? \q_recursion_tail \q_recursion_stop \ior_open:Nn \g_@@_data_ior { UnicodeData.txt } \ior_close:N \g_@@_data_ior \group_end: } % \end{macrocode} % % \begin{macrocode} % % \end{macrocode} % % \end{implementation} % % \PrintIndex