diff options
Diffstat (limited to 'Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx')
-rw-r--r-- | Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx | 824 |
1 files changed, 716 insertions, 108 deletions
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx b/Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx index 95f410bc1e2..66c1ef70ab8 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx @@ -43,15 +43,72 @@ % }^^A % } % -% \date{Released 2022-11-02} +% \date{Released 2022-12-17} % % \maketitle % % \begin{documentation} % % This module provides Unicode-specific functions along with loading data -% from a range of Unicode Consortium files. At present, it provides no -% public functions. +% from a range of Unicode Consortium files. Most of the code here is +% internal, but there are a small set of public functions. These work with +% Unicode \meta{codepoints} and are designed to give useable results with +% both Unicode-aware and $8$-bit engines. +% +% \begin{function}[EXP, added = 2022-10-09, updated = 2022-11-09] +% {\codepoint_generate:nn} +% \begin{syntax} +% \cs{codepoint_generate:nn} \Arg{codepoint} \Arg{catcode} +% \end{syntax} +% Generates one or more character tokens representing the \meta{codepoint}. +% With Unicode engines, exactly one character token will be generated, and +% this will have the \meta{catcode} specified as the second argument: +% \begin{itemize} +% \item $1$ (begin group) +% \item $2$ (end group) +% \item $3$ (math toggle) +% \item $4$ (alignment) +% \item $6$ (parameter) +% \item $7$ (math superscript) +% \item $8$ (math subscript) +% \item $10$ (space) +% \item $11$ (letter) +% \item $12$ (other) +% \item $13$ (active) +% \end{itemize} +% For $8$-bit engines, between one and four character tokens will be +% produced: these will be the bytes of the UTF-8 representation of the +% \meta{codepoint}. For all codepoints outside of the classical ASCII +% range, the generated character tokens will be active (category code +% $13$); for codepoints in the ASCII range, the given \meta{catcode} +% will be used. To allow the result of this function to be used +% inside a expansion context, the result is protected by \cs{exp_not:n}. +% \end{function} +% +% \begin{function}[EXP, added = 2022-10-09] +% {\codepoint_str_generate:n} +% \begin{syntax} +% \cs{codepoint_str_generate:n} \Arg{codepoint} +% \end{syntax} +% Generates one or more character tokens representing the \meta{codepoint}. +% With Unicode engines, exactly one character token will be generated. +% For $8$-bit engines, between one and four character tokens will be +% produced: these will be the bytes of the UTF-8 representation of the +% \meta{codepoint}. All of the generated character tokens will be of +% category code $12$, except any spaces (codepoint $32$), which will be +% category code $10$. +% \end{function} +% +% \begin{function}[added = 2022-10-09, EXP]{\codepoint_to_nfd:n} +% \begin{syntax} +% \cs{codepoint_to_nfd:n} \Arg{codepoint} +% \end{syntax} +% Converts the \meta{codepoint} to the Unicode Normalization +% Form Canonical Decomposition. The generated character(s) will have +% the current category code as they would if typed in directly for Unicode +% engines; for $8$-bit engines, active characters are used for all codepoints +% outside of the ASCII range. +% \end{function} % % \end{documentation} % @@ -64,122 +121,585 @@ % \end{macrocode} % % \begin{macrocode} -%<@@=char> +%<@@=codepoint> +% \end{macrocode} +% +% \subsection{User functions} +% +% \begin{macro}[EXP]{\codepoint_str_generate:n} +% \begin{macro}[EXP]{\@@_str_generate:nnnn} +% \begin{macro}[EXP]{\codepoint_generate:nn} +% \begin{macro}[EXP]{\@@_generate:nnnn} +% \begin{macro}[EXP]{\@@_generate:n} +% Conversion of a codepoint to a character (Unicode engines) or to one +% or more bytes ($8$-bit engines) is required. For loading the data, +% all that is needed is the form which creates strings: these are outside +% the group as they will also be used when looking up data in the hash +% table storage at point-of-use. Later, we will also need functions that +% can generate character tokens for document use: those are defined below, +% in the data recovery setup. +% \begin{macrocode} +\bool_lazy_or:nnTF + { \sys_if_engine_luatex_p: } + { \sys_if_engine_xetex_p: } + { + \cs_new:Npn \codepoint_str_generate:n #1 + { + \int_compare:nNnTF {#1} = { `\ } + { ~ } + { \char_generate:nn {#1} { 12 } } + } + \cs_new:Npn \codepoint_generate:nn #1#2 + { + \int_compare:nNnTF {#1} = { `\ } + { ~ } + { + \__kernel_exp_not:w \exp_after:wN \exp_after:wN \exp_after:wN + { \char_generate:nn {#1} {#2} } + } + } + } + { + \cs_new:Npn \codepoint_str_generate:n #1 + { + \int_compare:nNnTF {#1} = { `\ } + { ~ } + { + \use:e + { + \exp_not:N \@@_str_generate:nnnn + \__kernel_codepoint_to_bytes:n {#1} + } + } + } + \cs_new:Npn \@@_str_generate:nnnn #1#2#3#4 + { + \char_generate:nn {#1} { 12 } + \tl_if_blank:nF {#2} + { + \char_generate:nn {#2} { 12 } + \tl_if_blank:nF {#3} + { + \char_generate:nn {#3} { 12 } + \tl_if_blank:nF {#4} + { \char_generate:nn {#4} { 12 } } + } + } + } + \cs_new:Npn \codepoint_generate:nn #1#2 + { + \int_compare:nNnTF {#1} = { `\ } + { ~ } + { + \int_compare:nNnTF {#1} < { "80 } + { + \__kernel_exp_not:w \exp_after:wN \exp_after:wN \exp_after:wN + { \char_generate:nn {#1} {#2} } + } + { + \use:e + { + \exp_not:N \@@_generate:nnnn + \__kernel_codepoint_to_bytes:n {#1} + } + } + } + } + \cs_new:Npn \@@_generate:nnnn #1#2#3#4 + { + \__kernel_exp_not:w \exp_after:wN + { + \tex_expanded:D + { + \@@_generate:n {#1} + \@@_generate:n {#2} + \tl_if_blank:nF {#3} + { + \@@_generate:n {#3} + \tl_if_blank:nF {#4} + { \@@_generate:n {#4} } + } + } + } + } + \cs_new:Npn \@@_generate:n #1 + { + \__kernel_exp_not:w \exp_after:wN \exp_after:wN \exp_after:wN + { \char_generate:nn {#1} { 13 } } + } + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \begin{macro}[EXP]{\__kernel_codepoint_to_bytes:n} +% \begin{macro}[EXP]{\@@_to_bytes_auxi:n} +% \begin{macro}[EXP]{\@@_to_bytes_auxii:Nnn} +% \begin{macro}[EXP]{\@@_to_bytes_auxiii:n} +% \begin{macro}[EXP] +% { +% \@@_to_bytes_outputi:nw , +% \@@_to_bytes_outputii:nw , +% \@@_to_bytes_outputiii:nw , +% \@@_to_bytes_outputiv:nw +% } +% \begin{macro}[EXP] +% {\@@_to_bytes_output:nnn, \@@_to_bytes_output:fnn} +% \begin{macro}[EXP]{\@@_to_bytes_end:} +% This code converts a codepoint into the correct UTF-8 representation. +% In terms of the algorithm itself, see +% \url{https://en.wikipedia.org/wiki/UTF-8} for the octet pattern. +% \begin{macrocode} +\cs_new:Npn \__kernel_codepoint_to_bytes:n #1 + { + \exp_args:Nf \@@_to_bytes_auxi:n + { \int_eval:n {#1} } + } +\cs_new:Npn \@@_to_bytes_auxi:n #1 + { + \if_int_compare:w #1 > "80 \exp_stop_f: + \if_int_compare:w #1 < "800 \exp_stop_f: + \@@_to_bytes_outputi:nw + { \@@_to_bytes_auxii:Nnn C {#1} { 64 } } + \@@_to_bytes_outputii:nw + { \@@_to_bytes_auxiii:n {#1} } + \else: + \if_int_compare:w #1 < "10000 \exp_stop_f: + \@@_to_bytes_outputi:nw + { \@@_to_bytes_auxii:Nnn E {#1} { 64 * 64 } } + \@@_to_bytes_outputii:nw + { + \@@_to_bytes_auxiii:n + { \int_div_truncate:nn {#1} { 64 } } + } + \@@_to_bytes_outputiii:nw + { \@@_to_bytes_auxiii:n {#1} } + \else: + \@@_to_bytes_outputi:nw + { + \@@_to_bytes_auxii:Nnn F + {#1} { 64 * 64 * 64 } + } + \@@_to_bytes_outputii:nw + { + \@@_to_bytes_auxiii:n + { \int_div_truncate:nn {#1} { 64 * 64 } } + } + \@@_to_bytes_outputiii:nw + { + \@@_to_bytes_auxiii:n + { \int_div_truncate:nn {#1} { 64 } } + } + \@@_to_bytes_outputiv:nw + { \@@_to_bytes_auxiii:n {#1} } + \fi: + \fi: + \else: + \@@_to_bytes_outputi:nw {#1} + \fi: + \@@_to_bytes_end: { } { } { } { } + } +\cs_new:Npn \@@_to_bytes_auxii:Nnn #1#2#3 + { "#10 + \int_div_truncate:nn {#2} {#3} } +\cs_new:Npn \@@_to_bytes_auxiii:n #1 + { \int_mod:nn {#1} { 64 } + 128 } +\cs_new:Npn \@@_to_bytes_outputi:nw + #1 #2 \@@_to_bytes_end: #3 + { \@@_to_bytes_output:fnn { \int_eval:n {#1} } { } {#2} } +\cs_new:Npn \@@_to_bytes_outputii:nw + #1 #2 \@@_to_bytes_end: #3#4 + { \@@_to_bytes_output:fnn { \int_eval:n {#1} } { {#3} } {#2} } +\cs_new:Npn \@@_to_bytes_outputiii:nw + #1 #2 \@@_to_bytes_end: #3#4#5 + { + \@@_to_bytes_output:fnn + { \int_eval:n {#1} } { {#3} {#4} } {#2} + } +\cs_new:Npn \@@_to_bytes_outputiv:nw + #1 #2 \@@_to_bytes_end: #3#4#5#6 + { + \@@_to_bytes_output:fnn + { \int_eval:n {#1} } { {#3} {#4} {#5} } {#2} + } +\cs_new:Npn \@@_to_bytes_output:nnn #1#2#3 + { + #3 + \@@_to_bytes_end: #2 {#1} + } +\cs_generate_variant:Nn \@@_to_bytes_output:nnn { f } +\cs_new:Npn \@@_to_bytes_end: { } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \begin{macro}[EXP]{\codepoint_to_nfd:n} +% \begin{macro}[EXP]{\@@_to_nfd:nn} +% \begin{macro}[EXP]{\@@_to_nfd:nnn} +% \begin{macro}[EXP]{\@@_to_nfd:nnnn} +% \begin{macro}[EXP]{\@@_to_nfd_generate:nn} +% \begin{macro}[EXP]{\@@_to_nfd_generate:n} +% \begin{macro}[EXP]{\@@_to_nfd_generate:nnnn} +% Converted to NFD is a potentially-recursive process: the key is to +% check if we get the input codepoint back again. As far as possible, +% we use the same path for all engines. +% \begin{macrocode} +\cs_new:Npn \codepoint_to_nfd:n #1 + { \@@_to_nfd:nn {#1} { \char_value_catcode:n {#1} } } +\cs_new:Npn \@@_to_nfd:nn #1#2 + { + \exp_args:Ne \@@_to_nfd:nnn + { \@@_nfd:n {#1} } {#1} {#2} + } +\cs_new:Npn \@@_to_nfd:nnn #1#2#3 { \@@_to_nfd:nnnn #1 {#2} {#3} } +\cs_new:Npn \@@_to_nfd:nnnn #1#2#3#4 + { + \int_compare:nNnTF {#1} = {#3} + { \codepoint_generate:nn {#1} {#4} } + { + \@@_to_nfd:nn {#1} {#4} + \tl_if_blank:nF {#2} + { \@@_to_nfd:nn {#2} {#4} } + } + } % \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \subsection{Data loader} % % Text operations requires data from the Unicode Consortium. Data read into % Unicode engine formats is at best a small part of what we need, so there % is a loader here to set up the appropriate data structures. % +% Where we need data for most or all of the Unicode range, we use the two-stage +% table approach recommended by the Unicode Consortium and demonstrated in a +% model implementation in Python in +% \url{https://www.strchr.com/multi-stage_tables}. This approach uses the +% \texttt{intarray} (\texttt{fontdimen}-based) data type as it is fast for +% random access and avoids significant hash table usage. In contrast, where +% only a small subset of codepoints are required, storage as macros is +% preferable. There is also some consideration of the effort needed to load +% data: see for example the grapheme breaking information, which would be +% problematic to convert into a two-stage table but which can be used with +% reasonable performance in a small number of comma lists (at the cost that +% breaking at higher codepoint Hangul characters will be slightly slow). +% +% \begin{variable}{\c_@@_block_size_int} +% Choosing the block size for the blocks in the two-stage approach is +% non-trivial: depending on the data stored, the optimal size for +% memory usage will vary. At the same time, for us there is also the +% question of load-time: larger blocks require longer comma lists +% as intermediates, so are slower. As this is going to be needed +% to use the data, we set it up outside of the group for clarity. +% \begin{macrocode} +\int_const:Nn \c_@@_block_size_int { 64 } +% \end{macrocode} +% \end{variable} +% +% Parsing the data files can be the same way for all engines, but where they +% are stored as character tokens, the construction method depends on whether +% they are Unicode or $8$-bit internally. Parsing is therefore done by common +% functions, with some data storage using engine-specific auxiliaries. +% % As only the data needs to remain at the end of this process, everything % is set up inside a group. The only thing that is outside is creating a % stream: they are global anyway and it is best to force a stream for -% all engines. For performance reasons, some of the code here is very -% low-level: the material is read during loading \pkg{expl3} in package -% mode. -% -% Parsing the data files can be the same way for all engines, but the detail -% of data storage varies depending on whether they are Unicode or $8$-bit -% internally.Parsing is therefore done by common functions, with the data -% storage usign engine-specific auxiliaries. -% -% Conversion of a codepoint to a character (Unicode engines) or to one -% or more bytes ($8$-bit engines) is required. We might need those to be -% detokenized: if not, for Unicode engines they have the current category -% code. +% all engines. +% +% \begin{variable}{\g_@@_data_ior} % \begin{macrocode} \ior_new:N \g_@@_data_ior +% \end{macrocode} +% \end{variable} +% +% We need some setup for the two-part table approach. The number of blocks we +% need will be variable, but the resulting size of the stage one table +% is predictable. For performance reasons, we therefore create the stage one +% tables now so they can be used immediately, and will later rename them as a +% constant tables. For each two-stage table construction, we need a comma +% list to hold the partial block and a couple of integers to track where +% we are up to. To avoid burning registers, the latter are stored in macros +% and are \enquote{fake} integers. We also avoid any \texttt{new} functions, +% keeping as much as possible local. +% +% As we need both positive and negative values, case data requires one +% two-stage table for each transformation. In contrasts, general Unicode +% properties could be stored in one table with appropriate combination rules: +% that is not done at present but is likely to be added over time. Here, all +% that is needed is additional entries into the comma-list to create the +% structures. +% +% Notice that in the standard \pkg{expl3} way we are indexes position not +% offset: that does mean a little work later. +% \begin{macrocode} \group_begin: - \bool_lazy_or:nnTF - { \sys_if_engine_luatex_p: } - { \sys_if_engine_xetex_p: } + \clist_map_inline:nn + { uppercase , lowercase } { - \cs_set:Npn \@@_generate_other:n #1 - { \tex_detokenize:D \tex_expandafter:D { \tex_Uchar:D #1 } } - \cs_set:Npn \@@_generate:n #1 + \cs_set_nopar:cpn { l_@@_ #1 _block_clist } { } + \cs_set_nopar:cpn { l_@@_ #1 _block_tl } { 1 } + \cs_set_nopar:cpn { l_@@_ #1 _pos_tl } { 0 } + \intarray_new:cn { g_@@_ #1 _index_intarray } + { \int_div_truncate:nn { "110000 } \c_@@_block_size_int } + } +% \end{macrocode} +% We need an integer value when matching the current block to those we have +% already seen, and a way to track codepoints for handling ranges. Again, +% we avoid using up registers or creating global names. +% \begin{macrocode} + \cs_set_nopar:Npn \l_@@_next_codepoint_fint_tl { 0 } + \cs_set_nopar:Npn \l_@@_matched_block_tl { 0 } +% \end{macrocode} +% Parse the main Unicode data file and pull out the NFD and case changing +% data. The NFD data is stored on using the hash table approach and can yield +% a predictable number of code points: one or two. We also need the case data, +% which will be modified further below. To allow for finding rangs, the +% description of the codepoint needs to be carried forward. +% \begin{macrocode} + \cs_set_protected:Npn \@@_data_auxi:w + #1 ; #2 ; #3 ; #4 ; #5 ; #6 ; #7 ; #8 ; #9 ; + { + \tl_if_blank:nF {#6} { - \tex_unexpanded:D \exp_after:wN - { \tex_Ucharcat:D #1 ~ \tex_catcode:D #1 ~ } - } + \tl_if_head_eq_charcode:nNF {#6} < % > + { \@@_data_auxii:w #1 ; #6 ~ \q_stop } + } + \@@_data_auxiii:w #1 ; #2 ; } + \cs_set_protected:Npn \@@_data_auxii:w #1 ; #2 ~ #3 \q_stop { - \cs_set:Npn \@@_generate_other:n #1 + \tl_const:cx + { c_@@_nfd_ \codepoint_str_generate:n {"#1} _tl } { - \tex_detokenize:D \tex_expandafter:D - { \tex_expanded:D { \@@_generate:n {#1} } } + {"#2} + { \tl_if_blank:nF {#3} {"#3} } } - \cs_set:Npn \@@_generate:n #1 + } +% \end{macrocode} +% The case data is going to be stored as an offset from the parent character, +% rather than an absolute value. We therefore deal with that plus the situation +% where a codepoint has no mapping data in oen shot. +% \begin{macrocode} + \cs_set_protected:Npn \@@_data_auxiii:w + #1 ; #2 ; #3 ; #4 ; #5 ; #6 ; #7 ; #8 ~ \q_stop + { + \use:e { - \use:e - { - \exp_not:N \@@_generate:nnnn - \char_to_utfviii_bytes:n {#1} - } + \@@_data_auxiv:w + #1 ; #2 ; + \@@_data_offset:nn {#1} {#6} ; + \@@_data_offset:nn {#1} {#7} ; + #8; + } + } + \cs_set:Npn \@@_data_offset:nn #1#2 + { + \tl_if_blank:nTF {#2} + { 0 } + { \int_eval:n { "#2 - "#1 } } + } +% \end{macrocode} +% To deal with ranges, we track the position of the next codepoint expected. +% If there is a gap, we deal with that separately: it could be a range or +% an unused part of the Unicode space. As such, we deal with the current +% codepoint here whether or not there is range to fill in. Upper- and +% lowercase data go into the two-stage table, any titlecase exception is +% just stored in a macro. The data for the codepoint is added to the current +% block, and if that is now complete we move on to save the block. The +% case exceptions are all stored as codepoints, with a fixed number of +% balanced text as we know that there are never more than three. +% \begin{macrocode} + \cs_set_protected:Npn \@@_data_auxiv:w #1 ; #2 ; #3 ; #4 ; #5 ; + { + \int_compare:nNnT {"#1} > \l_@@_next_codepoint_fint_tl + { + \@@_data_auxvi:nnnw {#1} {#3} {#4} + #2 Last> \q_stop } - \cs_set:Npn \@@_generate:nnnn #1#2#3#4 + \@@_add:nn { uppercase } {#3} + \@@_add:nn { lowercase } {#4} + \int_compare:nNnF {#3} = { \@@_data_offset:nn {#1} {#5} } + { + \tl_const:cx + { c_@@_titlecase_ \codepoint_str_generate:n {"#1} _tl } + { {"#5} { } { } } + } + \tl_set:Nx \l_@@_next_codepoint_fint_tl + { \int_eval:n { "#1 + 1 } } + } + \cs_set_protected:Npn \@@_add:nn #1#2 + { + \clist_put_right:cn { l_@@_ #1 _block_clist } {#2} + \int_compare:nNnT { \clist_count:c { l_@@_ #1 _block_clist } } + = \c_@@_block_size_int + { \@@_save_blocks:nn {#1} { 1 } } + } +% \end{macrocode} +% Distinguish between a range and a gap, and pass on the appropriate value(s). +% \begin{macrocode} + \cs_set_protected:Npn \@@_data_auxvi:nnnw #1#2#3#4 Last> #5 \q_stop + { + \tl_if_blank:nTF {#5} + { + \@@_range:nnn {#1} { uppercase } { 0 } + \@@_range:nnn {#1} { lowercase } { 0 } + } + { + \@@_range:nnn {#1} { uppercase } {#2} + \@@_range:nnn {#1} { lowercase } {#3} + } + } +% \end{macrocode} +% Calculated the length of the range and the space remaining in the current +% block. +% \begin{macrocode} + \cs_set_protected:Npn \@@_range:nnn #1 + { + \exp_args:Nf \@@_range_aux:nnn + { \int_eval:n { "#1 - \l_@@_next_codepoint_fint_tl } } + } + \cs_set_protected:Npn \@@_range_aux:nnn #1#2 + { + \exp_args:Nf \@@_range:nnnn { - \tex_unexpanded:D \exp_after:wN \exp_after:wN \exp_after:wN - { \char_generate:nn {#1} { 13 } } - \tl_if_blank:nF {#2} + \int_min:nn + {#1} { - \tex_unexpanded:D \exp_after:wN \exp_after:wN \exp_after:wN - { \char_generate:nn {#2} { 13 } } - \tl_if_blank:nF {#3} - { - \tex_unexpanded:D \exp_after:wN \exp_after:wN \exp_after:wN - { \char_generate:nn {#3} { 13 } } - \tl_if_blank:nF {#4} - { - \tex_unexpanded:D - \exp_after:wN \exp_after:wN \exp_after:wN - { \char_generate:nn {#4} { 13 } } - } - } + \c_@@_block_size_int + - \clist_count:c { l_@@_ #2 _block_clist } } } + {#1} {#2} } % \end{macrocode} -% Parse the main Unicode data file for two things. First, we want the titlecase -% exceptions: the one-to-one lower- and uppercase mappings it contains are all -% be covered by the \TeX{} data. Second, we need normalization data: at present, -% just the canonical \textsc{nfd} mappings. Those all yield either one or two -% codepoints, so the split is relatively easy. +% Here we want to do three things: add to and possibly complete the current +% block, add complete blocks quickly, then finish up the range in a final +% open block. We need to avoid as far as possible avoid dealing with ever +% single codepoint, so the middle step is optimised. % \begin{macrocode} - \cs_set_protected:Npn \@@_data_auxi:w - #1 ; #2 ; #3 ; #4 ; #5 ; #6 ; #7 ; #8 ; #9 ; + \cs_set_protected:Npn \@@_range:nnnn #1#2#3#4 { - \tl_if_blank:nF {#6} - { - \tl_if_head_eq_charcode:nNF {#6} < % > - { \@@_data_auxii:w #1 ; #6 ~ \q_stop } + \prg_replicate:nn {#1} + { \clist_put_right:cn { l_@@_ #3 _block_clist } {#4} } + \int_compare:nNnT { \clist_count:c { l_@@_ #3 _block_clist } } + = \c_@@_block_size_int + { \@@_save_blocks:nn {#3} { 1 } } + \int_compare:nNnF + { \int_div_truncate:nn { #2 - #1 } \c_@@_block_size_int } = 0 + { + \tl_set:cx { l_@@_ #3 _block_clist } + { + \exp_args:NNe \use:nn \use_none:n + { \prg_replicate:nn { \c_@@_block_size_int } { , #4 } } + } + \@@_save_blocks:nn {#3} + { \int_div_truncate:nn { (#2 - #1) } \c_@@_block_size_int } } - \@@_data_auxiii:w #1 ; + \prg_replicate:nn + { \int_mod:nn { #2 - #1 } \c_@@_block_size_int } + { \clist_put_right:cx { l_@@_ #3 _block_clist } {#4} } } - \cs_set_protected:Npn \@@_data_auxii:w #1 ; #2 ~ #3 \q_stop +% \end{macrocode} +% To allow rapid comparison, each completed block is stored locally as a +% comma list: once all of the blocks have been created, they are converted +% into an \texttt{intarray} in one step. The aim here is to check the current +% block against those we've already used, and either match to an existing +% block or save a new block. +% \begin{macrocode} + \cs_set_protected:Npn \@@_save_blocks:nn #1#2 { - \tl_const:cx - { c_@@_nfd_ \@@_generate_other:n { "#1 } _tl } + \tl_set_eq:Nc \l_@@_matched_block_tl { l_@@_ #1 _block_tl } + \int_step_inline:nn { \tl_use:c { l_@@_ #1 _block_tl } - 1 } { - { \@@_generate:n { "#2 } } + \tl_if_eq:ccT { l_@@_ #1 _block_clist } + { l_@@_ #1 _block_ ##1 _clist } + { \tl_set:Nn \l_@@_matched_block_tl {##1} } + } + \int_compare:nNnT + { \tl_use:c { l_@@_ #1 _block_tl } } = \l_@@_matched_block_tl { - \tl_if_blank:nF {#3} - { \@@_generate:n { "#3 } } - } + \clist_set_eq:cc + { + l_@@_ #1 _block_ + \tl_use:c { l_@@_ #1 _block_tl } _clist + } + { l_@@_ #1 _block_clist } + \tl_set:cx { l_@@_ #1 _block_tl } + { \int_eval:n { \tl_use:c { l_@@_ #1 _block_tl } + 1 } } + } + \prg_replicate:nn {#2} + { + \tl_set:cx { l_@@_ #1 _pos_tl } + { \int_eval:n { \tl_use:c { l_@@_ #1 _pos_tl } + 1 } } + \exp_args:Nc \__kernel_intarray_gset:Nnn + { g_@@_ #1 _index_intarray } + { \tl_use:c { l_@@_ #1 _pos_tl } } + \l_@@_matched_block_tl + } + \clist_clear:c { l_@@_ #1 _block_clist } + } +% \end{macrocode} +% Close out the final block, rename the first stage table, then combine all +% of the block comma-lists into one large second-stage table with offsets. +% As we use an index not an offet, there is a little back-and-forward to do. +% \begin{macrocode} + \cs_set_protected:Npn \@@_finalise_blocks: + { + \clist_map_inline:nn { uppercase , lowercase } + { + \@@_range:nnn { 110000 } {##1} { 0 } + \@@_finalise_blocks:n {##1} } + } + \cs_set_protected:Npn \@@_finalise_blocks:n #1 + { + \cs_gset_eq:cc { c_@@_ #1 _index_intarray } { g_@@_ #1 _index_intarray } + \cs_undefine:c { g_@@_ #1 _index_intarray } + \intarray_new:cn { g_@@_ #1 _blocks_intarray } + { ( \tl_use:c { l_@@_ #1 _block_tl } - 1 ) * \c_@@_block_size_int } + \int_step_inline:nn { \tl_use:c { l_@@_ #1 _block_tl } - 1 } + { + \exp_args:Nv \@@_finalise_blocks:nnn + { l_@@_ #1 _block_ ##1 _clist } + {##1} {#1} + } + \cs_gset_eq:cc { c_@@_ #1 _blocks_intarray } + { g_@@_ #1 _blocks_intarray } + \cs_undefine:c { g_@@_ #1 _blocks_intarray } } - \cs_set_protected:Npn \@@_data_auxiii:w - #1 ; #2 ; #3 ; #4 ; #5 ; #6 ; #7 ~ \q_stop + \cs_set_protected:Npn \@@_finalise_blocks:nnn #1#2#3 { - \cs_set_nopar:Npn \l_@@_tmpa_tl {#7} - \reverse_if:N \if_meaning:w \l_@@_tmpa_tl \c_empty_tl - \cs_set_nopar:Npn \l_@@_tmpb_tl {#5} - \reverse_if:N \if_meaning:w \l_@@_tmpa_tl \l_@@_tmpb_tl - \tl_const:cx - { c_@@_titlecase_ \@@_generate_other:n { "#1 } _tl } - { \@@_generate:n { "#7 } } - \fi: - \fi: + \exp_args:Nnf \@@_finalise_blocks:nnnw { 1 } + { \int_eval:n { ( #2 - 1 ) * \c_@@_block_size_int } } + {#3} + #1 , \q_recursion_tail , \q_recursion_stop + } + \cs_set_protected:Npn \@@_finalise_blocks:nnnw #1#2#3#4 , + { + \quark_if_recursion_tail_stop:n {#4} + \intarray_gset:cnn { g_@@_ #3 _blocks_intarray } + { #1 + #2 } + {#4} + \exp_args:Nf \@@_finalise_blocks:nnnw + { \int_eval:n { #1 + 1 } } {#2} {#3} } +% \end{macrocode} +% With the setup done, read the main data file: it's easiest to do that as +% a token list with spaces retained. +% \begin{macrocode} \ior_open:Nn \g_@@_data_ior { UnicodeData.txt } \group_begin: \char_set_catcode_space:n { `\ }% @@ -190,8 +710,48 @@ \fi: \exp_after:wN \@@_data_auxi:w \l_@@_tmpa_tl \q_stop }% + \@@_finalise_blocks: \group_end: +\group_end: +% \end{macrocode} +% +% \begin{macro}[EXP]{\__kernel_codepoint_data:nn} +% \begin{macro}[EXP]{\@@_data:nnn} +% Recover data from a two-stage table: entirely generic as this applies to +% all tables (as we use the same block size for all of them). Notice that +% as we use indices not offsets we have to shuffle out-by-one issues. This +% function is needed \emph{before} loading the special casing data, as there +% we need to be able to check the standard case mappings. +% \begin{macrocode} +\cs_new:Npn \__kernel_codepoint_data:nn #1#2 + { + \exp_args:Nf \@@_data:nnn + { + \int_eval:n + { + \c_@@_block_size_int * + ( + \intarray_item:cn { c_@@_ #1 _index_intarray } + { + \int_div_truncate:nn {#2} + \c_@@_block_size_int + + 1 + } + - 1 + ) + } + } + {#2} {#1} + } +\cs_new:Npn \@@_data:nnn #1#2#3 + { + \intarray_item:cn { c_@@_ #3 _blocks_intarray } + { #1 + \int_mod:nn {#2} \c_@@_block_size_int + 1 } + } % \end{macrocode} +% \end{macro} +% \end{macro} +% % The other data files all use C-style comments so we have to worry about % |#| tokens (and reading as strings). The set up for case folding is in two % parts. For the basic (core) mappings, folding is the same as lower casing in @@ -199,15 +759,17 @@ % always store the result, splitting up the two or three code points in the input % as required. % \begin{macrocode} +\group_begin: \ior_open:Nn \g_@@_data_ior { CaseFolding.txt } \cs_set_protected:Npn \@@_data_auxi:w #1 ;~ #2 ;~ #3 ; #4 \q_stop { \if:w \tl_head:n { #2 ? } C \reverse_if:N \if_int_compare:w - \char_value_lccode:n {"#1} = "#3 ~ + \int_eval:n { \__kernel_codepoint_data:nn { lowercase } {"#1} + "#1 } + = "#3 ~ \tl_const:cx - { c_@@_foldcase_ \@@_generate_other:n { "#1 } _tl } - { \@@_generate:n { "#3 } } + { c_@@_casefold_ \codepoint_str_generate:n {"#1} _tl } + { {"#3} { } { } } \fi: \else: \if:w \tl_head:n { #2 ? } F @@ -215,25 +777,17 @@ \fi: \fi: } - \bool_lazy_or:nnF - { \sys_if_engine_luatex_p: } - { \sys_if_engine_xetex_p: } - { - \cs_set_protected:Npn \@@_data_auxi:w #1 ;~ #2 ;~ #3 ; #4 \q_stop - { - \if:w \tl_head:n { #2 ? } F - \@@_data_auxii:w #1 ~ #3 ~ \q_stop - \fi: - } - } +% \end{macrocode} +% Here, |#4| can have have a trailing space, so we tidy up a bit at the cost of +% speed for these small number of cases it applies to. +% \begin{macrocode} \cs_set_protected:Npn \@@_data_auxii:w #1 ~ #2 ~ #3 ~ #4 \q_stop { - \tl_const:cx { c_@@_foldcase_ \@@_generate_other:n { "#1 } _tl } + \tl_const:cx { c_@@_casefold_ \codepoint_str_generate:n {"#1} _tl } { - \@@_generate:n { "#2 } - \@@_generate:n { "#3 } - \tl_if_blank:nF {#4} - { \@@_generate:n { \int_value:w "#4 } } + {"#2} + {"#3} + { \tl_if_blank:nF {#4} { " \int_to_Hex:n {"#4} } } } } \ior_str_map_inline:Nn \g_@@_data_ior @@ -245,7 +799,7 @@ \ior_close:N \g_@@_data_ior % \end{macrocode} % For upper- and lowercasing special situations, there is a bit more to -% do as we also have title casing to consider, plus we need to stop part-way +% do as we also have titlecasing to consider, plus we need to stop part-way % through the file. % \begin{macrocode} \ior_open:Nn \g_@@_data_ior { SpecialCasing.txt } @@ -262,12 +816,11 @@ { \tl_if_empty:nF {#4} { - \tl_const:cx { c_@@_ #2 case_ \@@_generate_other:n { "#1 } _tl } + \tl_const:cx { c_@@_ #2 case_ \codepoint_str_generate:n {"#1} _tl } { - \@@_generate:n { "#3 } - \@@_generate:n { "#4 } - \tl_if_blank:nF {#5} - { \@@_generate:n { "#5 } } + {"#3} + {"#4} + { \tl_if_blank:nF {#5} {"#5} } } } } @@ -284,9 +837,64 @@ } \ior_close:N \g_@@_data_ior \group_end: - % \end{macrocode} % +% \begin{macro}[EXP]{\__kernel_codepoint_case:nn} +% \begin{macro}[EXP]{\@@_case:nnn} +% \begin{macro}[EXP] +% {\@@_uppercase:n, \@@_lowercase:n, \@@_titlecase:n, \@@_casefold:n} +% \begin{macro}[EXP]{\@_case:nn} +% With the core data files loaded, there is now a need to provide access to +% this information for other modules. That is done here such that case +% folding can also be covered. At this level, all that needs to be returned +% is the +% \begin{macrocode} +\cs_new:Npn \__kernel_codepoint_case:nn #1#2 + { + \exp_args:Ne \@@_case:nnn + { \codepoint_str_generate:n {#2} } {#1} {#2} + } +\cs_new:Npn \@@_case:nnn #1#2#3 + { + \cs_if_exist:cTF { c_@@_ #2 _ #1 _tl } + { + \tl_use:c + { c_@@_ #2 _ #1 _tl } + } + { \use:c { @@_ #2 :n } {#3} } + } +\cs_new:Npn \@@_uppercase:n { \@@_case:nn { uppercase } } +\cs_new:Npn \@@_lowercase:n { \@@_case:nn { lowercase } } +\cs_new:Npn \@@_titlecase:n { \@@_case:nn { uppercase } } +\cs_new:Npn \@@_casefold:n { \@@_case:nn { lowercase } } +\cs_new:Npn \@@_case:nn #1#2 + { + { \int_eval:n { \__kernel_codepoint_data:nn {#1} {#2} + #2 } } + { } + { } + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \begin{macro}[EXP]{\@@_nfd:n} +% \begin{macro}[EXP]{\@@_nfd:nn} +% A simple interface. +% \begin{macrocode} +\cs_new:Npn \@@_nfd:n #1 + { \exp_args:Ne \@@_nfd:nn { \codepoint_str_generate:n {#1} } {#1} } +\cs_new:Npn \@@_nfd:nn #1#2 + { + \tl_if_exist:cTF { c_@@_nfd_ #1 _tl } + { \tl_use:c { c_@@_nfd_ #1 _tl } } + { {#2} { } } + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% % \begin{macrocode} %<@@=text> % \end{macrocode} |