% \iffalse meta-comment
%
%% File: l3unicode.dtx
%
% Copyright (C) 2018-2022 The LaTeX Project
%
% It may be distributed and/or modified under the conditions of the
% LaTeX Project Public License (LPPL), either version 1.3c of this
% license or (at your option) any later version.  The latest version
% of this license is in the file
%
%    https://www.latex-project.org/lppl.txt
%
% This file is part of the "l3kernel bundle" (The Work in LPPL)
% and all files in that bundle must be distributed together.
%
% -----------------------------------------------------------------------
%
% The development version of the bundle can be found at
%
%    https://github.com/latex3/latex3
%
% for those people who are interested.
%
%<*driver>
\documentclass[full,kernel]{l3doc}
\begin{document}
  \DocInput{\jobname.dtx}
\end{document}
%</driver>
% \fi
%
% \title{^^A
%   The \pkg{l3unicode} package: Unicode support functions^^A
% }
%
% \author{^^A
%  The \LaTeX{} Project\thanks
%    {^^A
%      E-mail:
%        \href{mailto:latex-team@latex-project.org}
%          {latex-team@latex-project.org}^^A
%    }^^A
% }
%
% \date{Released 2022-11-02}
%
% \maketitle
%
% \begin{documentation}
%
% This module provides Unicode-specific functions along with loading data
% from a range of Unicode Consortium files. At present, it provides no
% public functions.
%
% \end{documentation}
%
% \begin{implementation}
%
% \section{\pkg{l3unicode} implementation}
%
%    \begin{macrocode}
%<*package>
%    \end{macrocode}
%
%    \begin{macrocode}
%<@@=char>
%    \end{macrocode}
%
% Text operations requires data from the Unicode Consortium. Data read into
% Unicode engine formats is at best a small part of what we need, so there
% is a loader here to set  up the appropriate data structures.
%
% As only the data needs to remain at the end of this process, everything
% is set up inside a group. The only thing that is outside is creating a
% stream: they are global anyway and it is best to force a stream for
% all engines. For performance reasons, some of the code here is very
% low-level: the material is read during loading \pkg{expl3} in package
% mode.
%
% Parsing the data files can be the same way for all engines, but the detail
% of data storage varies depending on whether they are Unicode or $8$-bit
% internally.Parsing is therefore done by common functions, with the data
% storage usign engine-specific auxiliaries.
%
% Conversion of a codepoint to a character (Unicode engines) or to one
% or more bytes ($8$-bit engines) is required. We might need those to be
% detokenized: if not, for Unicode engines they have the current category
% code.
%    \begin{macrocode}
\ior_new:N \g_@@_data_ior
\group_begin:
  \bool_lazy_or:nnTF
    { \sys_if_engine_luatex_p: }
    { \sys_if_engine_xetex_p: }
    {
      \cs_set:Npn \@@_generate_other:n #1
        { \tex_detokenize:D \tex_expandafter:D { \tex_Uchar:D #1 } }
      \cs_set:Npn \@@_generate:n #1
        {
          \tex_unexpanded:D \exp_after:wN
            { \tex_Ucharcat:D #1 ~ \tex_catcode:D #1 ~ }
        } 
    }
    {
      \cs_set:Npn \@@_generate_other:n #1
        {
          \tex_detokenize:D \tex_expandafter:D
            { \tex_expanded:D { \@@_generate:n {#1} } }
        }
      \cs_set:Npn \@@_generate:n #1
        {
          \use:e
            {
              \exp_not:N \@@_generate:nnnn
                \char_to_utfviii_bytes:n {#1}
            }
        }
      \cs_set:Npn \@@_generate:nnnn #1#2#3#4
        {
          \tex_unexpanded:D \exp_after:wN \exp_after:wN \exp_after:wN
            { \char_generate:nn {#1} { 13 } }
          \tl_if_blank:nF {#2}
            {
              \tex_unexpanded:D \exp_after:wN \exp_after:wN \exp_after:wN
                { \char_generate:nn {#2} { 13 } }
              \tl_if_blank:nF {#3}
                {
                  \tex_unexpanded:D \exp_after:wN \exp_after:wN \exp_after:wN
                    { \char_generate:nn {#3} { 13 } }
                  \tl_if_blank:nF {#4}
                    {
                      \tex_unexpanded:D
                        \exp_after:wN \exp_after:wN \exp_after:wN
                          { \char_generate:nn {#4} { 13 } }
                    }
                }
            }
        }
    }
%    \end{macrocode}
% Parse the main Unicode data file for two things. First, we want the titlecase
% exceptions: the one-to-one lower- and uppercase mappings it contains are all
% be covered by the \TeX{} data. Second, we need normalization data: at present,
% just the canonical \textsc{nfd} mappings. Those all yield either one or two
% codepoints, so the split is relatively easy.
%    \begin{macrocode}
  \cs_set_protected:Npn \@@_data_auxi:w
    #1 ; #2 ; #3 ; #4 ; #5 ; #6 ; #7 ; #8 ; #9 ;
    {
      \tl_if_blank:nF {#6}
        {
          \tl_if_head_eq_charcode:nNF {#6}  < % >
            { \@@_data_auxii:w #1 ; #6 ~ \q_stop }
        }
      \@@_data_auxiii:w #1 ;
    }
  \cs_set_protected:Npn \@@_data_auxii:w #1 ; #2 ~ #3 \q_stop
    {
      \tl_const:cx
        { c_@@_nfd_ \@@_generate_other:n { "#1 } _tl }
        {
          { \@@_generate:n { "#2 } }
          {
            \tl_if_blank:nF {#3}
              { \@@_generate:n { "#3 } }
            }
        }
    }
  \cs_set_protected:Npn \@@_data_auxiii:w
    #1 ; #2 ; #3 ; #4 ; #5 ; #6 ; #7 ~ \q_stop
    {
      \cs_set_nopar:Npn \l_@@_tmpa_tl {#7}
      \reverse_if:N \if_meaning:w \l_@@_tmpa_tl \c_empty_tl
        \cs_set_nopar:Npn \l_@@_tmpb_tl {#5}
        \reverse_if:N \if_meaning:w \l_@@_tmpa_tl \l_@@_tmpb_tl
          \tl_const:cx
            { c_@@_titlecase_ \@@_generate_other:n { "#1 } _tl }
            { \@@_generate:n { "#7 } }
        \fi:
      \fi:
    }
  \ior_open:Nn \g_@@_data_ior { UnicodeData.txt }
  \group_begin:
    \char_set_catcode_space:n { `\  }%
    \ior_map_variable:NNn \g_@@_data_ior \l_@@_tmpa_tl
      {%
        \if_meaning:w \l_@@_tmpa_tl \c_space_tl
          \exp_after:wN \ior_map_break:
        \fi:
        \exp_after:wN \@@_data_auxi:w \l_@@_tmpa_tl \q_stop
      }%
  \group_end:
%    \end{macrocode}
% The other data files all use C-style comments so we have to worry about
% |#| tokens (and reading as strings). The set up for case folding is in two
% parts. For the basic (core) mappings, folding is the same as lower casing in
% most positions so only store the differences. For the more complex foldings,
% always store the result, splitting up the two or three code points in the input
% as required.
%    \begin{macrocode}
  \ior_open:Nn \g_@@_data_ior { CaseFolding.txt }
  \cs_set_protected:Npn \@@_data_auxi:w #1 ;~ #2 ;~ #3 ; #4 \q_stop
    {
      \if:w \tl_head:n { #2 ? } C
        \reverse_if:N \if_int_compare:w
          \char_value_lccode:n {"#1} = "#3 ~
          \tl_const:cx
            { c_@@_foldcase_ \@@_generate_other:n { "#1 } _tl }
            { \@@_generate:n { "#3 } }
        \fi:
      \else:
        \if:w \tl_head:n { #2 ? } F
          \@@_data_auxii:w #1 ~ #3 ~ \q_stop
        \fi:
      \fi:
    }
  \bool_lazy_or:nnF
    { \sys_if_engine_luatex_p: }
    { \sys_if_engine_xetex_p: }
    {
      \cs_set_protected:Npn \@@_data_auxi:w #1 ;~ #2 ;~ #3 ; #4 \q_stop
        {
          \if:w \tl_head:n { #2 ? } F
            \@@_data_auxii:w #1 ~ #3 ~ \q_stop
          \fi:
        }
    }
  \cs_set_protected:Npn \@@_data_auxii:w #1 ~ #2 ~ #3 ~ #4 \q_stop
    {
      \tl_const:cx { c_@@_foldcase_ \@@_generate_other:n { "#1 } _tl }
        {
          \@@_generate:n { "#2 }
          \@@_generate:n { "#3 }
          \tl_if_blank:nF {#4}
            { \@@_generate:n { \int_value:w "#4 } }
        }
    }
  \ior_str_map_inline:Nn \g_@@_data_ior
    {
      \reverse_if:N \if:w \c_hash_str \tl_head:w #1 \c_hash_str \q_stop
        \@@_data_auxi:w #1 \q_stop
      \fi:
    }
  \ior_close:N \g_@@_data_ior
%    \end{macrocode}
% For upper- and lowercasing special situations, there is a bit more to
% do as we also have title casing to consider, plus we need to stop part-way
% through the file.
%    \begin{macrocode}
  \ior_open:Nn \g_@@_data_ior { SpecialCasing.txt }
  \cs_set_protected:Npn \@@_data_auxi:w
    #1 ;~ #2 ;~ #3 ;~ #4 ; #5 \q_stop
    {
      \use:n { \@@_data_auxii:w #1 ~ lower ~ #2 ~ } ~ \q_stop
      \use:n { \@@_data_auxii:w #1 ~ upper ~ #4 ~ } ~ \q_stop
      \str_if_eq:nnF {#3} {#4}
        { \use:n { \@@_data_auxii:w #1 ~ title ~ #3 ~ } ~ \q_stop }
    }
  \cs_set_protected:Npn \@@_data_auxii:w
    #1 ~ #2 ~ #3 ~ #4 ~ #5 \q_stop
    {
      \tl_if_empty:nF {#4}
        {
          \tl_const:cx { c_@@_ #2 case_ \@@_generate_other:n { "#1 } _tl }
            {
              \@@_generate:n { "#3 }
              \@@_generate:n { "#4 }
              \tl_if_blank:nF {#5}
                { \@@_generate:n { "#5 } }
            }
        }
    }
  \ior_str_map_inline:Nn \g_@@_data_ior
    {
      \str_if_eq:eeTF { \tl_head:w #1 \c_hash_str \q_stop } { \c_hash_str }
        {
          \str_if_eq:eeT
            {#1}
            { \c_hash_str \c_space_tl Conditional~Mappings }
            { \ior_map_break: }
        }
        { \@@_data_auxi:w #1 \q_stop }
    }
  \ior_close:N \g_@@_data_ior
\group_end:

%    \end{macrocode}
%
%    \begin{macrocode}
%<@@=text>
%    \end{macrocode}
%
%  Read the Unicode grapheme data. This is quite easy to handle and we only need
%  codepoints, not characters, so there is no need to worry about the engine in use.
%  As reading as a string is most convenient, we have to do some work to remove
%  spaces: the hardest part of the entire process!
%    \begin{macrocode}
\ior_new:N \g_@@_data_ior
\group_begin:
  \ior_open:Nn \g_@@_data_ior { GraphemeBreakProperty.txt }
  \cs_set_nopar:Npn \l_@@_tmpa_str { }
  \cs_set_nopar:Npn \l_@@_tmpb_str { }
  \cs_set_protected:Npn \@@_data_auxi:w #1 ;~ #2 ~ #3 \q_stop
    {
      \str_if_eq:VnF \l_@@_tmpb_str {#2}
        {
          \str_if_empty:NF \l_@@_tmpb_str
            {
              \clist_const:cx { c_@@_grapheme_ \l_@@_tmpb_str _clist }
                { \exp_after:wN \use_none:n \l_@@_tmpa_str }
              \cs_set_nopar:Npn \l_@@_tmpa_str { }
            }
          \cs_set_nopar:Npn \l_@@_tmpb_str {#2}
        }
      \@@_data_auxii:w #1 .. #1 .. #1 \q_stop
    }
  \cs_set_protected:Npn \@@_data_auxii:w #1 .. #2 .. #3 \q_stop
    {
      \cs_set_nopar:Npx \l_@@_tmpa_str
        {
          \l_@@_tmpa_str ,
          \tl_trim_spaces:n {#1} .. \tl_trim_spaces:n {#2}
        }
    }
  \ior_str_map_inline:Nn \g_@@_data_ior
    {
      \str_if_eq:eeF { \tl_head:w #1 \c_hash_str \q_stop } { \c_hash_str }
        {
          \tl_if_blank:nF {#1}
            { \@@_data_auxi:w #1 \q_stop }
        }
    }
  \ior_close:N \g_@@_data_ior
\group_end:    
%    \end{macrocode}
%
%    \begin{macrocode}
%</package>
%    \end{macrocode}
%
% \end{implementation}
%
% \PrintIndex
