% \iffalse meta-comment
%
%% File: l3unicode.dtx
%
% Copyright (C) 2018-2019 The LaTeX3 Project
%
% It may be distributed and/or modified under the conditions of the
% LaTeX Project Public License (LPPL), either version 1.3c of this
% license or (at your option) any later version.  The latest version
% of this license is in the file
%
%    https://www.latex-project.org/lppl.txt
%
% This file is part of the "l3kernel bundle" (The Work in LPPL)
% and all files in that bundle must be distributed together.
%
% -----------------------------------------------------------------------
%
% The development version of the bundle can be found at
%
%    https://github.com/latex3/latex3
%
% for those people who are interested.
%
%<*driver>
\documentclass[full,kernel]{l3doc}
\begin{document}
  \DocInput{\jobname.dtx}
\end{document}
%</driver>
% \fi
%
% \title{^^A
%   The \pkg{l3unicode} package: Unicode support functions^^A
% }
%
% \author{^^A
%  The \LaTeX3 Project\thanks
%    {^^A
%      E-mail:
%        \href{mailto:latex-team@latex-project.org}
%          {latex-team@latex-project.org}^^A
%    }^^A
% }
%
% \date{Released 2019-03-26}
%
% \maketitle
%
% \begin{documentation}
%
% This module provides Unicode-specific functions along with loading data
% from a range of Unicode Consortium files. At present, it provides no
% public functions.
%
% \end{documentation}
%
% \begin{implementation}
%
% \section{\pkg{l3unicode} implementation}
%
%    \begin{macrocode}
%<*initex|package>
%    \end{macrocode}
%
%    \begin{macrocode}
%<@@=char>
%    \end{macrocode}
%
% Case changing both for strings and \enquote{text} requires data from
% the Unicode Consortium. Some of this is build in to the format (as
% \tn{lccode} and \tn{uccode} values) but this covers only the simple
% one-to-one situations and does not fully handle for example case folding.
%
% As only the data needs to remain at the end of this process, everything
% is set up inside a group. The only thing that is outside is creating a
% stream: they are global anyway and it is best to force a stream for
% all engines. For performance reasons, some of the code here is very
% low-level: the material is read during loading \pkg{expl3} in package
% mode.
%    \begin{macrocode}
\ior_new:N \g_@@_data_ior
\bool_lazy_or:nnTF { \sys_if_engine_luatex_p: } { \sys_if_engine_xetex_p: }
  {
    \group_begin:
%    \end{macrocode}
%   Set up a private copy of the char-generation primitive.
%    \begin{macrocode}
      \cs_set_eq:NN \@@_generate:w \tex_Uchar:D
%    \end{macrocode}
%   A fast local implementation for generating characters.
%    \begin{macrocode}
      \cs_set:Npx \@@_generate:n #1
        {
          \sys_if_engine_luatex:TF
            {
              \exp_not:N \lua_now:e
                {
                  l3kernel.charcat
                    (
                      \exp_not:N \tex_number:D #1 ,
                      \exp_not:N \tex_the:D \tex_catcode:D #1
                    )
                }
            }
            {
              \exp_not:N \tex_Ucharcat:D
                #1 ~
                \tex_catcode:D #1 ~
            }
        } 
%    \end{macrocode}
% Parse the main Unicode data file for title case exceptions (the one-to-one
% lower and upper case mappings it contains are all be covered by the \TeX{}
% data). There are no comments in the main data file so this can be done using
% a standard mapping and no checks.
%    \begin{macrocode}
      \ior_open:Nn \g_@@_data_ior { UnicodeData.txt }
      \cs_set_protected:Npn \@@_data_auxi:w
        #1 ; #2 ; #3 ; #4 ; #5 ; #6 ; #7 ; #8 ; #9 ;
        { \@@_data_auxii:w #1 ; }
      \cs_set_protected:Npn \@@_data_auxii:w
        #1 ; #2 ; #3 ; #4 ; #5 ; #6 ; #7 \q_stop
        {
          \cs_set_nopar:Npn \l_@@_tmpa_tl {#7}
          \reverse_if:N \if_meaning:w \l_@@_tmpa_tl \c_empty_tl
            \cs_set_nopar:Npn \l_@@_tmpb_tl {#5}
            \reverse_if:N \if_meaning:w \l_@@_tmpa_tl \l_@@_tmpb_tl
              \tl_const:cx
                { c_@@_mixed_case_ \@@_generate:w "#1 _tl }
                { \@@_generate:n { "#7 } }
            \fi:
          \fi:
        }
      \ior_map_variable:NNn \g_@@_data_ior \l_@@_tmpa_tl
        {
          \if_meaning:w \l_@@_tmpa_tl \c_space_tl
            \exp_after:wN \ior_map_break:
          \fi:
          \exp_after:wN \@@_data_auxi:w \l_@@_tmpa_tl \q_stop
        }
      \ior_close:N \g_@@_data_ior
%    \end{macrocode}
% The other data files all use C-style comments so we have to worry about
% |#| tokens (and reading as strings). The set up for case folding is in two
% parts. For the basic (core) mappings, folding is the same as lower casing in
% most positions so only store the differences. For the more complex foldings,
% always store the result, splitting up the two or three code points in the input
% as required.
%    \begin{macrocode}
      \ior_open:Nn \g_@@_data_ior { CaseFolding.txt }
      \cs_set_protected:Npn \@@_data_auxi:w #1 ;~ #2 ;~ #3 ; #4 \q_stop
        {
          \if:w \tl_head:n { #2 ? } C
            \reverse_if:N \if_int_compare:w
              \char_value_lccode:n {"#1} = "#3 ~
              \tl_const:cx
                { c_@@_fold_case_ \@@_generate:w "#1 _tl }
                { \@@_generate:n { "#3 } }
            \fi:
          \else:
            \if:w \tl_head:n { #2 ? } F
              \@@_data_auxii:w #1 ~ #3 ~ \q_stop
            \fi:
          \fi:
        }
      \cs_set_protected:Npn \@@_data_auxii:w #1 ~ #2 ~ #3 ~ #4 \q_stop
        {
          \tl_const:cx { c_@@_fold_case_ \@@_generate:w "#1 _tl }
            {
              \@@_generate:n { "#2 }
              \@@_generate:n { "#3 }
              \tl_if_blank:nF {#4}
                { \@@_generate:n { \int_value:w "#4 } }
            }
        }
      \ior_str_map_inline:Nn \g_@@_data_ior
        {
          \reverse_if:N \if:w \c_hash_str \tl_head:w #1 \c_hash_str \q_stop
            \@@_data_auxi:w #1 \q_stop
          \fi:
        }
      \ior_close:N \g_@@_data_ior
%    \end{macrocode}
% For upper and lower casing special situations, there is a bit more to
% do as we also have title casing to consider, plus we need to stop part-way
% through the file.
%    \begin{macrocode}
      \ior_open:Nn \g_@@_data_ior { SpecialCasing.txt }
      \cs_set_protected:Npn \@@_data_auxi:w
        #1 ;~ #2 ;~ #3 ;~ #4 ; #5 \q_stop
        {
          \use:n { \@@_data_auxii:w #1 ~ lower ~ #2 ~ } ~ \q_stop
          \use:n { \@@_data_auxii:w #1 ~ upper ~ #4 ~ } ~ \q_stop
          \str_if_eq:nnF {#3} {#4}
            { \use:n { \@@_data_auxii:w #1 ~ mixed ~ #3 ~ } ~ \q_stop }
        }
      \cs_set_protected:Npn \@@_data_auxii:w
        #1 ~ #2 ~ #3 ~ #4 ~ #5 \q_stop
        {
          \tl_if_empty:nF {#4}
            {
              \tl_const:cx { c_@@_ #2 _case_ \@@_generate:w "#1 _tl }
                {
                  \@@_generate:n { "#3 }
                  \@@_generate:n { "#4 }
                  \tl_if_blank:nF {#5}
                    { \@@_generate:n { "#5 } }
                }
            }
        }
      \ior_str_map_inline:Nn \g_@@_data_ior
        {
          \str_if_eq:eeTF
            { \tl_head:w #1 \c_hash_str \q_stop }
            { \c_hash_str }
            {
              \str_if_eq:eeT
                {#1}
                { \c_hash_str \c_space_tl Conditional~Mappings }
                { \ior_map_break: }
            }
            { \@@_data_auxi:w #1 \q_stop }
        }
      \ior_close:N \g_@@_data_ior
    \group_end:
  }
%    \end{macrocode}
% For the $8$-bit engines, the above is skipped but there is still some
% set up required. As case changing can only be applied to bytes, and
% they have to be in the ASCII range, we define a series of data stores
% to represent them, and the data are used such that only these are
% ever case-changed. We do open and close one file to force allocation of
% a read: this keeps all engines in line.
%    \begin{macrocode}
  {
    \group_begin:
      \cs_set_protected:Npn \@@_tmp:NN #1#2
        {
          \quark_if_recursion_tail_stop:N #2
          \tl_const:cn { c_@@_upper_case_ #2 _tl } {#1}
          \tl_const:cn { c_@@_lower_case_ #1 _tl } {#2}
          \tl_const:cn { c_@@_fold_case_  #1 _tl } {#2}
          \@@_tmp:NN
        }
      \@@_tmp:NN
        AaBbCcDdEeFfGgHhIiJjKkLlMmNnOoPpQqRrSsTtUuVvWwXxYyZz
        ? \q_recursion_tail \q_recursion_stop
      \ior_open:Nn \g_@@_data_ior { UnicodeData.txt }
      \ior_close:N \g_@@_data_ior
    \group_end:
  }
%    \end{macrocode}
%
%    \begin{macrocode}
%</initex|package>
%    \end{macrocode}
%
% \end{implementation}
%
% \PrintIndex