1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
|
% \iffalse meta-comment
%
%% File: l3unicode.dtx
%
% Copyright (C) 2018-2019 The LaTeX3 Project
%
% It may be distributed and/or modified under the conditions of the
% LaTeX Project Public License (LPPL), either version 1.3c of this
% license or (at your option) any later version. The latest version
% of this license is in the file
%
% https://www.latex-project.org/lppl.txt
%
% This file is part of the "l3kernel bundle" (The Work in LPPL)
% and all files in that bundle must be distributed together.
%
% -----------------------------------------------------------------------
%
% The development version of the bundle can be found at
%
% https://github.com/latex3/latex3
%
% for those people who are interested.
%
%<*driver>
\documentclass[full,kernel]{l3doc}
\begin{document}
\DocInput{\jobname.dtx}
\end{document}
%</driver>
% \fi
%
% \title{^^A
% The \pkg{l3unicode} package: Unicode support functions^^A
% }
%
% \author{^^A
% The \LaTeX3 Project\thanks
% {^^A
% E-mail:
% \href{mailto:latex-team@latex-project.org}
% {latex-team@latex-project.org}^^A
% }^^A
% }
%
% \date{Released 2019-08-25}
%
% \maketitle
%
% \begin{documentation}
%
% This module provides Unicode-specific functions along with loading data
% from a range of Unicode Consortium files. At present, it provides no
% public functions.
%
% \end{documentation}
%
% \begin{implementation}
%
% \section{\pkg{l3unicode} implementation}
%
% \begin{macrocode}
%<*initex|package>
% \end{macrocode}
%
% \begin{macrocode}
%<@@=char>
% \end{macrocode}
%
% Case changing both for strings and \enquote{text} requires data from
% the Unicode Consortium. Some of this is build in to the format (as
% \tn{lccode} and \tn{uccode} values) but this covers only the simple
% one-to-one situations and does not fully handle for example case folding.
%
% As only the data needs to remain at the end of this process, everything
% is set up inside a group. The only thing that is outside is creating a
% stream: they are global anyway and it is best to force a stream for
% all engines. For performance reasons, some of the code here is very
% low-level: the material is read during loading \pkg{expl3} in package
% mode.
% \begin{macrocode}
\ior_new:N \g_@@_data_ior
\bool_lazy_or:nnTF { \sys_if_engine_luatex_p: } { \sys_if_engine_xetex_p: }
{
\group_begin:
% \end{macrocode}
% Set up a private copy of the char-generation primitive.
% \begin{macrocode}
\cs_set_eq:NN \@@_generate:w \tex_Uchar:D
% \end{macrocode}
% A fast local implementation for generating characters.
% \begin{macrocode}
\cs_set:Npx \@@_generate:n #1
{
\sys_if_engine_luatex:TF
{
\exp_not:N \lua_now:e
{
l3kernel.charcat
(
\exp_not:N \tex_number:D #1 ,
\exp_not:N \tex_the:D \tex_catcode:D #1
)
}
}
{
\exp_not:N \tex_Ucharcat:D
#1 ~
\tex_catcode:D #1 ~
}
}
% \end{macrocode}
% Parse the main Unicode data file for title case exceptions (the one-to-one
% lower and upper case mappings it contains are all be covered by the \TeX{}
% data). There are no comments in the main data file so this can be done using
% a standard mapping and no checks.
% \begin{macrocode}
\ior_open:Nn \g_@@_data_ior { UnicodeData.txt }
\cs_set_protected:Npn \@@_data_auxi:w
#1 ; #2 ; #3 ; #4 ; #5 ; #6 ; #7 ; #8 ; #9 ;
{ \@@_data_auxii:w #1 ; }
\cs_set_protected:Npn \@@_data_auxii:w
#1 ; #2 ; #3 ; #4 ; #5 ; #6 ; #7 \q_stop
{
\cs_set_nopar:Npn \l_@@_tmpa_tl {#7}
\reverse_if:N \if_meaning:w \l_@@_tmpa_tl \c_empty_tl
\cs_set_nopar:Npn \l_@@_tmpb_tl {#5}
\reverse_if:N \if_meaning:w \l_@@_tmpa_tl \l_@@_tmpb_tl
\tl_const:cx
{ c_@@_mixed_case_ \@@_generate:w "#1 _tl }
{ \@@_generate:n { "#7 } }
\fi:
\fi:
}
\ior_map_variable:NNn \g_@@_data_ior \l_@@_tmpa_tl
{
\if_meaning:w \l_@@_tmpa_tl \c_space_tl
\exp_after:wN \ior_map_break:
\fi:
\exp_after:wN \@@_data_auxi:w \l_@@_tmpa_tl \q_stop
}
\ior_close:N \g_@@_data_ior
% \end{macrocode}
% The other data files all use C-style comments so we have to worry about
% |#| tokens (and reading as strings). The set up for case folding is in two
% parts. For the basic (core) mappings, folding is the same as lower casing in
% most positions so only store the differences. For the more complex foldings,
% always store the result, splitting up the two or three code points in the input
% as required.
% \begin{macrocode}
\ior_open:Nn \g_@@_data_ior { CaseFolding.txt }
\cs_set_protected:Npn \@@_data_auxi:w #1 ;~ #2 ;~ #3 ; #4 \q_stop
{
\if:w \tl_head:n { #2 ? } C
\reverse_if:N \if_int_compare:w
\char_value_lccode:n {"#1} = "#3 ~
\tl_const:cx
{ c_@@_fold_case_ \@@_generate:w "#1 _tl }
{ \@@_generate:n { "#3 } }
\fi:
\else:
\if:w \tl_head:n { #2 ? } F
\@@_data_auxii:w #1 ~ #3 ~ \q_stop
\fi:
\fi:
}
\cs_set_protected:Npn \@@_data_auxii:w #1 ~ #2 ~ #3 ~ #4 \q_stop
{
\tl_const:cx { c_@@_fold_case_ \@@_generate:w "#1 _tl }
{
\@@_generate:n { "#2 }
\@@_generate:n { "#3 }
\tl_if_blank:nF {#4}
{ \@@_generate:n { \int_value:w "#4 } }
}
}
\ior_str_map_inline:Nn \g_@@_data_ior
{
\reverse_if:N \if:w \c_hash_str \tl_head:w #1 \c_hash_str \q_stop
\@@_data_auxi:w #1 \q_stop
\fi:
}
\ior_close:N \g_@@_data_ior
% \end{macrocode}
% For upper and lower casing special situations, there is a bit more to
% do as we also have title casing to consider, plus we need to stop part-way
% through the file.
% \begin{macrocode}
\ior_open:Nn \g_@@_data_ior { SpecialCasing.txt }
\cs_set_protected:Npn \@@_data_auxi:w
#1 ;~ #2 ;~ #3 ;~ #4 ; #5 \q_stop
{
\use:n { \@@_data_auxii:w #1 ~ lower ~ #2 ~ } ~ \q_stop
\use:n { \@@_data_auxii:w #1 ~ upper ~ #4 ~ } ~ \q_stop
\str_if_eq:nnF {#3} {#4}
{ \use:n { \@@_data_auxii:w #1 ~ mixed ~ #3 ~ } ~ \q_stop }
}
\cs_set_protected:Npn \@@_data_auxii:w
#1 ~ #2 ~ #3 ~ #4 ~ #5 \q_stop
{
\tl_if_empty:nF {#4}
{
\tl_const:cx { c_@@_ #2 _case_ \@@_generate:w "#1 _tl }
{
\@@_generate:n { "#3 }
\@@_generate:n { "#4 }
\tl_if_blank:nF {#5}
{ \@@_generate:n { "#5 } }
}
}
}
\ior_str_map_inline:Nn \g_@@_data_ior
{
\str_if_eq:eeTF
{ \tl_head:w #1 \c_hash_str \q_stop }
{ \c_hash_str }
{
\str_if_eq:eeT
{#1}
{ \c_hash_str \c_space_tl Conditional~Mappings }
{ \ior_map_break: }
}
{ \@@_data_auxi:w #1 \q_stop }
}
\ior_close:N \g_@@_data_ior
\group_end:
}
% \end{macrocode}
% For the $8$-bit engines, the above is skipped but there is still some
% set up required. As case changing can only be applied to bytes, and
% they have to be in the ASCII range, we define a series of data stores
% to represent them, and the data are used such that only these are
% ever case-changed. We do open and close one file to force allocation of
% a read: this keeps all engines in line.
% \begin{macrocode}
{
\group_begin:
\cs_set_protected:Npn \@@_tmp:NN #1#2
{
\quark_if_recursion_tail_stop:N #2
\tl_const:cn { c_@@_upper_case_ #2 _tl } {#1}
\tl_const:cn { c_@@_lower_case_ #1 _tl } {#2}
\tl_const:cn { c_@@_fold_case_ #1 _tl } {#2}
\@@_tmp:NN
}
\@@_tmp:NN
AaBbCcDdEeFfGgHhIiJjKkLlMmNnOoPpQqRrSsTtUuVvWwXxYyZz
? \q_recursion_tail \q_recursion_stop
\ior_open:Nn \g_@@_data_ior { UnicodeData.txt }
\ior_close:N \g_@@_data_ior
\group_end:
}
% \end{macrocode}
%
% \begin{macrocode}
%</initex|package>
% \end{macrocode}
%
% \end{implementation}
%
% \PrintIndex
|
