1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
|
-- lua-uni-normalize.lua
-- Copyright 2020 Marcel Krüger
--
-- This work may be distributed and/or modified under the
-- conditions of the LaTeX Project Public License, either version 1.3
-- of this license or (at your option) any later version.
-- The latest version of this license is in
-- http://www.latex-project.org/lppl.txt
-- and version 1.3 or later is part of all distributions of LaTeX
-- version 2005/12/01 or later.
--
-- This work has the LPPL maintenance status `maintained'.
--
-- The Current Maintainer of this work is Marcel Krüger
-- Provide all four kinds of Unicode normalization
local newtable = lua.newtable
local move = table.move
local char = utf8.char
local codes = utf8.codes
local unpack = table.unpack
kpse.set_program_name'kpsewhich'
local ccc, composition_mapping, decomposition_mapping, compatibility_mapping do
local function doubleset(ts, key, v1, kind, v2)
ts[1][key] = v1
ts[3][key] = v2
if not kind then
ts[2][key] = v2
end
return ts
end
local p = require'lua-uni-parse'
local l = lpeg
local Cnil = l.Cc(nil)
local letter = lpeg.R('AZ', 'az')
ccc, decomposition_mapping, compatibility_mapping
= unpack(p.parse_file('UnicodeData', l.Cf(
l.Ct(l.Ct'' * l.Ct'' * l.Ct'') * (
l.Cg(p.fields(p.codepoint,
p.ignore_field,
p.ignore_field,
'0' * Cnil + p.number,
p.ignore_field,
('<' * l.C(letter^1) * '> ' + Cnil)
* l.Ct(p.codepoint * (' ' * p.codepoint)^0)^-1,
p.ignore_line)) + p.eol
)^0 * -1, doubleset)))
composition_mapping = {}
local composition_exclusions = { [0x00958] = true, [0x00959] = true,
[0x0095A] = true, [0x0095B] = true, [0x0095C] = true, [0x0095D] = true,
[0x0095E] = true, [0x0095F] = true, [0x009DC] = true, [0x009DD] = true,
[0x009DF] = true, [0x00A33] = true, [0x00A36] = true, [0x00A59] = true,
[0x00A5A] = true, [0x00A5B] = true, [0x00A5E] = true, [0x00B5C] = true,
[0x00B5D] = true, [0x00F43] = true, [0x00F4D] = true, [0x00F52] = true,
[0x00F57] = true, [0x00F5C] = true, [0x00F69] = true, [0x00F76] = true,
[0x00F78] = true, [0x00F93] = true, [0x00F9D] = true, [0x00FA2] = true,
[0x00FA7] = true, [0x00FAC] = true, [0x00FB9] = true, [0x0FB1D] = true,
[0x0FB1F] = true, [0x0FB2A] = true, [0x0FB2B] = true, [0x0FB2C] = true,
[0x0FB2D] = true, [0x0FB2E] = true, [0x0FB2F] = true, [0x0FB30] = true,
[0x0FB31] = true, [0x0FB32] = true, [0x0FB33] = true, [0x0FB34] = true,
[0x0FB35] = true, [0x0FB36] = true, [0x0FB38] = true, [0x0FB39] = true,
[0x0FB3A] = true, [0x0FB3B] = true, [0x0FB3C] = true, [0x0FB3E] = true,
[0x0FB40] = true, [0x0FB41] = true, [0x0FB43] = true, [0x0FB44] = true,
[0x0FB46] = true, [0x0FB47] = true, [0x0FB48] = true, [0x0FB49] = true,
[0x0FB4A] = true, [0x0FB4B] = true, [0x0FB4C] = true, [0x0FB4D] = true,
[0x0FB4E] = true,
[0x02ADC] = true, [0x1D15E] = true, [0x1D15F] = true, [0x1D160] = true,
[0x1D161] = true, [0x1D162] = true, [0x1D163] = true, [0x1D164] = true,
[0x1D1BB] = true, [0x1D1BC] = true, [0x1D1BD] = true, [0x1D1BE] = true,
[0x1D1BF] = true, [0x1D1C0] = true,
}
for cp, decomp in next, decomposition_mapping do
if #decomp > 1 and not (composition_exclusions[cp] or ccc[decomp[1]]) then
local mapping = composition_mapping[decomp[1]]
if not mapping then
mapping = {}
composition_mapping[decomp[1]] = mapping
end
mapping[decomp[2]] = cp
end
end
local function fixup_decomp(decomp)
local first = decomp[1]
local first_decomp = decomposition_mapping[first]
if not first_decomp then return false end
if fixup_decomp(first_decomp) then
print('nested', first)
end
move(decomp, 2, #decomp, #first_decomp + 1)
move(first_decomp, 1, #first_decomp, 1, decomp)
return true
end
-- Fixup stage
for cp, decomp in next, decomposition_mapping do
if fixup_decomp(decomp) then
-- print(':(', cp)
end
end
-- NFKD edition
local DEBUG = false
local function fixup_decomp(orig, decomp)
local work
local shared = decomposition_mapping[orig] == decomp
local j = 0
for i = 1, #decomp do
local cp = decomp[i]
local cp_decomp = compatibility_mapping[cp]
if cp_decomp then
if shared then
local old = decomp
decomp = {}
compatibility_mapping[orig] = decomp
move(old, 1, #old, 1, decomp)
end
decomp[i] = cp_decomp
j = j + #cp_decomp
work = true
else
j = j + 1
end
end
if not work then return decomp end
for i = #decomp, 1, -1 do
local v = decomp[i]
if type(v) == 'number' then
decomp[j] = v
j = j - 1
else
local count = #v
move(v, 1, count, j - count + 1, decomp)
j = j - count
end
end
assert(j == 0)
return decomp
end
-- Fixup stage
for cp, decomp in next, compatibility_mapping do
fixup_decomp(cp, decomp)
end
end
local function ccc_reorder(codepoints, i, j, k)
if k >= j then return end
local first = codepoints[k]
local first_ccc = ccc[first]
if not first_ccc then
return ccc_reorder(codepoints, k+1, j, k+1)
end
local new_pos = k
local cur_ccc
repeat
new_pos = new_pos + 1
if new_pos > j then break end
local cur = codepoints[new_pos]
cur_ccc = ccc[cur]
until (not cur_ccc) or (cur_ccc >= first_ccc)
new_pos = new_pos - 1
if new_pos == k then
return ccc_reorder(codepoints, i, j, k+1)
end
move(codepoints, k+1, new_pos, k)
codepoints[new_pos] = first
return ccc_reorder(codepoints, i, j, k == i and i or k-1)
end
local result_table = {}
local function get_string()
local result_table = result_table
local s = char(unpack(result_table))
for i=1,#result_table do
result_table[i] = nil
end
return s
end
function to_nfd_table(s, decomposition_mapping)
local new_codepoints = result_table
local j = 1
for _, c in codes(s) do
local decomposed = decomposition_mapping[c]
if decomposed then
move(decomposed, 1, #decomposed, j, new_codepoints)
j = j + #decomposed
elseif c >= 0xAC00 and c <= 0xD7A3 then
c = c - 0xAC00
local tIndex = c % 28
c = c // 28
local vIndex = c % 21
local lIndex = c // 21
new_codepoints[j] = 0x1100 + lIndex
new_codepoints[j+1] = 0x1161 + vIndex
if tIndex == 0 then
j = j + 2
else
new_codepoints[j+2] = 0x11A7 + tIndex
j = j + 3
end
else
new_codepoints[j] = c
j = j + 1
end
end
ccc_reorder(new_codepoints, 1, #new_codepoints, 1)
end
local function to_nfd(s)
to_nfd_table(s, decomposition_mapping)
return get_string()
end
local function to_nfkd(s)
to_nfd_table(s, compatibility_mapping)
return get_string()
end
local function to_nfc_generic(s, decomposition_mapping)
to_nfd_table(s, decomposition_mapping)
local codepoints = result_table
local starter, lookup, last_ccc, lvt
local j = 1
for i, c in ipairs(codepoints) do
local cur_ccc = ccc[c]
if lookup then
if (cur_ccc == nil) == (cur_ccc == last_ccc) then -- unblocked
local composed = lookup[c]
if composed then
codepoints[starter] = composed
lookup = composition_mapping[composed]
goto CONTINUE
end
end
elseif lvt then
if lvt == 1 then
if c >= 0x1161 and c <= 0x1175 then
lvt = 2
codepoints[starter] = ((codepoints[starter] - 0x1100) * 21 + c - 0x1161) * 28 + 0xAC00
goto CONTINUE
end
else -- if lvt == 2 then
if c >= 0x11A8 and c <= 0x11C2 then
lvt = nil
codepoints[starter] = codepoints[starter] + c - 0x11A7
goto CONTINUE
end
end
end
codepoints[j] = c
lvt = nil
if not cur_ccc then
starter = j
lookup = composition_mapping[c]
if not lookup and c >= 0x1100 and c <= 0x1112 then
lvt = 1
end
end
j = j + 1
last_ccc = cur_ccc
::CONTINUE::
end
for i = j,#codepoints do codepoints[i] = nil end
return get_string()
end
local function to_nfc(s)
return to_nfc_generic(s, decomposition_mapping)
end
local function to_nfkc(s)
return to_nfc_generic(s, compatibility_mapping)
end
-- allowed_characters only works reliably if it's closed under canonical decomposition mappings
-- but it should fail in reasonable ways as long as it's at least closed under full canonical decompositions
local function nodes_to_nfc(head, f, allowed_characters, preserve_attr)
if not head then return head end
local tmp_node = node.new'temp'
-- This is more complicated since we want to ensure that nodes (including their attributes and properties) are preserved whenever possible
--
-- We use three passes:
-- 1. Decompose composition exclusions etc.
local n = head
while n do
local char = node.is_char(n, f)
if char then
local decomposed = decomposition_mapping[char]
if decomposed then
local compose_lookup = composition_mapping[decomposed[1]]
if not (compose_lookup and compose_lookup[decomposed[2]]) then
local available = true
if allowed_characters then
-- This is probably buggy for werd fonts
for i=1, #decomposed do
if not allowed_characters[decomposed[i]] then
available = false
break
end
end
end
if available then
-- Here we never want to compose again, so we can decompose directly
n.char = decomposed[1]
for i=2, #decomposed do
local nn = node.copy(n)
nn.char = decomposed[i]
node.insert_after(head, n, nn)
n = nn
end
end
end
end
end
n = n.next
end
-- 2. Reorder marks
local last_ccc
n = head
local prev = head.prev
tmp_node.next, head.prev = head, tmp_node
while n do
local char = node.is_char(n, f)
if char then
local this_ccc = ccc[char]
if last_ccc and this_ccc and last_ccc > this_ccc then
local nn = n
while nn ~= tmp_node do
nn = nn.prev
local nn_char = node.is_char(nn, f)
if not nn_char then break end
local nn_ccc = ccc[nn_char]
if not nn_ccc or nn_ccc <= this_ccc then break end
end
local before, after = n.prev, n.next
node.insert_after(head, nn, n)
before.next = after
if after then after.prev = before end
n = after
else
n = n.next
last_ccc = this_ccc
end
else
n = n.next
last_ccc = nil
end
end
head, head.prev = tmp_node.next, prev
-- 3. The rest: Maybe decompose and then compose again
local starter_n, starter, lookup
local starter_decomposition
local last_ccc
local i -- index into starter_decomposition
local i_ccc
n = head
node.insert_after(head, nil, tmp_node)
repeat
local char = node.is_char(n, f)
local this_ccc = ccc[char] or 300
while i and i_ccc <= this_ccc do
local new_starter = lookup and lookup[starter_decomposition[i]]
if new_starter and (not allowed_characters or allowed_characters[new_starter]) then
starter = new_starter
starter_n.char = starter
lookup = composition_mapping[starter]
else
local nn = node.copy(starter_n)
nn.char = starter_decomposition[i]
node.insert_before(head, n, nn)
last_ccc = i_ccc
end
i = i + 1
local i_char = starter_decomposition[i]
if i_char then
i_ccc = ccc[starter_decomposition[i]] or 300
else
i = nil
end
end
if char then
if lookup and (this_ccc == 300) == (this_ccc == last_ccc) then
local new_starter = lookup[char]
if new_starter and (not allowed_characters or allowed_characters[new_starter]) and (not preserve_attr or starter_n.attr == n.attr) then
local last = n.prev
node.remove(head, n)
node.free(n)
n = last
starter = new_starter
starter_n.char, char = starter, starter
lookup = composition_mapping[starter]
else
last_ccc = this_ccc
end
-- Now handle Hangul syllables. We never decompose them since we would just recompose them anyway and they are starters
elseif not lookup and this_ccc == 300 and last_ccc == 300 then
if starter >= 0x1100 and starter <= 0x1112 and char >= 0x1161 and char <= 0x1175 then -- L + V -> LV
local new_starter = ((starter - 0x1100) * 21 + char - 0x1161) * 28 + 0xAC00
if (not allowed_characters or allowed_characters[new_starter]) and (not preserve_attr or starter_n.attr == n.attr) then
node.remove(head, n)
node.free(n)
starter = starter
starter_n.char, char = starter, starter
lookup = composition_mapping[starter]
n = starter_n
end
elseif char >= 0x11A8 and char <= 0x11C2 and starter >= 0xAC00 and starter <= 0xD7A3 and (starter-0xAC00) % 28 == 0 then -- LV + T -> LVT
local new_starter = starter + char - 0x11A7
if (not allowed_characters or allowed_characters[new_starter]) and (not preserve_attr or starter_n.attr == n.attr) then
node.remove(head, n)
node.free(n)
starter = new_starter
starter_n.char, char = starter, starter
lookup = composition_mapping[starter]
n = starter_n
end
end
else
last_ccc = this_ccc
end
if this_ccc == 300 then
starter_n = n
starter_decomposition = decomposition_mapping[char]
if allowed_characters and starter_decomposition then
for i=1, #starter_decomposition do
if not allowed_characters[starter_decomposition[i]] then
starter_decomposition = nil
break
end
end
end
starter = starter_decomposition and starter_decomposition[1] or char
starter_n.char = starter
lookup = composition_mapping[starter]
if starter_decomposition then
i, i_ccc = 2, ccc[starter_decomposition[2]] or 300
else
i, i_ccc = nil
end
end
else
starter, lookup, last_ccc, last_decomposition, i, i_ccc = nil
end
if n == tmp_node then
node.remove(head, tmp_node)
break
end
n = n.next
until false
node.free(tmp_node)
return head
end
local todirect, tonode = node.direct.todirect, node.direct.tonode
return {
NFD = to_nfd,
NFC = to_nfc,
NFKD = to_nfkd,
NFKC = to_nfkc,
node = {
NFC = nodes_to_nfc,
},
direct = {
NFC = function(head, ...) return todirect(nodes_to_nfc(tonode(head), ...)) end,
},
}
-- print(require'inspect'{to_nfd{0x1E0A}, to_nfc{0x1E0A}})
-- print(require'inspect'{to_nfd{0x1100, 0x1100, 0x1161, 0x11A8}, to_nfc{0x1100, 0x1100, 0x1161, 0x11A8}})
|