summaryrefslogtreecommitdiff
path: root/macros/luatex/latex/uninormalize/unicode-normalization.lua
blob: b8a58b63bb68927f8e6cc8a8504f8fe57bc19dad (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
-- char-def now contains all necessary fields, no need for a custom file
if not characters then
  require "char-def"
end

if not unicode then require('unicode') end
unicode.conformance = unicode.conformance or { }

unicharacters = unicharacters or {}
uni = unicode.utf8
unidata = characters.data

function printf(s, ...) print(string.format(s, ...)) end
-- function debug(s, ...) io.write("DEBUG: ", string.format(s, ...), "\n") end
function warn(s, ...) io.write("Warning: ", string.format(s, ...), "\n") end

function md5sum(any) return md5.hex(md5.sum(any)) end

-- Rehash the character data
unicharacters.combinee = { }
unicharacters.context = unicharacters.context or { }
local charu = unicode.utf8.char

function unicharacters.context.rehash2()
  for ucode, udata in pairs(unidata) -- *not* ipairs :-)
  do
    local sp = udata.specials
    if sp then
      if sp[1] == 'char' then
        -- local ucode = udata.unicodeslot
        local entry = { combinee = sp[2], combining = sp[3], combined = ucode }
        if not unicharacters.combinee[sp[2]]
        then unicharacters.combinee[sp[2]] = { } end
        local n = #unicharacters.combinee[sp[2]]
        unicharacters.combinee[sp[2]][n+1] = entry
      end
    end
    -- copy context's combining field to combclass field
    -- this field was in the custom copy of the char-def.lua that we no longer use
    udata.combclass = udata.combining
  end
end


unicharacters.context.rehash2()
combdata = unicharacters.combinee

--[[ function unicode.conformance.is_hangul(ucode)
  return ucode >= 0xAC00 and ucode <= 0xD7A3
end ]] -- Make it local for the moment
local function is_hangul(char)
  return char >= 0xAC00 and char <= 0xD7A3
end

local function is_jamo(char)
  if char < 0x1100 then return false
  elseif char < 0x1160 then return 'choseong'
  elseif char < 0x11A8 then return 'jungseong'
  elseif char < 0x11FA then return 'jongseong'
  else return false
  end
end

local function decompose(ucode, compat) -- if compat then compatibility
  local invbuf = { }
  local sp = unidata[ucode].specials
  if not sp
  then return { ucode } else
    if compat then compat = (sp[1] == 'compat') else compat = false end
    while sp[1] == 'char' or compat do
      head, tail = sp[2], sp[3]
      if not tail then invbuf[#invbuf + 1] = head break end -- singleton
      invbuf[#invbuf + 1] = tail
        sp = unidata[head].specials
        if not sp then invbuf[#invbuf + 1] = head sp = { } end
      -- end -- not unidata[head]
    end -- while sp[1] == 'char' or compat
  end -- not sp

  local seq = { }
  for i = #invbuf, 1, -1
  do seq[#seq + 1] = invbuf[i]
  end
  return seq
end

local function canon(seq) -- Canonical reordering
  if #seq < 3 then return seq end
  local c1, c2, buf
  -- I'd never thought I'd implement an actual bubble sort some day ;-)
  for k = #seq - 1, 1, -1 do
    for i = 2, k do -- was k - 1!  Argh!
      c1 = unidata[seq[i]].combclass
      c2 = unidata[seq[i+1]].combclass
      if c1 and c2 then
        if c1 > c2 then
          buf = seq[i]
          seq[i] = seq[i+1]
          seq[i+1] = buf
        end
      end
    end
  end
  return seq
end

if not math.div then -- from l-math.lua
  function math.div(n, m)
    return math.floor(n/m)
  end
end

local SBase, LBase, VBase, TBase = 0xAC00, 0x1100, 0x1161, 0x11A7
local LCount, VCount, TCount = 19, 21, 28
local NCount = VCount * TCount
local SCount = TCount * NCount

local function decompose_hangul(ucode) -- assumes input is really a Hangul
  local SIndex = ucode - SBase
  local L = LBase + math.div(SIndex, NCount)
  local V = VBase + math.div((SIndex % NCount), TCount)
  local T = TBase + SIndex % TCount
  if T == TBase then T = nil end
  return { L, V, T }
end

-- To NFK?D.
function toNF_D_or_KD(unistring, compat)
  local nfd, seq = { }, { }
  for uchar in uni.gmatch(unistring, '.') do
    local ucode = uni.byte(uchar)
    if is_hangul(ucode) then
      seq = decompose_hangul(ucode)
      for  _, c in ipairs(seq)
      do nfd[#nfd + 1] = c
      end
      seq = { }
    elseif not unidata[ucode]
    then nfd[#nfd + 1] = ucode else
      local ccc = unidata[ucode].combclass
      if not ccc or ccc == 0 then
        seq = canon(seq)
        for _, c in ipairs(seq) do nfd[#nfd + 1] = c end
        seq = decompose(ucode, compat)
      else seq[#seq + 1] = ucode
      end -- not ccc or ccc == 0
    end -- if is_hangul(ucode) / elseif not unidata[ucode]
  end -- for uchar in uni.gmatch(unistring, ".")

  if #seq > 0 then
    seq = canon(seq)
    for _, c in ipairs(seq) do nfd[#nfd + 1] = c end
  end

  local nfdstr = ""
  for _, chr in ipairs(nfd)
  do nfdstr = string.format("%s%s", nfdstr, uni.char(chr)) end
  return nfdstr, nfd
end

function unicode.conformance.toNFD(unistring)
  return toNF_D_or_KD(unistring, false)
end

function unicode.conformance.toNFKD(unistring)
  return toNF_D_or_KD(unistring, true)
end

local function compose(seq)
  local base = seq[1]
  if not combdata[base] then return seq else
    local i = 2
    while i <= #seq do -- can I play with 'i' in a for loop?
      local cbng = seq[i]
      local cccprev
      if unidata[seq[i-1]] then cccprev = unidata[seq[i-1]].combclass end
      if not cccprev then cccprev = -1 end
      if unidata[cbng].combclass > cccprev then
        if not combdata[base] then return seq else
          for _, cbdata in ipairs(combdata[base]) do
            if cbdata.combining == cbng then
              seq[1] = cbdata.combined
              base = seq[1]
              for k = i, #seq - 1
              do seq[k] = seq[k+1]
              end -- for k = i, #seq - 1
              seq[#seq] = nil
              i = i - 1
            end -- if cbdata.combining == cbng
          end -- for _, cbdata in ipairs(combdata[base])
        end -- if unidata[cbng.combclass > cccprev
      end -- if not combdata[base]
    i = i + 1
    end -- while i <= #seq
  end -- if not combdata[base]
  return seq
end

-- To NFC from NFD.
-- Does not yet take all the composition exclusions in account
-- (missing types 1 and 2 as defined by UAX #15 X6)
function unicode.conformance.toNFC_fromNFD(nfd)
  local nfc = { }
  local seq = { }
  for uchar in uni.gmatch(nfd, '.') do
    local ucode = uni.byte(uchar)
    if not unidata[ucode]
    then nfc[#nfc + 1] = ucode else
      local cb = unidata[ucode].combclass
      if not cb or (cb == 0) then
        -- if seq ~= { } then -- Dubious ...
        if #seq > 0 then
          seq = compose(seq) -- There was a check for #seq == 1 here
          for i = 1, #seq do nfc[#nfc + 1] = seq[i] end
        end -- #seq > 0
        seq = { ucode } 
      else seq[#seq + 1] = ucode --[[ Maybe check if seq is not empty ... ]]
      end -- not cb or cb == 0
    end
  end

  seq = compose(seq)
  for i = 1, #seq do nfc[#nfc + 1] = seq[i] end

  local nfcstr = ""
  for _, chr in ipairs(nfc)
  do nfcstr = string.format("%s%s", nfcstr, uni.char(chr)) end
  return nfcstr, nfc
end

local function cancompose(seq, compat)
  local dec = { } -- new table to hold the decomposed sequence

  local shift
  if #seq >= 2 then -- let's do it the brutal way :-)
    if is_jamo(seq[1]) == 'choseong' and
       is_jamo(seq[2]) == 'jungseong' then
      LIndex = seq[1] - LBase
      VIndex = seq[2] - VBase
      if #seq == 2 or is_jamo(seq[3]) ~= 'jongseong' then
        TIndex = 0
        shift = 1
      else
        TIndex = seq[3] - TBase
        shift = 2
      end
      seq[1] = (LIndex * VCount + VIndex) * TCount + TIndex + SBase
      for i = 2, #seq -- this shifts and shrinks the table at the same time
      do seq[i] = seq[i + shift]
      end
    end
  end

  dec[1] = seq[1]
  for i = 2, #seq do
    local u = seq[i]
    local sp = unidata[u].specials
    if sp then
      if compat then compat = (sp[1] == 'compat') else compat = false end
      if (sp[1] == 'char') or compat then
        for i = 2, #sp
        do dec[#dec + 1] = sp[i]
        end
      end
    else dec[#dec + 1] = u
    end
  end -- we have the fully decomposed sequence; now sort it

  for i = #dec - 1, 2, -1 do -- bubble sort!
    for j = 2, #dec - 1 do
      local u = dec[j]
      local ccc1 = unidata[u].combclass
      local v = dec[j+1]
      local ccc2 = unidata[v].combclass
      if ccc1 > ccc2 then -- swap
        dec[j+1] = u
        dec[j] = v
      end
    end
  end -- dec sorted; now recursively compose

  local base, i, n = dec[1], 2, #dec
  local cbd = combdata[base]
  local incr_i = true
  while i <= n do
    local cbg = dec[i]
    if cbd then
      for _, cb in ipairs(cbd) do
        if cb.combining == cbg then
      -- NO :-) -- if cbd[cbg] then -- base and cbg combine; compose
          dec[1] = cb.combined
          base = dec[1]
          cbd = combdata[base]
          for j = i, n-1 -- shift table elements right of i
          do dec[j] = dec[j+1] end
          dec[n] = nil
          n = n-1 -- table has shrunk by 1, and i doesn't grow
          incr_i = false
        end
      end
    end
    if incr_i then i = i + 1
    else incr_i = true end
  end -- we're finally through! return
  return dec
end

function toNF_C_or_KC(unistring, compat)
  if unistring == "" then return "" end
  local nfc, seq = "", { }
  local start, space = true, ""
  for uchar in uni.gmatch(unistring, '.') do
    local ucode = uni.byte(uchar)
    if start then space = ", " start = true end
    if not unidata[ucode] then -- unknown to the UCD, will not compose
      nfc = string.format("%s%s", nfc, uchar)
    else
      local ccc = unidata[ucode].combclass 
      if not (ccc or is_jamo(ucode) == 'jongseong'
              or is_jamo(ucode) == 'jungseong')
         or ccc == 0 or (is_jamo(ucode) == 'choseong') then
      -- and is actually good :-) -- Well, yes and no ;-)
        if #seq == 0 then -- add ucode and go to next item of the loop
          seq = { ucode }
        else -- seq contains unicharacters, try and compose them
          if #seq == 1 then nfc = string.format("%s%s", nfc, uni.char(seq[1]))
          else dec = cancompose(seq, compat)
            for _, c in ipairs(dec) -- add the whole sequence to nfc
            do nfc = string.format("%s%s", nfc, uni.char(c)) end
          end
          seq = { ucode } -- don't forget to reinitialize seq with current char
        end
      else -- not ccc or ccc == 0 and is_choseong:
           -- character is combining, add it to seq
        seq[#seq + 1] = ucode
      end
    end
  end
  if #seq > 0 then dec = cancompose(seq, compat) end
  for _, c in ipairs(dec)
  do nfc = string.format("%s%s", nfc, uni.char(c)) end
  return nfc
end

function unicode.conformance.toNFC(unistring)
  return toNF_C_or_KC(unistring, false)
end

function unicode.conformance.toNFKC(unistring)
  return toNF_C_or_KC(unistring, true)
end