From 96c21343055c250b69580b63f5726ff626328a73 Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Tue, 29 Dec 2020 22:01:58 +0000 Subject: uninormalize (29dec20) git-svn-id: svn://tug.org/texlive/trunk@57257 c570f23f-e606-0410-a88d-b1316a301751 --- .../uninormalize/unicode-normalization.lua | 351 +++++++++++++++++++++ 1 file changed, 351 insertions(+) create mode 100644 Master/texmf-dist/tex/lualatex/uninormalize/unicode-normalization.lua (limited to 'Master/texmf-dist/tex/lualatex/uninormalize/unicode-normalization.lua') diff --git a/Master/texmf-dist/tex/lualatex/uninormalize/unicode-normalization.lua b/Master/texmf-dist/tex/lualatex/uninormalize/unicode-normalization.lua new file mode 100644 index 00000000000..b8a58b63bb6 --- /dev/null +++ b/Master/texmf-dist/tex/lualatex/uninormalize/unicode-normalization.lua @@ -0,0 +1,351 @@ +-- char-def now contains all necessary fields, no need for a custom file +if not characters then + require "char-def" +end + +if not unicode then require('unicode') end +unicode.conformance = unicode.conformance or { } + +unicharacters = unicharacters or {} +uni = unicode.utf8 +unidata = characters.data + +function printf(s, ...) print(string.format(s, ...)) end +-- function debug(s, ...) io.write("DEBUG: ", string.format(s, ...), "\n") end +function warn(s, ...) io.write("Warning: ", string.format(s, ...), "\n") end + +function md5sum(any) return md5.hex(md5.sum(any)) end + +-- Rehash the character data +unicharacters.combinee = { } +unicharacters.context = unicharacters.context or { } +local charu = unicode.utf8.char + +function unicharacters.context.rehash2() + for ucode, udata in pairs(unidata) -- *not* ipairs :-) + do + local sp = udata.specials + if sp then + if sp[1] == 'char' then + -- local ucode = udata.unicodeslot + local entry = { combinee = sp[2], combining = sp[3], combined = ucode } + if not unicharacters.combinee[sp[2]] + then unicharacters.combinee[sp[2]] = { } end + local n = #unicharacters.combinee[sp[2]] + unicharacters.combinee[sp[2]][n+1] = entry + end + end + -- copy context's combining field to combclass field + -- this field was in the custom copy of the char-def.lua that we no longer use + udata.combclass = udata.combining + end +end + + +unicharacters.context.rehash2() +combdata = unicharacters.combinee + +--[[ function unicode.conformance.is_hangul(ucode) + return ucode >= 0xAC00 and ucode <= 0xD7A3 +end ]] -- Make it local for the moment +local function is_hangul(char) + return char >= 0xAC00 and char <= 0xD7A3 +end + +local function is_jamo(char) + if char < 0x1100 then return false + elseif char < 0x1160 then return 'choseong' + elseif char < 0x11A8 then return 'jungseong' + elseif char < 0x11FA then return 'jongseong' + else return false + end +end + +local function decompose(ucode, compat) -- if compat then compatibility + local invbuf = { } + local sp = unidata[ucode].specials + if not sp + then return { ucode } else + if compat then compat = (sp[1] == 'compat') else compat = false end + while sp[1] == 'char' or compat do + head, tail = sp[2], sp[3] + if not tail then invbuf[#invbuf + 1] = head break end -- singleton + invbuf[#invbuf + 1] = tail + sp = unidata[head].specials + if not sp then invbuf[#invbuf + 1] = head sp = { } end + -- end -- not unidata[head] + end -- while sp[1] == 'char' or compat + end -- not sp + + local seq = { } + for i = #invbuf, 1, -1 + do seq[#seq + 1] = invbuf[i] + end + return seq +end + +local function canon(seq) -- Canonical reordering + if #seq < 3 then return seq end + local c1, c2, buf + -- I'd never thought I'd implement an actual bubble sort some day ;-) + for k = #seq - 1, 1, -1 do + for i = 2, k do -- was k - 1! Argh! + c1 = unidata[seq[i]].combclass + c2 = unidata[seq[i+1]].combclass + if c1 and c2 then + if c1 > c2 then + buf = seq[i] + seq[i] = seq[i+1] + seq[i+1] = buf + end + end + end + end + return seq +end + +if not math.div then -- from l-math.lua + function math.div(n, m) + return math.floor(n/m) + end +end + +local SBase, LBase, VBase, TBase = 0xAC00, 0x1100, 0x1161, 0x11A7 +local LCount, VCount, TCount = 19, 21, 28 +local NCount = VCount * TCount +local SCount = TCount * NCount + +local function decompose_hangul(ucode) -- assumes input is really a Hangul + local SIndex = ucode - SBase + local L = LBase + math.div(SIndex, NCount) + local V = VBase + math.div((SIndex % NCount), TCount) + local T = TBase + SIndex % TCount + if T == TBase then T = nil end + return { L, V, T } +end + +-- To NFK?D. +function toNF_D_or_KD(unistring, compat) + local nfd, seq = { }, { } + for uchar in uni.gmatch(unistring, '.') do + local ucode = uni.byte(uchar) + if is_hangul(ucode) then + seq = decompose_hangul(ucode) + for _, c in ipairs(seq) + do nfd[#nfd + 1] = c + end + seq = { } + elseif not unidata[ucode] + then nfd[#nfd + 1] = ucode else + local ccc = unidata[ucode].combclass + if not ccc or ccc == 0 then + seq = canon(seq) + for _, c in ipairs(seq) do nfd[#nfd + 1] = c end + seq = decompose(ucode, compat) + else seq[#seq + 1] = ucode + end -- not ccc or ccc == 0 + end -- if is_hangul(ucode) / elseif not unidata[ucode] + end -- for uchar in uni.gmatch(unistring, ".") + + if #seq > 0 then + seq = canon(seq) + for _, c in ipairs(seq) do nfd[#nfd + 1] = c end + end + + local nfdstr = "" + for _, chr in ipairs(nfd) + do nfdstr = string.format("%s%s", nfdstr, uni.char(chr)) end + return nfdstr, nfd +end + +function unicode.conformance.toNFD(unistring) + return toNF_D_or_KD(unistring, false) +end + +function unicode.conformance.toNFKD(unistring) + return toNF_D_or_KD(unistring, true) +end + +local function compose(seq) + local base = seq[1] + if not combdata[base] then return seq else + local i = 2 + while i <= #seq do -- can I play with 'i' in a for loop? + local cbng = seq[i] + local cccprev + if unidata[seq[i-1]] then cccprev = unidata[seq[i-1]].combclass end + if not cccprev then cccprev = -1 end + if unidata[cbng].combclass > cccprev then + if not combdata[base] then return seq else + for _, cbdata in ipairs(combdata[base]) do + if cbdata.combining == cbng then + seq[1] = cbdata.combined + base = seq[1] + for k = i, #seq - 1 + do seq[k] = seq[k+1] + end -- for k = i, #seq - 1 + seq[#seq] = nil + i = i - 1 + end -- if cbdata.combining == cbng + end -- for _, cbdata in ipairs(combdata[base]) + end -- if unidata[cbng.combclass > cccprev + end -- if not combdata[base] + i = i + 1 + end -- while i <= #seq + end -- if not combdata[base] + return seq +end + +-- To NFC from NFD. +-- Does not yet take all the composition exclusions in account +-- (missing types 1 and 2 as defined by UAX #15 X6) +function unicode.conformance.toNFC_fromNFD(nfd) + local nfc = { } + local seq = { } + for uchar in uni.gmatch(nfd, '.') do + local ucode = uni.byte(uchar) + if not unidata[ucode] + then nfc[#nfc + 1] = ucode else + local cb = unidata[ucode].combclass + if not cb or (cb == 0) then + -- if seq ~= { } then -- Dubious ... + if #seq > 0 then + seq = compose(seq) -- There was a check for #seq == 1 here + for i = 1, #seq do nfc[#nfc + 1] = seq[i] end + end -- #seq > 0 + seq = { ucode } + else seq[#seq + 1] = ucode --[[ Maybe check if seq is not empty ... ]] + end -- not cb or cb == 0 + end + end + + seq = compose(seq) + for i = 1, #seq do nfc[#nfc + 1] = seq[i] end + + local nfcstr = "" + for _, chr in ipairs(nfc) + do nfcstr = string.format("%s%s", nfcstr, uni.char(chr)) end + return nfcstr, nfc +end + +local function cancompose(seq, compat) + local dec = { } -- new table to hold the decomposed sequence + + local shift + if #seq >= 2 then -- let's do it the brutal way :-) + if is_jamo(seq[1]) == 'choseong' and + is_jamo(seq[2]) == 'jungseong' then + LIndex = seq[1] - LBase + VIndex = seq[2] - VBase + if #seq == 2 or is_jamo(seq[3]) ~= 'jongseong' then + TIndex = 0 + shift = 1 + else + TIndex = seq[3] - TBase + shift = 2 + end + seq[1] = (LIndex * VCount + VIndex) * TCount + TIndex + SBase + for i = 2, #seq -- this shifts and shrinks the table at the same time + do seq[i] = seq[i + shift] + end + end + end + + dec[1] = seq[1] + for i = 2, #seq do + local u = seq[i] + local sp = unidata[u].specials + if sp then + if compat then compat = (sp[1] == 'compat') else compat = false end + if (sp[1] == 'char') or compat then + for i = 2, #sp + do dec[#dec + 1] = sp[i] + end + end + else dec[#dec + 1] = u + end + end -- we have the fully decomposed sequence; now sort it + + for i = #dec - 1, 2, -1 do -- bubble sort! + for j = 2, #dec - 1 do + local u = dec[j] + local ccc1 = unidata[u].combclass + local v = dec[j+1] + local ccc2 = unidata[v].combclass + if ccc1 > ccc2 then -- swap + dec[j+1] = u + dec[j] = v + end + end + end -- dec sorted; now recursively compose + + local base, i, n = dec[1], 2, #dec + local cbd = combdata[base] + local incr_i = true + while i <= n do + local cbg = dec[i] + if cbd then + for _, cb in ipairs(cbd) do + if cb.combining == cbg then + -- NO :-) -- if cbd[cbg] then -- base and cbg combine; compose + dec[1] = cb.combined + base = dec[1] + cbd = combdata[base] + for j = i, n-1 -- shift table elements right of i + do dec[j] = dec[j+1] end + dec[n] = nil + n = n-1 -- table has shrunk by 1, and i doesn't grow + incr_i = false + end + end + end + if incr_i then i = i + 1 + else incr_i = true end + end -- we're finally through! return + return dec +end + +function toNF_C_or_KC(unistring, compat) + if unistring == "" then return "" end + local nfc, seq = "", { } + local start, space = true, "" + for uchar in uni.gmatch(unistring, '.') do + local ucode = uni.byte(uchar) + if start then space = ", " start = true end + if not unidata[ucode] then -- unknown to the UCD, will not compose + nfc = string.format("%s%s", nfc, uchar) + else + local ccc = unidata[ucode].combclass + if not (ccc or is_jamo(ucode) == 'jongseong' + or is_jamo(ucode) == 'jungseong') + or ccc == 0 or (is_jamo(ucode) == 'choseong') then + -- and is actually good :-) -- Well, yes and no ;-) + if #seq == 0 then -- add ucode and go to next item of the loop + seq = { ucode } + else -- seq contains unicharacters, try and compose them + if #seq == 1 then nfc = string.format("%s%s", nfc, uni.char(seq[1])) + else dec = cancompose(seq, compat) + for _, c in ipairs(dec) -- add the whole sequence to nfc + do nfc = string.format("%s%s", nfc, uni.char(c)) end + end + seq = { ucode } -- don't forget to reinitialize seq with current char + end + else -- not ccc or ccc == 0 and is_choseong: + -- character is combining, add it to seq + seq[#seq + 1] = ucode + end + end + end + if #seq > 0 then dec = cancompose(seq, compat) end + for _, c in ipairs(dec) + do nfc = string.format("%s%s", nfc, uni.char(c)) end + return nfc +end + +function unicode.conformance.toNFC(unistring) + return toNF_C_or_KC(unistring, false) +end + +function unicode.conformance.toNFKC(unistring) + return toNF_C_or_KC(unistring, true) +end -- cgit v1.2.3