summaryrefslogtreecommitdiff
path: root/macros/luatex/generic/luatexko/luatexko-normalize.lua
diff options
context:
space:
mode:
Diffstat (limited to 'macros/luatex/generic/luatexko/luatexko-normalize.lua')
-rw-r--r--macros/luatex/generic/luatexko/luatexko-normalize.lua360
1 files changed, 7 insertions, 353 deletions
diff --git a/macros/luatex/generic/luatexko/luatexko-normalize.lua b/macros/luatex/generic/luatexko/luatexko-normalize.lua
index 243762aa2f..ef8edbd587 100644
--- a/macros/luatex/generic/luatexko/luatexko-normalize.lua
+++ b/macros/luatex/generic/luatexko/luatexko-normalize.lua
@@ -13,8 +13,8 @@
luatexbase.provides_module({
name = "luatexko-normalize",
- version = "2.8",
- date = "2020/03/20",
+ version = "2.9",
+ date = "2020/07/27",
author = "Dohyun Kim, Soojin Nam",
description = "Hangul normalization",
license = "LPPL v1.3+",
@@ -30,278 +30,8 @@ local tableinsert = table.insert
local tableappend = table.append
local tableunpack = table.unpack
-local chanjatohanja = {
- [0xF900] = {0x8C48, 0xFE00},
- [0xF901] = {0x66F4, 0xFE00},
- [0xF902] = {0x8ECA, 0xFE00},
- [0xF903] = {0x8CC8, 0xFE00},
- [0xF904] = {0x6ED1, 0xFE00},
- [0xF905] = {0x4E32, 0xFE00},
- [0xF906] = {0x53E5, 0xFE00},
- [0xF907] = {0x9F9C, 0xFE00},
- [0xF908] = {0x9F9C, 0xFE01},
- [0xF909] = {0x5951, 0xFE00},
- [0xF90A] = {0x91D1, 0xFE00},
- [0xF90B] = {0x5587, 0xFE00},
- [0xF90C] = {0x5948, 0xFE00},
- [0xF90D] = {0x61F6, 0xFE00},
- [0xF90E] = {0x7669, 0xFE00},
- [0xF90F] = {0x7F85, 0xFE00},
- [0xF910] = {0x863F, 0xFE00},
- [0xF911] = {0x87BA, 0xFE00},
- [0xF912] = {0x88F8, 0xFE00},
- [0xF913] = {0x908F, 0xFE00},
- [0xF914] = {0x6A02, 0xFE00},
- [0xF915] = {0x6D1B, 0xFE00},
- [0xF916] = {0x70D9, 0xFE00},
- [0xF917] = {0x73DE, 0xFE00},
- [0xF918] = {0x843D, 0xFE00},
- [0xF919] = {0x916A, 0xFE00},
- [0xF91A] = {0x99F1, 0xFE00},
- [0xF91B] = {0x4E82, 0xFE00},
- [0xF91C] = {0x5375, 0xFE00},
- [0xF91D] = {0x6B04, 0xFE00},
- [0xF91E] = {0x721B, 0xFE00},
- [0xF91F] = {0x862D, 0xFE00},
- [0xF920] = {0x9E1E, 0xFE00},
- [0xF921] = {0x5D50, 0xFE00},
- [0xF922] = {0x6FEB, 0xFE00},
- [0xF923] = {0x85CD, 0xFE00},
- [0xF924] = {0x8964, 0xFE00},
- [0xF925] = {0x62C9, 0xFE00},
- [0xF926] = {0x81D8, 0xFE00},
- [0xF927] = {0x881F, 0xFE00},
- [0xF928] = {0x5ECA, 0xFE00},
- [0xF929] = {0x6717, 0xFE00},
- [0xF92A] = {0x6D6A, 0xFE00},
- [0xF92B] = {0x72FC, 0xFE00},
- [0xF92C] = {0x90CE, 0xFE00},
- [0xF92D] = {0x4F86, 0xFE00},
- [0xF92E] = {0x51B7, 0xFE00},
- [0xF92F] = {0x52DE, 0xFE00},
- [0xF930] = {0x64C4, 0xFE00},
- [0xF931] = {0x6AD3, 0xFE00},
- [0xF932] = {0x7210, 0xFE00},
- [0xF933] = {0x76E7, 0xFE00},
- [0xF934] = {0x8001, 0xFE00},
- [0xF935] = {0x8606, 0xFE00},
- [0xF936] = {0x865C, 0xFE00},
- [0xF937] = {0x8DEF, 0xFE00},
- [0xF938] = {0x9732, 0xFE00},
- [0xF939] = {0x9B6F, 0xFE00},
- [0xF93A] = {0x9DFA, 0xFE00},
- [0xF93B] = {0x788C, 0xFE00},
- [0xF93C] = {0x797F, 0xFE00},
- [0xF93D] = {0x7DA0, 0xFE00},
- [0xF93E] = {0x83C9, 0xFE00},
- [0xF93F] = {0x9304, 0xFE00},
- [0xF940] = {0x9E7F, 0xFE00},
- [0xF941] = {0x8AD6, 0xFE00},
- [0xF942] = {0x58DF, 0xFE00},
- [0xF943] = {0x5F04, 0xFE00},
- [0xF944] = {0x7C60, 0xFE00},
- [0xF945] = {0x807E, 0xFE00},
- [0xF946] = {0x7262, 0xFE00},
- [0xF947] = {0x78CA, 0xFE00},
- [0xF948] = {0x8CC2, 0xFE00},
- [0xF949] = {0x96F7, 0xFE00},
- [0xF94A] = {0x58D8, 0xFE00},
- [0xF94B] = {0x5C62, 0xFE00},
- [0xF94C] = {0x6A13, 0xFE00},
- [0xF94D] = {0x6DDA, 0xFE00},
- [0xF94E] = {0x6F0F, 0xFE00},
- [0xF94F] = {0x7D2F, 0xFE00},
- [0xF950] = {0x7E37, 0xFE00},
- [0xF951] = {0x964B, 0xFE00},
- [0xF952] = {0x52D2, 0xFE00},
- [0xF953] = {0x808B, 0xFE00},
- [0xF954] = {0x51DC, 0xFE00},
- [0xF955] = {0x51CC, 0xFE00},
- [0xF956] = {0x7A1C, 0xFE00},
- [0xF957] = {0x7DBE, 0xFE00},
- [0xF958] = {0x83F1, 0xFE00},
- [0xF959] = {0x9675, 0xFE00},
- [0xF95A] = {0x8B80, 0xFE00},
- [0xF95B] = {0x62CF, 0xFE00},
- [0xF95C] = {0x6A02, 0xFE01},
- [0xF95D] = {0x8AFE, 0xFE00},
- [0xF95E] = {0x4E39, 0xFE00},
- [0xF95F] = {0x5BE7, 0xFE00},
- [0xF960] = {0x6012, 0xFE00},
- [0xF961] = {0x7387, 0xFE00},
- [0xF962] = {0x7570, 0xFE00},
- [0xF963] = {0x5317, 0xFE00},
- [0xF964] = {0x78FB, 0xFE00},
- [0xF965] = {0x4FBF, 0xFE00},
- [0xF966] = {0x5FA9, 0xFE00},
- [0xF967] = {0x4E0D, 0xFE00},
- [0xF968] = {0x6CCC, 0xFE00},
- [0xF969] = {0x6578, 0xFE00},
- [0xF96A] = {0x7D22, 0xFE00},
- [0xF96B] = {0x53C3, 0xFE00},
- [0xF96C] = {0x585E, 0xFE00},
- [0xF96D] = {0x7701, 0xFE00},
- [0xF96E] = {0x8449, 0xFE00},
- [0xF96F] = {0x8AAA, 0xFE00},
- [0xF970] = {0x6BBA, 0xFE00},
- [0xF971] = {0x8FB0, 0xFE00},
- [0xF972] = {0x6C88, 0xFE00},
- [0xF973] = {0x62FE, 0xFE00},
- [0xF974] = {0x82E5, 0xFE00},
- [0xF975] = {0x63A0, 0xFE00},
- [0xF976] = {0x7565, 0xFE00},
- [0xF977] = {0x4EAE, 0xFE00},
- [0xF978] = {0x5169, 0xFE00},
- [0xF979] = {0x51C9, 0xFE00},
- [0xF97A] = {0x6881, 0xFE00},
- [0xF97B] = {0x7CE7, 0xFE00},
- [0xF97C] = {0x826F, 0xFE00},
- [0xF97D] = {0x8AD2, 0xFE00},
- [0xF97E] = {0x91CF, 0xFE00},
- [0xF97F] = {0x52F5, 0xFE00},
- [0xF980] = {0x5442, 0xFE00},
- [0xF981] = {0x5973, 0xFE00},
- [0xF982] = {0x5EEC, 0xFE00},
- [0xF983] = {0x65C5, 0xFE00},
- [0xF984] = {0x6FFE, 0xFE00},
- [0xF985] = {0x792A, 0xFE00},
- [0xF986] = {0x95AD, 0xFE00},
- [0xF987] = {0x9A6A, 0xFE00},
- [0xF988] = {0x9E97, 0xFE00},
- [0xF989] = {0x9ECE, 0xFE00},
- [0xF98A] = {0x529B, 0xFE00},
- [0xF98B] = {0x66C6, 0xFE00},
- [0xF98C] = {0x6B77, 0xFE00},
- [0xF98D] = {0x8F62, 0xFE00},
- [0xF98E] = {0x5E74, 0xFE00},
- [0xF98F] = {0x6190, 0xFE00},
- [0xF990] = {0x6200, 0xFE00},
- [0xF991] = {0x649A, 0xFE00},
- [0xF992] = {0x6F23, 0xFE00},
- [0xF993] = {0x7149, 0xFE00},
- [0xF994] = {0x7489, 0xFE00},
- [0xF995] = {0x79CA, 0xFE00},
- [0xF996] = {0x7DF4, 0xFE00},
- [0xF997] = {0x806F, 0xFE00},
- [0xF998] = {0x8F26, 0xFE00},
- [0xF999] = {0x84EE, 0xFE00},
- [0xF99A] = {0x9023, 0xFE00},
- [0xF99B] = {0x934A, 0xFE00},
- [0xF99C] = {0x5217, 0xFE00},
- [0xF99D] = {0x52A3, 0xFE00},
- [0xF99E] = {0x54BD, 0xFE00},
- [0xF99F] = {0x70C8, 0xFE00},
- [0xF9A0] = {0x88C2, 0xFE00},
- [0xF9A1] = {0x8AAA, 0xFE01},
- [0xF9A2] = {0x5EC9, 0xFE00},
- [0xF9A3] = {0x5FF5, 0xFE00},
- [0xF9A4] = {0x637B, 0xFE00},
- [0xF9A5] = {0x6BAE, 0xFE00},
- [0xF9A6] = {0x7C3E, 0xFE00},
- [0xF9A7] = {0x7375, 0xFE00},
- [0xF9A8] = {0x4EE4, 0xFE00},
- [0xF9A9] = {0x56F9, 0xFE00},
- [0xF9AA] = {0x5BE7, 0xFE01},
- [0xF9AB] = {0x5DBA, 0xFE00},
- [0xF9AC] = {0x601C, 0xFE00},
- [0xF9AD] = {0x73B2, 0xFE00},
- [0xF9AE] = {0x7469, 0xFE00},
- [0xF9AF] = {0x7F9A, 0xFE00},
- [0xF9B0] = {0x8046, 0xFE00},
- [0xF9B1] = {0x9234, 0xFE00},
- [0xF9B2] = {0x96F6, 0xFE00},
- [0xF9B3] = {0x9748, 0xFE00},
- [0xF9B4] = {0x9818, 0xFE00},
- [0xF9B5] = {0x4F8B, 0xFE00},
- [0xF9B6] = {0x79AE, 0xFE00},
- [0xF9B7] = {0x91B4, 0xFE00},
- [0xF9B8] = {0x96B8, 0xFE00},
- [0xF9B9] = {0x60E1, 0xFE00},
- [0xF9BA] = {0x4E86, 0xFE00},
- [0xF9BB] = {0x50DA, 0xFE00},
- [0xF9BC] = {0x5BEE, 0xFE00},
- [0xF9BD] = {0x5C3F, 0xFE00},
- [0xF9BE] = {0x6599, 0xFE00},
- [0xF9BF] = {0x6A02, 0xFE02},
- [0xF9C0] = {0x71CE, 0xFE00},
- [0xF9C1] = {0x7642, 0xFE00},
- [0xF9C2] = {0x84FC, 0xFE00},
- [0xF9C3] = {0x907C, 0xFE00},
- [0xF9C4] = {0x9F8D, 0xFE00},
- [0xF9C5] = {0x6688, 0xFE00},
- [0xF9C6] = {0x962E, 0xFE00},
- [0xF9C7] = {0x5289, 0xFE00},
- [0xF9C8] = {0x677B, 0xFE00},
- [0xF9C9] = {0x67F3, 0xFE00},
- [0xF9CA] = {0x6D41, 0xFE00},
- [0xF9CB] = {0x6E9C, 0xFE00},
- [0xF9CC] = {0x7409, 0xFE00},
- [0xF9CD] = {0x7559, 0xFE00},
- [0xF9CE] = {0x786B, 0xFE00},
- [0xF9CF] = {0x7D10, 0xFE00},
- [0xF9D0] = {0x985E, 0xFE00},
- [0xF9D1] = {0x516D, 0xFE00},
- [0xF9D2] = {0x622E, 0xFE00},
- [0xF9D3] = {0x9678, 0xFE00},
- [0xF9D4] = {0x502B, 0xFE00},
- [0xF9D5] = {0x5D19, 0xFE00},
- [0xF9D6] = {0x6DEA, 0xFE00},
- [0xF9D7] = {0x8F2A, 0xFE00},
- [0xF9D8] = {0x5F8B, 0xFE00},
- [0xF9D9] = {0x6144, 0xFE00},
- [0xF9DA] = {0x6817, 0xFE00},
- [0xF9DB] = {0x7387, 0xFE01},
- [0xF9DC] = {0x9686, 0xFE00},
- [0xF9DD] = {0x5229, 0xFE00},
- [0xF9DE] = {0x540F, 0xFE00},
- [0xF9DF] = {0x5C65, 0xFE00},
- [0xF9E0] = {0x6613, 0xFE00},
- [0xF9E1] = {0x674E, 0xFE00},
- [0xF9E2] = {0x68A8, 0xFE00},
- [0xF9E3] = {0x6CE5, 0xFE00},
- [0xF9E4] = {0x7406, 0xFE00},
- [0xF9E5] = {0x75E2, 0xFE00},
- [0xF9E6] = {0x7F79, 0xFE00},
- [0xF9E7] = {0x88CF, 0xFE00},
- [0xF9E8] = {0x88E1, 0xFE00},
- [0xF9E9] = {0x91CC, 0xFE00},
- [0xF9EA] = {0x96E2, 0xFE00},
- [0xF9EB] = {0x533F, 0xFE00},
- [0xF9EC] = {0x6EBA, 0xFE00},
- [0xF9ED] = {0x541D, 0xFE00},
- [0xF9EE] = {0x71D0, 0xFE00},
- [0xF9EF] = {0x7498, 0xFE00},
- [0xF9F0] = {0x85FA, 0xFE00},
- [0xF9F1] = {0x96A3, 0xFE00},
- [0xF9F2] = {0x9C57, 0xFE00},
- [0xF9F3] = {0x9E9F, 0xFE00},
- [0xF9F4] = {0x6797, 0xFE00},
- [0xF9F5] = {0x6DCB, 0xFE00},
- [0xF9F6] = {0x81E8, 0xFE00},
- [0xF9F7] = {0x7ACB, 0xFE00},
- [0xF9F8] = {0x7B20, 0xFE00},
- [0xF9F9] = {0x7C92, 0xFE00},
- [0xF9FA] = {0x72C0, 0xFE00},
- [0xF9FB] = {0x7099, 0xFE00},
- [0xF9FC] = {0x8B58, 0xFE00},
- [0xF9FD] = {0x4EC0, 0xFE00},
- [0xF9FE] = {0x8336, 0xFE00},
- [0xF9FF] = {0x523A, 0xFE00},
- [0xFA00] = {0x5207, 0xFE00},
- [0xFA01] = {0x5EA6, 0xFE00},
- [0xFA02] = {0x62D3, 0xFE00},
- [0xFA03] = {0x7CD6, 0xFE00},
- [0xFA04] = {0x5B85, 0xFE00},
- [0xFA05] = {0x6D1E, 0xFE00},
- [0xFA06] = {0x66B4, 0xFE00},
- [0xFA07] = {0x8F3B, 0xFE00},
- [0xFA08] = {0x884C, 0xFE00},
- [0xFA09] = {0x964D, 0xFE00},
- [0xFA0A] = {0x898B, 0xFE00},
- [0xFA0B] = {0x5ED3, 0xFE00},
- [0xFA2E] = {0x90DE, 0xFE00},
- [0xFA2F] = {0x96B7, 0xFE00},
-}
+local normalize = require'lua-uni-normalize'
+
local jamotocjamo = {
ccho = {
[0x1100] = 0x3131,
@@ -407,14 +137,6 @@ local function is_hangul (c)
return c >= 0xAC00 and c <= 0xD7A3
end
-local function is_modern_cho (c)
- return c >= 0x1100 and c <= 0x1112
-end
-
-local function is_modern_jung (c)
- return c >= 0x1161 and c <= 0x1175
-end
-
local function is_modern_jong (c)
return c >= 0x11A8 and c <= 0x11C2
end
@@ -428,16 +150,6 @@ local function is_jongsong (c)
return is_modern_jong(c) or is_old_jong(c)
end
-local function jamo2syllable (t) -- table -> integer
- local cho, jung, jong = tableunpack(t)
- local s = (cho - 0x1100) * 21
- s = (s + jung - 0x1161) * 28
- if jong then
- s = s + jong - 0x11A7
- end
- return s + 0xAC00
-end
-
local function syllable2jamo (s) -- integer -> table
local t = {}
s = s - 0xAC00
@@ -450,17 +162,7 @@ local function syllable2jamo (s) -- integer -> table
return t
end
-local function hanguldecompose (buffer)
- local t = {}
- for _, c in utf8codes(buffer) do
- if is_hangul(c) then
- tableappend(t, syllable2jamo(c))
- else
- tableinsert(t, c)
- end
- end
- return utf8char(tableunpack(t))
-end
+local hanguldecompose = normalize.NFD
-- LV | LVT, T -> L, V, T+
local function flush_syllable_jong (t, s)
@@ -487,30 +189,6 @@ local function compose_hanguldecompose (buffer) -- string -> table
return t
end
--- L, V, [T & ^OT]? -> LV | LVT
-local function flush_syllable (t, s)
- if #s == 2 or #s == 3 and is_modern_jong( s[3] ) then
- tableinsert(t, jamo2syllable(s))
- else
- tableappend(t, s)
- end
- return t, {}
-end
-
-local function compose_modern_hangul (ot)
- local t, s = {}, {}
- for _, c in ipairs(ot) do
- if #s == 1 and is_modern_jung(c) or #s >= 2 and is_jongsong(c) then
- tableinsert(s, c)
- else
- t, s = flush_syllable(t, s)
- tableinsert(is_modern_cho(c) and s or t, c)
- end
- end
- t = flush_syllable(t, s)
- return t
-end
-
-- L, VF -> CL
local function flush_cjamocho (t, s)
if #s == 2 then
@@ -559,36 +237,12 @@ local function compose_jamo_jungsong (ot)
return t
end
--- CHJ -> HJ, VS
-local function flush_compat_hanja (t, s)
- if #s == 1 then
- tableappend(t, chanjatohanja[ s[1] ])
- else
- tableappend(t, s)
- end
- return t, {}
-end
-
-local function compose_compat_hanja (ot)
- local t, s = {}, {}
- for _, c in ipairs(ot) do
- if #s == 1 and c >= 0xFE00 and c <= 0xFE02 then
- tableinsert(s, c)
- else
- t, s = flush_compat_hanja(t, s)
- tableinsert(chanjatohanja[c] and s or t, c)
- end
- end
- t = flush_compat_hanja(t, s)
- return t
-end
-
local function hangulcompose (buffer)
+ buffer = normalize.NFC(buffer)
+
local t = compose_hanguldecompose(buffer)
- t = compose_modern_hangul(t)
t = compose_jamo_chosong (t)
t = compose_jamo_jungsong(t)
- t = compose_compat_hanja (t)
return utf8char(tableunpack(t))
end