luatexko (10jul21)

git-svn-id: svn://tug.org/texlive/trunk@59892 c570f23f-e606-0410-a88d-b1316a301751

8 files changed, 145 insertions, 67 deletions

diff --git a/Master/texmf-dist/doc/luatex/luatexko/ChangeLog b/Master/texmf-dist/doc/luatex/luatexko/ChangeLog
index 07a26993c61..62e73af1e49 100644
--- a/Master/texmf-dist/doc/luatex/luatexko/ChangeLog
+++ b/Master/texmf-dist/doc/luatex/luatexko/ChangeLog
@@ -1,3 +1,15 @@
+2021-07-10	Dohyun Kim <nomos at ktug org>
+
+	Version 3.3
+
+	* luatexko.sty:
+	- respect 'para/end' hook
+	- allow linebreak before CJK Unified Ideographs Extension G
+
+	* luatexko.lua:
+	- normalize node sequence 'LV, T'
+	- fix a bug wrt detection of HB fonts
+
 2021-06-10	Dohyun Kim <nomos at ktug org>
 
 	Version 3.2
diff --git a/Master/texmf-dist/doc/luatex/luatexko/README b/Master/texmf-dist/doc/luatex/luatexko/README
index f3e1feb636d..fd2b7fd95bc 100644
--- a/Master/texmf-dist/doc/luatex/luatexko/README
+++ b/Master/texmf-dist/doc/luatex/luatexko/README
@@ -1,4 +1,4 @@
-LuaTeX-ko Package version 3.2 (2021/06/10)
+LuaTeX-ko Package version 3.3 (2021/07/10)
 ===========================================
 
 This is a Lua(La)TeX macro package that supports typesetting Korean
diff --git a/Master/texmf-dist/doc/luatex/luatexko/luatexko-doc.pdf b/Master/texmf-dist/doc/luatex/luatexko/luatexko-doc.pdf
index 7b4c4336e64..033ac28c5b9 100644
--- a/Master/texmf-dist/doc/luatex/luatexko/luatexko-doc.pdf
+++ b/Master/texmf-dist/doc/luatex/luatexko/luatexko-doc.pdf
diff --git a/Master/texmf-dist/doc/luatex/luatexko/luatexko-doc.tex b/Master/texmf-dist/doc/luatex/luatexko/luatexko-doc.tex
index 9153b572ed9..8b32a6f49e8 100644
--- a/Master/texmf-dist/doc/luatex/luatexko/luatexko-doc.tex
+++ b/Master/texmf-dist/doc/luatex/luatexko/luatexko-doc.tex
@@ -142,7 +142,7 @@
 \author{Dohyun Kim \normalsize |<nomos at ktug org>| \and
         Soojin Nam \normalsize |<jsunam at gmail com>| \and
   \normalsize <\url{http://github.com/dohyunkim/luatexko}>}
-\date{Version 3.2\quad 2021/06/10}
+\date{Version 3.3\quad 2021/07/10}
 \maketitle
 
 \begin{quote}
diff --git a/Master/texmf-dist/tex/luatex/luatexko/luatexko-normalize.lua b/Master/texmf-dist/tex/luatex/luatexko/luatexko-normalize.lua
index 96a0d12cf3d..961d581c798 100644
--- a/Master/texmf-dist/tex/luatex/luatexko/luatexko-normalize.lua
+++ b/Master/texmf-dist/tex/luatex/luatexko/luatexko-normalize.lua
@@ -13,8 +13,8 @@
 
 luatexbase.provides_module({
   name        = "luatexko-normalize",
-  version     = "3.2",
-  date        = "2021/06/10",
+  version     = "3.3",
+  date        = "2021/07/10",
   author      = "Dohyun Kim, Soojin Nam",
   description = "Hangul normalization",
   license     = "LPPL v1.3+",
diff --git a/Master/texmf-dist/tex/luatex/luatexko/luatexko-uhc2utf8.lua b/Master/texmf-dist/tex/luatex/luatexko/luatexko-uhc2utf8.lua
index 2c5fd64f9e5..1a48a486935 100644
--- a/Master/texmf-dist/tex/luatex/luatexko/luatexko-uhc2utf8.lua
+++ b/Master/texmf-dist/tex/luatex/luatexko/luatexko-uhc2utf8.lua
@@ -13,8 +13,8 @@
 
 luatexbase.provides_module({
   name        = "luatexko-uhc2utf8",
-  version     = "3.2",
-  date        = "2021/06/10",
+  version     = "3.3",
+  date        = "2021/07/10",
   author      = "Dohyun Kim, Soojin Nam",
   description = "UHC (CP949) input encoding",
   license     = "LPPL v1.3+",
diff --git a/Master/texmf-dist/tex/luatex/luatexko/luatexko.lua b/Master/texmf-dist/tex/luatex/luatexko/luatexko.lua
index aa1aa4149f4..6b82a190f53 100644
--- a/Master/texmf-dist/tex/luatex/luatexko/luatexko.lua
+++ b/Master/texmf-dist/tex/luatex/luatexko/luatexko.lua
@@ -13,8 +13,8 @@
 
 luatexbase.provides_module {
   name        = 'luatexko',
-  date        = '2021/06/10',
-  version     = '3.2',
+  date        = '2021/07/10',
+  version     = '3.3',
   description = 'typesetting Korean with LuaTeX',
   author      = 'Dohyun Kim, Soojin Nam',
   license     = 'LPPL v1.3+',
@@ -184,7 +184,12 @@ local os2tag = harfbuzz and harfbuzz.Tag.new"OS/2"
 local fontoptions = {
   is_not_harf = setmetatable( {}, { __index = function (t, fid)
     if fid then
-      local bool = has_harf_data(fid) and false or true
+      local bool
+      if has_harf_data(fid) then
+        bool = false
+      else
+        bool = true
+      end
       t[fid] = bool
       return bool
     end
@@ -415,8 +420,8 @@ end
 
 local function is_hanja (c)
   return c >= 0x3400 and c <= 0xA4C6
-  or     c >= 0xF900 and c <= 0xFAD9
-  or     c >= 0x20000 and c <= 0x2FFFD
+  or     c >= 0xF900 and c <= 0xFAFF
+  or     c >= 0x20000 and c <= 0x3FFFD
   or     c >= 0x2E81 and c <= 0x2FD5
 end
 
@@ -447,16 +452,18 @@ local function is_compat_jamo (c)
   return c >= 0x3131 and c <= 0x318E
 end
 
-local function is_unicode_var_sel (c)
-  return c >= 0xFE00  and c <= 0xFE0F
-  or     c >= 0xE0100 and c <= 0xE01EF
-end
-
-local function is_cjk_combining (c)
+local function is_combining (c)
   return c >= 0x302A and c <= 0x302F
-  or     c >= 0x3099 and c <= 0x309C
-  or     c >= 0xFF9E and c <= 0xFF9F
-  or     is_unicode_var_sel(c)
+  or     c == 0x3099 or  c == 0x309A
+  -- variation selectors
+  or     c >= 0xFE00  and c <= 0xFE0F
+  or     c >= 0xE0100 and c <= 0xE01EF
+  -- others (probably non-cjk)
+  or     c >= 0x0300 and c <= 0x036F
+  or     c >= 0x1AB0 and c <= 0x1AFF
+  or     c >= 0x1DC0 and c <= 0x1DFF
+  or     c >= 0x20D0 and c <= 0x20FF
+  or     c >= 0xFE20 and c <= 0xFE2F
 end
 
 local function is_noncjk_char (c)
@@ -465,12 +472,12 @@ local function is_noncjk_char (c)
   or     c >= 0x61 and c <= 0x7A
   or     c >= 0xC0 and c <= 0xD6
   or     c >= 0xD8 and c <= 0xF6
-  or     c >= 0xF8 and c <= 0x10FC
-  or     c >= 0x1200 and c <= 0x1FFE
-  or     c >= 0xA4D0 and c <= 0xA877
-  or     c >= 0xAB01 and c <= 0xABBF
-  or     c >= 0xFB00 and c <= 0xFDFD
-  or     c >= 0xFE70 and c <= 0xFEFC
+  or     c >= 0xF8 and c <= 0x10FF
+  or     c >= 0x1200 and c <= 0x1FFF
+  or     c >= 0xA4D0 and c <= 0xA95F
+  or     c >= 0xA980 and c <= 0xABFF
+  or     c >= 0xFB00 and c <= 0xFDFF
+  or     c >= 0xFE70 and c <= 0xFEFF
 end
 
 local function is_kana (c)
@@ -564,17 +571,19 @@ local breakable_after = setmetatable({
   [0x2013] = true, [0x2014] = true, [0x25A1] = true, [0x25CB] = true,
   [0x2E80] = true, [0x3003] = true, [0x3005] = true, [0x3007] = true,
   [0x301C] = true, [0x3035] = true, [0x303B] = true, [0x303C] = true,
+  [0x309B] = true, [0x309C] = true,
   [0x309D] = true, [0x309E] = true, [0x30A0] = true, [0x30FC] = true,
   [0x30FD] = true, [0x30FE] = true, [0xFE13] = true, [0xFE14] = true,
   [0xFE32] = true, [0xFE50] = true, [0xFE51] = true, [0xFE52] = true,
   [0xFE54] = true, [0xFE55] = true, [0xFE57] = true, [0xFE57] = true,
   [0xFE58] = true, [0xFE5A] = true, [0xFE5C] = true, [0xFE5E] = true,
   [0xFF1E] = true, [0xFF5E] = true, [0xFF70] = true, [0x226B] = true, -- ≫
+  [0xFF9E] = true, [0xFF9F] = true,
 },{ __index = function (_,c)
   return is_hangul_jamo(c) -- chosong also is breakable_after
   or     is_noncjk_char(c)
   or     is_hanja(c)
-  or     is_cjk_combining(c)
+  or     is_combining(c)
   or     is_kana(c)
   or     charclass[c] >= 2
 end })
@@ -600,6 +609,8 @@ local breakable_before = setmetatable({
   [0xFF6B] = 1000, [0xFF6C] = 1000, [0xFF6D] = 1000, [0xFF6E] = 1000,
   [0xFF6F] = 1000,  [0x1B150] = 1000, [0x1B151] = 1000, [0x1B152] = 1000,
   [0x1B164] = 1000, [0x1B165] = 1000, [0x1B166] = 1000, [0x1B167] = 1000,
+  -- nonstarter
+  [0xA015] = false, -- YI SYLLABLE WU
 },{ __index = function(_,c)
   return is_hangul(c)
   or     is_compat_jamo(c)
@@ -614,7 +625,7 @@ luatexko.breakablebefore = breakable_before
 local function is_cjk_char (c)
   return is_hangul_jamo(c)
   or     is_hanja(c)
-  or     is_cjk_combining(c)
+  or     hangul_tonemark[c]
   or     is_kana(c)
   or     charclass[c] >= 1
   or     rawget(breakable_before, c) and c >= 0x2000
@@ -696,22 +707,33 @@ local function process_fonts (head)
 
         local c = curr.char
 
-        if is_cjk_combining(c) then
+        local done
+        if is_combining(c) then
           local p = getprev(curr)
-          if p.id == glyphid and curr.font ~= p.font then
-            hangul_space_skip(curr, p.font)
-            curr.font = p.font
+          if p and p.id == glyphid then
+            if curr.font ~= p.font then
+              hangul_space_skip(curr, p.font)
+              curr.font = p.font
+            end
             curr.lang = p.lang
-          end
 
-          if not active_processes.reorderTM and
-             hangul_tonemark[c] and
-             fontoptions.is_not_harf[curr.font] and
-             fontoptions.is_hangulscript[curr.font] then
-            luatexko.activate("reorderTM") -- activate reorderTM here
+            if hangul_tonemark[c] then
+              if not active_processes.reorderTM and
+                 fontoptions.is_not_harf[curr.font] and
+                 fontoptions.is_hangulscript[curr.font] then
+                luatexko.activate("reorderTM") -- activate reorderTM here
+              end
+
+              set_attribute(curr, unicodeattr, c)
+            else
+              curr.attr = p.attr -- inherit previous attr including unicodeattr
+            end
+            done = true
           end
+        end
+
+        if not done then
 
-        else
           if curr.subtype == 1 and curr.lang ~= nohyphen and is_cjk_char(c) then
             curr.lang = langkor -- suppress hyphenation of cjk chars
           end
@@ -740,14 +762,13 @@ local function process_fonts (head)
               end
             end
           end
+          set_attribute(curr, unicodeattr, c)
         end
-
-        set_attribute(curr, unicodeattr, c)
       end
     elseif id == discid then
-      curr.pre     = process_fonts(curr.pre)
-      curr.post    = process_fonts(curr.post)
-      curr.replace = process_fonts(curr.replace)
+      process_fonts(curr.pre)
+      process_fonts(curr.post)
+      process_fonts(curr.replace)
     elseif id == mathid then
       curr = end_of_math(curr)
     elseif id      == whatsitid  and
@@ -761,7 +782,6 @@ local function process_fonts (head)
     end
     curr = getnext(curr)
   end
-  return head
 end
 
 -- linebreak
@@ -787,7 +807,7 @@ local function hbox_char_font (box, init, glyfonly)
     local id = curr.id
     if id == glyphid then
       local c = has_attribute(curr, unicodeattr) or curr.char
-      if c and not is_cjk_combining(c) then
+      if c and not is_combining(c) then
         return c, curr.font
       end
     elseif curr.list then
@@ -866,7 +886,7 @@ local function process_linebreak (head, par)
     local id = curr.id
     if id == glyphid then
       local c = has_attribute(curr, unicodeattr) or curr.char
-      if c and not is_unicode_var_sel(c) then
+      if c and not is_combining(curr.char) then -- we are in pre-shaping stage
         local old = has_attribute(curr, classicattr)
         head, pc, pcl = maybe_linebreak(head, curr, pc, pcl, c, old, curr.font, par)
       end
@@ -921,7 +941,7 @@ local function process_interhangul (head, par)
     local id = curr.id
     if id == glyphid then
       local c = has_attribute(curr, unicodeattr) or curr.char
-      if c and not is_unicode_var_sel(c) then
+      if c and not is_combining(curr.char) then -- we are in pre-shaping stage
         head, pc = do_interhangul_option(head, curr, pc, c, curr.font, par)
 
         if is_jungsong(c) or is_jongsong(c) or hangul_tonemark[c] then
@@ -983,7 +1003,7 @@ local function process_interlatincjk (head, par)
     local id = curr.id
     if id == glyphid then
       local c = has_attribute(curr, unicodeattr) or curr.char
-      if c and not is_unicode_var_sel(c) then
+      if c and not is_combining(curr.char) then -- we are in pre-shaping stage
         head, pc, pf, pcl = do_interlatincjk_option(head, curr, pc, pf, pcl, c, curr.font, par)
         pc = breakable_after[c] and pc or 0
       end
@@ -1095,7 +1115,7 @@ local function process_remove_spaces (head)
               local vchar, vfont
               if id == glyphid and v.lang ~= nohyphen then
                 local c = has_attribute(v, unicodeattr) or v.char or 0
-                if is_unicode_var_sel(c) then
+                if is_combining(c) then
                   v = getprev(v) or v
                 end
                 vchar, vfont = has_attribute(v, unicodeattr) or v.char, v.font
@@ -1335,18 +1355,23 @@ local function process_dotemph (head)
     elseif curr.id == glyphid then
       local dotattr = has_attribute(curr, dotemphattr)
       if dotattr and dotemphbox[dotattr] then
-        local c = has_attribute(curr, unicodeattr) or curr.char
-        if is_hangul(c)      or
-           is_compat_jamo(c) or
-           is_chosong(c)     or
-           is_hanja(c)       or
-           is_kana(c)        then
-
-          -- harf font: skip reordered tone mark
-          if harf_reordered_tonemark(curr) then
-            curr = getnext(curr)
+
+        local ok
+        if hangul_tonemark[curr.char] and harf_reordered_tonemark(curr) then
+          curr = getnext(curr)
+          if is_hangul_jamo(has_attribute(curr, unicodeattr) or curr.char) then
+            ok = true
           end
+        else
+          if not is_combining(curr.char) then -- bypass unicodeattr inherited
+            local c = has_attribute(curr, unicodeattr) or curr.char
+            if is_hangul(c) or is_compat_jamo(c) or is_chosong(c) or is_hanja(c) or is_kana(c) then
+              ok = true
+            end
+          end
+        end
 
+        if ok then
           local currwd = curr.width
           if currwd >= fontoptions.en_size[curr.font] then
             local box = nodecopy(dotemphbox[dotattr]).list
@@ -2016,6 +2041,7 @@ local function process_fake_slant_corr (head) -- for font fallback
             else
               break
             end
+          elseif p.id == whatsitid then
           else
             break
           end
@@ -2076,18 +2102,55 @@ local function process_fake_slant_font (fontdata, fsl)
   end
 end
 
+-- AC00 11A8, AC00 11F0 ...
+-- these should not happen in KS-observing documents
+-- but HarfBuzz supports these sequences anyway.
+local function normalize_syllable_TC (head)
+  local curr = head
+  while curr do
+    if curr.id == glyphid then
+      local c, f = curr.char, curr.font
+      if is_hangul(c) and (c - 0xAC00) % 28 == 0 then
+        local t = getnext(curr)
+        if t then
+          if t.id == glyphid then
+            local tc, tf = t.char, t.font
+            if is_jongsong(tc) and f == tf then
+              if tc <= 0x11C2 then
+                curr.char = c + tc - 0x11A7
+                noderemove(head, t)
+                nodefree(t)
+              else
+                c = (c - 0xAC00) // 28
+                curr.char = c // 21 + 0x1100
+                local v = nodecopy(curr)
+                v.char = c % 21 + 0x1161
+                insert_after(head, curr, v)
+                curr = t
+              end
+            end
+          else
+            curr = t
+          end
+        end
+      end
+    end
+    curr = getnext(curr)
+  end
+end
+
 -- wrap up
 
 add_to_callback ("hyphenate",
 function(head)
-  local head = process_fonts(head)
+  normalize_syllable_TC(head)
+  process_fonts(head)
   lang.hyphenate(head)
 end,
 "luatexko.hyphenate.fonts_and_languages")
 
-local pass_fun = function(...) return ... end
-create_callback("luatexko_prelinebreak_first",  "data", pass_fun)
-create_callback("luatexko_prelinebreak_second", "data", pass_fun)
+create_callback("luatexko_prelinebreak_first",  "data", function(...) return ... end)
+create_callback("luatexko_prelinebreak_second", "data", function(...) return ... end)
 
 add_to_callback("pre_shaping_filter", function(h, gc)
   local par = gc == ""
diff --git a/Master/texmf-dist/tex/luatex/luatexko/luatexko.sty b/Master/texmf-dist/tex/luatex/luatexko/luatexko.sty
index b3f78f4ba42..0cfb6da6a9b 100644
--- a/Master/texmf-dist/tex/luatex/luatexko/luatexko.sty
+++ b/Master/texmf-dist/tex/luatex/luatexko/luatexko.sty
@@ -14,7 +14,7 @@
 \ifdefined\luatexkohangulfontattr \endinput\fi
 \ifdefined\ProvidesPackage
   \NeedsTeXFormat{LaTeX2e}[2021/06/01]
-  \ProvidesPackage{luatexko}[2021/06/10 v3.2 typesetting Korean with LuaTeX]
+  \ProvidesPackage{luatexko}[2021/07/10 v3.3 typesetting Korean with LuaTeX]
   \RequirePackage{luatexbase}
   \RequirePackage{fontspec}[2020/02/03]
 \else
@@ -351,6 +351,7 @@
   \fi
   \typesetvertical }
 \protected\def\endvertical{%
+  \ifinner\else \par\fi
   \egroup
   \luatexkorotatebox\z@
   \box\z@ }
@@ -381,6 +382,7 @@
   \fi
   \typesetmodern }
 \protected\def\endhorizontal{%
+  \ifinner\else \par\fi
   \egroup
   \luatexkounrotatebox\z@
   \raise\luatexkohorizboxmoveright
@@ -464,7 +466,7 @@
       \unexpanded\expandafter{\plainoutput}}%
     \def\pagebody{\setbox\z@\vbox to\hsize{\boxmaxdepth=\maxdepth \pagecontents}%
       \luatexkorotatebox\z@\box\z@}%
-    \maxdepth\z@ \typesetvertical
+    \typesetvertical
     \let\verticaltypesetting\relax % prevent multiple running
   }
   \def\beginverticaltypesetting{\vfill\supereject \begingroup \verticaltypesetting}
@@ -539,7 +541,7 @@
     \expandafter\textwidth\expandafter\textheight
     \@outputpage
   }%
-  \maxdepth\z@ \typesetvertical
+  \typesetvertical
   \let\verticaltypesetting\relax % prevent multiple running
 }
 \def\endverticaltypesetting{%
@@ -964,6 +966,7 @@
     \def\는{는}\def\은{은}\def\을{을}\def\를{를}\def\와{와}\def\과{과}%
     \def\가{가}\def\이{이}\def\라{라}\def\으{으}\def\로{로}%
     \def\hellipsis{...}}}{}
+  \@ifpackageloaded{pxrubrica}{\let\ruby\jruby}{}
   \if@hangul \RequirePackage{konames-utf}\fi
 }
 % misc