CTAN sync 202004010302

2 files changed, 352 insertions, 72 deletions

diff --git a/macros/luatex/latex/lua-ul/lua-ul.dtx b/macros/luatex/latex/lua-ul/lua-ul.dtx
index 9573212ba2..b4c33d7b13 100644
--- a/macros/luatex/latex/lua-ul/lua-ul.dtx
+++ b/macros/luatex/latex/lua-ul/lua-ul.dtx
@@ -168,9 +168,8 @@
 %
 % The final argument contains the actual leader command. You should omit the final glue normally passed to \verb+\leaders+,
 % so e.g.\ write \verb+\leaders\hbox{ . }+ without appending \verb+\hfill+ or \verb+\hskip1pt+ etc.
-% The height and depth of your leaders is ignored to ensure that adding underlines etc. does not change the general text layout.
-% It is your responsibility to ensure that you are not too high or too low and intercept other lines. On the other hand,
-% running dimensions work fine if you use a rule.
+% In most cases, the height and depth of your underlines is passed on to TeX to ensure that a deep underline does not interept other lines.
+% On the other hand, running dimensions work fine if you use a rule.
 %
 % For example, the special underline commands demonstrated above are implemented as
 %
@@ -259,7 +258,7 @@ local function filtered_append_to_vlist_filter(box,
     current = box
   end
   return call_callback("append_to_vlist_filter",
-                       box, locationcode, prevdepth, mirrored)
+                       current, locationcode, prevdepth, mirrored)
 end
 
 callback_define('append_to_vlist_filter',
@@ -287,15 +286,75 @@ luatexbase.create_callback('pre_append_to_vlist_filter',
 %<*luacode>
 % \fi
 %    \begin{macrocode}
+local unset_t = node.id'unset'
 local hlist_t = node.id'hlist'
 local vlist_t = node.id'vlist'
 local kern_t = node.id'kern'
 local glue_t = node.id'glue'
 
+local properties = node.direct.get_properties_table()
+
+local has_attribute = node.direct.has_attribute
+local set_attribute = node.direct.set_attribute
+local dimensions = node.direct.dimensions
+local flush_node = node.direct.flush_node
+local getboth = node.direct.getboth
+local getfield = node.direct.getfield
+local getglue = node.direct.getglue
+local getleader = node.direct.getleader
+local getlist = node.direct.getlist
+local setheight = node.direct.setheight
+local setdepth = node.direct.setdepth
+local getheight = node.direct.getheight
+local getdepth = node.direct.getdepth
+local getnext = node.direct.getnext
+local getshift = node.direct.getshift
+local insert_after = node.direct.insert_after
+local insert_before = node.direct.insert_before
+local nodecopy = node.direct.copy
+local nodenew = node.direct.new
+local setboth = node.direct.setboth
+local setlink = node.direct.setlink
+local hpack = node.direct.hpack
+local setfield = node.direct.setfield
+local slide = node.direct.slide
+local setglue = node.direct.setglue
+local setnext = node.direct.setnext
+local setshift = node.direct.setshift
+local todirect = node.direct.todirect
+local tonode = node.direct.tonode
+local traverse = node.direct.traverse
+local traverse_id = node.direct.traverse_id
+local traverse_list = node.direct.traverse_list
+% The following two are needed to deal with unset nodes
+local getList = function(n) return getfield(n, 'list') end
+local setList = function(n, h) return setfield(n, 'list', h) end
+
+local tokennew = token.new
+local set_lua = token.set_lua
+local scan_keyword = token.scan_keyword
+local scan_list = token.scan_list
+local scan_int = token.scan_int
+local put_next = token.put_next
+local texerror = tex.error
+
 local char_given = token.command_id'char_given'
 
 local underlineattrs = {}
 local underline_types = {}
+local underline_strict_flag = {}
+local underline_over_flag = {}
+
+local vmode do
+  for k, v in pairs(tex.getmodevalues()) do
+  if v == "vertical" then
+      vmode = k
+      break
+    end
+  end
+end
+local texnest = tex.nest
+
 local saved_values = {}
 local function new_underline_type()
   for i=1,#underlineattrs do
@@ -303,30 +362,58 @@ local function new_underline_type()
     saved_values[i] = tex.attribute[attr]
     tex.attribute[attr] = -0x7FFFFFFF
   end
-  local b = token.scan_list()
+  local strict_flag = scan_keyword'strict'
+  local over_flag = scan_keyword'over'
+  local b = todirect(scan_list())
   for i=1,#underlineattrs do
     tex.attribute[underlineattrs[i]] = saved_values[i]
   end
-  local lead = b.head
-  if not lead.leader then
-    tex.error("Leader required", {"An underline type has to \z
+  local lead = getlist(b)
+  if not getleader(lead) then
+    texerror("Leader required", {"An underline type has to \z
       be defined by leader. You should use one of the", "commands \z
       \\leaders, \\cleaders, or \\xleader, or \\gleaders here."})
   else
-    if lead.next then
-    tex.error("Too many nodes", {"An underline type can only be \z
-        defined by a single leaders specification,", "not by \z
-        multiple nodes. Maybe you supplied an additional glue?",
-        "Anyway, the additional nodes will be ignored"})
+    local after = getnext(lead)
+    if after then
+      texerror("Too many nodes", {"An underline type can only be \z
+          defined by a single leaders specification,", "not by \z
+          multiple nodes. Maybe you supplied an additional glue?",
+          "Anyway, the additional nodes will be ignored"})
+      setnext(lead, nil)
     end
     table.insert(underline_types, lead)
-    b.head = lead.next
-    node.flush_node(b)
+    setList(b, after)
+    flush_node(b)
   end
-  token.put_next(token.new(#underline_types, char_given))
+  put_next(tokennew(#underline_types, char_given))
+  underline_strict_flag[#underline_types] = strict_flag
+  underline_over_flag[#underline_types] = over_flag
 end
+%    \end{macrocode}
+% In \verb+append_to_vlist_filter+ we can not access the list attributes,
+% so we just take the current ones. They might be incorrect if the attribute
+% changes in the vlist, so we record the original value in a property then.
+%    \begin{macrocode}
 local function set_underline()
-  local j
+  local j, props
+  for i=texnest.ptr,0,-1 do
+    local mode = texnest[i].mode
+    if mode == vmode or mode == -vmode then
+      local head = todirect(texnest[i].head)
+      local head_props = properties[head]
+      if not head_props then
+        head_props = {}
+        properties[head] = head_props
+      end
+      props = head_props.luaul_attributes
+      if not props then
+        props = {}
+        head_props.luaul_attributes = props
+        break
+      end
+    end
+  end
   for i=1,#underlineattrs do
     local attr = underlineattrs[i]
     if tex.attribute[attr] == -0x7FFFFFFF then
@@ -339,13 +426,14 @@ local function set_underline()
         "luaul" .. tostring(#underlineattrs+1))
     underlineattrs[#underlineattrs+1] = j
   end
-  tex.attribute[j] = token.scan_int()
+  props[j] = props[j] or -0x7FFFFFFF
+  tex.attribute[j] = scan_int()
 end
 %    \end{macrocode}
 % \changes{0.0.2}{2020-03-15}{Add command to disable active underlining}
 %    \begin{macrocode}
 local function reset_underline()
-  local reset_all = token.scan_keyword'*'
+  local reset_all = scan_keyword'*'
   local j
   for i=1,#underlineattrs do
     local attr = underlineattrs[i]
@@ -359,9 +447,9 @@ local function reset_underline()
   end
   if not j then
     if not reset_all then
-      tex.error("No underline active", {"You tried to disable \z
-            underlining but underlining was not active",
-            "in the first place. Maybe you wanted to ensure that \z
+      texerror("No underline active", {"You tried to disable \z
+            underlining but underlining was not active in the first",
+            "place. Maybe you wanted to ensure that \z
             no underling can be active anymore?", "Then you should \z
             append a *."})
     end
@@ -370,7 +458,6 @@ local function reset_underline()
   tex.attribute[j] = -0x7FFFFFFF
 end
 local functions = lua.get_functions_table()
-local set_lua = token.set_lua
 local new_underline_type_func =
     luatexbase.new_luafunction"luaul.new_underline_type"
 local set_underline_func =
@@ -384,77 +471,269 @@ functions[new_underline_type_func] = new_underline_type
 functions[set_underline_func] = set_underline
 functions[reset_underline_func] = reset_underline
 
-local add_underline_h
-local function add_underline_v(head, attr)
-  for n in node.traverse(head) do
-    if head.id == hlist_t then
-      add_underline_h(n, attr)
-    elseif head.id == vlist_t then
-      add_underline_v(n.head, attr)
+%    \end{macrocode}
+% \changes{0.0.3}{2020-03-17}{Use glue instead of kern to allow handling
+% unset lists}
+% A little helper to measure box contents and creating a glue
+% node with inverted dimensions.
+%    \begin{macrocode}
+local stretch_fi = {}
+local shrink_fi = {}
+local function fil_levels(n)
+  for i=0,4 do
+    stretch_fi[i], shrink_fi[i] = 0, 0
+  end
+  for n in traverse_id(glue_t, n) do
+    local w, st, sh, sto, sho = getglue(n)
+    stretch_fi[sto] = stretch_fi[sto] + st
+    shrink_fi[sho] = shrink_fi[sho] + sh
+  end
+  local stretch, shrink = 0, 0
+  for i=0,4 do
+    if stretch_fi[i] ~= 0 then
+      stretch = i
+    end
+    if shrink_fi[i] ~= 0 then
+      shrink = i
+    end
+  end
+  return stretch, shrink
+end
+local function new_glue_neg_dimensions(n, t,
+                                       stretch_order, shrink_order)
+  local g = nodenew(glue_t)
+  local w = -dimensions(n, t)
+  setglue(g, w)
+%  setglue(g, -dimensions(n, t), 0, 0, stretch_order, shrink_order)
+  setnext(g, n)
+  setglue(g, w, -dimensions(1, 1, stretch_order, g, t),
+                   dimensions(1, 2, shrink_order, g, t),
+                   stretch_order, shrink_order)
+  setnext(g, nil)
+  return g
+end
+
+%    \end{macrocode}
+% \changes{0.0.3}{2020-03-17}{Make streight lines over hboxes}
+% \changes{0.0.4}{2020-03-31}{Consistently respect height and depth}
+% Now the actual undelining
+%    \begin{macrocode}
+local add_underline_hlist, add_underline_hbox, add_underline_vbox
+local function add_underline_vlist(head, attr, outervalue)
+  local iter, state, n = traverse_list(head) -- FIXME: unset nodes
+  local t
+  n, t = iter(state, n)
+  while n ~= nil do
+    local real_new_value = has_attribute(n, attr)
+    local new_value = real_new_value ~= outervalue
+                        and real_new_value or nil
+    if underline_strict_flag[new_value] or not new_value then
+      if t == hlist_t then
+        add_underline_hbox(n, attr, real_new_value)
+      elseif t == vlist_t then
+        add_underline_vbox(n, attr, real_new_value)
+      end
+      n, t = iter(state, n)
+    elseif real_new_value <= 0 then
+      n, t = iter(state, n)
+    else
+      local nn
+      nn, t = iter(state, n)
+      local prev, next = getboth(n)
+      setboth(n, nil, nil)
+      local shift = getshift(n)
+      setshift(n, 0)
+      local new_list = hpack((add_underline_hlist(n, attr)))
+      setheight(new_list, getheight(n))
+      setdepth(new_list, getdepth(n))
+      setshift(new_list, shift)
+      setlink(prev, new_list, next)
+      set_attribute(new_list, attr, 0)
+      if n == head then
+        head = new_list
+      end
+      n = nn
     end
   end
+  return head
+end
+function add_underline_vbox(head, attr, outervalue)
+  if outervalue and outervalue <= 0 then return end
+  setList(head, add_underline_vlist(getList(head), attr, outervalue))
+  set_attribute(head, attr, outervalue and -outervalue or 0)
 end
-function add_underline_h(head, attr)
-  local used = false
-  node.slide(head.head)
+function add_underline_hlist(head, attr, outervalue)
+  local max_height, max_depth
+  slide(head)
   local last_value
   local first
-  for n in node.traverse(head.head) do
-    local new_value = node.has_attribute(n, attr)
-    if n.id == hlist_t then
-      new_value = nil
-      add_underline_h(n, attr)
-    elseif n.id == vlist_t then
-      new_value = nil
-      add_underline_v(n.head, attr)
-    elseif n.id == kern_t and n.subtype == 0 then
-      if n.next and not node.has_attribute(n.next, attr) then
+  local shrink_order, stretch_order
+  for n, id, subtype in traverse(head) do
+    local real_new_value = has_attribute(n, attr)
+    local new_value
+    if real_new_value then
+      if real_new_value > 0 then
+        set_attribute(n, attr, -real_new_value)
+        new_value = real_new_value ~= outervalue
+                      and real_new_value or nil
+      end
+    else
+      set_attribute(n, attr, 0)
+    end
+    if id == hlist_t then
+      if underline_strict_flag[new_value]
+          or subtype == 3 or not new_value then
+        add_underline_hbox(n, attr, real_new_value)
+        new_value = nil
+      end
+    elseif id == vlist_t then
+      if underline_strict_flag[new_value] or not new_value then
+        add_underline_vbox(n, attr, real_new_value)
         new_value = nil
+      end
+    elseif id == kern_t and subtype == 0 then
+      local after = getnext(n)
+      if after then
+        local next_value = has_attribute(after, attr)
+        if next_value == outervalue or not next_value then
+          new_value = nil
+        else
+          new_value = last_value
+        end
       else
         new_value = last_value
       end
-    elseif n.id == glue_t and (
-        n.subtype == 8 or
-        n.subtype == 9 or
-        n.subtype == 15 or
+    elseif id == glue_t and (
+        subtype == 8 or
+        subtype == 9 or
+        subtype == 15 or
     false) then
       new_value = nil
     end
     if last_value ~= new_value then
+      if not stretch_order then
+        stretch_order, shrink_order = fil_levels(head)
+      end
       if last_value then
-        local width = node.rangedimensions(head, first, n)
-        local kern = node.new(kern_t)
-        kern.kern = -width
-        local lead = node.copy(underline_types[last_value])
-        lead.width = width
-        head.head = node.insert_before(head.head, first, lead)
-        node.insert_after(head, lead, kern)
+        local glue = new_glue_neg_dimensions(first, n,
+            stretch_order, shrink_order)
+        local w, st, sh = getglue(glue)
+        local lead = nodecopy(underline_types[last_value])
+        setglue(lead, -w, -st, -sh, stretch_order, shrink_order)
+        if underline_over_flag[last_value] then
+          head = insert_before(head, n, glue)
+          insert_after(head, glue, lead)
+        else
+          head = insert_before(head, first, lead)
+          insert_after(head, lead, glue)
+        end
       end
       if new_value then
         first = n
+        local box = getleader(underline_types[new_value])
+        if not max_height or getheight(box) > max_height then
+          max_height = getheight(box)
+        end
+        if not max_depth or getdepth(box) > max_depth then
+          max_depth = getdepth(box)
+        end
       end
       last_value = new_value
     end
   end
   if last_value then
-    local width = node.rangedimensions(head, first)
-    local kern = node.new(kern_t)
-    kern.kern = -width
-    local lead = node.copy(underline_types[last_value])
-    lead.width = width
-    head.head = node.insert_before(head.head, first, lead)
-    node.insert_after(head, lead, kern)
+    local glue = new_glue_neg_dimensions(first, nil,
+        stretch_order, shrink_order)
+    local w, st, sh = getglue(glue)
+    local lead = nodecopy(underline_types[last_value])
+    setglue(lead, -w, -st, -sh, stretch_order, shrink_order)
+    if underline_over_flag[last_value] then
+      insert_before(head, nil, glue)
+      insert_after(head, glue, lead)
+    else
+      head = insert_before(head, first, lead)
+      insert_after(head, lead, glue)
+    end
   end
+  return head, max_height, max_depth
 end
-local function filter(b, loc, prev, mirror)
-  for i = 1, #underlineattrs do
-    add_underline_v(b, underlineattrs[i])
+function add_underline_hbox(head, attr, outervalue, set_height_depth)
+  if outervalue and outervalue <= 0 then return end
+  local new_head, new_height, new_depth
+      = add_underline_hlist(getList(head), attr, outervalue)
+  setList(head, new_head)
+  if set_height_depth then
+    if new_height and getheight(head) < new_height then
+      setheight(head, new_height)
+    end
+    if new_depth and getdepth(head) < new_depth then
+      setdepth(head, new_depth)
+    end
   end
-  return true
+  set_attribute(head, attr, outervalue and -outervalue or 0)
 end
 require'pre_append_to_vlist_filter'
 luatexbase.add_to_callback('pre_append_to_vlist_filter',
-                           filter, 'add underlines to list')
+    function(b, loc, prev, mirror)
+      local props = properties[todirect(texnest.top.head)]
+      props = props and props.luaul_attributes
+      b = todirect(b)
+      if loc == "post_linebreak" then
+        for i = 1, #underlineattrs do
+          local attr = underlineattrs[i]
+          local current = props and props[attr] or tex.attribute[attr]
+          if current == -0x7FFFFFFF then
+            current = nil
+          end
+          add_underline_hbox(b, underlineattrs[i], current, true)
+        end
+      else
+        for i = 1, #underlineattrs do
+          local attr = underlineattrs[i]
+          local current = props and props[attr] or tex.attribute[attr]
+          local b_attr = has_attribute(b, attr)
+          if b_attr and b_attr ~= current then
+            local shift = getshift(b)
+            setshift(b, 0)
+            b = hpack((add_underline_hlist(b, attr)))
+            setshift(b, shift)
+            set_attribute(b, attr, 0)
+          end
+        end
+      end
+      return tonode(b)
+    end, 'add underlines to list')
+luatexbase.add_to_callback('hpack_filter',
+    function(head, group, size, pack, dir, attr)
+      head = todirect(head)
+      for i = 1, #underlineattrs do
+        local ulattr = underlineattrs[i]
+        local current
+        for n in node.traverse(attr) do
+          if n.number == ulattr then
+            current = n.value
+          end
+        end
+        head = add_underline_hlist(head, ulattr, current)
+      end
+      return tonode(head)
+    end, 'add underlines to list')
+luatexbase.add_to_callback('vpack_filter',
+    function(head, group, size, pack, maxdepth, dir, attr)
+%      if true then return head end
+      head = todirect(head)
+      for i = 1, #underlineattrs do
+        local ulattr = underlineattrs[i]
+        local current
+        for n in node.traverse(attr) do
+          if n.number == ulattr then
+            current = n.value
+          end
+        end
+        head = add_underline_vlist(head, ulattr, current)
+      end
+      return tonode(head)
+    end, 'add underlines to list')
 %    \end{macrocode}
 % \iffalse
 %</luacode>
@@ -469,7 +748,7 @@ luatexbase.add_to_callback('pre_append_to_vlist_filter',
 \NeedsTeXFormat{LaTeX2e}
 \ProvidesPackage
   {lua-ul}
-  [2020/03/15 v0.0.2 Underlining and related functionality for LuaTeX]
+  [2020/03/31 v0.0.4 Underlining and related functionality for LuaTeX]
 
 % \fi
 % Only \LuaLaTeX{} is supported.
@@ -516,13 +795,14 @@ luatexbase.add_to_callback('pre_append_to_vlist_filter',
 %    \end{macrocode}
 % The main macro.
 %    \begin{macrocode}
-\NewDocumentCommand\newunderlinetype{mO{\luaul@defaultcontext}m}{%
-  \newcommand#1{}% "Reserve" the name
-  \protected\def#1{%
+\NewDocumentCommand\newunderlinetype
+    { E{*}{{}} m O{\luaul@defaultcontext} m }{%
+  \newcommand#2{}% "Reserve" the name
+  \protected\def#2{%
     \expandafter\luaul@maybedefineuse
-      \expanded{{\csstring#1@@#2}}%
+      \expanded{{\csstring#2@@#3}}%
       {\LuaULSetUnderline
-        \LuaULNewUnderlineType\hbox{#3\hskip0pt}%
+        \LuaULNewUnderlineType#1\hbox{#4\hskip0pt}%
   }}%
 }
 \ifluaul@predefined
diff --git a/macros/luatex/latex/lua-ul/lua-ul.pdf b/macros/luatex/latex/lua-ul/lua-ul.pdf
index 257d7fee40..66d1f8551e 100644
--- a/macros/luatex/latex/lua-ul/lua-ul.pdf
+++ b/macros/luatex/latex/lua-ul/lua-ul.pdf