make4ht (23jan20)

git-svn-id: svn://tug.org/texlive/trunk@53514 c570f23f-e606-0410-a88d-b1316a301751

11 files changed, 307 insertions, 68 deletions

diff --git a/Master/texmf-dist/scripts/make4ht/domfilters/make4ht-booktabs.lua b/Master/texmf-dist/scripts/make4ht/domfilters/make4ht-booktabs.lua
new file mode 100644
index 00000000000..edec8adbd8b
--- /dev/null
+++ b/Master/texmf-dist/scripts/make4ht/domfilters/make4ht-booktabs.lua
@@ -0,0 +1,82 @@
+
+local function find_cmidrules(current_rows)
+  -- save rows with cmidrules here
+  local matched_rows = {}
+  local continue = false
+  for row_no, row in ipairs(current_rows) do
+    local columnposition = 1
+    local matched_cmidrule = false
+
+    for _, col in ipairs(row:query_selector("td")) do
+      -- keep track of culumns
+      local span = tonumber(col:get_attribute("colspan")) or 1
+      local cmidrule = col:query_selector(".cmidrule")
+      -- column contain cmidrule
+      if #cmidrule > 0 then
+        -- remove any child elements, we don't need them anymore
+        col._children = {}
+        -- only one cmidrule can be on each row, save the position, column span and all attributes
+        matched_rows[row_no] = {attributes = col._attr, column = columnposition, span = span, continue = continue}
+        matched_cmidrule = true
+      end
+      columnposition = columnposition + span
+    end
+    if matched_cmidrule then
+      -- save the row number of the first cmidrule on the current row
+      continue = continue or row_no
+    else
+      continue = false
+    end
+
+  end
+  -- save the table rows count, so we can loop over them sequentially later
+  matched_rows.length = #current_rows
+  return matched_rows
+end
+
+local function update_row(current_rows, match, newspan, i)
+  local row_to_update = current_rows[match.continue]
+  -- insert spanning column if necessary
+  if newspan > 0 then
+    local td = row_to_update:create_element("td", {colspan=tostring(newspan), span="nazdar"})
+    row_to_update:add_child_node(td)
+  end
+  -- insert the rule column
+  local td = row_to_update:create_element("td", match.attributes)
+  row_to_update:add_child_node(td)
+  -- remove unnecessary row
+  current_rows[i]:remove_node()
+end
+
+local function join_rows(matched_rows,current_rows)
+  for i = 1, matched_rows.length do
+    local match = matched_rows[i]
+    if match then
+      -- we only need to process rows that place subsequent cmidrules on the same row
+      local continue = match.continue
+      if continue then
+        local prev_row = matched_rows[continue]
+        -- find column where the previous cmidrule ends
+        local prev_end = prev_row.column + prev_row.span
+        local newspan = match.column - prev_end 
+        update_row(current_rows, match, newspan, i)
+        -- update the current row position
+        prev_row.column = match.column
+        prev_row.span = match.span
+      end
+    end
+  end
+end
+
+local function process_booktabs(dom)
+  local tables = dom:query_selector("table")
+  for _, tbl in ipairs(tables) do
+    local current_rows = tbl:query_selector("tr")
+    local matched_rows = find_cmidrules(current_rows)
+    join_rows(matched_rows, current_rows)
+  end
+  return dom
+end
+
+return process_booktabs
+
diff --git a/Master/texmf-dist/scripts/make4ht/domfilters/make4ht-joincharacters.lua b/Master/texmf-dist/scripts/make4ht/domfilters/make4ht-joincharacters.lua
index 5830eae78e6..4493aef023a 100644
--- a/Master/texmf-dist/scripts/make4ht/domfilters/make4ht-joincharacters.lua
+++ b/Master/texmf-dist/scripts/make4ht/domfilters/make4ht-joincharacters.lua
@@ -3,14 +3,35 @@ local log = logging.new("joincharacters")
 local charclasses = {
   span=true,
   mn = true,
-  mi = true
 }
 
+local function update_mathvariant(curr)
+  -- when we join several <mi> elements, they will be rendered incorrectly
+  -- we must set the mathvariant attribute
+  local parent = curr:get_parent()
+  -- set mathvariant only if it haven't been set by the parent element
+  if not parent:get_attribute("mathvariant") then
+    -- curr._attr = curr._attr or {}
+    local mathvariant = "italic"
+    -- the joined elements don't have attributes
+    curr._attr = curr._attr or {}
+    curr:set_attribute("mathvariant", mathvariant)
+  end
+end
+
+local table_count = function(tbl)
+  local tbl = tbl or {}
+  local i = 0
+  for k,v in pairs(tbl) do i = i + 1 end
+  return i
+end
+
+
 local has_matching_attributes = function (el, next_el)
   local el_attr = el._attr or {}
   local next_attr = next_el._attr or {}
   -- if the number of attributes doesn't match, elements don't match
-  if #next_attr ~= #el_attr then return false end
+  if table_count(next_attr) ~= table_count(el_attr) then return false end
   for k, v in pairs(el_attr) do
     -- if any attribute doesn't match, elements don't match
     if v~=next_attr[k] then return false end
@@ -18,6 +39,7 @@ local has_matching_attributes = function (el, next_el)
   return true
 end
 
+
 local function join_characters(obj,par)
   -- join adjanced span and similar elements inserted by 
   -- tex4ht to just one object.
@@ -25,45 +47,48 @@ local function join_characters(obj,par)
   local options = get_filter_settings "joincharacters"
   local charclasses = options.charclasses or par.charclasses or charclasses
 
-
-  obj:traverse_elements(function(el)
-    local get_name = function(curr) 
-      return string.lower(curr:get_element_name())
-    end
-    local get_class = function(next_el)
-      return next_el:get_attribute("class") or next_el:get_attribute("mathvariant")
-    end
-    local is_span = function(next_el)
-      return charclasses[get_name(next_el)]
+  local get_name = function(curr) 
+    return string.lower(curr:get_element_name())
+  end
+  local get_class = function(next_el)
+    return next_el:get_attribute("class") or next_el:get_attribute("mathvariant")
+  end
+  local is_span = function(next_el)
+    return charclasses[get_name(next_el)]
+  end
+  local join_elements = function(el, next_el)
+    -- it the following element match, copy it's children to the current element
+    for _, child in ipairs(next_el:get_children()) do
+      el:add_child_node(child)
     end
+    -- remove the next element
+    next_el:remove_node()
+  end
 
-    local function get_next(curr, class)
-      local next_el = curr:get_next_node()
-      if next_el and next_el:is_element() and is_span(next_el) then
-        return next_el
-        -- if the next node is space followed by a matching element, we should add this space
-      elseif next_el and next_el:is_text() and get_next(next_el, class) then
-        local text = next_el._text 
-        -- match only text containing just whitespace
-        if text:match("^%s+$") then return next_el end
-      end
+  local function get_next(curr, class)
+    local next_el = curr:get_next_node()
+    if next_el and next_el:is_element() and is_span(next_el) then
+      return next_el
+      -- if the next node is space followed by a matching element, we should add this space
+    elseif next_el and next_el:is_text() and get_next(next_el, class) then
+      local text = next_el._text 
+      -- match only text containing just whitespace
+      if text:match("^%s+$") then return next_el end
     end
+  end
+
+  obj:traverse_elements(function(el)
     -- loop over all elements and test if the current element is in a list of
     -- processed elements (charclasses)
     if is_span(el) then
       local next_el = get_next(el)
       -- loop over the following elements and test whether they are of the same type
       -- as the current one
-      while  next_el do
+      while next_el do
         -- save the next element because we will remove it later
         local real_next = get_next(next_el)
         if get_name(el) == get_name(next_el) and has_matching_attributes(el,next_el) and not el:get_attribute("id") then
-          -- it the following element match, copy it's children to the current element
-          for _, child in ipairs(next_el:get_children()) do
-            el:add_child_node(child)
-          end
-          -- remove the next element
-          next_el:remove_node()
+          join_elements(el, next_el)
           -- add the whitespace
         elseif next_el:is_text() then
           local s = next_el._text
@@ -80,6 +105,36 @@ local function join_characters(obj,par)
     end
 
   end)
+  -- process <mi> elements
+  obj:traverse_elements(function(el)
+    local function get_next_mi(curr)
+      local next_el = curr:get_next_node()
+      if next_el and next_el:is_element() then
+        return next_el
+      end
+    end
+    local function has_no_attributes(x)
+      return table_count(x._attr) == 0
+    end
+
+    -- join only subsequential <mi> elements with no attributes
+    if get_name(el) == "mi" and has_no_attributes(el) then
+      local next_el = get_next_mi(el)
+      while next_el do
+        local real_next = get_next_mi(next_el)
+        if get_name(next_el) == "mi" and has_no_attributes(next_el) then
+          join_elements(el, next_el)
+          -- set math variant to italic 
+          -- (if the parent <mstyle> element doesn't set it to something else)
+          update_mathvariant(el)
+        else
+          -- break the loop otherwise
+          real_next = nil
+        end
+        next_el = real_next
+      end
+    end
+  end)
   -- join text nodes in an element into one
   obj:traverse_elements(function(el)
     -- save the text
diff --git a/Master/texmf-dist/scripts/make4ht/domfilters/make4ht-tablerows.lua b/Master/texmf-dist/scripts/make4ht/domfilters/make4ht-tablerows.lua
index 155c24529ac..3e3931b368a 100644
--- a/Master/texmf-dist/scripts/make4ht/domfilters/make4ht-tablerows.lua
+++ b/Master/texmf-dist/scripts/make4ht/domfilters/make4ht-tablerows.lua
@@ -1,5 +1,16 @@
 local log = logging.new ("tablerows")
 return function(dom)
+  local has_child_elements = function(child)
+    -- detect if the element contains child elements
+    local child_elements = 0
+    local children = child:get_children()
+    for _, el in ipairs(children) do
+      local step = el:is_element() and 1 or 0
+      -- log:info("element name", el._name)
+      child_elements = child_elements + step
+    end
+    return child_elements > 0
+  end
   local is_empty_row = function(row)
     local not_empty = false
     local element_count = 0
@@ -11,7 +22,8 @@ return function(dom)
       if child:is_element() then 
         element_count = element_count + 1
         -- empty rows contain only one element, it is not empty otherwise
-        if element_count > 1 then return false end
+        if element_count > 1 or has_child_elements(child) then return false end
+
         -- detect if it contains only whitespace
         not_empty = child:get_text():gsub("%s","") ~= "" or not_empty
       end
diff --git a/Master/texmf-dist/scripts/make4ht/extensions/make4ht-ext-common_domfilters.lua b/Master/texmf-dist/scripts/make4ht/extensions/make4ht-ext-common_domfilters.lua
index ec687cd042d..63232b22cec 100644
--- a/Master/texmf-dist/scripts/make4ht/extensions/make4ht-ext-common_domfilters.lua
+++ b/Master/texmf-dist/scripts/make4ht/extensions/make4ht-ext-common_domfilters.lua
@@ -26,7 +26,7 @@ function M.modify_build(make)
     make:match("4om$", process, {charclasses= charclasses})
     count = 2
   else
-    local process = filter {"fixinlines", "idcolons", "joincharacters", "mathmlfixes", "tablerows"}
+    local process = filter {"fixinlines", "idcolons", "joincharacters", "mathmlfixes", "tablerows","booktabs"}
     make:match("html$", process)
     count = 1
   end
diff --git a/Master/texmf-dist/scripts/make4ht/extensions/make4ht-ext-detect_engine.lua b/Master/texmf-dist/scripts/make4ht/extensions/make4ht-ext-detect_engine.lua
new file mode 100644
index 00000000000..8bb91d3a697
--- /dev/null
+++ b/Master/texmf-dist/scripts/make4ht/extensions/make4ht-ext-detect_engine.lua
@@ -0,0 +1,91 @@
+-- support magic comments used by TeXShop and TeXWorks to detect used engine and format
+--
+local M = {}
+local log = logging.new("detect engine")
+local htlatex = require "make4ht-htlatex"
+
+-- we must change build sequence when Plain TeX is requested
+local change_table = {
+  tex = {
+    htlatex = "etex",
+    command = htlatex.httex
+  }, 
+  pdftex = {
+    htlatex = "etex",
+    command = htlatex.httex
+  },
+  etex = {
+    htlatex = "etex",
+    command = htlatex.httex
+  },
+  luatex = {
+    htlatex = "dviluatex",
+    command = htlatex.httex
+  },
+  xetex = {
+    htlatex = "xetex -no-pdf",
+    command = htlatex.httex
+  },
+  xelatex = {
+    htlatex = "xelatex -no-pdf",
+  },
+  lualatex = {
+    htlatex = "dvilualatex",
+  },
+  pdflatex = {
+    htlatex = "latex"
+  }
+
+}
+
+local function find_magic_program(filename)
+  -- find the magic line containing program name
+  local get_comment = function(line)
+    return line:match("%s*%%%s*(.+)")
+  end
+  local empty_line = function(line) return line:match("^%s*$") end
+  for line in io.lines(filename) do
+    local comment = get_comment(line)
+    -- read line after line from the file, break the processing after first non comment or non empty line
+    if not comment and not empty_line(line) then return nil, "Cannot find program name" end
+    comment = comment or "" -- comment is nil for empty lines
+    local program = comment:match("!%s*[Tt][Ee][Xx].-program%s*=%s*([^%s]+)")
+    if program then return program:lower() end
+  end
+end
+
+-- update htlatex entries with detected program
+local function update_build_sequence(program, build_seq)
+  -- handle Plain TeX
+  local replaces = change_table[program] or {}
+  local is_xetex = program:match("xe") -- we must handle xetex in tex4ht
+  for pos, entry in ipairs(build_seq) do
+    if entry.name == "htlatex" then
+      -- handle httex
+      entry.command = replaces.command or entry.command
+      local params = entry.params or {}
+      params.htlatex = replaces.htlatex or params.htlatex
+      entry.params = params
+    elseif is_xetex and entry.name == "tex4ht" then
+      -- tex4ht must process .xdv file if the TeX file was compiled by XeTeX
+      entry.params.tex4ht_par = entry.params.tex4ht_par .. " -.xdv"
+    end
+  end
+end
+
+
+function M.modify_build(make)
+  -- find magic comments in the TeX file
+  local build_seq = make.build_seq
+  local tex_file = make.params.tex_file
+  local program, msg = find_magic_program(tex_file)
+  if program then
+    log:info("Found program name", program)
+    update_build_sequence(program, build_seq)
+  else
+    log:warning("Cannot find magic line with the program name")
+  end
+  return make
+end
+
+return M
diff --git a/Master/texmf-dist/scripts/make4ht/make4ht b/Master/texmf-dist/scripts/make4ht/make4ht
index 10311088d78..d90bd5b9704 100755
--- a/Master/texmf-dist/scripts/make4ht/make4ht
+++ b/Master/texmf-dist/scripts/make4ht/make4ht
@@ -29,7 +29,7 @@ make4ht [options] filename ["tex4ht.sty op." "tex4ht op." "t4ht op" "latex op"]
 
 -- set version number. the template should be replaced by the
 -- actual version number by the build script
-local version = "v0.3c"
+local version = "v0.3d"
 mkparams.version_number = version
 
 local args = mkparams.get_args()
diff --git a/Master/texmf-dist/scripts/make4ht/make4ht-htlatex.lua b/Master/texmf-dist/scripts/make4ht/make4ht-htlatex.lua
index 214972e8bb9..f141070406b 100755
--- a/Master/texmf-dist/scripts/make4ht/make4ht-htlatex.lua
+++ b/Master/texmf-dist/scripts/make4ht/make4ht-htlatex.lua
@@ -51,11 +51,17 @@ Make.latex_command = "${htlatex} --interaction=${interaction} ${latex_par} '\\ma
 "\\documentstyle[tex4ht]}}}\\makeatother\\HCode ${tex4ht_sty_par}.a.b.c."..
 "\\input \"\\detokenize{${tex_file}}\"'"
 
+Make.plain_command = '${htlatex} --interaction=${interaction} ${latex_par}' ..
+"'\\def\\Link#1.a.b.c.{\\expandafter\\def\\csname tex4ht\\endcsname{\\expandafter\\def\\csname tex4ht\\endcsname{#1,html}\\input tex4ht.sty }}" ..
+"\\def\\HCode{\\futurelet\\HCode\\HChar}\\def\\HChar{\\ifx\"\\HCode\\def\\HCode\"##1\"{\\Link##1}\\expandafter\\HCode\\else\\expandafter\\Link\\fi}" ..
+"\\HCode ${tex4ht_sty_par}.a.b.c.\\input \"\\detokenize{${tex_file}}\"'"
+
 
 local m = {}
 
-function m.htlatex(par)
-  local command = Make.latex_command
+function m.htlatex(par, latex_command)
+  -- latex_command can be also plain_command for Plain TeX
+  local command = latex_command or Make.latex_command
   local devnull = " > /dev/null 2>&1"
   if os.type == "windows" then
     command = command:gsub("'",'')
@@ -72,4 +78,15 @@ function m.htlatex(par)
   return Make.testlogfile(par)
 end
 
+function m.httex(par)
+  local newpar = {}
+  for k,v in pairs(par) do newpar[k] = v end
+  -- change executable name from *latex to *tex
+  newpar.htlatex = newpar.htlatex:gsub("latex", "tex")
+  -- plain tex command doesn't support etex extensions
+  -- which are necessary for TeX4ht. just quick hack to fix this
+  if newpar.htlatex == "tex" then newpar.htlatex = "etex" end
+  return m.htlatex(newpar, Make.plain_command)
+end
+
 return m
diff --git a/Master/texmf-dist/scripts/make4ht/make4ht-indexing.lua b/Master/texmf-dist/scripts/make4ht/make4ht-indexing.lua
index 5801df509bc..072536556a8 100755
--- a/Master/texmf-dist/scripts/make4ht/make4ht-indexing.lua
+++ b/Master/texmf-dist/scripts/make4ht/make4ht-indexing.lua
@@ -177,6 +177,9 @@ local run_indexing_command = function(command, par)
   if not status then xindylog:warning(msg) end
   -- remove the temporary idx file
   os.remove(newidxfile)
+  -- null the indfile, it is necessary in order to support
+  -- multiple indices
+  par.indfile = nil
 end
 
 M.get_utf8 = get_utf8
diff --git a/Master/texmf-dist/scripts/make4ht/make4ht-odtfilter.lua b/Master/texmf-dist/scripts/make4ht/make4ht-odtfilter.lua
deleted file mode 100755
index 97f0b8df670..00000000000
--- a/Master/texmf-dist/scripts/make4ht/make4ht-odtfilter.lua
+++ /dev/null
@@ -1,33 +0,0 @@
-local mkutils = require "mkutils"
-local zip = require "zip"
-
-
--- use function to change contents of the ODT file
-local function update_odt(odtfilename, file_path, fn)
-  -- get name of the odt file
-  local odtname = mkutils.remove_extension(odtfilename) .. ".odt"
-  -- open and read contents of the requested file inside ODT file
-  local odtfile = zip.open(odtname)
-  local local_file = odtfile:open(file_path)
-  local content = local_file:read("*all")
-  local_file:close()
-  odtfile:close()
-  -- update the content using user function
-  content = fn(content)
-  -- write the updated file
-  local local_file_file  = io.open(file_path,"w")
-  local_file_file:write(content)
-  local_file_file:close()
-  os.execute("zip " .. odtname .. " " .. file_path)
-  os.remove(file_path)
-end
-
-Make:match("tmp$", function(name, par)
-  update_odt(name, "content.xml", function(content)
-    return content:gsub("%&%#x([A-Fa-f0-9]+);", function(entity)
-      -- convert hexadecimal entity to Unicode
-      print(entity,utfchar(tonumber(entity, 16)))
-      return utfchar(tonumber(entity, 16))
-    end)
-  end)
-end)
diff --git a/Master/texmf-dist/scripts/make4ht/mkparams.lua b/Master/texmf-dist/scripts/make4ht/mkparams.lua
index eff8c99670b..0254de9ac23 100755
--- a/Master/texmf-dist/scripts/make4ht/mkparams.lua
+++ b/Master/texmf-dist/scripts/make4ht/mkparams.lua
@@ -134,6 +134,11 @@ local function handle_jobname(input, args)
   return latex_params, input
 end
 
+local function tex_file_not_exits(tex_file)
+  -- try to find the input file, return false if we cannot find it
+  return not (kpse.find_file(tex_file, "tex") or kpse.find_file(tex_file .. ".tex", "tex"))
+end
+
 -- use standard input instead of file if the filename is just `-`
 -- return the filename and status if it is a tmp name
 local function handle_input_file(filename)
@@ -191,6 +196,10 @@ local function process_args(args)
 
 	local compiler = args.lua and "dvilualatex" or args.xetex and "xelatex --no-pdf" or "latex"
   local tex_file, is_tmp_file = handle_input_file(args.filename)
+  -- test if the file exists
+  if not is_tmp_file and tex_file_not_exits(tex_file) then
+    log:warning("Cannot find input file: " .. tex_file)
+  end
 	local input = mkutils.remove_extension(tex_file)
   -- the output file name can be influneced using -jobname parameter passed to the TeX engine
 	local latex_params, input = handle_jobname(input, args) 
diff --git a/Master/texmf-dist/scripts/make4ht/mkutils.lua b/Master/texmf-dist/scripts/make4ht/mkutils.lua
index 944ab99cc74..a4f16fb71ad 100755
--- a/Master/texmf-dist/scripts/make4ht/mkutils.lua
+++ b/Master/texmf-dist/scripts/make4ht/mkutils.lua
@@ -338,7 +338,10 @@ env.Make:add("test","test the variables:  ${tex4ht_sty_par} ${htlatex} ${input}
 local htlatex = require "make4ht-htlatex"
 env.Make:add("htlatex", htlatex.htlatex
 ,{correct_exit=0})
-
+env.Make:add("htttex", htlatex.httex, {
+  htlatex = "etex",
+  correct_exit=0
+})
 
 env.Make:add("latexmk", function(par)
   local settings = get_filter_settings "htlatex" or {}