make4ht (3may18)

git-svn-id: svn://tug.org/texlive/trunk@47598 c570f23f-e606-0410-a88d-b1316a301751

12 files changed, 422 insertions, 15 deletions

diff --git a/Master/texmf-dist/doc/support/make4ht/README b/Master/texmf-dist/doc/support/make4ht/README
index 924456189d8..6b9ba2e30cf 100644
--- a/Master/texmf-dist/doc/support/make4ht/README
+++ b/Master/texmf-dist/doc/support/make4ht/README
@@ -111,7 +111,7 @@ the output format name:
 
 Available extensions:
 
-latexmk
+latexmk_build
 
 :    use `Latexmk` for \LaTeX\ compilation.
 
@@ -135,6 +135,10 @@ mathjaxnode
      convert from MathML code to HTML + CSS or SVG. See [the available
      settings](#mathjaxsettings).
 
+staticsite
+
+:    build the document in form suitable for static site generators like [Jekyll](https://jekyllrb.com/).
+
 # Build files
 
 `make4ht` supports build files. These are `Lua` scripts which can adjust
@@ -366,6 +370,10 @@ mathjaxnode
      convert from MathML code to HTML + CSS or SVG. See [the available
      settings](#mathjaxsettings).
 
+staticsite
+
+:    create HTML file in format suitable for static site generators such as [Jekyll](https://jekyllrb.com/)
+
 svg-height
 
 :    some  SVG images produced by `dvisvgm` seem to have wrong dimensions. This filter
@@ -733,6 +741,43 @@ sentence\_match
 
 :  Lua pattern used to match a sentence. Default value: `"([^%.^%?^!]*)([%.%?!]?)"`.
 
+### The `staticsite` filter and extension
+
+site\_root 
+
+:  directory where generated files should be copied.
+
+map
+
+:  table where keys are patterns that match filenames, value contains destination directoryfor matched files, relative to the `site_root` (it is possible to use `..` to swich to parent directory).
+
+file\_pattern 
+
+:  pattern used for filename generation. It is possible to use string templates and format strings for `os.date` function. Default value of `%Y-%m-%d-${input}` creates names in the form of `YYYY-MM-DD-file_name`.
+
+header
+
+:  table with variables to be set in the YAML header in HTML files. If the table value is a function, it is executed with current parameters and HTML page DOM object as arguments.
+
+Example:
+
+
+    local outdir = os.getenv "blog_root" 
+    
+    filter_settings "staticsite" {
+      site_root = outdir, 
+      map = {
+        [".css$"] = "../css/"
+      },
+      header = {
+         layout="post",
+         date = function(parameters, dom)
+           return os.date("!%Y-%m-%d %T", parameters.time)
+         end
+      }
+    }
+
+
 # Configuration file {#configfile}
 
 It is possible to globally modify the build settings using the configuration
diff --git a/Master/texmf-dist/doc/support/make4ht/changelog.tex b/Master/texmf-dist/doc/support/make4ht/changelog.tex
index 26614e59491..5200e488573 100644
--- a/Master/texmf-dist/doc/support/make4ht/changelog.tex
+++ b/Master/texmf-dist/doc/support/make4ht/changelog.tex
@@ -3,6 +3,71 @@
 
 \begin{itemize}
 \item
+  2018/05/03
+
+  \begin{itemize}
+  \tightlist
+  \item
+    renamed \texttt{latexmk} extension to \texttt{latexmk\_build}, due
+    to clash in TL
+  \end{itemize}
+\item
+  2018/04/18
+
+  \begin{itemize}
+  \tightlist
+  \item
+    \texttt{staticsite} extension:
+
+    \begin{itemize}
+    \tightlist
+    \item
+      make YAML header configurable
+    \item
+      set the \texttt{time} and \texttt{updated} headers
+    \end{itemize}
+  \item
+    don't override existing tables in \texttt{filter\_settings}
+  \end{itemize}
+\item
+  2018/04/17
+
+  \begin{itemize}
+  \tightlist
+  \item
+    done first version of \texttt{staticsite} extension
+  \end{itemize}
+\item
+  2018/04/16
+
+  \begin{itemize}
+  \tightlist
+  \item
+    check for Git repo in the Makefile, don't run Git commands outside
+    of repo
+  \end{itemize}
+\item
+  2018/04/15
+
+  \begin{itemize}
+  \tightlist
+  \item
+    added \texttt{staticsite} filter
+  \item
+    working on \texttt{staticsite} extension
+  \end{itemize}
+\item
+  2018/04/13
+
+  \begin{itemize}
+  \tightlist
+  \item
+    use \texttt{ipairs} instead of \texttt{pairs} to traverse lists of
+    images and image match functions
+  \item
+    load extensions in the correct order
+  \end{itemize}
+\item
   2018/04/09
 
   \begin{itemize}
diff --git a/Master/texmf-dist/doc/support/make4ht/make4ht-doc.pdf b/Master/texmf-dist/doc/support/make4ht/make4ht-doc.pdf
index fa9039865ce..910126cb313 100644
--- a/Master/texmf-dist/doc/support/make4ht/make4ht-doc.pdf
+++ b/Master/texmf-dist/doc/support/make4ht/make4ht-doc.pdf
diff --git a/Master/texmf-dist/doc/support/make4ht/readme.tex b/Master/texmf-dist/doc/support/make4ht/readme.tex
index df67a151b87..261e3f2b882 100644
--- a/Master/texmf-dist/doc/support/make4ht/readme.tex
+++ b/Master/texmf-dist/doc/support/make4ht/readme.tex
@@ -151,7 +151,7 @@ The extensions can be enabled or disabled by appending
 Available extensions:
 
 \begin{description}
-\item[latexmk]
+\item[latexmk\_build]
 use \texttt{Latexmk} for \LaTeX~compilation.
 \item[tidy]
 clean the \texttt{HTML} files using the \texttt{tidy} command.
@@ -165,6 +165,9 @@ Clean the HTML file using DOM filters. It is more powerful than
 use \href{https://github.com/pkra/mathjax-node-page/}{mathjax-node-page}
 to convert from MathML code to HTML + CSS or SVG. See
 \protect\hyperlink{mathjaxsettings}{the available settings}.
+\item[staticsite]
+build the document in form suitable for static site generators like
+\href{https://jekyllrb.com/}{Jekyll}.
 \end{description}
 
 \hypertarget{build-files}{%
@@ -389,6 +392,9 @@ internal links, which results in validation error.
 use \href{https://github.com/pkra/mathjax-node-page/}{mathjax-node-page}
 to convert from MathML code to HTML + CSS or SVG. See
 \protect\hyperlink{mathjaxsettings}{the available settings}.
+\item[staticsite]
+create HTML file in format suitable for static site generators such as
+\href{https://jekyllrb.com/}{Jekyll}
 \item[svg-height]
 some SVG images produced by \texttt{dvisvgm} seem to have wrong
 dimensions. This filter tries to set the correct image size.
@@ -815,6 +821,48 @@ Lua pattern used to match a sentence. Default value:
 \texttt{"({[}\^{}\%.\^{}\%?\^{}!{]}*)({[}\%.\%?!{]}?)"}.
 \end{description}
 
+\hypertarget{the-staticsite-filter-and-extension}{%
+\subsubsection{\texorpdfstring{The \texttt{staticsite} filter and
+extension}{The staticsite filter and extension}}\label{the-staticsite-filter-and-extension}}
+
+\begin{description}
+\item[site\_root]
+directory where generated files should be copied.
+\item[map]
+table where keys are patterns that match filenames, value contains
+destination directoryfor matched files, relative to the
+\texttt{site\_root} (it is possible to use \texttt{..} to swich to
+parent directory).
+\item[file\_pattern]
+pattern used for filename generation. It is possible to use string
+templates and format strings for \texttt{os.date} function. Default
+value of \texttt{\%Y-\%m-\%d-\$\{input\}} creates names in the form of
+\texttt{YYYY-MM-DD-file\_name}.
+\item[header]
+table with variables to be set in the YAML header in HTML files. If the
+table value is a function, it is executed with current parameters and
+HTML page DOM object as arguments.
+\end{description}
+
+Example:
+
+\begin{verbatim}
+local outdir = os.getenv "blog_root" 
+
+filter_settings "staticsite" {
+  site_root = outdir, 
+  map = {
+    [".css$"] = "../css/"
+  },
+  header = {
+     layout="post",
+     date = function(parameters, dom)
+       return os.date("!%Y-%m-%d %T", parameters.time)
+     end
+  }
+}
+\end{verbatim}
+
 \hypertarget{configfile}{%
 \section{Configuration file}\label{configfile}}
 
diff --git a/Master/texmf-dist/scripts/make4ht/extensions/latexmk.lua b/Master/texmf-dist/scripts/make4ht/extensions/latexmk_build.lua
index bf4dcdde5a4..bf4dcdde5a4 100644
--- a/Master/texmf-dist/scripts/make4ht/extensions/latexmk.lua
+++ b/Master/texmf-dist/scripts/make4ht/extensions/latexmk_build.lua
diff --git a/Master/texmf-dist/scripts/make4ht/extensions/staticsite.lua b/Master/texmf-dist/scripts/make4ht/extensions/staticsite.lua
new file mode 100644
index 00000000000..fcb6c09e17d
--- /dev/null
+++ b/Master/texmf-dist/scripts/make4ht/extensions/staticsite.lua
@@ -0,0 +1,135 @@
+local M = {}
+local filter = require "make4ht-filter"
+local mkutils = require "mkutils"
+
+-- get the published file name
+local function get_slug(settings)
+  local published_name = mkutils.remove_extension(settings.tex_file) .. ".published"
+  local config = get_filter_settings "staticsite"
+  local file_pattern = config.file_pattern or "%Y-%m-%d-${input}"
+  local time = os.time()
+
+  -- we must save the published date, so the subsequent compilations at different days
+  -- use the same name
+  if mkutils.file_exists(published_name) then
+    local f = io.open(published_name, "r")
+    local readtime  = f:read("*line")
+    time = tonumber(readtime)
+    print("Already pubslished", slug)
+    f:close()
+  else
+    -- escape 
+    -- slug must contain the unescaped input name
+    local f = io.open(published_name, "w")
+    f:write(time)
+    f:close()
+  end
+  -- set the updated and publishing times
+  local updated
+  -- the updated time will be set only when it is more than one day from the published time
+  local newtime = os.time()
+  if (newtime - time) > (24 * 3600) then updated = newtime end
+  filter_settings "staticsite" {
+    header = {
+      time = time,
+      updated = updated
+    }
+  }
+
+  -- make the output file name in the format YYYY-MM-DD-old-filename.html
+  local slug = os.date(file_pattern,time) % settings
+  return slug
+end
+
+
+-- it is necessary to set correct -jobname in latex_par parameters field
+-- in order to the get correct HTML file name
+local function update_jobname(slug, latex_par)
+  local latex_par = latex_par or ""
+  if latex_par:match("%-jobname") then
+    local firstchar=latex_par:match("%-jobname=.")
+    local replace_pattern="%-jobname=[^%s]+"
+    if firstchar == "'" or firstchar=='"' then
+      replace_pattern = "%-jobname=".. firstchar .."[^%"..firstchar.."]+"
+    end
+    
+    return latex_par:gsub(replace_pattern, "-jobname=".. slug)
+  else
+    return latex_par .. "-jobname="..slug
+  end
+end
+
+-- execute the function passed as parameter only once, when the file matching
+-- starts
+local function insert_filter(make, pattern, fn)
+  local insert_executed = false
+  table.insert(make.matches, 1, {
+    pattern=pattern,
+    params = {},
+    command = function()
+      if not insert_executed  then
+        fn()
+      end
+      insert_executed = true
+    end
+  })
+end
+
+
+local function copy_files(filename, par)
+  local function prepare_path(dir, subdir)
+    local path = dir .. "/" .. subdir .. "/" .. filename
+    return path:gsub("//", "/")
+  end
+  -- get extension settings
+  local site_settings = get_filter_settings "staticsite"
+  local site_root = site_settings.site_root
+  local map = site_settings.map or {}
+  -- default path without subdir, will be used if the file is not matched
+  -- by any pattern in the map
+  local path = prepare_path(site_root, "")
+  for pattern, destination in pairs(map) do
+    if filename:match(pattern) then
+      path = prepare_path(site_root, destination)
+      break
+    end
+  end
+  -- it is possible to use string extrapolation in path, for example for slug
+  mkutils.copy(filename, path % par)
+end
+
+function M.modify_build(make)
+  -- it is necessary to insert the filters for YAML header and file copying as last matches
+  -- we use an bogus match which will be executed only once as the very first one to insert
+  -- the filters
+  -- I should make filter from this
+  local process = filter {
+    "staticsite"
+  }
+  local settings = make.params
+  -- get the published file name
+  local slug = get_slug(settings)
+  for _, cmd in ipairs(make.build_seq) do
+    -- all commands must use the published file name
+    cmd.params.input = slug
+    cmd.params.latex_par = update_jobname(slug, cmd.params.latex_par)
+  end
+
+  local quotepattern = '(['..("%^$().[]*+-?"):gsub("(.)", "%%%1")..'])'
+  local mainfile = string.gsub(slug, quotepattern, "%%%1")
+
+  -- run the following code once in the first match on the first file
+  insert_filter(make, ".*", function()
+    -- for _, match in ipairs(make.matches) do
+    --   match.params.outdir = outdir
+    --   print(match.pattern, match.params.outdir)
+    -- end
+    -- make the YAML header only for the main HTML file
+    make:match(mainfile .. ".html", process)
+    make:match(".*", copy_files, {slug=slug})
+  end)
+
+  return make
+end
+
+return M
diff --git a/Master/texmf-dist/scripts/make4ht/filters/make4ht-staticsite.lua b/Master/texmf-dist/scripts/make4ht/filters/make4ht-staticsite.lua
new file mode 100644
index 00000000000..6378f0440aa
--- /dev/null
+++ b/Master/texmf-dist/scripts/make4ht/filters/make4ht-staticsite.lua
@@ -0,0 +1,98 @@
+local domobj = require "luaxml-domobject"
+-- save the header settings in YAML format
+local function make_yaml(tbl, level)
+  local t = {}
+  local level = level or 0
+  local indent = string.rep("  ", level)
+  -- indentation for multilen strings
+  local str_indent = string.rep("  ", level + 1)
+  for k,v in pairs(tbl) do
+    if type(v)=="string" then
+      -- detect multiline strings
+      if v:match("\n") then
+        table.insert(t, string.format(indent .. "%s: |", k))
+        table.insert(t, str_indent .. (v:gsub("\n", "\n".. str_indent)))
+      else
+        v = v:gsub("'", "''")
+        table.insert(t, string.format(indent .. "%s: '%s'", k,v))
+      end
+    elseif type(v) == "table" then
+      table.insert(t,string.format(indent .. "%s:", k))
+      -- we need to differently process array and hash table
+      -- we don't support mixing types
+      if #v > 0 then
+        for x,y in ipairs(v) do
+          if type(y) == "string" then
+            -- each string can be printed on it's own line
+            table.insert(t, indent .. string.format("- '%s'", y))
+          else
+            -- subtables need to be indented
+            -- table.insert(t, indent .. "-")
+            local subtable = make_yaml(y, level + 1)
+            -- we must insert dash at a correct place
+            local insert_dash = subtable:gsub("^(%s*)%s%s", "%1- ")
+            table.insert(t, insert_dash)
+          end
+        end
+      else
+        -- print indented table
+        table.insert(t, make_yaml(v,level + 1))
+      end
+    else
+      -- convert numbers and other values to string
+      table.insert(t, string.format(indent .. "%s: %s", k,tostring(v)))
+    end
+    
+  end
+  return table.concat(t,  "\n")
+end
+
+local function update_properties(properties, dom)
+  -- enable properties update from the config or build file
+  local settings = get_filter_settings "staticsite" or {}
+  local header = settings.header or {}
+  for field, rule in pairs(header) do
+    -- it is possible to pass function as a rule, it will be executed with properties as a parameter
+    if type(rule) == "function" then
+      properties[field] = rule(properties, dom)
+    else
+      -- otherwise set properties
+      properties[field] = rule
+    end
+  end
+  return properties
+end
+
+local function get_header(tbl)
+  local yaml = make_yaml(tbl)
+  return "---\n".. yaml.. "\n---\n"
+end
+
+return function(s,par)
+  print(os.getenv "blog_home")
+  local dom = domobj.parse(s)
+  local properties = {}
+  local head = dom:query_selector("head")[1]
+  properties.title = head:query_selector("title")[1]:get_text()
+  local styles = {}
+  for _, link in ipairs(head:query_selector("link")) do
+    local typ = link:get_attribute("type")
+    if typ == "text/css" then 
+      table.insert(styles, link:get_attribute("href"))
+    end
+  end
+  properties.styles = styles
+  local metas = {}
+  for _, meta in ipairs(head:query_selector("meta")) do
+    print(meta:serialize())
+    table.insert(metas, {charset= meta:get_attribute("charset"), content = meta:get_attribute("content"), property = meta:get_attribute("property"), name = meta:get_attribute("name")})
+  end
+  properties.meta = metas
+  properties = update_properties(properties, dom)
+
+
+  local body = dom:query_selector("body")[1]
+  print(get_header(properties))
+  -- return s
+  return get_header(properties) .. body:serialize():gsub("<body.->", ""):gsub("</body>", "")
+end
diff --git a/Master/texmf-dist/scripts/make4ht/make4ht b/Master/texmf-dist/scripts/make4ht/make4ht
index 0da3c30549c..10b277d2a73 100755
--- a/Master/texmf-dist/scripts/make4ht/make4ht
+++ b/Master/texmf-dist/scripts/make4ht/make4ht
@@ -27,7 +27,7 @@ make4ht [options] filename ["tex4ht.sty op." "tex4ht op." "t4ht op" "latex op"]
 
 -- set version number. the template should be replaced by the
 -- actual version number by the build script
-local version = "v0.2"
+local version = "0.2a"
 mkparams.version_number = version
 
 local args = mkparams.get_args()
diff --git a/Master/texmf-dist/scripts/make4ht/make4ht-filterlib.lua b/Master/texmf-dist/scripts/make4ht/make4ht-filterlib.lua
index 33cb3530a85..3c2e63cf69d 100755
--- a/Master/texmf-dist/scripts/make4ht/make4ht-filterlib.lua
+++ b/Master/texmf-dist/scripts/make4ht/make4ht-filterlib.lua
@@ -6,7 +6,7 @@ function M.load_filters(filters, load_filter)
 	if type(filters) == "string" then
 		table.insert(sequence,load_filter(filters))
 	elseif type(filters) == "table" then
-		for _,n in pairs(filters) do
+		for _,n in ipairs(filters) do
 			if type(n) == "string" then
 				table.insert(sequence,load_filter(n))
 			elseif type(n) == "function" then
diff --git a/Master/texmf-dist/scripts/make4ht/make4ht-lib.lua b/Master/texmf-dist/scripts/make4ht/make4ht-lib.lua
index 918343f20ca..2ada398f869 100755
--- a/Master/texmf-dist/scripts/make4ht/make4ht-lib.lua
+++ b/Master/texmf-dist/scripts/make4ht/make4ht-lib.lua
@@ -81,9 +81,9 @@ Make.image_convert =  function(self, images)
 		end
 		image_patterns[i].params = p
 	end
-	for _,i in pairs(images) do
+	for _,i in ipairs(images) do
 		local output = i.output
-		for _, x in pairs(image_patterns) do
+		for _, x in ipairs(image_patterns) do
 			local pattern = x.pattern
 			if output:match(pattern) then
 				local command = x.command
@@ -107,7 +107,7 @@ end
 Make.file_matches = function(self, files)
 	local statuses = {}
 	-- First make params for all matchers
-	for k,v in pairs(self.matches) do
+	for k,v in ipairs(self.matches) do
 		local v = self.matches[k].params or {}
 		local p = self.params or {}
 		for i,j in pairs(p) do
diff --git a/Master/texmf-dist/scripts/make4ht/mkutils.lua b/Master/texmf-dist/scripts/make4ht/mkutils.lua
index 88b456fe9da..7dab423bc9b 100755
--- a/Master/texmf-dist/scripts/make4ht/mkutils.lua
+++ b/Master/texmf-dist/scripts/make4ht/mkutils.lua
@@ -22,6 +22,19 @@ getmetatable("").__add = addProperty
 --print( "${name} is ${value}" % {name = "foo", value = "bar"} )
 -- Outputs "foo is bar"
 
+
+-- merge two tables recursively
+function merge(t1, t2)
+  for k, v in pairs(t2) do
+    if (type(v) == "table") and (type(t1[k] or false) == "table") then
+      merge(t1[k], t2[k])
+    else
+      t1[k] = v
+    end
+  end
+  return t1
+end
+
 function string:split(sep)
 	local sep, fields = sep or ":", {}
 	local pattern = string.format("([^%s]+)", sep)
@@ -157,7 +170,7 @@ function mkdirectories(dirs)
 	if type(dirs) ~="table" then
 		return false, "mkdirectories: dirs is not table"
 	end
-	for _,d in pairs(dirs) do
+	for _,d in ipairs(dirs) do
 		local stat,msg = lfs.mkdir(d)
 		if not stat then return false, "makedirectories error: "..msg end
 		lfs.chdir(d)
@@ -269,10 +282,7 @@ function env.filter_settings(name)
   local filters = settings.filter or {}
   local filter_options = filters[name] or {}
   return function(par)
-    for k,v in pairs(par) do
-      filter_options[k] = v
-    end
-    filters[name] = filter_options
+    filters[name] = merge(filter_options, par)
     settings.filter = filters
   end
 end
@@ -458,13 +468,19 @@ end
 function load_extensions(extensions, format)
   local module_names = {}
   local extension_table = {}
+  local extension_sequence = {}
   -- process the extension table. it contains type field, which can enable or
   -- diable the extension
   for _, v in ipairs(extensions) do
     local enable = v.type == "+" and true or nil
-    module_names[v.name] = enable
+    -- don't load extensions multiple times
+    if not module_names[v.name] then
+      module_names[v.name] = enable
+      -- load extenisons in a correct order
+      table.insert(extension_sequence, v.name)
+    end
   end
-  for name, _ in pairs(module_names) do
+  for _, name in ipairs(extension_sequence) do
     local extension = load_extension(name,format)
     if extension then
       table.insert(extension_table, extension)
diff --git a/Master/tlpkg/libexec/ctan2tds b/Master/tlpkg/libexec/ctan2tds
index a86072177f4..68b91d056aa 100755
--- a/Master/tlpkg/libexec/ctan2tds
+++ b/Master/tlpkg/libexec/ctan2tds
@@ -1535,7 +1535,7 @@ $standardfmt='latex';
 
 # packages which need special .tex/.sty files installed
 $standardtex
-   = '(\.(.bx|4ht|cls|clo|cmap|code\.tex|def|fd|fontspec|ldf|lua|sty|trsl)'
+   = '(\.(.bx|4ht|cls|clo|cmap|code\.tex|def|fd|fontspec|ldf|sty|trsl)'
     . '|.*[^c]\.cfg)$'; # not ltxdoc.cfg
 %specialtex = (
  '2up',         '2up\.tex|' . $standardtex,