From d20905e40a1f1c00c8b4d268ecaafcef47953e7a Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Thu, 3 May 2018 21:04:48 +0000 Subject: make4ht (3may18) git-svn-id: svn://tug.org/texlive/trunk@47598 c570f23f-e606-0410-a88d-b1316a301751 --- Master/texmf-dist/doc/support/make4ht/README | 47 ++++++- .../texmf-dist/doc/support/make4ht/changelog.tex | 65 ++++++++++ .../texmf-dist/doc/support/make4ht/make4ht-doc.pdf | Bin 98011 -> 101276 bytes Master/texmf-dist/doc/support/make4ht/readme.tex | 50 +++++++- .../scripts/make4ht/extensions/latexmk.lua | 31 ----- .../scripts/make4ht/extensions/latexmk_build.lua | 31 +++++ .../scripts/make4ht/extensions/staticsite.lua | 135 +++++++++++++++++++++ .../scripts/make4ht/filters/make4ht-staticsite.lua | 98 +++++++++++++++ Master/texmf-dist/scripts/make4ht/make4ht | 2 +- .../scripts/make4ht/make4ht-filterlib.lua | 2 +- Master/texmf-dist/scripts/make4ht/make4ht-lib.lua | 6 +- Master/texmf-dist/scripts/make4ht/mkutils.lua | 30 +++-- 12 files changed, 452 insertions(+), 45 deletions(-) delete mode 100644 Master/texmf-dist/scripts/make4ht/extensions/latexmk.lua create mode 100644 Master/texmf-dist/scripts/make4ht/extensions/latexmk_build.lua create mode 100644 Master/texmf-dist/scripts/make4ht/extensions/staticsite.lua create mode 100644 Master/texmf-dist/scripts/make4ht/filters/make4ht-staticsite.lua (limited to 'Master/texmf-dist') diff --git a/Master/texmf-dist/doc/support/make4ht/README b/Master/texmf-dist/doc/support/make4ht/README index 924456189d8..6b9ba2e30cf 100644 --- a/Master/texmf-dist/doc/support/make4ht/README +++ b/Master/texmf-dist/doc/support/make4ht/README @@ -111,7 +111,7 @@ the output format name: Available extensions: -latexmk +latexmk_build : use `Latexmk` for \LaTeX\ compilation. @@ -135,6 +135,10 @@ mathjaxnode convert from MathML code to HTML + CSS or SVG. See [the available settings](#mathjaxsettings). +staticsite + +: build the document in form suitable for static site generators like [Jekyll](https://jekyllrb.com/). + # Build files `make4ht` supports build files. These are `Lua` scripts which can adjust @@ -366,6 +370,10 @@ mathjaxnode convert from MathML code to HTML + CSS or SVG. See [the available settings](#mathjaxsettings). +staticsite + +: create HTML file in format suitable for static site generators such as [Jekyll](https://jekyllrb.com/) + svg-height : some SVG images produced by `dvisvgm` seem to have wrong dimensions. This filter @@ -733,6 +741,43 @@ sentence\_match : Lua pattern used to match a sentence. Default value: `"([^%.^%?^!]*)([%.%?!]?)"`. +### The `staticsite` filter and extension + +site\_root + +: directory where generated files should be copied. + +map + +: table where keys are patterns that match filenames, value contains destination directoryfor matched files, relative to the `site_root` (it is possible to use `..` to swich to parent directory). + +file\_pattern + +: pattern used for filename generation. It is possible to use string templates and format strings for `os.date` function. Default value of `%Y-%m-%d-${input}` creates names in the form of `YYYY-MM-DD-file_name`. + +header + +: table with variables to be set in the YAML header in HTML files. If the table value is a function, it is executed with current parameters and HTML page DOM object as arguments. + +Example: + + + local outdir = os.getenv "blog_root" + + filter_settings "staticsite" { + site_root = outdir, + map = { + [".css$"] = "../css/" + }, + header = { + layout="post", + date = function(parameters, dom) + return os.date("!%Y-%m-%d %T", parameters.time) + end + } + } + + # Configuration file {#configfile} It is possible to globally modify the build settings using the configuration diff --git a/Master/texmf-dist/doc/support/make4ht/changelog.tex b/Master/texmf-dist/doc/support/make4ht/changelog.tex index 26614e59491..5200e488573 100644 --- a/Master/texmf-dist/doc/support/make4ht/changelog.tex +++ b/Master/texmf-dist/doc/support/make4ht/changelog.tex @@ -2,6 +2,71 @@ \section{Changelog}\label{changelog}} \begin{itemize} +\item + 2018/05/03 + + \begin{itemize} + \tightlist + \item + renamed \texttt{latexmk} extension to \texttt{latexmk\_build}, due + to clash in TL + \end{itemize} +\item + 2018/04/18 + + \begin{itemize} + \tightlist + \item + \texttt{staticsite} extension: + + \begin{itemize} + \tightlist + \item + make YAML header configurable + \item + set the \texttt{time} and \texttt{updated} headers + \end{itemize} + \item + don't override existing tables in \texttt{filter\_settings} + \end{itemize} +\item + 2018/04/17 + + \begin{itemize} + \tightlist + \item + done first version of \texttt{staticsite} extension + \end{itemize} +\item + 2018/04/16 + + \begin{itemize} + \tightlist + \item + check for Git repo in the Makefile, don't run Git commands outside + of repo + \end{itemize} +\item + 2018/04/15 + + \begin{itemize} + \tightlist + \item + added \texttt{staticsite} filter + \item + working on \texttt{staticsite} extension + \end{itemize} +\item + 2018/04/13 + + \begin{itemize} + \tightlist + \item + use \texttt{ipairs} instead of \texttt{pairs} to traverse lists of + images and image match functions + \item + load extensions in the correct order + \end{itemize} \item 2018/04/09 diff --git a/Master/texmf-dist/doc/support/make4ht/make4ht-doc.pdf b/Master/texmf-dist/doc/support/make4ht/make4ht-doc.pdf index fa9039865ce..910126cb313 100644 Binary files a/Master/texmf-dist/doc/support/make4ht/make4ht-doc.pdf and b/Master/texmf-dist/doc/support/make4ht/make4ht-doc.pdf differ diff --git a/Master/texmf-dist/doc/support/make4ht/readme.tex b/Master/texmf-dist/doc/support/make4ht/readme.tex index df67a151b87..261e3f2b882 100644 --- a/Master/texmf-dist/doc/support/make4ht/readme.tex +++ b/Master/texmf-dist/doc/support/make4ht/readme.tex @@ -151,7 +151,7 @@ The extensions can be enabled or disabled by appending Available extensions: \begin{description} -\item[latexmk] +\item[latexmk\_build] use \texttt{Latexmk} for \LaTeX~compilation. \item[tidy] clean the \texttt{HTML} files using the \texttt{tidy} command. @@ -165,6 +165,9 @@ Clean the HTML file using DOM filters. It is more powerful than use \href{https://github.com/pkra/mathjax-node-page/}{mathjax-node-page} to convert from MathML code to HTML + CSS or SVG. See \protect\hyperlink{mathjaxsettings}{the available settings}. +\item[staticsite] +build the document in form suitable for static site generators like +\href{https://jekyllrb.com/}{Jekyll}. \end{description} \hypertarget{build-files}{% @@ -389,6 +392,9 @@ internal links, which results in validation error. use \href{https://github.com/pkra/mathjax-node-page/}{mathjax-node-page} to convert from MathML code to HTML + CSS or SVG. See \protect\hyperlink{mathjaxsettings}{the available settings}. +\item[staticsite] +create HTML file in format suitable for static site generators such as +\href{https://jekyllrb.com/}{Jekyll} \item[svg-height] some SVG images produced by \texttt{dvisvgm} seem to have wrong dimensions. This filter tries to set the correct image size. @@ -815,6 +821,48 @@ Lua pattern used to match a sentence. Default value: \texttt{"({[}\^{}\%.\^{}\%?\^{}!{]}*)({[}\%.\%?!{]}?)"}. \end{description} +\hypertarget{the-staticsite-filter-and-extension}{% +\subsubsection{\texorpdfstring{The \texttt{staticsite} filter and +extension}{The staticsite filter and extension}}\label{the-staticsite-filter-and-extension}} + +\begin{description} +\item[site\_root] +directory where generated files should be copied. +\item[map] +table where keys are patterns that match filenames, value contains +destination directoryfor matched files, relative to the +\texttt{site\_root} (it is possible to use \texttt{..} to swich to +parent directory). +\item[file\_pattern] +pattern used for filename generation. It is possible to use string +templates and format strings for \texttt{os.date} function. Default +value of \texttt{\%Y-\%m-\%d-\$\{input\}} creates names in the form of +\texttt{YYYY-MM-DD-file\_name}. +\item[header] +table with variables to be set in the YAML header in HTML files. If the +table value is a function, it is executed with current parameters and +HTML page DOM object as arguments. +\end{description} + +Example: + +\begin{verbatim} +local outdir = os.getenv "blog_root" + +filter_settings "staticsite" { + site_root = outdir, + map = { + [".css$"] = "../css/" + }, + header = { + layout="post", + date = function(parameters, dom) + return os.date("!%Y-%m-%d %T", parameters.time) + end + } +} +\end{verbatim} + \hypertarget{configfile}{% \section{Configuration file}\label{configfile}} diff --git a/Master/texmf-dist/scripts/make4ht/extensions/latexmk.lua b/Master/texmf-dist/scripts/make4ht/extensions/latexmk.lua deleted file mode 100644 index bf4dcdde5a4..00000000000 --- a/Master/texmf-dist/scripts/make4ht/extensions/latexmk.lua +++ /dev/null @@ -1,31 +0,0 @@ --- use Latexmk in first LaTeX call --- only in the first call, because we don't need to execute biber, etc. in the subsequent --- LaTeX calls, these are only for resolving the cross-references -local M = {} -function M.modify_build(make) - local used = false - local first - local build_seq = make.build_seq - -- find first htlatex call in the build sequence - for pos,v in ipairs(build_seq) do - if v.name == "htlatex" and not first then - first = pos - end - end - -- if htlatex was found - if first then - -- add dummy latexmk call to the build sequence - make:latexmk {} - -- replace name, command and type in the first htlatex - -- call with values from the dummy latexmk call - local replaced = build_seq[first] - local latexmk = build_seq[#build_seq] - replaced.name = latexmk.name - replaced.command = latexmk.command - replaced.type = latexmk.type - -- remove the dummy latexmk - table.remove(build_seq) - end - return make -end -return M diff --git a/Master/texmf-dist/scripts/make4ht/extensions/latexmk_build.lua b/Master/texmf-dist/scripts/make4ht/extensions/latexmk_build.lua new file mode 100644 index 00000000000..bf4dcdde5a4 --- /dev/null +++ b/Master/texmf-dist/scripts/make4ht/extensions/latexmk_build.lua @@ -0,0 +1,31 @@ +-- use Latexmk in first LaTeX call +-- only in the first call, because we don't need to execute biber, etc. in the subsequent +-- LaTeX calls, these are only for resolving the cross-references +local M = {} +function M.modify_build(make) + local used = false + local first + local build_seq = make.build_seq + -- find first htlatex call in the build sequence + for pos,v in ipairs(build_seq) do + if v.name == "htlatex" and not first then + first = pos + end + end + -- if htlatex was found + if first then + -- add dummy latexmk call to the build sequence + make:latexmk {} + -- replace name, command and type in the first htlatex + -- call with values from the dummy latexmk call + local replaced = build_seq[first] + local latexmk = build_seq[#build_seq] + replaced.name = latexmk.name + replaced.command = latexmk.command + replaced.type = latexmk.type + -- remove the dummy latexmk + table.remove(build_seq) + end + return make +end +return M diff --git a/Master/texmf-dist/scripts/make4ht/extensions/staticsite.lua b/Master/texmf-dist/scripts/make4ht/extensions/staticsite.lua new file mode 100644 index 00000000000..fcb6c09e17d --- /dev/null +++ b/Master/texmf-dist/scripts/make4ht/extensions/staticsite.lua @@ -0,0 +1,135 @@ +local M = {} +local filter = require "make4ht-filter" +local mkutils = require "mkutils" + +-- get the published file name +local function get_slug(settings) + local published_name = mkutils.remove_extension(settings.tex_file) .. ".published" + local config = get_filter_settings "staticsite" + local file_pattern = config.file_pattern or "%Y-%m-%d-${input}" + local time = os.time() + + -- we must save the published date, so the subsequent compilations at different days + -- use the same name + if mkutils.file_exists(published_name) then + local f = io.open(published_name, "r") + local readtime = f:read("*line") + time = tonumber(readtime) + print("Already pubslished", slug) + f:close() + else + -- escape + -- slug must contain the unescaped input name + local f = io.open(published_name, "w") + f:write(time) + f:close() + end + -- set the updated and publishing times + local updated + -- the updated time will be set only when it is more than one day from the published time + local newtime = os.time() + if (newtime - time) > (24 * 3600) then updated = newtime end + filter_settings "staticsite" { + header = { + time = time, + updated = updated + } + } + + -- make the output file name in the format YYYY-MM-DD-old-filename.html + local slug = os.date(file_pattern,time) % settings + return slug +end + + +-- it is necessary to set correct -jobname in latex_par parameters field +-- in order to the get correct HTML file name +local function update_jobname(slug, latex_par) + local latex_par = latex_par or "" + if latex_par:match("%-jobname") then + local firstchar=latex_par:match("%-jobname=.") + local replace_pattern="%-jobname=[^%s]+" + if firstchar == "'" or firstchar=='"' then + replace_pattern = "%-jobname=".. firstchar .."[^%"..firstchar.."]+" + end + + return latex_par:gsub(replace_pattern, "-jobname=".. slug) + else + return latex_par .. "-jobname="..slug + end +end + +-- execute the function passed as parameter only once, when the file matching +-- starts +local function insert_filter(make, pattern, fn) + local insert_executed = false + table.insert(make.matches, 1, { + pattern=pattern, + params = {}, + command = function() + if not insert_executed then + fn() + end + insert_executed = true + end + }) +end + + +local function copy_files(filename, par) + local function prepare_path(dir, subdir) + local path = dir .. "/" .. subdir .. "/" .. filename + return path:gsub("//", "/") + end + -- get extension settings + local site_settings = get_filter_settings "staticsite" + local site_root = site_settings.site_root + local map = site_settings.map or {} + -- default path without subdir, will be used if the file is not matched + -- by any pattern in the map + local path = prepare_path(site_root, "") + for pattern, destination in pairs(map) do + if filename:match(pattern) then + path = prepare_path(site_root, destination) + break + end + end + -- it is possible to use string extrapolation in path, for example for slug + mkutils.copy(filename, path % par) +end + +function M.modify_build(make) + -- it is necessary to insert the filters for YAML header and file copying as last matches + -- we use an bogus match which will be executed only once as the very first one to insert + -- the filters + -- I should make filter from this + local process = filter { + "staticsite" + } + local settings = make.params + -- get the published file name + local slug = get_slug(settings) + for _, cmd in ipairs(make.build_seq) do + -- all commands must use the published file name + cmd.params.input = slug + cmd.params.latex_par = update_jobname(slug, cmd.params.latex_par) + end + + local quotepattern = '(['..("%^$().[]*+-?"):gsub("(.)", "%%%1")..'])' + local mainfile = string.gsub(slug, quotepattern, "%%%1") + + -- run the following code once in the first match on the first file + insert_filter(make, ".*", function() + -- for _, match in ipairs(make.matches) do + -- match.params.outdir = outdir + -- print(match.pattern, match.params.outdir) + -- end + -- make the YAML header only for the main HTML file + make:match(mainfile .. ".html", process) + make:match(".*", copy_files, {slug=slug}) + end) + + return make +end + +return M diff --git a/Master/texmf-dist/scripts/make4ht/filters/make4ht-staticsite.lua b/Master/texmf-dist/scripts/make4ht/filters/make4ht-staticsite.lua new file mode 100644 index 00000000000..6378f0440aa --- /dev/null +++ b/Master/texmf-dist/scripts/make4ht/filters/make4ht-staticsite.lua @@ -0,0 +1,98 @@ +local domobj = require "luaxml-domobject" +-- save the header settings in YAML format +local function make_yaml(tbl, level) + local t = {} + local level = level or 0 + local indent = string.rep(" ", level) + -- indentation for multilen strings + local str_indent = string.rep(" ", level + 1) + for k,v in pairs(tbl) do + if type(v)=="string" then + -- detect multiline strings + if v:match("\n") then + table.insert(t, string.format(indent .. "%s: |", k)) + table.insert(t, str_indent .. (v:gsub("\n", "\n".. str_indent))) + else + v = v:gsub("'", "''") + table.insert(t, string.format(indent .. "%s: '%s'", k,v)) + end + elseif type(v) == "table" then + table.insert(t,string.format(indent .. "%s:", k)) + -- we need to differently process array and hash table + -- we don't support mixing types + if #v > 0 then + for x,y in ipairs(v) do + if type(y) == "string" then + -- each string can be printed on it's own line + table.insert(t, indent .. string.format("- '%s'", y)) + else + -- subtables need to be indented + -- table.insert(t, indent .. "-") + local subtable = make_yaml(y, level + 1) + -- we must insert dash at a correct place + local insert_dash = subtable:gsub("^(%s*)%s%s", "%1- ") + table.insert(t, insert_dash) + end + end + else + -- print indented table + table.insert(t, make_yaml(v,level + 1)) + end + else + -- convert numbers and other values to string + table.insert(t, string.format(indent .. "%s: %s", k,tostring(v))) + end + + end + return table.concat(t, "\n") +end + +local function update_properties(properties, dom) + -- enable properties update from the config or build file + local settings = get_filter_settings "staticsite" or {} + local header = settings.header or {} + for field, rule in pairs(header) do + -- it is possible to pass function as a rule, it will be executed with properties as a parameter + if type(rule) == "function" then + properties[field] = rule(properties, dom) + else + -- otherwise set properties + properties[field] = rule + end + end + return properties +end + +local function get_header(tbl) + local yaml = make_yaml(tbl) + return "---\n".. yaml.. "\n---\n" +end + +return function(s,par) + print(os.getenv "blog_home") + local dom = domobj.parse(s) + local properties = {} + local head = dom:query_selector("head")[1] + properties.title = head:query_selector("title")[1]:get_text() + local styles = {} + for _, link in ipairs(head:query_selector("link")) do + local typ = link:get_attribute("type") + if typ == "text/css" then + table.insert(styles, link:get_attribute("href")) + end + end + properties.styles = styles + local metas = {} + for _, meta in ipairs(head:query_selector("meta")) do + print(meta:serialize()) + table.insert(metas, {charset= meta:get_attribute("charset"), content = meta:get_attribute("content"), property = meta:get_attribute("property"), name = meta:get_attribute("name")}) + end + properties.meta = metas + properties = update_properties(properties, dom) + + + local body = dom:query_selector("body")[1] + print(get_header(properties)) + -- return s + return get_header(properties) .. body:serialize():gsub("", ""):gsub("", "") +end diff --git a/Master/texmf-dist/scripts/make4ht/make4ht b/Master/texmf-dist/scripts/make4ht/make4ht index 0da3c30549c..10b277d2a73 100755 --- a/Master/texmf-dist/scripts/make4ht/make4ht +++ b/Master/texmf-dist/scripts/make4ht/make4ht @@ -27,7 +27,7 @@ make4ht [options] filename ["tex4ht.sty op." "tex4ht op." "t4ht op" "latex op"] -- set version number. the template should be replaced by the -- actual version number by the build script -local version = "v0.2" +local version = "0.2a" mkparams.version_number = version local args = mkparams.get_args() diff --git a/Master/texmf-dist/scripts/make4ht/make4ht-filterlib.lua b/Master/texmf-dist/scripts/make4ht/make4ht-filterlib.lua index 33cb3530a85..3c2e63cf69d 100755 --- a/Master/texmf-dist/scripts/make4ht/make4ht-filterlib.lua +++ b/Master/texmf-dist/scripts/make4ht/make4ht-filterlib.lua @@ -6,7 +6,7 @@ function M.load_filters(filters, load_filter) if type(filters) == "string" then table.insert(sequence,load_filter(filters)) elseif type(filters) == "table" then - for _,n in pairs(filters) do + for _,n in ipairs(filters) do if type(n) == "string" then table.insert(sequence,load_filter(n)) elseif type(n) == "function" then diff --git a/Master/texmf-dist/scripts/make4ht/make4ht-lib.lua b/Master/texmf-dist/scripts/make4ht/make4ht-lib.lua index 918343f20ca..2ada398f869 100755 --- a/Master/texmf-dist/scripts/make4ht/make4ht-lib.lua +++ b/Master/texmf-dist/scripts/make4ht/make4ht-lib.lua @@ -81,9 +81,9 @@ Make.image_convert = function(self, images) end image_patterns[i].params = p end - for _,i in pairs(images) do + for _,i in ipairs(images) do local output = i.output - for _, x in pairs(image_patterns) do + for _, x in ipairs(image_patterns) do local pattern = x.pattern if output:match(pattern) then local command = x.command @@ -107,7 +107,7 @@ end Make.file_matches = function(self, files) local statuses = {} -- First make params for all matchers - for k,v in pairs(self.matches) do + for k,v in ipairs(self.matches) do local v = self.matches[k].params or {} local p = self.params or {} for i,j in pairs(p) do diff --git a/Master/texmf-dist/scripts/make4ht/mkutils.lua b/Master/texmf-dist/scripts/make4ht/mkutils.lua index 88b456fe9da..7dab423bc9b 100755 --- a/Master/texmf-dist/scripts/make4ht/mkutils.lua +++ b/Master/texmf-dist/scripts/make4ht/mkutils.lua @@ -22,6 +22,19 @@ getmetatable("").__add = addProperty --print( "${name} is ${value}" % {name = "foo", value = "bar"} ) -- Outputs "foo is bar" + +-- merge two tables recursively +function merge(t1, t2) + for k, v in pairs(t2) do + if (type(v) == "table") and (type(t1[k] or false) == "table") then + merge(t1[k], t2[k]) + else + t1[k] = v + end + end + return t1 +end + function string:split(sep) local sep, fields = sep or ":", {} local pattern = string.format("([^%s]+)", sep) @@ -157,7 +170,7 @@ function mkdirectories(dirs) if type(dirs) ~="table" then return false, "mkdirectories: dirs is not table" end - for _,d in pairs(dirs) do + for _,d in ipairs(dirs) do local stat,msg = lfs.mkdir(d) if not stat then return false, "makedirectories error: "..msg end lfs.chdir(d) @@ -269,10 +282,7 @@ function env.filter_settings(name) local filters = settings.filter or {} local filter_options = filters[name] or {} return function(par) - for k,v in pairs(par) do - filter_options[k] = v - end - filters[name] = filter_options + filters[name] = merge(filter_options, par) settings.filter = filters end end @@ -458,13 +468,19 @@ end function load_extensions(extensions, format) local module_names = {} local extension_table = {} + local extension_sequence = {} -- process the extension table. it contains type field, which can enable or -- diable the extension for _, v in ipairs(extensions) do local enable = v.type == "+" and true or nil - module_names[v.name] = enable + -- don't load extensions multiple times + if not module_names[v.name] then + module_names[v.name] = enable + -- load extenisons in a correct order + table.insert(extension_sequence, v.name) + end end - for name, _ in pairs(module_names) do + for _, name in ipairs(extension_sequence) do local extension = load_extension(name,format) if extension then table.insert(extension_table, extension) -- cgit v1.2.3