From 9dec02fb03afe114f68ce75e31f6c5f8ee0adcf8 Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Fri, 24 Aug 2018 22:19:06 +0000 Subject: make4ht (24aug18) git-svn-id: svn://tug.org/texlive/trunk@48477 c570f23f-e606-0410-a88d-b1316a301751 --- Master/texmf-dist/doc/support/make4ht/README | 27 +++ .../texmf-dist/doc/support/make4ht/changelog.tex | 129 +++++++++++ .../texmf-dist/doc/support/make4ht/make4ht-doc.pdf | Bin 101505 -> 106622 bytes Master/texmf-dist/doc/support/make4ht/readme.tex | 36 +++ .../scripts/make4ht/extensions/dvisvgm_hashes.lua | 254 ++++++++++++++++++++ .../filters/make4ht-entities-to-unicode.lua | 13 ++ Master/texmf-dist/scripts/make4ht/formats/odt.lua | 257 ++++++++++++++++++++- Master/texmf-dist/scripts/make4ht/make4ht | 7 +- .../scripts/make4ht/make4ht-dvireader.lua | 182 +++++++++++++++ Master/texmf-dist/scripts/make4ht/mkparams.lua | 2 + Master/texmf-dist/scripts/make4ht/mkutils.lua | 70 ++++-- 11 files changed, 953 insertions(+), 24 deletions(-) create mode 100644 Master/texmf-dist/scripts/make4ht/extensions/dvisvgm_hashes.lua create mode 100644 Master/texmf-dist/scripts/make4ht/filters/make4ht-entities-to-unicode.lua create mode 100755 Master/texmf-dist/scripts/make4ht/make4ht-dvireader.lua (limited to 'Master') diff --git a/Master/texmf-dist/doc/support/make4ht/README b/Master/texmf-dist/doc/support/make4ht/README index 6b9ba2e30cf..c878fdb1db4 100644 --- a/Master/texmf-dist/doc/support/make4ht/README +++ b/Master/texmf-dist/doc/support/make4ht/README @@ -119,6 +119,11 @@ tidy : clean the `HTML` files using the `tidy` command. +dvisvgm_hashes + +: efficient generation of SVG pictures using Dvisvgm. It can utilize +multiple processor cores and generates only changed images. + common\_filters : clean the output HTML files using filters. @@ -777,6 +782,20 @@ Example: } } +### The `dvisvgm_hashes` extension + +options + +: command line options for Dvisvgm. The default value is `-n --exact -c 1.15,1.15`. + +cpu_cnt + +: number of processor cores used for conversion. The extension tries to detect the available cores automatically by default. + +parallel_size + +: number of pages used in each Dvisvgm call. The extension detects changed pages in the DVI file and construct multiple calls to Dvisvgm with only changed pages. + # Configuration file {#configfile} @@ -883,6 +902,14 @@ Another option is to add a space before `tex4ht` options: The former way is preferable, though. +## Filenames containing spaces + +`tex4ht` cannot handle filenames containing spaces. `make4ht` thus replaces spaces in input file names with underscores, so generated XML files use underscores instead of spaces as well. + +## Filenames containing non-ASCII characters + +The `odt` output doesn't support accented filenames, it is best to stick to ASCII characters in filenames. + # License Permission is granted to copy, distribute and/or modify this software diff --git a/Master/texmf-dist/doc/support/make4ht/changelog.tex b/Master/texmf-dist/doc/support/make4ht/changelog.tex index ac9e2aa6e33..ec2c521110f 100644 --- a/Master/texmf-dist/doc/support/make4ht/changelog.tex +++ b/Master/texmf-dist/doc/support/make4ht/changelog.tex @@ -2,6 +2,135 @@ \section{Changelog}\label{changelog}} \begin{itemize} +\item + 2018/08/23 + + \begin{itemize} + \tightlist + \item + released version 0.2c + \end{itemize} +\item + 2018/08/21 + + \begin{itemize} + \tightlist + \item + added processor core detection on Windows + \item + make processor number configurable + \item + updated the documentation. + \end{itemize} +\item + 2018/08/20 + + \begin{itemize} + \tightlist + \item + added \texttt{dvisvgm\_hashes} extension + \end{itemize} +\item + 2018/07/03 + + \begin{itemize} + \tightlist + \item + create the \texttt{mimetype} file to achieve the ODT file validity + \end{itemize} +\item + 2018/07/02 + + \begin{itemize} + \tightlist + \item + disabled conversion of XML entities for \&, \textless{} and + \textgreater{} characters back to Unicode, because it breaks XML + validity + \end{itemize} +\item + 2018/06/27 + + \begin{itemize} + \tightlist + \item + fixed root dir detection + \end{itemize} +\item + 2018/06/26 + + \begin{itemize} + \tightlist + \item + added code for detection of TeX distribution root for Miktex and TL + \end{itemize} +\item + 2018/06/25 + + \begin{itemize} + \tightlist + \item + moved call to \texttt{xtpipes} from \texttt{t4ht} to the + \texttt{ODT} format drives. This should fix issues with path + expansion in \texttt{tex4ht.env} in TeX distributions. + \end{itemize} +\item + 2018/06/22 + + \begin{itemize} + \tightlist + \item + added \texttt{mkutils.find\_zip} function. It detects \texttt{zip} + or \texttt{miktex-zip} executables + \end{itemize} +\item + 2018/06/19 + + \begin{itemize} + \tightlist + \item + added new filter: \texttt{entities-to-unicode}. It converts XML + entites for Unicode characters back to Unicode. + \item + execute \texttt{entities-to-unicode} filter on text and math files + in the ODT output. + \end{itemize} +\item + 2018/06/12 + + \begin{itemize} + \tightlist + \item + added support for direct \texttt{ODT} file packing + \end{itemize} +\item + 2018/06/11 + + \begin{itemize} + \tightlist + \item + new function available for formats, \texttt{format.modify\_build} + \item + function \texttt{mkutils.delete\_dir} for directory removal + \item + function \texttt{mkutils.mv} for file moving + \item + started on packing of the \texttt{ODT} files directly by the format, + instead of \texttt{t4ht} + \end{itemize} +\item + 2018/06/08 + + \begin{itemize} + \tightlist + \item + added support for filenames containing spaces + \item + added support for filenames containing non-ascii characters + \item + don't require sudo for the installation, let the user to install + symbolic links to \texttt{\$PATH} + \end{itemize} \item 2018/05/03 diff --git a/Master/texmf-dist/doc/support/make4ht/make4ht-doc.pdf b/Master/texmf-dist/doc/support/make4ht/make4ht-doc.pdf index bc684e17682..d25efb855d9 100644 Binary files a/Master/texmf-dist/doc/support/make4ht/make4ht-doc.pdf and b/Master/texmf-dist/doc/support/make4ht/make4ht-doc.pdf differ diff --git a/Master/texmf-dist/doc/support/make4ht/readme.tex b/Master/texmf-dist/doc/support/make4ht/readme.tex index 261e3f2b882..20bf33575d1 100644 --- a/Master/texmf-dist/doc/support/make4ht/readme.tex +++ b/Master/texmf-dist/doc/support/make4ht/readme.tex @@ -155,6 +155,9 @@ Available extensions: use \texttt{Latexmk} for \LaTeX~compilation. \item[tidy] clean the \texttt{HTML} files using the \texttt{tidy} command. +\item[dvisvgm\_hashes] +efficient generation of SVG pictures using Dvisvgm. It can utilize +multiple processor cores and generates only changed images. \item[common\_filters] clean the output HTML files using filters. \item[common\_domfilters] @@ -863,6 +866,23 @@ filter_settings "staticsite" { } \end{verbatim} +\hypertarget{the-dvisvgm_hashes-extension}{% +\subsubsection{\texorpdfstring{The \texttt{dvisvgm\_hashes} +extension}{The dvisvgm\_hashes extension}}\label{the-dvisvgm_hashes-extension}} + +\begin{description} +\item[options] +command line options for Dvisvgm. The default value is +\texttt{-n\ -\/-exact\ -c\ 1.15,1.15}. +\item[cpu\_cnt] +number of processor cores used for conversion. The extension tries to +detect the available cores automatically by default. +\item[parallel\_size] +number of pages used in each Dvisvgm call. The extension detects changed +pages in the DVI file and construct multiple calls to Dvisvgm with only +changed pages. +\end{description} + \hypertarget{configfile}{% \section{Configuration file}\label{configfile}} @@ -999,6 +1019,22 @@ make4ht hello.tex "customcfg,charset=utf-8" " -cunihtf -utf8" -d foo The former way is preferable, though. +\hypertarget{filenames-containing-spaces}{% +\subsection{Filenames containing +spaces}\label{filenames-containing-spaces}} + +\texttt{tex4ht} cannot handle filenames containing spaces. +\texttt{make4ht} thus replaces spaces in input file names with +underscores, so generated XML files use underscores instead of spaces as +well. + +\hypertarget{filenames-containing-non-ascii-characters}{% +\subsection{Filenames containing non-ASCII +characters}\label{filenames-containing-non-ascii-characters}} + +The \texttt{odt} output doesn't support accented filenames, it is best +to stick to ASCII characters in filenames. + \hypertarget{license}{% \section{License}\label{license}} diff --git a/Master/texmf-dist/scripts/make4ht/extensions/dvisvgm_hashes.lua b/Master/texmf-dist/scripts/make4ht/extensions/dvisvgm_hashes.lua new file mode 100644 index 00000000000..ac25e10cf5d --- /dev/null +++ b/Master/texmf-dist/scripts/make4ht/extensions/dvisvgm_hashes.lua @@ -0,0 +1,254 @@ +local dvireader = require "make4ht-dvireader" +local mkutils = require "mkutils" +local filter = require "make4ht-filter" + + +local M = {} +-- mapping between tex4ht image names and hashed image names +local output_map = {} +local dvisvgm_options = "-n --exact -c 1.15,1.15" +local parallel_size = 64 +-- local parallel_size = 3 + +local function make_hashed_name(base, hash) + return base .. "-" ..hash..".svg" +end + +-- detect the number of available processors +local cpu_cnt = 3 -- set a reasonable default for non-Linux systems + +if os.name == 'linux' then + cpu_cnt = 0 + local cpuinfo=assert(io.open('/proc/cpuinfo', 'r')) + for line in cpuinfo:lines() do + if line:match('^processor') then + cpu_cnt = cpu_cnt + 1 + end + end + -- set default number of threds if no CPU core have been found + if cpu_cnt == 0 then cpu_cnt = 1 end + cpuinfo:close() +elseif os.name == 'cygwin' or os.type == 'windows' then + -- windows has NUMBER_OF_PROCESSORS environmental value + local nop = os.getenv('NUMBER_OF_PROCESSORS') + if tonumber(nop) then + cpu_cnt = nop + end +end + + + +-- process output of dvisvgm and find output page numbers and corresponding files +local function get_generated_pages(output, pages) + local pages = pages or {} + local pos = 1 + local pos, finish, page = string.find(output, "processing page (%d+)", pos) + while(pos) do + pos, finish, file = string.find(output, "output written to ([^\n]+)", finish) + pages[tonumber(page)] = file + pos, finish, page = string.find(output, "processing page (%d+)", finish) + end + return pages +end + +local function make_ranges(pages) + local newpages = {} + local start, stop + for i=1,#pages do + local current = pages[i] + local next_el = pages[i+1] or current + 100 -- just select a big number + local diff = next_el - current + if diff == 1 then + if not start then start = current end + else + local element + if start then + element = start .. "-" .. current + else + element = current + end + newpages[#newpages+1] = element + start = nil + end + end + return newpages +end + +local function read_log(dvisvgmlog) + local f = io.open(dvisvgmlog, "r") + if not f then return nil, "Cannot read dvisvgm log" end + local output = f:read("*all") + f:close() + return output +end + +-- test the existence of GNU Make, which can execute tasks in parallel +local function test_make() + local make = io.popen("make -v", "r") + if not make then return false end + local content = make:read("*all") + make:close() + return true +end + +local function save_file(filename, text) + local f = io.open(filename, "w") + f:write(text) + f:close() +end + + +local function make_makefile_command(idvfile, page_sequences) + local logs = {} + local all = {} -- list of targets in the "all:" makefile target + local targets = {} + local basename = idvfile:gsub(".idv$", "") + local makefilename = basename .. "-images" .. ".mk" + -- build make targets + for i, ranges in ipairs(page_sequences) do + local target = basename .. "-" .. i + local logfile = target .. ".dlog" + logs[#logs + 1] = logfile + all[#all+1] = target + local chunk = target .. ":\n\tdvisvgm -v4 " .. dvisvgm_options .. " -p " .. ranges .. " " .. idvfile .. " 2> " .. logfile .. "\n" + targets[#targets + 1] = chunk + end + -- construct makefile and save it + local makefile = "all: " .. table.concat(all, " ") .. "\n\n" .. table.concat(targets, "\n") + save_file(makefilename, makefile) + local command = "make -j" .. cpu_cnt .." -f " .. makefilename + return command, logs +end + +local function prepare_command(idvfile, pages) + local logs = {} + if #pages > parallel_size and test_make() then + local page_sequences = {} + for i=1, #pages, parallel_size do + local current_pages = {} + for x = i, i+parallel_size -1 do + current_pages[#current_pages + 1] = pages[x] + end + table.insert(page_sequences,table.concat(make_ranges(current_pages), ",")) + end + return make_makefile_command(idvfile, page_sequences) + end + -- else + local pagesequence = table.concat(make_ranges(pages), ",") + -- the stderr from dvisvgm must be redirected and postprocessed + local dvisvgmlog = idvfile:gsub("idv$", "dlog") + -- local dvisvgm = io.popen("dvisvgm -v4 -n --exact -c 1.15,1.15 -p " .. pagesequence .. " " .. idvfile, "r") + local command = "dvisvgm -v4 " .. dvisvgm_options .. " -p " .. pagesequence .. " " .. idvfile .. " 2> " .. dvisvgmlog + return command, {dvisvgmlog} + -- end +end + +local function execute_dvisvgm(idvfile, pages) + if #pages < 1 then return nil, "No pages to convert" end + local command, logs = prepare_command(idvfile, pages) + print(command) + os.execute(command) + local generated_pages = {} + for _, dvisvgmlog in ipairs(logs) do + local output = read_log(dvisvgmlog) + generated_pages = get_generated_pages(output, generated_pages) + end + return generated_pages +end + +local function get_dvi_pages(arg) + -- list of pages to convert in this run + local to_convert = {} + local idv_file = arg.input .. ".idv" + -- set extension options + local extoptions = mkutils.get_filter_settings "dvisvgm_hashes" or {} + dvisvgm_options = arg.options or extoptions.options or dvisvgm_options + parallel_size = arg.parallel_size or extoptions.parallel_size or parallel_size + cpu_cnt = arg.cpu_cnt or extoptions.cpu_cnt or cpu_cnt + local f = io.open(idv_file, "r") + if not f then return nil, "Cannot open idv file: " .. idv_file end + local content = f:read("*all") + f:close() + local dvi_pages = dvireader.get_pages(content) + -- we must find page numbers and output name sfor the generated images + local lg = mkutils.parse_lg(arg.input ..".lg") + for _, name in ipairs(lg.images) do + local page = tonumber(name.page) + local hash = dvi_pages[page] + local tex4ht_name = name.output + local output_name = make_hashed_name(arg.input, hash) + output_map[tex4ht_name] = output_name + if not mkutils.file_exists(output_name) then + print(output_name) + to_convert[#to_convert+1] = page + end + end + local generated_files, msg = execute_dvisvgm(idv_file, to_convert) + if not generated_files then + return nil, msg + end + + -- rename the generated files to the hashed filenames + for page, file in pairs(generated_files) do + os.rename(file, make_hashed_name(arg.input, dvi_pages[page])) + end + +end + +function M.test(format) + -- ODT format doesn't support SVG + if format == "odt" then return false end + return true +end + +function M.modify_build(make) + -- this must be used in the .mk4 file as + -- Make:dvisvgm_hashes {} + make:add("dvisvgm_hashes", function(arg) + get_dvi_pages(arg) + end, + { + }) + + -- insert dvisvgm_hashes command at the end of the build sequence -- it needs to be called after t4ht + make:dvisvgm_hashes {} + + -- replace original image names with hashed names + local executed = false + make:match(".*", function(arg) + if not executed then + executed = true + local lgfiles = make.lgfile.files + for i, filename in ipairs(lgfiles) do + local replace = output_map[filename] + if replace then + lgfiles[i] = replace + end + end + end + end) + + -- fix src attributes + local process = filter { + function(str) + return str:gsub('src="([^"]+)', function(filename) + local newname = output_map[filename] or filename + print("newname", newname) + return 'src="'.. newname + end) + end + } + + make:match("htm.?$", process) + + -- disable the image processing + for _,v in ipairs(make.build_seq) do + if v.name == "t4ht" then + v.params.t4ht_par = v.params.t4ht_par .. " -p" + end + end + make:image(".", function() return "" end) + return make +end + +return M diff --git a/Master/texmf-dist/scripts/make4ht/filters/make4ht-entities-to-unicode.lua b/Master/texmf-dist/scripts/make4ht/filters/make4ht-entities-to-unicode.lua new file mode 100644 index 00000000000..f93e776717f --- /dev/null +++ b/Master/texmf-dist/scripts/make4ht/filters/make4ht-entities-to-unicode.lua @@ -0,0 +1,13 @@ +-- convert Unicode characters encoded as XML entities back to Unicode + +-- list of disabled characters +local disabled = { ["&"] = "&", ["<"] = "<", [">"] = ">" } +local utfchar = unicode.utf8.char +return function(content) + return content:gsub("%&%#x([A-Fa-f0-9]+);", function(entity) + -- convert hexadecimal entity to Unicode + local newchar = utfchar(tonumber(entity, 16)) + -- we don't want to break XML validity with forbidden characters + return disabled[newchar] or newchar + end) +end diff --git a/Master/texmf-dist/scripts/make4ht/formats/odt.lua b/Master/texmf-dist/scripts/make4ht/formats/odt.lua index 66105166f32..d5a6b0c9ba5 100644 --- a/Master/texmf-dist/scripts/make4ht/formats/odt.lua +++ b/Master/texmf-dist/scripts/make4ht/formats/odt.lua @@ -1,11 +1,266 @@ local M = {} local mkutils = require "mkutils" +local lfs = require "lfs" +local os = require "os" +local kpse = require "kpse" +local filter = require "make4ht-filter" + function M.prepare_parameters(settings, extensions) settings.tex4ht_sty_par = settings.tex4ht_sty_par ..",ooffice" settings.tex4ht_par = settings.tex4ht_par .. " ooffice/! -cmozhtf" - settings.t4ht_par = settings.t4ht_par .. " -cooxtpipes -coo " + -- settings.t4ht_par = settings.t4ht_par .. " -cooxtpipes -coo " + -- settings.t4ht_par = settings.t4ht_par .. " -cooxtpipes " settings = mkutils.extensions_prepare_parameters(extensions, settings) return settings end + +-- object for working with the ODT file +local Odtfile = {} +Odtfile.__index = Odtfile + +Odtfile.new = function(archivename) + local self = setmetatable({}, Odtfile) + -- create temporary directory + local tmpname = os.tmpname() + tmpname = tmpname:match("([a-zA-Z0-9_%-]+)$") + local status, msg = lfs.mkdir(tmpname) + if not status then return nil, msg end + -- make picture dir + lfs.mkdir(tmpname .. "/Pictures") + self.archivelocation = tmpname + self.name = archivename + return self +end + +function Odtfile:copy(src, dest) + mkutils.cp(src, self.archivelocation .. "/" .. dest) +end + +function Odtfile:move(src, dest) + mkutils.mv(src, self.archivelocation .. "/" .. dest) +end + +function Odtfile:create_dir(dir) + local currentdir = lfs.currentdir() + lfs.chdir(self.archivelocation) + lfs.mkdir(dir) + lfs.chdir(currentdir) +end + +function Odtfile:make_mimetype() + self.mimetypename = "mimetype" + local m = io.open(self.mimetypename, "w") + m:write("application/vnd.oasis.opendocument.text") + m:close() +end + +function Odtfile:remove_mimetype() + os.remove(self.mimetypename) +end + + +function Odtfile:pack() + local currentdir = lfs.currentdir() + local zip_command = mkutils.find_zip() + lfs.chdir(self.archivelocation) + -- make temporary mime type file + self:make_mimetype() + os.execute(zip_command .. " -q0X " .. self.name .. " " .. self.mimetypename) + -- remove it, so the next command doesn't overwrite it + self:remove_mimetype() + os.execute(zip_command .." -r " .. self.name .. " *") + lfs.chdir(currentdir) + mkutils.cp(self.archivelocation .. "/" .. self.name, self.name) + mkutils.delete_dir(self.archivelocation) +end + +-- find if tex4ht.jar exists in a path +local function find_tex4ht_jar(path) + local jar_file = path .. "/tex4ht/bin/tex4ht.jar" + return mkutils.file_exists(jar_file) +end + +-- return value of TEXMFROOT variable if it exists and if tex4ht.jar can be located inside +local function get_texmfroot() + -- user can set TEXMFROOT environmental variable as the last resort + local root_directories = {kpse.var_value("TEXMFROOT"), kpse.var_value("TEXMFDIST"), os.getenv("TEXMFROOT")} + for _, root in ipairs(root_directories) do + if root then + if find_tex4ht_jar(root) then return root end + -- TeX live locates files in texmf-dist subdirectory, but Miktex doesn't + local path = root .. "/texmf-dist" + if find_tex4ht_jar(path) then return path end + end + end +end + +-- Miktex doesn't seem to set TeX variables such as TEXMFROOT +-- we will try to find the TeX root using trick with locating package in TeX root +-- there is a danger that this file is located in TEXMFHOME, the location will fail then +local function find_texmfroot() + local tex4ht_path = kpse.find_file("tex4ht.sty") + if tex4ht_path then + local path = tex4ht_path:gsub("/tex/generic/tex4ht/tex4ht.sty$","") + if find_tex4ht_jar(path) then return path end + end + return nil +end + +-- call xtpipes from Lua +local function call_xtpipes(make) + -- we must find root of the TeX distribution + local selfautoparent = get_texmfroot() or find_texmfroot() + if selfautoparent then + -- make pattern using TeX distro path + local pattern = string.format("java -classpath %s/tex4ht/bin/tex4ht.jar xtpipes -i %s/tex4ht/xtpipes/ -o ${outputfile} ${filename}", selfautoparent, selfautoparent) + -- call xtpipes on a temporary file + local matchfunction = function(filename) + -- move the matched file to a temporary file, xtpipes will write it back to the original file + local basename = mkutils.remove_extension(filename) + local tmpfile = basename ..".tmp" + mkutils.mv(filename, tmpfile) + local command = pattern % {filename = tmpfile, outputfile = filename} + print(command) + local status = os.execute(command) + if status > 0 then + -- if xtpipes failed to process the file, it may mean that it was bad-formed xml + -- we can try to make it well-formed using Tidy + local tidy_command = "tidy -utf8 -xml -asxml -q -o ${filename} ${tmpfile}" % {tmpfile = tmpfile, filename = filename} + print("xtpipes failed trying tidy") + print(tidy_command) + local status = os.execute(tidy_command) + if status > 0 then + -- if tidy failed as well, just use the original file + -- it will probably produce corrupted ODT file though + print("Tidy failed as well") + mkutils.mv(tmpfile, filename) + end + end + end + make:match("4oo", matchfunction) + make:match("4om", matchfunction) + -- is is necessary to execute the above matches as first in the build file + local matches = make.matches + -- move last match to a first place + local function move_matches() + local last = matches[#matches] + table.insert(matches, 1, last) + matches[#matches] = nil + end + -- we need to move last two matches, for 4oo and 4om files + move_matches() + move_matches() + else + print "Cannot locate xtpipes. Try to set TEXMFROOT variable to a root directory of your TeX distribution" + end +end + +-- sort output files according to their extensions +local function prepare_output_files(lgfiles) + local groups = {} + for _, name in ipairs(lgfiles) do + local basename, extension = name:match("(.-)%.([^%.]+)$") + local group = groups[extension] or {} + table.insert(group, basename) + groups[extension] = group + print(basename, extension) + end + return groups +end + +-- execute function on all files in the group +-- function fn takes current filename and table with various attributes +local function exec_group(groups, name, fn) + for _, basename in ipairs(groups[name] or {}) do + fn{basename = basename, extension=name, filename = basename .. "." .. name} + end +end + +function M.modify_build(make) + local executed = false + -- execute xtpipes from the build file, instead of t4ht. this fixes issues with wrong paths + -- expanded in tex4ht.env in Miktex or Debian + call_xtpipes(make) + -- convert XML entities for Unicoe characters produced by Xtpipes to characters + local fixentities = filter {"entities-to-unicode"} + make:match("4oo", fixentities) + make:match("4om", fixentities) + -- build the ODT file. This match must be executed as a last one + -- this will be executed as a first match, just to find the last filename + -- in the lgfile + make:match(".*", function() + -- execute it only once + if not executed then + -- this is list of processed files + local lgfiles = make.lgfile.files + -- find the last one + local lastfile = lgfiles[#lgfiles] .."$" + -- make match for the last file + -- odt packing will be done here + make:match(lastfile, function() + local groups = prepare_output_files(make.lgfile.files) + local basename = groups.odt[1] + local odtname = basename .. ".odt" + local odt,msg = Odtfile.new(odtname) + if not odt then + print("Cannot create ODT file: " .. msg) + end + -- helper function for simple file moving + local function move_file(group, dest) + exec_group(groups, group, function(par) + odt:move("${filename}" % par, dest) + end) + end + + -- the document text + exec_group(groups, "4oo", function(par) + odt:move("${filename}" % par, "content.xml") + odt:create_dir("Pictures") + end) + + -- manifest + exec_group(groups, "4of", function(par) + odt:create_dir("META-INF") + odt:move("${filename}" % par, "META-INF/manifest.xml") + end) + + -- math + exec_group(groups, "4om", function(par) + odt:create_dir(par.basename) + odt:move("${filename}" % par, "${basename}/content.xml" % par) + -- copy the settings file to math subdir + local settings = groups["4os"][1] + odt:copy(settings .. ".4os", "${basename}/settings.xml" % par) + end) + + -- these files are created only once, so it doesn't matter that they are + -- copied to one file + move_file("4os", "settings.xml") + move_file("4ot", "meta.xml") + move_file("4oy", "styles.xml") + + -- pictures + exec_group(groups, "4og", function(par) + -- add support for images in the TEXMF tree + if not mkutils.file_exists(par.basename) then + par.basename = kpse.find_file(par.basename, "graphic/figure") + if not par.basename then return nil, "Cannot find picture" end + end + -- the Pictues dir is flat, without subdirs + odt:copy("${basename}" % par, "Pictures") + end) + + -- remove some spurious file + exec_group(groups, "4od", function(par) + os.remove(par.filename) + end) + + odt:pack() + end) + end + executed = true + end) + return make +end return M diff --git a/Master/texmf-dist/scripts/make4ht/make4ht b/Master/texmf-dist/scripts/make4ht/make4ht index 9f9caed2e95..c72fe69ece2 100755 --- a/Master/texmf-dist/scripts/make4ht/make4ht +++ b/Master/texmf-dist/scripts/make4ht/make4ht @@ -27,7 +27,7 @@ make4ht [options] filename ["tex4ht.sty op." "tex4ht op." "t4ht op" "latex op"] -- set version number. the template should be replaced by the -- actual version number by the build script -local version = "v0.2b" +local version = "v0.2c" mkparams.version_number = version local args = mkparams.get_args() @@ -92,6 +92,11 @@ make:t4ht {ext = ext} if #extensions > 0 then make = mkutils.extensions_modify_build(extensions, make) end + +-- allow output formats to modify the build process at the end +make = formatter.modify_build(make) or make + + make:match("tmp$", function() return false,"tmp file" end) make:match(".*",function(filename,par) local outdir = '' --par["outdir"] and par["outdir"] .."/" or '' diff --git a/Master/texmf-dist/scripts/make4ht/make4ht-dvireader.lua b/Master/texmf-dist/scripts/make4ht/make4ht-dvireader.lua new file mode 100755 index 00000000000..f5988ee96c4 --- /dev/null +++ b/Master/texmf-dist/scripts/make4ht/make4ht-dvireader.lua @@ -0,0 +1,182 @@ +-- This is not actually full DVI reader. It just calculates hash for each page, +-- so it can be detected if it changed between compilations and needs to be +-- converted to image using Dvisvgm or Dvipng +-- +-- information about DVI format is from here: https://web.archive.org/web/20070403030353/http://www.math.umd.edu/~asnowden/comp-cont/dvi.html +-- +local M + +-- the file after post_post is filled with bytes 223 +local endfill = 223 + +-- numbers of bytes for each data type in DVI file +local int = 4 +local byte = 1 +local sixteen = 2 + +local function read_char(str, pos) + if pos and pos > string.len(str) then return nil end + return string.sub(str, pos, pos + 1) +end + +local function read_byte(str, pos) + return string.byte(read_char(str, pos)) +end + +-- DVI file format uses signed big endian integers. This code doesn't take into account +-- the sign, so it will return incorrect result for negative numbers. It doesn't matter +-- for the original purpose of this library, but it should be fixed for general use. +local function read_integer(str, pos) + local first = read_byte(str, pos) + local num = first * (256 ^ 3) + num = read_byte(str, pos + 1) * (256 ^ 2) + num + num = read_byte(str, pos + 2) * 256 + num + num = read_byte(str, pos + 3) + num + return num +end + +local function read_sixteen(str, pos) + local num = read_byte(str, pos) * 256 + num = read_byte(str, pos + 1) + num + return num +end + +-- select reader function with number of bytes of an argument +local readers = { + [byte] = read_byte, + [int] = read_integer, + [sixteen] = read_sixteen +} + + +local opcodes = { + post_post = { + opcode = 249, args = { + {name="q", type = int}, -- postamble address + {name="i", type = byte} + } + }, + post = { + opcode = 248, + args = { + {name="p", type = int}, -- address of the last page + {name="num", type = int}, + {name="den", type = int}, + {name="mag", type = int}, + {name="l", type = int}, + {name="u", type = int}, + {name="s", type = sixteen}, + {name="t", type = sixteen}, + } + }, + bop = { + opcode = 139, + args = { + {name="c0", type=int}, + {name="c1", type=int}, + {name="c2", type=int}, + {name="c3", type=int}, + {name="c4", type=int}, + {name="c5", type=int}, + {name="c6", type=int}, + {name="c7", type=int}, + {name="c8", type=int}, + {name="c9", type=int}, + {name="p", type=int}, -- previous page + } + } +} + +local function read_arguments(str, pos, args) + local t = {} + for _, v in ipairs(args) do + local fn = readers[v.type] + t[v.name] = fn(str, pos) + -- seek the position. v.type contains size of the current data type in bytes + pos = pos + v.type + end + return t +end + +local function read_opcode(opcode, str, pos) + local format = opcodes[opcode] + if not format then return nil, "Cannot find opcode format: " .. opcode end + -- check that opcode byte in the current position is the same as required opcode + local op = read_byte(str, pos) + if op ~= format.opcode then return nil, "Wrong opcode " .. op " at position " .. pos end + return read_arguments(str, pos+1, format.args) +end + +-- find the postamble address +local function get_postamble_addr(dvicontent) + local pos = string.len(dvicontent) + local last = read_char(dvicontent, pos) + -- skip endfill bytes at the end of file + while string.byte(last) == endfill do + pos = pos - 1 + last = read_char(dvicontent, pos) + end + -- first read post_post to get address of the postamble + local post_postamble, msg = read_opcode("post_post", dvicontent, pos-5) + if not post_postamble then return nil, msg end + -- return the postamble address + return post_postamble.q + 1 + -- return read_opcode("post", dvicontent, post_postamble.q + 1) + +end + +local function read_page(str, start, stop) + local function get_end_of_page(str, pos) + if read_byte(str, pos) == 140 then -- end of page + return pos + end + return get_end_of_page(str, pos - 1) + end + -- we reached the end of file + if start == 2^32-1 then return nil end + local current_page = read_opcode("bop", str, start + 1) + if not current_page then return nil end + local endofpage = get_end_of_page(str, stop) + -- get the page contents, but skip all parameters, because they can change + -- (especially pointer to the previous page) + local page = str:sub(start + 46, endofpage) + local page_obj = { + number = current_page.c0, -- the page number + hash = md5.sumhexa(page) -- hash the page contents + } + return page_obj, current_page.p, start +end + +local function get_pages(dvicontent) + local pages = {} + local postamble_pos = get_postamble_addr(dvicontent) + local postamble = read_opcode("post", dvicontent, postamble_pos) + local next_page_pos = postamble.p + local page, previous_page = nil, postamble_pos + local page_sequence = {} + while next_page_pos do + page, next_page_pos, previous_page = read_page(dvicontent, next_page_pos, previous_page) + page_sequence[#page_sequence+1] = page + end + + -- reorder pages + for _, v in ipairs(page_sequence) do + pages[v.number] = v.hash + end + return pages + +end + +-- if arg[1] then +-- local f = io.open(arg[1], "r") +-- local dvicontent = f:read("*all") +-- f:close() +-- local pages = get_pages(dvicontent) +-- for k,v in pairs(pages) do +-- print(k,v) +-- end +-- end + +return { + get_pages = get_pages +} diff --git a/Master/texmf-dist/scripts/make4ht/mkparams.lua b/Master/texmf-dist/scripts/make4ht/mkparams.lua index f5df4849966..6f293c8862e 100755 --- a/Master/texmf-dist/scripts/make4ht/mkparams.lua +++ b/Master/texmf-dist/scripts/make4ht/mkparams.lua @@ -118,6 +118,8 @@ local function process_args(args) if not latex_cli_params:match("%-jobname") then -- we must strip out directories from jobname when full path to document is given input = input:match("([^%/^%\\]+)$") + -- input also cannot contain spaces, replace them with underscores + input = input:gsub("%s", "_") table.insert(latex_params,"-jobname="..input) else -- when user specifies -jobname, we must change name of the input file, diff --git a/Master/texmf-dist/scripts/make4ht/mkutils.lua b/Master/texmf-dist/scripts/make4ht/mkutils.lua index 59da85bc74e..63924eeb50e 100755 --- a/Master/texmf-dist/scripts/make4ht/mkutils.lua +++ b/Master/texmf-dist/scripts/make4ht/mkutils.lua @@ -118,6 +118,20 @@ function cp(src,dest) os.execute(command) end +function mv(src, dest) + local mv_func = os.type == "unix" and "mv " or "move " + local command = string.format('%s "%s" "%s"', mv_func, src, dest) + -- fix windows paths + if mv_func == "move" then command = command:gsub("/",'\\') end + print("Move: ".. command) + os.execute(command) +end + +function delete_dir(path) + local cmd = os.type == "unix" and "rm -rd " or "rd /s/q " + os.execute(cmd .. path) +end + local used_dir = {} function prepare_path(path) @@ -209,15 +223,26 @@ function copy(filename,outfilename) return true end +-- find the zip command +function find_zip() + if io.popen("zip -v","r"):close() then + return "zip" + elseif io.popen("miktex-zip -v","r"):close() then + return "miktex-zip" + end + -- we cannot find the zip command + return "zip" +end + -- Config loading local function run(untrusted_code, env) - if untrusted_code:byte(1) == 27 then return nil, "binary bytecode prohibited" end - local untrusted_function = nil + if untrusted_code:byte(1) == 27 then return nil, "binary bytecode prohibited" end + local untrusted_function = nil untrusted_function, message = load(untrusted_code, nil, "t",env) - if not untrusted_function then return nil, message end - if not setfenv then setfenv = function(a,b) return true end end - setfenv(untrusted_function, env) - return pcall(untrusted_function) + if not untrusted_function then return nil, message end + if not setfenv then setfenv = function(a,b) return true end end + setfenv(untrusted_function, env) + return pcall(untrusted_function) end local main_settings = {} @@ -283,9 +308,9 @@ function env.filter_settings(name) end end env.Font = function(s) - local font_name = s["name"] - if not font_name then return nil, "Cannot find font name" end - env.settings.fonts[font_name] = s + local font_name = s["name"] + if not font_name then return nil, "Cannot find font name" end + env.settings.fonts[font_name] = s end env.Make = make4ht.Make @@ -294,22 +319,22 @@ env.Make:add("test","test the variables: ${tex4ht_sty_par} ${htlatex} ${input} -- this function reads the LaTeX log file and tries to detect fatal errors in the compilation local function testlogfile(par) - local logfile = par.input .. ".log" - local f = io.open(logfile,"r") - if not f then - print("Make4ht: cannot open log file "..logfile) - return 1 - end - local len = f:seek("end") + local logfile = par.input .. ".log" + local f = io.open(logfile,"r") + if not f then + print("Make4ht: cannot open log file "..logfile) + return 1 + end + local len = f:seek("end") -- test only the end of the log file, no need to run search functions on everything local newlen = len - 1256 -- but the value to seek must be greater than 0 newlen = (newlen > 0) and newlen or 0 - f:seek("set", newlen) - local text = f:read("*a") - f:close() - if text:match("No pages of output") or text:match("TeX capacity exceeded, sorry") then return 1 end - return 0 + f:seek("set", newlen) + local text = f:read("*a") + f:close() + if text:match("No pages of output") or text:match("TeX capacity exceeded, sorry") then return 1 end + return 0 end @@ -327,7 +352,7 @@ Make.latex_command = "${htlatex} ${latex_par} '\\makeatletter".. "\\HCode\\expandafter\\def\\csname tex4ht\\endcsname{#1,html}\\def".. "\\HCode####1{\\documentstyle[tex4ht,}\\@ifnextchar[{\\HCode}{".. "\\documentstyle[tex4ht]}}}\\makeatother\\HCode ${tex4ht_sty_par}.a.b.c.".. -"\\input ${tex_file}'" +"\\input \"\\detokenize{${tex_file}}\"'" env.Make:add("htlatex",function(par) local command = Make.latex_command @@ -402,6 +427,7 @@ function load_output_format(format_name) local format = assert(require(format_library)) if format then format.prepare_extensions = format.prepare_extensions or function(extensions) return extensions end + format.modify_build = format.modify_build or function(make) return make end end return format end -- cgit v1.2.3