summaryrefslogtreecommitdiff
path: root/support/make4ht/mkutils.lua
blob: ec5158d9d20d7f52687ea94720d0f01e44a63ce3 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
module(...,package.seeall)

local log = logging.new("mkutils")

local make4ht = require("make4ht-lib")
local mkparams = require("mkparams")
local indexing = require("make4ht-indexing")
--template engine
function interp(s, tab)
  local tab = tab or {}
  return (s:gsub('($%b{})', function(w) return tab[w:sub(3, -2)] or w end))
end
--print( interp("${name} is ${value}", {name = "foo", value = "bar"}) )

function addProperty(s,prop)
  if prop ~=nil then
    return s .." "..prop
  else
    return s
  end
end
getmetatable("").__mod = interp
getmetatable("").__add = addProperty 

--print( "${name} is ${value}" % {name = "foo", value = "bar"} )
-- Outputs "foo is bar"

function is_url(path)
  return path:match("^%a+://")
end


-- merge two tables recursively
function merge(t1, t2)
  for k, v in pairs(t2) do
    if (type(v) == "table") and (type(t1[k] or false) == "table") then
      merge(t1[k], t2[k])
    else
      t1[k] = v
    end
  end
  return t1
end

function string:split(sep)
  local sep, fields = sep or ":", {}
  local pattern = string.format("([^%s]+)", sep)
  self:gsub(pattern, function(c) fields[#fields+1] = c end)
  return fields
end

function remove_extension(path)
  local found, len, remainder = string.find(path, "^(.*)%.[^%.]*$")
  if found then
    return remainder
  else
    return path
  end
end

-- 
-- check if file exists
function file_exists(file)
  local f = io.open(file, "rb")
  if f then f:close() end
  return f ~= nil
end

-- check if Lua module exists
-- source: https://stackoverflow.com/a/15434737/2467963
function isModuleAvailable(name)
  if package.loaded[name] then
    return true
  else
    for _, searcher in ipairs(package.searchers or package.loaders) do
      local loader = searcher(name)
      if type(loader) == 'function' then
        package.preload[name] = loader
        return true
      end
    end
    return false
  end
end


-- searching for converted images
function parse_lg(filename, builddir)
  log:info("Parse LG")
  local dir = builddir~="" and builddir .. "/" or ""
  local outputimages,outputfiles,status={},{},nil
  local fonts, used_fonts = {},{}
  if not file_exists(filename) then
    log:warning("Cannot read log file: "..filename)
  else
    local usedfiles={}
    for line in io.lines(filename) do
      --- needs --- pokus.idv[1] ==> pokus0x.png --- 
      -- line:gsub("needs --- (.+?)[([0-9]+) ==> ([%a%d%p%.%-%_]*)",function(name,page,k) table.insert(outputimages,k)end)
      line:gsub("needs %-%-%- (.+)%[([0-9]+)%] ==> (.*) %-%-%-",
      function(file,page,output) 
        local rec = {
          source=file,
          page=page,
          output=dir..output
        }
        table.insert(outputimages,rec)
      end
      )
      line:gsub("File: (.*)",  function(k)
        k = dir .. k
        if not usedfiles[k] then
          table.insert(outputfiles,k)
          usedfiles[k] = true
        end
      end)
      line:gsub("htfcss: ([^%s]+)(.*)",function(k,r)
        local fields = {}
        r:gsub("[%s]*([^%:]+):[%s]*([^;]+);",function(c,v)
          fields[c] = v
        end)
        fonts[k] = fields
      end)

      line:gsub('Font("([^"]+)","([%d]+)","([%d]+)","([%d]+)"',function(n,s1,s2,s3)
        table.insert(used_fonts,{n,s1,s2,s3})
      end)

    end
    status=true
  end
  return {files = outputfiles, images = outputimages},status
end


-- 
local cp_func = os.type == "unix" and "cp" or "copy"
-- maybe it would be better to actually move the files
-- in reality it isn't.
-- local cp_func = os.type == "unix" and "mv" or "move"
function cp(src,dest)
  if is_url(src) then
    log.info(src .. " is a URL, will leave as is")
    return
  end
  if not file_exists(src) then
    -- try to find file using kpse library if it cannot be found
    src = kpse.find_file(src) or src
  end
  local command = string.format('%s "%s" "%s"', cp_func, src, dest)
  if cp_func == "copy" then command = command:gsub("/",'\\') end
  log:info("Copy: "..command)
  if not file_exists(src) then
    log:error("File " .. src .. " doesn't exist")
  end
  os.execute(command)
end

function mv(src, dest)
  local mv_func = os.type == "unix" and "mv " or "move "
  local command = string.format('%s "%s" "%s"', mv_func, src, dest)
  -- fix windows paths
  if mv_func == "move" then command = command:gsub("/",'\\') end
  log:info("Move: ".. command)
  os.execute(command)
end

function delete_dir(path)
  local cmd = os.type == "unix" and "rm -rd " or "rd /s/q "
  os.execute(cmd .. path)
end

local used_dir = {}

function prepare_path(path)
  --local dirs = path:split("/")
  local dirs = {}
  if path:match("^/") then dirs = {""}
  elseif path:match("^~") then
    local home = os.getenv "HOME"
    dirs = home:split "/"
    path = path:gsub("^~/","")
    table.insert(dirs,1,"")
  end
  if path:match("/$")then path = path .. " " end
  for _,d in pairs(path:split "/") do
    table.insert(dirs,d)
  end
  table.remove(dirs,#dirs)
  return dirs,table.concat(dirs,"/")
end

-- Find which part of path already exists
-- and which directories have to be created
function find_directories(dirs, pos)
  local pos = pos or #dirs
  -- we tried whole path and no dir exist
  if pos < 1 then return dirs end
  local path = ""
  -- in the case of unix absolute path, empty string is inserted in dirs
  if pos == 1 and dirs[pos] == "" then
    path = "/"
  else
    path = table.concat(dirs,"/", 1,pos) .. "/"
  end
  if not lfs.chdir(path)  then -- recursion until we succesfully changed dir
  -- or there are no elements in the dir table
  return find_directories(dirs,pos - 1)
elseif pos ~= #dirs then -- if we succesfully changed dir
  -- and we have dirs to create
  local p = {}
  for i = pos+1, #dirs do
    table.insert(p, dirs[i])
  end
  return p
else  -- whole path exists
  return {}
end
end

function mkdirectories(dirs)
  if type(dirs) ~="table" then
    return false, "mkdirectories: dirs is not table"
  end
  local path = ""
  for _,d in ipairs(dirs) do
    path = path .. d .. "/"
    local stat,msg = lfs.mkdir(path)
    if not stat then return false, "makedirectories error: "..msg end
  end
  return true
end

function make_path(path)
  -- we must create the build dir if it doesn't exist
  local cwd = lfs.currentdir()
  -- add dummy /foo dir. it won't be created, but without that, the top-level dir wouldn't be created
  local parts = mkutils.prepare_path(path .. "/foo")
  local to_create = mkutils.find_directories(parts)
  mkutils.mkdirectories(to_create)
  -- change back to the original dir
  lfs.chdir(cwd)
end

function file_in_builddir(filename, par)
  if par.builddir and par.builddir ~= "" then
    return par.builddir .. "/" .. filename
  end
  return filename
end

function copy_filter(src,dest, filter)
  local src_f=io.open(src,"rb")
  local dst_f=io.open(dest,"w")
  local contents = src_f:read("*all")
  local filter = filter or function(s) return s end
  src_f:close()
  dst_f:write(filter(contents))
  dst_f:close()
end



function copy(filename,outfilename)
  local currdir = lfs.currentdir()
  if filename == outfilename then return true end
  local parts, path = prepare_path(outfilename)
  if not used_dir[path] then 
    local to_create, msg = find_directories(parts)
    if not to_create then
      log:warning(msg)
      return false
    end
    used_dir[path] = true
    local stat, msg = mkdirectories(to_create)
    if not stat then log:warning(msg) end
  end
  lfs.chdir(currdir)
  cp(filename, path)
  return true
end

function execute(command)
  local f = io.popen(command, "r")
  local output = f:read("*all")
  -- rc will contain return codes of the executed command
  local rc =  {f:close()}
  -- the status code is on the third position 
  -- https://stackoverflow.com/a/14031974/2467963
  local status = rc[3]
  -- print the command line output only when requested through
  -- log  level
  log:output(output)
  return status, output
end

-- find the zip command
function find_zip()
  if io.popen("zip -v","r"):close() then
    return "zip"
  elseif io.popen("miktex-zip -v","r"):close() then
    return "miktex-zip"
  end
  -- we cannot find the zip command
  return "zip"
end

-- Config loading
local function run(untrusted_code, env)
  if untrusted_code:byte(1) == 27 then return nil, "binary bytecode prohibited" end
  local untrusted_function = nil
  untrusted_function, message = load(untrusted_code, nil, "t",env)
  if not untrusted_function then return nil, message end
  if not setfenv then setfenv = function(a,b) return true end end
  setfenv(untrusted_function, env)
  return pcall(untrusted_function)
end

local main_settings = {}
main_settings.fonts = {}
-- use global environment in the build file
-- it used to be sandboxed, but it proved not to be useful at all
local env = _G ---{}

-- explicitly enale some functions and modules in the sandbox
-- Function declarations:
env.pairs  = pairs
env.ipairs = ipairs
env.print  = print
env.split  = split
env.string = string
env.table  = table
env.copy   = copy
env.tonumber = tonumber
env.tostring = tostring
env.mkdirectories = mkdirectories
env.require = require
env.texio  = texio
env.type   = type
env.lfs    = lfs
env.os     = os
env.io     = io
env.math   = math
env.unicode = unicode
env.logging = logging


-- it is necessary to use the settings table
-- set in the Make environment by mkutils
function env.set_settings(par)
  local settings = env.settings
  for k,v in pairs(par) do
    settings[k] = v
  end
end

-- Add a value to the current settings
function env.settings_add(par)
  local settings = env.settings
  for k,v in pairs(par) do
    local oldval = settings[k] or ""
    settings[k] = oldval .. v
  end
end

function env.get_filter_settings(name)
  local settings = env.settings
  -- local settings = self.params
  local filters = settings.filter or {}
  local filter_options = filters[name] or {}
  return filter_options
end

function env.filter_settings(name)
  -- local settings = Make.params
  local settings = env.settings
  local filters = settings.filter or {}
  local filter_options = filters[name] or {}
  return function(par)
    filters[name] = merge(filter_options, par)
    settings.filter = filters
  end
end
env.Font   = function(s)
  local font_name = s["name"]
  if not font_name then return nil, "Cannot find font name" end
  env.settings.fonts[font_name] = s
end

env.Make   = make4ht.Make
env.Make.params = env.settings
env.Make:add("test","test the variables:  ${tex4ht_sty_par} ${htlatex} ${input} ${config}")

local htlatex = require "make4ht-htlatex"
env.Make:add("htlatex", htlatex.htlatex
,{correct_exit=0})
env.Make:add("httex", htlatex.httex, {
  htlatex = "etex",
  correct_exit=0
})

env.Make:add("latexmk", function(par)
  local settings = get_filter_settings "htlatex" or {}
  par.interaction = par.interaction or settings.interaction or "batchmode"
  local command = Make.latex_command 
  -- add " %O " after the engine name. it should be filled by latexmk
  command = command:gsub("%s", " %%O ", 1)
  par.expanded = command % par
  -- quotes in latex_command must be escaped, they cause Latexmk error
  par.expanded = par.expanded:gsub('"', '\\"')
  local newcommand = 'latexmk  -pdf- -ps- -auxdir=${builddir} -outdir=${builddir} -latex="${expanded}" -dvi -jobname=${input} ${tex_file}' % par
  log:info("LaTeX call: " .. newcommand)
  os.execute(newcommand)
  return Make.testlogfile(par)
end, {correct_exit= 0})



-- env.Make:add("tex4ht","tex4ht ${tex4ht_par} \"${input}.${dvi}\"", nil, 1)
env.Make:add("tex4ht",function(par)
  -- detect if svg output is used
  -- if yes, we need to pass the -g.svg option to tex4ht command
  -- to support svg images for character pictures
  local logfile = mkutils.file_in_builddir(par.input .. ".log", par)
  if file_exists(logfile) then
    for line in io.lines(logfile) do
      local options = line:match("TeX4ht package options:(.+)")
      if options then
        log:info(options)
        if options:match("svg") then
          par.tex4ht_par = (par.tex4ht_par or "") .. " -g.svg"
        end
        break
      end
    end
  end
  local cwd = lfs.currentdir()
  if par.builddir~="" then
      lfs.chdir(par.builddir)
  end
  local command = "tex4ht ${tex4ht_par} \"${input}.${dvi}\"" % par
  log:info("executing: " .. command)
  local status, output = execute(command)
  lfs.chdir(cwd)
  return status, output
end
, nil, 1)
env.Make:add("t4ht", function(par)
    par.ext = "dvi"
    local cwd = lfs.currentdir()
    if par.builddir ~= "" then
        lfs.chdir(par.builddir)
    end
    local command = "t4ht ${t4ht_par} \"${input}.${ext}\"" % par
    log:info("executing: " .. command)
    execute(command)
    lfs.chdir(cwd)
end
)

env.Make:add("clean", function(par)
  -- remove all functions that process produced files
  -- we will provide only one function, that remove all of them
  Make.matches = {}
  local main_name = mkutils.file_in_builddir( par.input, par)
  local remove_file = function(filename)
    if file_exists(filename) then
      log:info("removing file: " .. filename)
      os.remove(filename)
    end
  end
  -- try to find if the last converted file was in the ODT format
  local lg_name =  main_name .. ".lg"
  local lg_file = parse_lg(lg_name, par.builddir)
  local is_odt = false
  if lg_file and lg_file.files then
    for _, x in ipairs(lg_file.files) do
      is_odt = x:match("odt$") or is_odt
    end
  end
  if is_odt then
    Make:match("4om$",function(filename)
      -- math temporary file
      local to_remove = filename:gsub("4om$", "tmp")
      remove_file(to_remove)
      return false
    end)
    Make:match("4og$", remove_file)
  end
  Make:match("tmp$", function()
    -- remove temporary and auxilary files
    for _,ext in ipairs {"aux", "xref", "tmp", "4tc", "4ct", "idv", "lg","dvi", "log", "ncx", "idx", "ind"} do
      remove_file(main_name .. "." .. ext)
    end
  end)
  Make:match(".*", function(filename, par)
    -- remove only files that start with the input file basename
    -- this should prevent removing of images. this also means that
    -- images shouldn't be names as <filename>-hello.png for example
    if filename:find(main_name, 1,true) then
      -- log:info("Matched file", filename)
      remove_file(filename)
    end
  end)

end)

-- enable extension in the config file
-- the following two functions must be here and not in make4ht-lib.lua
-- because of the access to env.settings
env.Make.enable_extension = function(self,name)
  table.insert(env.settings.extensions, {type="+", name=name})
end

-- disable extension in the config file
env.Make.disable_extension = function(self,name)
  table.insert(env.settings.extensions, {type="-", name=name})
end

function load_config(settings, config_name)
  local settings = settings or main_settings
  -- the extensions requested from the command line should take precedence over
  -- extensions enabled in the config file
  local saved_extensions = settings.extensions
  settings.extensions = {}
  env.settings = settings
  env.mode = settings.mode
  if config_name and not file_exists(config_name) then
    config_name = kpse.find_file(config_name, 'texmfscripts') or config_name
  end
  local f = io.open(config_name,"r")
  if not f then 
    log:info("Cannot open config file", config_name)
    return  env
  end
  log:info("Using build file", config_name)
  local code = f:read("*all")
  local fn, msg = run(code,env)
  if not fn then log:warning(msg) end
  assert(fn)
  -- reload extensions from command line arguments for the "format" parameter
  for _,v in ipairs(saved_extensions) do
    table.insert(settings.extensions, v)
  end
  return env
end

env.Make:add("xindy", function(par)
  local xindylog = logging.new "xindy"
  local settings = get_filter_settings "xindy" or {}
  par.encoding  = settings.encoding or  par.encoding or "utf8"
  par.language = settings.language or par.language or "english"
  local modules = settings.modules or par.modules or {}
  local t = {}
  for k,v in ipairs(modules) do
    xindylog:debug("Loading module: " ..v)
    t[#t+1] = "-M ".. v
  end
  par.moduleopt = table.concat(t, " ")
  return  indexing.run_indexing_command("texindy -L ${language} -C ${encoding} ${moduleopt} -o ${indfile} ${newidxfile}", par)
end, {})

env.Make:add("makeindex", function(par)
  local makeindxcall = "makeindex ${options} -t ${ilgfile} -o ${indfile} ${newidxfile}"
  local settings = get_filter_settings "makeindex" or {}
  par.options = settings.options or par.options  or ""
  par.ilgfile = par.input .. ".ilg" 
  local status = indexing.run_indexing_command(makeindxcall, par)
  return status
end, {})

env.Make:add("xindex", function(par)
  local xindex_call = "xindex -l ${language} ${options} -o ${indfile} ${newidxfile}"
  local settings = get_filter_settings "xindex" or {}
  par.options = settings.options or par.options  or ""
  par.language = settings.language or par.language or "en"
  local status = indexing.run_indexing_command(xindex_call, par)
  return status
end, {})



local function find_lua_file(name)
  local extension_path = name:gsub("%.", "/") .. ".lua"
  return kpse.find_file(extension_path, "lua")
end

-- for the BibLaTeX support
env.Make:add("biber", "biber ${input}")
env.Make:add("bibtex", "bibtex ${input}")
env.Make:add("pythontex", "pythontex ${input}")

--- load the output format plugins
function load_output_format(format_name)
  local format_library =  "make4ht.formats.make4ht-"..format_name
  local is_format_file = find_lua_file(format_library)
  if is_format_file then 
    local format = assert(require(format_library))
    if format then
      format.prepare_extensions = format.prepare_extensions or function(extensions) return extensions end
      format.modify_build = format.modify_build or function(make) return make end
    end
    return format
  end
end

--- Execute the prepare_parameters function in list of extensions
function extensions_prepare_parameters(extensions, parameters)
  for _, ext in ipairs(extensions) do
    -- execute the extension only if it contains prepare_parameters function
    local fn = ext.prepare_parameters
    if fn then
      parameters = fn(parameters)
    end
  end
  return parameters
end

--- Modify the build sequence using extensions
-- @param extensions list of extensions 
-- @make  Make object
function extensions_modify_build(extensions, make)
  for _, ext in ipairs(extensions) do
    local fn = ext.modify_build
    if fn then
      make = fn(make)
    end
  end
  return make
end


--- load one extension
-- @param name  extension name
-- @param format current output format
function load_extension(name,format)
  -- first test if the extension exists
  local extension_library = "make4ht.extensions.make4ht-ext-" .. name
  local is_extension_file = find_lua_file(extension_library)
  -- don't try to load the extension if it doesn't exist
  if not is_extension_file then return nil, "cannot fint extension " .. name  end
  local extension = nil
  local local_extension_path = package.searchpath(extension_library, package.path)
  if local_extension_path then
      extension = dofile(local_extension_path)
  else
      extension = require("make4ht.extensions.make4ht-ext-".. name)
  end
  -- extensions can test if the current output format is supported
  local test = extension.test
  if test then
    if test(format) then 
      return extension
    end
    -- if the test fail return nil
    return nil, "extension " .. name .. " is not supported in the " .. format .. " format"
  end
  -- if the extension doesn't provide the test function, we will assume that
  -- it supports every output format
  return extension
end

--- load extensions
-- @param extensions table created by mkparams.get_format_extensions function
-- @param format  output type format. extensions may support only certain file
-- formats
function load_extensions(extensions, format)
  local module_names = {}
  local extension_table = {}
  local extension_sequence = {}
  -- process the extension table. it contains type field, which can enable or
  -- diable the extension
  for _, v in ipairs(extensions) do
    local enable = v.type == "+" and true or nil
    -- load extenisons in a correct order
    -- don't load extensions multiple times
    if enable and not module_names[v.name] then
      table.insert(extension_sequence, v.name)
    end
    -- the last extension request can disable it
    module_names[v.name] = enable
  end
  for _, name in ipairs(extension_sequence) do
    -- the extension can be inserted into the extension_sequence, but disabled
    -- later.
    if module_names[name] == true then
      local extension, msg= load_extension(name,format)
      if extension then
        log:info("Load extension", name)
        table.insert(extension_table, extension)
      else
        log:warning("Cannot load extension: ".. name)
        log:warning(msg)
      end
    end
  end
  return extension_table
end

--- add new extensions to a list of loaded extensions
-- @param added  string with extensions to be added in the form +ext1+ext2
function add_extensions(added, extensions)
  local _, newextensions = mkparams.get_format_extensions("dummyfmt" .. added)
  -- insert new extension at the beginning, in order to support disabling using
  -- the -f option
  for _, x in ipairs(extensions or {}) do table.insert(newextensions, x) end
  return newextensions
end

-- I don't know if this is clean, but settings functions won't be available
-- for filters and extensions otherwise
for k,v in pairs(env) do _G[k] = v end