diff options
author | Karl Berry <karl@freefriends.org> | 2021-05-05 19:44:19 +0000 |
---|---|---|
committer | Karl Berry <karl@freefriends.org> | 2021-05-05 19:44:19 +0000 |
commit | 04c30db0018970c69516d748c7b4509240857b65 (patch) | |
tree | 60c80f16c8a85519e3e0af6981047bfc3fcd5d29 /Master/texmf-dist/scripts | |
parent | 4754ab4e3ed4d6763f26591c2c84be9dfcf54664 (diff) |
l3build (5may21)
git-svn-id: svn://tug.org/texlive/trunk@59091 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/scripts')
17 files changed, 552 insertions, 465 deletions
diff --git a/Master/texmf-dist/scripts/l3build/l3build-arguments.lua b/Master/texmf-dist/scripts/l3build/l3build-arguments.lua index aa2f086d616..80f19d66ad5 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-arguments.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-arguments.lua @@ -1,6 +1,6 @@ --[[ -File l3build-arguments.lua Copyright (C) 2018-2020 The LaTeX3 Project +File l3build-arguments.lua Copyright (C) 2018-2020 The LaTeX Project It may be distributed and/or modified under the conditions of the LaTeX Project Public License (LPPL), either version 1.3c of this @@ -170,7 +170,7 @@ local function argparse() end long_options[k] = k end - local args = args + local arg = arg -- arg[1] is a special case: must be a command or "-h"/"--help" -- Deal with this by assuming help and storing only apparently-valid -- input @@ -239,21 +239,21 @@ local function argparse() if optarg then local opt = "-" .. (match(a, "^%-%-") and "-" or "") .. opt stderr:write("Value not allowed for option " .. opt .."\n") - return {"help"} + return { target = "help" } end else if not optarg then optarg = arg[i + 1] if not optarg then stderr:write("Missing value for option " .. a .."\n") - return {"help"} + return { target = "help" } end i = i + 1 end end else stderr:write("Unknown option " .. a .."\n") - return {"help"} + return { target = "help" } end -- Store the result if optarg then diff --git a/Master/texmf-dist/scripts/l3build/l3build-aux.lua b/Master/texmf-dist/scripts/l3build/l3build-aux.lua index 8fa4b0cd82c..12fe704af49 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-aux.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-aux.lua @@ -1,6 +1,6 @@ --[[ -File l3build-aux.lua Copyright (C) 2018-2020 The LaTeX3 Project +File l3build-aux.lua Copyright (C) 2018-2020 The LaTeX Project It may be distributed and/or modified under the conditions of the LaTeX Project Public License (LPPL), either version 1.3c of this @@ -22,6 +22,8 @@ for those people who are interested. --]] +-- local safety guards and shortcuts + local match = string.match local pairs = pairs @@ -29,97 +31,137 @@ local print = print local lookup = kpse.lookup +local os_time = os.time -- -- Auxiliary functions which are used by more than one main function -- -function normalise_epoch() +---Convert the given `epoch` to a number. +---@param epoch string +---@return number +---@see l3build.lua +---@usage private? +function normalise_epoch(epoch) + assert(epoch, 'normalize_epoch argument must not be nil') -- If given as an ISO date, turn into an epoch number local y, m, d = match(epoch, "^(%d%d%d%d)-(%d%d)-(%d%d)$") if y then - epoch = - os_time({year = y, month = m, day = d, hour = 0, sec = 0, isdst = nil}) - - os_time({year = 1970, month = 1, day = 1, hour = 0, sec = 0, isdst = nil}) + return os_time({ + year = y, month = m, day = d, + hour = 0, sec = 0, isdst = nil + }) - os_time({ + year = 1970, month = 1, day = 1, + hour = 0, sec = 0, isdst = nil + }) elseif match(epoch, "^%d+$") then - epoch = tonumber(epoch) + return tonumber(epoch) else - epoch = 0 + return 0 end end -function setepoch() - return +---Returns the CLI command (ending with `os_concat`) to set the epoch +---when forcecheckepoch is true, a void string otherwise. +---Will be run while checking or typesetting +---@param epoch string +---@param force boolean +---@return string +---@see check, typesetting +---@usage private? +function set_epoch_cmd(epoch, force) + return force and ( os_setenv .. " SOURCE_DATE_EPOCH=" .. epoch .. os_concat .. os_setenv .. " SOURCE_DATE_EPOCH_TEX_PRIMITIVES=1" .. os_concat .. os_setenv .. " FORCE_SOURCE_DATE=1" .. os_concat + ) or "" end -local function getscriptname() +---Returns the script name depending on the calling sequence. +---`l3build ...` -> full path of `l3build.lua` in the TDS +---When called via `texlua l3build.lua ...`, `l3build.lua` is resolved to either +---`./l3build.lua` or the full path of `l3build.lua` in the TDS. +---`texlua l3build.lua` -> `/Library/TeX/texbin/l3build.lua` or `./l3build.lua` +---@return string +local function get_script_name() if match(arg[0], "l3build$") or match(arg[0], "l3build%.lua$") then return lookup("l3build.lua") else - return arg[0] + return arg[0] -- Why no lookup here? end end --- Do some subtarget for all modules in a bundle -function call(dirs, target, opts) - -- Turn the option table into a string - local opts = opts or options - local s = "" +-- Performs the task named target given modules in a bundle. +---A module is the path of a directory relative to the main one. +---Uses `run` to launch a command. +---@param modules table List of modules. +---@param target string +---@param opts table +---@return number 0 on proper termination, a non 0 error code otherwise. +---@see many places, including latex2e/build.lua +---@usage Public +function call(modules, target, opts) + -- Turn the option table into a CLI option string + opts = opts or options + local cli_opts = "" for k,v in pairs(opts) do - if k ~= "names" and k ~= "target" then -- Special cases - local t = option_list[k] or { } - local arg = "" + if k ~= "names" and k ~= "target" then -- Special cases, TODO enhance the design to remove the need for this comment + local t = option_list[k] or {} + local value = "" if t["type"] == "string" then - arg = arg .. "=" .. v - end - if t["type"] == "table" then + value = value .. "=" .. v + elseif t["type"] == "table" then for _,a in pairs(v) do - if arg == "" then - arg = "=" .. a -- Add the initial "=" here + if value == "" then + value = "=" .. a -- Add the initial "=" here else - arg = arg .. "," .. a + value = value .. "," .. a end end end - s = s .. " --" .. k .. arg + cli_opts = cli_opts .. " --" .. k .. value end end - if opts["names"] then - for _,v in pairs(opts["names"]) do - s = s .. " " .. v + if opts.names then + for _, name in pairs(opts.names) do + cli_opts = cli_opts .. " " .. name end end - local scriptname = getscriptname() - for _,i in ipairs(dirs) do - local text = " for module " .. i - if i == "." and opts["config"] then + local script_name = get_script_name() + for _, module in ipairs(modules) do + local text + if module == "." and opts["config"] and #opts["config"]>0 then text = " with configuration " .. opts["config"][1] + else + text = " for module " .. module end print("Running l3build with target \"" .. target .. "\"" .. text ) - local errorlevel = run( - i, - "texlua " .. scriptname .. " " .. target .. s + local error_level = run( + module, + "texlua " .. script_name .. " " .. target .. cli_opts ) - if errorlevel ~= 0 then - return errorlevel + if error_level ~= 0 then + return error_level end end return 0 end --- Unpack files needed to support testing/typesetting/unpacking -function depinstall(deps) - local errorlevel - for _,i in ipairs(deps) do - print("Installing dependency: " .. i) - errorlevel = run(i, "texlua " .. getscriptname() .. " unpack -q") - if errorlevel ~= 0 then - return errorlevel +---Unpack the given dependencies. +---A dependency is the path of a directory relative to the main one. +---@param deps table regular array of dependencies. +---@return number 0 on proper termination, a non 0 error code otherwise. +---@see stdmain, check, unpack, typesetting +---@usage Private? +function dep_install(deps) + local error_level + for _, dep in ipairs(deps) do + print("Installing dependency: " .. dep) + error_level = run(dep, "texlua " .. get_script_name() .. " unpack -q") + if error_level ~= 0 then + return error_level end end return 0 diff --git a/Master/texmf-dist/scripts/l3build/l3build-check.lua b/Master/texmf-dist/scripts/l3build/l3build-check.lua index ff022c60793..dce9e68c6ab 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-check.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-check.lua @@ -1,6 +1,6 @@ --[[ -File l3build-check.lua Copyright (C) 2018-2020 The LaTeX3 Project +File l3build-check.lua Copyright (C) 2018-2020 The LaTeX Project It may be distributed and/or modified under the conditions of the LaTeX Project Public License (LPPL), either version 1.3c of this @@ -34,7 +34,7 @@ local luatex_version = status.luatex_version local len = string.len local char = string.char -local format = string.format +local str_format = string.format local gmatch = string.gmatch local gsub = string.gsub local match = string.match @@ -59,7 +59,7 @@ function checkinit() cleandir(testdir) cleandir(resultdir) end - depinstall(checkdeps) + dep_install(checkdeps) -- Copy dependencies to the test directory itself: this makes the paths -- a lot easier to manage, and is important for dealing with the log and -- with file input/output tests @@ -85,7 +85,7 @@ function checkinit() return checkinit_hook() end -checkinit_hook = checkinit_hook or function() return 0 end +function checkinit_hook() return 0 end local function rewrite(source,result,processor,...) local file = assert(open(source,"rb")) @@ -102,7 +102,7 @@ end -- the 'business' part from the tests and removes system-dependent stuff local function normalize_log(content,engine,errlevels) local maxprintline = maxprintline - if match(engine,"^lua") or match(engine,"^harf") then + if (match(engine,"^lua") or match(engine,"^harf")) and luatex_version < 113 then maxprintline = maxprintline + 1 -- Deal with an out-by-one error end local function killcheck(line) @@ -134,7 +134,7 @@ local function normalize_log(content,engine,errlevels) not match(line, "%.%.%.$") then return "", (lastline or "") .. line end - local line = (lastline or "") .. line + line = (lastline or "") .. line lastline = "" -- Zap ./ at begin of filename line = gsub(line, "%(%.%/", "(") @@ -223,15 +223,19 @@ local function normalize_log(content,engine,errlevels) -- Remove '\displace 0.0' lines in (u)pTeX if match(line,"^%.*\\displace 0%.0$") then return "" - end - end - -- Remove the \special line that in DVI mode keeps PDFs comparable + end + end + -- Deal with Lua function calls + if match(line, "^Lua function") then + line = gsub(line,"= %d+$","= ...") + end + -- Remove the \special line that in DVI mode keeps PDFs comparable if match(line, "^%.*\\special%{pdf: docinfo << /Creator") or match(line, "^%.*\\special%{ps: /setdistillerparams") or match(line, "^%.*\\special%{! <</........UUID") then return "" end - -- Remove \special lines for DVI .pro files + -- Remove \special lines for DVI .pro files if match(line, "^%.*\\special%{header=") then return "" end @@ -251,7 +255,7 @@ local function normalize_log(content,engine,errlevels) match(line, "^used file >") or match(line, "^used option >") or match(line, "^used structure >") then - return "" + return "" end -- The first time a new font is used by LuaTeX, it shows up -- as being cached: make it appear loaded every time @@ -265,7 +269,7 @@ local function normalize_log(content,engine,errlevels) -- tidy up to match pdfTeX if an ASCII engine is in use if next(asciiengines) then for i = 128, 255 do - line = gsub(line, utf8_char(i), "^^" .. format("%02x", i)) + line = gsub(line, utf8_char(i), "^^" .. str_format("%02x", i)) end end return line, lastline @@ -341,7 +345,7 @@ local function normalize_lua_log(content,luatex) l, m .. " (%-?)%d+%.%d+", m .. " %1" - .. format( + .. str_format( "%.3f", match(line, m .. " %-?(%d+%.%d+)") or 0 ) @@ -370,20 +374,20 @@ local function normalize_lua_log(content,luatex) -- This block only applies to the output of LuaTeX itself, -- hence needing a flag to skip the case of the reference log if luatex and - tonumber(luatex_version) >= 107 and - match(line, "^%.*\\kern") then - -- Re-insert the space in explicit kerns - if match(line, "kern%-?%d+%.%d+ *$") then - line = gsub(line, "kern", "kern ") - elseif match(line, "%(accent%)$") then - line = gsub(line, "kern", "kern ") - line = gsub(line, "%(accent%)$", "(for accent)") - elseif match(line, "%(italic%)$") then - line = gsub(line, "kern", "kern ") - line = gsub(line, " %(italic%)$", "") - else - line = gsub(line, " %(font%)$", "") - end + tonumber(luatex_version) >= 107 and + match(line, "^%.*\\kern") then + -- Re-insert the space in explicit kerns + if match(line, "kern%-?%d+%.%d+ *$") then + line = gsub(line, "kern", "kern ") + elseif match(line, "%(accent%)$") then + line = gsub(line, "kern", "kern ") + line = gsub(line, "%(accent%)$", "(for accent)") + elseif match(line, "%(italic%)$") then + line = gsub(line, "kern", "kern ") + line = gsub(line, " %(italic%)$", "") + else + line = gsub(line, " %(font%)$", "") + end end -- Changes in PDF specials line = gsub(line, "\\pdfliteral origin", "\\pdfliteral") @@ -394,8 +398,8 @@ local function normalize_lua_log(content,luatex) -- 'Recover' some discretionary data if match(lastline, "^%.+\\discretionary %(penalty 50%)$") and match(line, boxprefix(lastline) .. "%.= ") then - line = gsub(line," %(font%)$","") - return gsub(line, "%.= ", ""),"" + line = gsub(line," %(font%)$","") + return gsub(line, "%.= ", ""),"" end -- Where the last line was a discretionary, looks for the -- info one level in about what it represents @@ -406,13 +410,13 @@ local function normalize_lua_log(content,luatex) local prefix = boxprefix(lastline) if match(line, prefix .. "%.") or match(line, prefix .. "%|") then - if match(lastline, " replacing $") and - not dropping then - -- Modify the return line - return gsub(line, "^%.", ""), lastline, true - else - return "", lastline, true - end + if match(lastline, " replacing $") and + not dropping then + -- Modify the return line + return gsub(line, "^%.", ""), lastline, true + else + return "", lastline, true + end else if dropping then -- End of a \discretionary block @@ -431,7 +435,7 @@ local function normalize_lua_log(content,luatex) end end -- Look for another form of \discretionary, replacing a "-" - pattern = "^%.+\\discretionary replacing *$" + local pattern = "^%.+\\discretionary replacing *$" if match(line, pattern) then return "", line else @@ -461,7 +465,7 @@ local function normalize_lua_log(content,luatex) match(line, "^%.+\\localbrokenpenalty=0$") or match(line, "^%.+\\localleftbox=null$") or match(line, "^%.+\\localrightbox=null$") then - return "", "" + return "", "" end -- Older LuaTeX versions set the above up as a whatsit -- (at some stage this can therefore go) @@ -522,7 +526,7 @@ local function normalize_pdf(content) if binary then new_content = new_content .. "[BINARY STREAM]" .. os_newline else - new_content = new_content .. stream_content .. line .. os_newline + new_content = new_content .. stream_content .. line .. os_newline end binary = false else @@ -541,7 +545,7 @@ local function normalize_pdf(content) stream = true stream_content = "stream" .. os_newline elseif not match(line, "^ *$") and - not match(line,"^%%%%Invocation") and + not match(line,"^%%%%Invocation") and not match(line,"^%%%%%+") then line = gsub(line,"%/ID( ?)%[<[^>]+><[^>]+>]","/ID%1[<ID-STRING><ID-STRING>]") new_content = new_content .. line .. os_newline @@ -550,10 +554,19 @@ local function normalize_pdf(content) return new_content end +function rewrite_log(source, result, engine, errlevels) + return rewrite(source, result, normalize_log, engine, errlevels) +end + +function rewrite_pdf(source, result, engine, errlevels) + return rewrite(source, result, normalize_pdf, engine, errlevels) +end + -- Run one test which may have multiple engine-dependent comparisons -- Should create a difference file for each failed test function runcheck(name, hide) - if not testexists(name) then + local test_filename, kind = testexists(name) + if not test_filename then print("Failed to find input for test " .. name) return 1 end @@ -562,9 +575,10 @@ function runcheck(name, hide) checkengines = options["engine"] end -- Used for both .lvt and .pvt tests - local function check_and_diff(ext,engine,comp,pdftest) - runtest(name,engine,hide,ext,pdftest,true) - local errorlevel = comp(name,engine) + local test_type = test_types[kind] + local function check_and_diff(engine) + runtest(name, engine, hide, test_type.test, test_type, true) + local errorlevel = base_compare(test_type,name,engine) if errorlevel == 0 then return errorlevel end @@ -579,12 +593,7 @@ function runcheck(name, hide) local errorlevel = 0 for _,engine in pairs(checkengines) do setup_check(name,engine) - local errlevel = 0 - if fileexists(testfiledir .. "/" .. name .. pvtext) then - errlevel = check_and_diff(pvtext,engine,compare_pdf,true) - else - errlevel = check_and_diff(lvtext,engine,compare_tlg) - end + local errlevel = check_and_diff(engine) if errlevel ~= 0 and options["halt-on-error"] then return 1 end @@ -598,64 +607,73 @@ end function setup_check(name, engine) local testname = name .. "." .. engine - local tlgfile = locate( - {testfiledir, unpackdir}, - {testname .. tlgext, name .. tlgext} - ) - local tpffile = locate( - {testfiledir, unpackdir}, - {testname .. tpfext, name .. tpfext} - ) - -- Attempt to generate missing reference file from expectation - if not (tlgfile or tpffile) then - if not locate({unpackdir, testfiledir}, {name .. lveext}) then - print( - "Error: failed to find " .. tlgext .. ", " .. tpfext .. " or " - .. lveext .. " file for " .. name .. "!" + local found + for _, kind in ipairs(test_order) do + local reference_ext = test_types[kind].reference + local reference_file = locate( + {testfiledir, unpackdir}, + {testname .. reference_ext, name .. reference_ext} + ) + if reference_file then + found = true + -- Install comparison file found + cp( + match(reference_file, ".*/(.*)"), + match(reference_file, "(.*)/.*"), + testdir ) - exit(1) end - runtest(name, engine, true, lveext) - ren(testdir, testname .. logext, testname .. tlgext) - else - -- Install comparison files found - for _,v in pairs({tlgfile, tpffile}) do - if v then - cp( - match(v, ".*/(.*)"), - match(v, "(.*)/.*"), - testdir - ) - end + end + if found then + return + end + -- Attempt to generate missing reference file from expectation + for _, kind in ipairs(test_order) do + local test_type = test_types[kind] + local exp_ext = test_type.expectation + local expectation_file = exp_ext and locate( + {testfiledir, unpackdir}, + {name .. exp_ext} + ) + if expectation_file then + found = true + runtest(name, engine, true, exp_ext, test_type) + ren(testdir, testname .. test_type.generated, testname .. test_type.reference) end end + if found then + return + end + print( + "Error: failed to find any reference or expectation file for " + .. name .. "!" + ) + exit(1) end -function compare_pdf(name,engine,cleanup) +function base_compare(test_type,name,engine,cleanup) local testname = name .. "." .. engine - local difffile = testdir .. "/" .. testname .. pdfext .. os_diffext - local pdffile = testdir .. "/" .. testname .. pdfext - local tpffile = locate({testdir}, {testname .. tpfext, name .. tpfext}) - if not tpffile then + local difffile = testdir .. "/" .. testname.. os_diffext + local genfile = testdir .. "/" .. testname .. test_type.generated + local reffile = locate({testdir}, {testname .. test_type.reference, name .. test_type.reference}) + if not reffile then return 1 end + local compare = test_type.compare + if compare then + return compare(difffile, reffile, genfile, cleanup, name, engine) + end local errorlevel = execute(os_diffexe .. " " - .. normalize_path(tpffile .. " " .. pdffile .. " > " .. difffile)) + .. normalize_path(reffile .. " " .. genfile .. " > " .. difffile)) if errorlevel == 0 or cleanup then remove(difffile) end return errorlevel end -function compare_tlg(name,engine,cleanup) +function compare_tlg(difffile, tlgfile, logfile, cleanup, name, engine) local errorlevel local testname = name .. "." .. engine - local difffile = testdir .. "/" .. testname .. os_diffext - local logfile = testdir .. "/" .. testname .. logext - local tlgfile = locate({testdir}, {testname .. tlgext, name .. tlgext}) - if not tlgfile then - return 1 - end -- Do additional log formatting if the engine is LuaTeX, there is no -- LuaTeX-specific .tlg file and the default engine is not LuaTeX if (match(engine,"^lua") or match(engine,"^harf")) @@ -687,22 +705,22 @@ end -- Run one of the test files: doesn't check the result so suitable for -- both creating and verifying -function runtest(name, engine, hide, ext, pdfmode, breakout) +function runtest(name, engine, hide, ext, test_type, breakout) local lvtfile = name .. (ext or lvtext) cp(lvtfile, fileexists(testfiledir .. "/" .. lvtfile) and testfiledir or unpackdir, testdir) local checkopts = checkopts - local engine = engine or stdengine + engine = engine or stdengine local binary = engine local format = gsub(engine,"tex$",checkformat) -- Special binary/format combos - if specialformats[checkformat] and next(specialformats[checkformat]) then - local t = specialformats[checkformat] - if t[engine] and next(t[engine]) then - local t = t[engine] - binary = t.binary or binary - checkopts = t.options or checkopts - format = t.format or format + local special_check = specialformats[checkformat] + if special_check and next(special_check) then + local engine_info = special_check[engine] + if engine_info then + binary = engine_info.binary or binary + format = engine_info.format or format + checkopts = engine_info.options or checkopts end end -- Finalise format string @@ -710,7 +728,7 @@ function runtest(name, engine, hide, ext, pdfmode, breakout) format = " --fmt=" .. format end -- Special casing for XeTeX engine - if match(engine, "xetex") and not pdfmode then + if match(engine, "xetex") and test_type.generated ~= pdfext then checkopts = checkopts .. " -no-pdf" end -- Special casing for ConTeXt @@ -721,10 +739,8 @@ function runtest(name, engine, hide, ext, pdfmode, breakout) function setup(file) return ' "' .. file .. '" ' end end local basename = testdir .. "/" .. name - local logfile = basename .. logext - local newfile = basename .. "." .. engine .. logext - local pdffile = basename .. pdfext - local npffile = basename .. "." .. engine .. pdfext + local gen_file = basename .. test_type.generated + local new_file = basename .. "." .. engine .. test_type.generated local asciiopt = "" for _,i in ipairs(asciiengines) do if binary == i then @@ -737,7 +753,7 @@ function runtest(name, engine, hide, ext, pdfmode, breakout) rm(testdir,filetype) end -- Ensure there is no stray .log file - rm(testdir,name .. logext) + rmfile(testdir,name .. logext) local errlevels = {} local localtexmf = "" if texmfdir and texmfdir ~= "" and direxists(texmfdir) then @@ -760,7 +776,7 @@ function runtest(name, engine, hide, ext, pdfmode, breakout) -- Allow for local texmf files os_setenv .. " TEXMFCNF=." .. os_pathsep .. os_concat .. - (forcecheckepoch and setepoch() or "") .. + set_epoch_cmd(epoch, forcecheckepoch) .. -- Ensure lines are of a known length os_setenv .. " max_print_line=" .. maxprintline .. os_concat .. @@ -773,47 +789,34 @@ function runtest(name, engine, hide, ext, pdfmode, breakout) ) -- Break the loop if the result is stable if breakout and i < checkruns then - if pdfmode then + if test_type.generated == pdfext then if fileexists(testdir .. "/" .. name .. dviext) then dvitopdf(name, testdir, engine, hide) end - rewrite(pdffile,npffile,normalize_pdf) - if compare_pdf(name,engine,true) == 0 then - break - end - else - rewrite(logfile,newfile,normalize_log,engine,errlevels) - if compare_tlg(name,engine,true) == 0 then - break - end + end + test_type.rewrite(gen_file,new_file,engine,errlevels) + if base_compare(test_type,name,engine,true) == 0 then + break end end end - if pdfmode and fileexists(testdir .. "/" .. name .. dviext) then - dvitopdf(name, testdir, engine, hide) - end - if pdfmode then + if test_type.generated == pdfext then + if fileexists(testdir .. "/" .. name .. dviext) then + dvitopdf(name, testdir, engine, hide) + end cp(name .. pdfext,testdir,resultdir) ren(resultdir,name .. pdfext,name .. "." .. engine .. pdfext) - rewrite(pdffile,npffile,normalize_pdf) - else - rewrite(logfile,newfile,normalize_log,engine,errlevels) end + test_type.rewrite(gen_file,new_file,engine,errlevels) -- Store secondary files for this engine for _,filetype in pairs(auxfiles) do for _,file in pairs(filelist(testdir, filetype)) do - if match(file,"^" .. name .. ".[^.]+$") then - local ext = match(file, "%.[^.]+$") - if ext ~= lvtext and - ext ~= tlgext and - ext ~= lveext and - ext ~= logext then - local newname = gsub(file,"(%.[^.]+)$","." .. engine .. "%1") - if fileexists(testdir,newname) then - rm(testdir,newname) - end - ren(testdir,file,newname) + if match(file,"^" .. name .. "%.[^.]+$") then + local newname = gsub(file,"(%.[^.]+)$","." .. engine .. "%1") + if fileexists(testdir .. "/" .. newname) then + rmfile(testdir,newname) end + ren(testdir,file,newname) end end end @@ -821,14 +824,25 @@ function runtest(name, engine, hide, ext, pdfmode, breakout) end -- A hook to allow additional tasks to run for the tests -runtest_tasks = runtest_tasks or function(name,run) +function runtest_tasks(name,run) return "" end -- Look for a test: could be in the testfiledir or the unpackdir function testexists(test) - return(locate({testfiledir, unpackdir}, - {test .. lvtext, test .. pvtext})) + local filenames = {} + for i, kind in ipairs(test_order) do + filenames[i] = test .. test_types[kind].test + end + local found = locate({testfiledir, unpackdir}, filenames) + if found then + for i, kind in ipairs(test_order) do + local filename = filenames[i] + if found:sub(-#filename) == filename then + return found, kind + end + end + end end function check(names) @@ -844,36 +858,42 @@ function check(names) names = names or { } -- No names passed: find all test files if not next(names) then - local excludenames = { } - for _,glob in pairs(excludetests) do - for _,name in pairs(filelist(testfiledir, glob .. lvtext)) do - excludenames[jobname(name)] = true - end - for _,name in pairs(filelist(unpackdir, glob .. lvtext)) do - excludenames[jobname(name)] = true - end - for _,name in pairs(filelist(testfiledir, glob .. pvtext)) do - excludenames[jobname(name)] = true - end - end - local function addname(name) - if not excludenames[jobname(name)] then - insert(names,jobname(name)) - end - end - for _,glob in pairs(includetests) do - for _,name in pairs(filelist(testfiledir, glob .. lvtext)) do - addname(name) - end - for _,name in pairs(filelist(testfiledir, glob .. pvtext)) do - addname(name) + for _, kind in ipairs(test_order) do + local ext = test_types[kind].test + local excludepatterns = { } + local num_exclude = 0 + for _,glob in pairs(excludetests) do + num_exclude = num_exclude+1 + excludepatterns[num_exclude] = glob_to_pattern(glob .. ext) end - for _,name in pairs(filelist(unpackdir, glob .. lvtext)) do - if fileexists(testfiledir .. "/" .. name) then - print("Duplicate test file: " .. i) - return 1 + for _,glob in pairs(includetests) do + for _,name in pairs(filelist(testfiledir, glob .. ext)) do + local exclude + for i=1, num_exclude do + if match(name, excludepatterns[i]) then + exclude = true + break + end + end + if not exclude then + insert(names,jobname(name)) + end + end + for _,name in pairs(filelist(unpackdir, glob .. ext)) do + local exclude + for i=1, num_exclude do + if not match(name, excludepatterns[i]) then + exclude = true + break + end + end + if not exclude then + if fileexists(testfiledir .. "/" .. name) then + return 1 + end + insert(names,jobname(name)) + end end - addname(name) end end sort(names) @@ -904,24 +924,17 @@ function check(names) end end end - -- https://stackoverflow.com/a/32167188 - local function shuffle(tbl) - local len, random = #tbl, rnd - for i = len, 2, -1 do - local j = random(1, i) - tbl[i], tbl[j] = tbl[j], tbl[i] - end - return tbl - end if options["shuffle"] then - names = shuffle(names) + -- https://stackoverflow.com/a/32167188 + for i = #names, 2, -1 do + local j = rnd(1, i) + names[i], names[j] = names[j], names[i] + end end -- Actually run the tests print("Running checks on") - local i = 0 - for _,name in ipairs(names) do - i = i + 1 - print(" " .. name .. " (" .. i.. "/" .. #names ..")") + for i, name in ipairs(names) do + print(" " .. name .. " (" .. i .. "/" .. #names ..")") local errlevel = runcheck(name, hide) -- Return value must be 1 not errlevel if errlevel ~= 0 then @@ -988,39 +1001,31 @@ function save(names) return 1 end for _,name in pairs(names) do - if testexists(name) then - for _,engine in pairs(engines) do - local testengine = ((engine == stdengine and "") or "." .. engine) - local function save_test(test_ext,gen_ext,out_ext,pdfmode) - local out_file = name .. testengine .. out_ext - local gen_file = name .. "." .. engine .. gen_ext - print("Creating and copying " .. out_file) - runtest(name,engine,false,test_ext,pdfmode) - ren(testdir,gen_file,out_file) - cp(out_file,testdir,testfiledir) - if fileexists(unpackdir .. "/" .. out_file) then - print("Saved " .. out_ext - .. " file overrides unpacked version of the same name") - return 1 - end - return 0 - end - local errorlevel - if fileexists(testfiledir .. "/" .. name .. lvtext) then - errorlevel = save_test(lvtext,logext,tlgext) - else - errorlevel = save_test(pvtext,pdfext,tpfext,true) - end - if errorlevel ~=0 then return errorlevel end - end - elseif locate({unpackdir, testfiledir}, {name .. lveext}) then - print("Saved " .. tlgext .. " file overrides a " - .. lveext .. " file of the same name") - return 1 - else + local test_filename, kind = testexists(name) + if not test_filename then print('Test "' .. name .. '" not found') return 1 end + local test_type = test_types[kind] + if locate({unpackdir, testfiledir}, {name .. test_type.expectation}) then + print("Saved " .. test_type.test .. " file would override a " + .. test_type.expectation .. " file of the same name") + return 1 + end + for _,engine in pairs(engines) do + local testengine = engine == stdengine and "" or ("." .. engine) + local out_file = name .. testengine .. test_type.reference + local gen_file = name .. "." .. engine .. test_type.generated + print("Creating and copying " .. out_file) + runtest(name, engine, false, test_type.test, test_type) + ren(testdir, gen_file, out_file) + cp(out_file, testdir, testfiledir) + if fileexists(unpackdir .. "/" .. test_type.reference) then + print("Saved " .. test_type.reference + .. " file overrides unpacked version of the same name") + return 1 + end + end end return 0 end diff --git a/Master/texmf-dist/scripts/l3build/l3build-clean.lua b/Master/texmf-dist/scripts/l3build/l3build-clean.lua index 27ed42be9c8..f548a392eb9 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-clean.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-clean.lua @@ -1,6 +1,6 @@ --[[ -File l3build-clean.lua Copyright (C) 2018,2020 The LaTeX3 Project +File l3build-clean.lua Copyright (C) 2018,2020 The LaTeX Project It may be distributed and/or modified under the conditions of the LaTeX Project Public License (LPPL), either version 1.3c of this @@ -22,36 +22,46 @@ for those people who are interested. --]] +local pairs = pairs +local ipairs = ipairs +local insert = table.insert + -- Remove all generated files function clean() -- To make sure that distribdir never contains any stray subdirs, -- it is entirely removed then recreated rather than simply deleting -- all of the files - local errorlevel = - rmdir(distribdir) + - mkdir(distribdir) + - cleandir(localdir) + - cleandir(testdir) + - cleandir(typesetdir) + - cleandir(unpackdir) + local errorlevel = rmdir(distribdir) + + mkdir(distribdir) + + cleandir(localdir) + + cleandir(testdir) + + cleandir(typesetdir) + + cleandir(unpackdir) if errorlevel ~= 0 then return errorlevel end - local clean_list = { } for _,dir in pairs(remove_duplicates({maindir,sourcefiledir,docfiledir})) do + local clean_list = {} + local flags = {} for _,glob in pairs(cleanfiles) do - for file,_ in pairs(tree(dir,glob)) do - clean_list[file] = true + for _,p in ipairs(tree(dir,glob)) do + insert(clean_list, p.src) + flags[p.src] = true end end for _,glob in pairs(sourcefiles) do - for file,_ in pairs(tree(dir,glob)) do - clean_list[file] = nil + for _,p in ipairs(tree(dir,glob)) do + flags[p.src] = nil end end - for file,_ in pairs(clean_list) do - errorlevel = rm(dir,file) - if errorlevel ~= 0 then return errorlevel end + for i = #clean_list, 1, -1 do + local p_src = clean_list[i] + if flags[p_src] then + errorlevel = rm(dir,p_src) + if errorlevel ~= 0 then + return errorlevel + end + end end end @@ -63,10 +73,8 @@ function bundleclean() for _,i in ipairs(cleanfiles) do errorlevel = rm(currentdir, i) + errorlevel end - return ( - errorlevel + - rmdir(ctandir) + - rmdir(tdsdir) - ) + return errorlevel + + rmdir(ctandir) + + rmdir(tdsdir) end diff --git a/Master/texmf-dist/scripts/l3build/l3build-ctan.lua b/Master/texmf-dist/scripts/l3build/l3build-ctan.lua index df1daf97287..fd1491dd584 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-ctan.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-ctan.lua @@ -1,6 +1,6 @@ --[[ -File l3build-ctan.lua Copyright (C) 2018-2020 The LaTeX3 Project +File l3build-ctan.lua Copyright (C) 2018-2020 The LaTeX Project It may be distributed and/or modified under the conditions of the LaTeX Project Public License (LPPL), either version 1.3c of this @@ -38,12 +38,12 @@ function copyctan() end else for _,filetype in pairs(files) do - for file,_ in pairs(tree(source,filetype)) do - local path = splitpath(file) + for _,p in ipairs(tree(source,filetype)) do + local path = dirname(p.src) local ctantarget = ctandir .. "/" .. ctanpkg .. "/" .. source .. "/" .. path mkdir(ctantarget) - cp(file,source,ctantarget) + cp(p.src,source,ctantarget) end end end diff --git a/Master/texmf-dist/scripts/l3build/l3build-file-functions.lua b/Master/texmf-dist/scripts/l3build/l3build-file-functions.lua index 7d13fc933ef..6ca1cc60111 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-file-functions.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-file-functions.lua @@ -1,6 +1,6 @@ --[[ -File l3build-file-functions.lua Copyright (C) 2018-2020 The LaTeX3 Project +File l3build-file-functions.lua Copyright (C) 2018-2020 The LaTeX Project It may be distributed and/or modified under the conditions of the LaTeX Project Public License (LPPL), either version 1.3c of this @@ -36,7 +36,6 @@ local execute = os.execute local exit = os.exit local getenv = os.getenv local remove = os.remove -local os_time = os.time local os_type = os.type local luatex_revision = status.luatex_revision @@ -171,14 +170,19 @@ function normalize_path(path) end -- Return an absolute path from a relative one +-- Due to chdir, path must exist and be accessible. function abspath(path) local oldpwd = currentdir() - chdir(path) - local result = currentdir() - chdir(oldpwd) - return escapepath(gsub(result, "\\", "/")) + local ok, msg = chdir(path) + if ok then + local result = currentdir() + chdir(oldpwd) + return escapepath(gsub(result, "\\", "/")) + end + error(msg) end +-- TODO: Fix the cross platform problem function escapepath(path) if os_type == "windows" then local path,count = gsub(path,'"','') @@ -210,22 +214,25 @@ end -- Copy files 'quietly' function cp(glob, source, dest) local errorlevel - for i,_ in pairs(tree(source, glob)) do - local source = source .. "/" .. i + for _,p in ipairs(tree(source, glob)) do + -- p_src is a path relative to `source` whereas + -- p_cwd is the counterpart relative to the current working directory if os_type == "windows" then - if attributes(source)["mode"] == "directory" then + if attributes(p.cwd, "mode") == "directory" then errorlevel = execute( - 'xcopy /y /e /i "' .. unix_to_win(source) .. '" "' - .. unix_to_win(dest .. '/' .. i) .. '" > nul' - ) + 'xcopy /y /e /i "' .. unix_to_win(p.cwd) .. '" "' + .. unix_to_win(dest .. '/' .. p.src) .. '" > nul' + ) and 0 or 1 else errorlevel = execute( - 'xcopy /y "' .. unix_to_win(source) .. '" "' + 'xcopy /y "' .. unix_to_win(p.cwd) .. '" "' .. unix_to_win(dest .. '/') .. '" > nul' - ) + ) and 0 or 1 end else - errorlevel = execute("cp -RLf '" .. source .. "' '" .. dest .. "'") + errorlevel = execute( + "cp -RLf '" .. p.cwd .. "' '" .. dest .. "'" + ) and 0 or 1 end if errorlevel ~=0 then return errorlevel @@ -255,7 +262,7 @@ function fileexists(file) f:close() return true else - return false + return false -- also file exits and is not readable end end @@ -283,54 +290,73 @@ function filelist(path, glob) return files end --- Does what filelist does, but can also glob subdirectories. In the returned --- table, the keys are paths relative to the given starting path, the values --- are their counterparts relative to the current working directory. -function tree(path, glob) +---@class tree_entry_t +---@field src string path relative to the source directory +---@field cwd string path counterpart relative to the current working directory + +---Does what filelist does, but can also glob subdirectories. +---In the returned table, the keys are paths relative to the given source path, +---the values are their counterparts relative to the current working directory. +---@param src_path string +---@param glob string +---@return table<integer,tree_entry_t> +function tree(src_path, glob) local function cropdots(path) - return gsub(gsub(path, "^%./", ""), "/%./", "/") + return path:gsub( "^%./", ""):gsub("/%./", "/") end + src_path = cropdots(src_path) + glob = cropdots(glob) local function always_true() return true end local function is_dir(file) - return attributes(file)["mode"] == "directory" + return attributes(file, "mode") == "directory" end - local dirs = {["."] = cropdots(path)} - for pattern, criterion in gmatch(cropdots(glob), "([^/]+)(/?)") do - local criterion = criterion == "/" and is_dir or always_true - function fill(path, dir, table) - for _, file in ipairs(filelist(dir, pattern)) do - local fullpath = path .. "/" .. file - if file ~= "." and file ~= ".." and - fullpath ~= builddir - then - local fulldir = dir .. "/" .. file - if criterion(fulldir) then - table[fullpath] = fulldir + ---@type table<integer,tree_entry_t> + local result = { { + src = ".", + cwd = src_path, + } } + for glob_part, sep in glob:gmatch("([^/]+)(/?)/*") do + local accept = sep == "/" and is_dir or always_true + ---Feeds the given table according to `glob_part` + ---@param p tree_entry_t path counterpart relative to the current working directory + ---@param table table + local function fill(p, table) + for _,file in ipairs(filelist(p.cwd, glob_part)) do + if file ~= "." and file ~= ".." then + local pp = { + src = p.src .. "/" .. file, + cwd = p.cwd .. "/" .. file, + } + if pp.cwd ~= builddir -- TODO: ensure that `builddir` is properly formatted + and accept(pp.cwd) + then + insert(table, pp) end end end end - local newdirs = {} - if pattern == "**" then + local new_result = {} + if glob_part == "**" then + local i = 1 while true do - path, dir = next(dirs) - if not path then + local p = result[i] + i = i + 1 + if not p then break end - dirs[path] = nil - newdirs[path] = dir - fill(path, dir, dirs) + insert(new_result, p) -- shorter path + fill(p, result) -- after longer end else - for path, dir in pairs(dirs) do - fill(path, dir, newdirs) + for _,p in ipairs(result) do + fill(p, new_result) end end - dirs = newdirs + result = new_result end - return dirs + return result end function remove_duplicates(a) @@ -353,7 +379,7 @@ function mkdir(dir) if os_type == "windows" then -- Windows (with the extensions) will automatically make directory trees -- but issues a warning if the dir already exists: avoid by including a test - local dir = unix_to_win(dir) + dir = unix_to_win(dir) return execute( "if not exist " .. dir .. "\\nul " .. "mkdir " .. dir ) @@ -364,10 +390,10 @@ end -- Rename function ren(dir, source, dest) - local dir = dir .. "/" + dir = dir .. "/" if os_type == "windows" then - local source = gsub(source, "^%.+/", "") - local dest = gsub(dest, "^%.+/", "") + source = gsub(source, "^%.+/", "") + dest = gsub(dest, "^%.+/", "") return execute("ren " .. unix_to_win(dir) .. source .. " " .. dest) else return execute("mv " .. dir .. source .. " " .. dir .. dest) @@ -376,8 +402,8 @@ end -- Remove file(s) based on a glob function rm(source, glob) - for i,_ in pairs(tree(source, glob)) do - rmfile(source,i) + for _,p in ipairs(tree(source, glob)) do + rmfile(source,p.src) end -- os.remove doesn't give a sensible errorlevel return 0 diff --git a/Master/texmf-dist/scripts/l3build/l3build-help.lua b/Master/texmf-dist/scripts/l3build/l3build-help.lua index 055fc1a9186..6431ac547ef 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-help.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-help.lua @@ -1,6 +1,6 @@ --[[ -File l3build-help.lua Copyright (C) 2018,2020 The LaTeX3 Project +File l3build-help.lua Copyright (C) 2018,2020 The LaTeX Project It may be distributed and/or modified under the conditions of the LaTeX Project Public License (LPPL), either version 1.3c of this @@ -32,14 +32,14 @@ function version() "\n" .. "l3build: A testing and building system for LaTeX\n\n" .. "Release " .. release_date .. "\n" .. - "Copyright (C) 2014-2020 The LaTeX3 Project" + "Copyright (C) 2014-2020 The LaTeX Project" ) end function help() local function setup_list(list) local longest = 0 - for k,v in pairs(list) do + for k,_ in pairs(list) do if k:len() > longest then longest = k:len() end @@ -70,7 +70,7 @@ function help() end print("") print("Valid options are:") - local longest,t = setup_list(option_list) + longest,t = setup_list(option_list) for _,k in ipairs(t) do local opt = option_list[k] local filler = rep(" ", longest - k:len() + 1) @@ -87,5 +87,5 @@ function help() print("") print("Repository : https://github.com/latex3/l3build") print("Bug tracker : https://github.com/latex3/l3build/issues") - print("Copyright (C) 2014-2020 The LaTeX3 Project") + print("Copyright (C) 2014-2020 The LaTeX Project") end diff --git a/Master/texmf-dist/scripts/l3build/l3build-install.lua b/Master/texmf-dist/scripts/l3build/l3build-install.lua index 77c15fd0e25..f4839fdca77 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-install.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-install.lua @@ -1,6 +1,6 @@ --[[ -File l3build-install.lua Copyright (C) 2018-2020 The LaTeX3 Project +File l3build-install.lua Copyright (C) 2018-2020 The LaTeX Project It may be distributed and/or modified under the conditions of the LaTeX Project Public License (LPPL), either version 1.3c of this @@ -37,7 +37,9 @@ local insert = table.insert local function gethome() set_program("latex") - return abspath(options["texmfhome"] or var_value("TEXMFHOME")) + local result = options["texmfhome"] or var_value("TEXMFHOME") + mkdir(result) + return abspath(result) end function uninstall() @@ -68,15 +70,15 @@ function uninstall() -- Any script man files need special handling local manfiles = { } for _,glob in pairs(scriptmanfiles) do - for file,_ in pairs(tree(docfiledir,glob)) do + for _,p in ipairs(tree(docfiledir,glob)) do -- Man files should have a single-digit extension: the type - local installdir = gethome() .. "/doc/man/man" .. match(file,".$") - if fileexists(installdir .. "/" .. file) then + local installdir = gethome() .. "/doc/man/man" .. match(p.src,".$") + if fileexists(installdir .. "/" .. p.src) then if options["dry-run"] then - insert(manfiles,"man" .. match(file,".$") .. "/" .. - select(2,splitpath(file))) + insert(manfiles,"man" .. match(p.src,".$") .. "/" .. + select(2,splitpath(p.src))) else - errorlevel = errorlevel + rm(installdir,file) + errorlevel = errorlevel + rm(installdir,p.src) end end end @@ -97,7 +99,7 @@ function uninstall() if errorlevel ~= 0 then return errorlevel end -- Finally, clean up special locations for _,location in ipairs(tdslocations) do - local path,glob = splitpath(location) + local path = dirname(location) errorlevel = zapdir(path) if errorlevel ~= 0 then return errorlevel end end @@ -127,9 +129,9 @@ function install_files(target,full,dry_run) -- Generate a file list and include the directory for _,glob_table in pairs(files) do for _,glob in pairs(glob_table) do - for file,_ in pairs(tree(source,glob)) do + for _,p in ipairs(tree(source,glob)) do -- Just want the name - local path,filename = splitpath(file) + local path,filename = splitpath(p.src) local sourcepath = "/" if path == "." then sourcepaths[filename] = source @@ -140,11 +142,11 @@ function install_files(target,full,dry_run) end local matched = false for _,location in ipairs(tdslocations) do - local path,glob = splitpath(location) - local pattern = glob_to_pattern(glob) + local l_dir,l_glob = splitpath(location) + local pattern = glob_to_pattern(l_glob) if match(filename,pattern) then - insert(paths,path) - insert(filenames,path .. sourcepath .. filename) + insert(paths,l_dir) + insert(filenames,l_dir .. sourcepath .. filename) matched = true break end @@ -161,20 +163,20 @@ function install_files(target,full,dry_run) -- The target is only created if there are actual files to install if next(filenames) then if not dry_run then - for _,path in pairs(paths) do - local dir = target .. "/" .. path - if not cleanpaths[dir] then - errorlevel = cleandir(dir) + for _,path in ipairs(paths) do + local target_path = target .. "/" .. path + if not cleanpaths[target_path] then + errorlevel = cleandir(target_path) if errorlevel ~= 0 then return errorlevel end end - cleanpaths[dir] = true + cleanpaths[target_path] = true end end - for _,file in ipairs(filenames) do + for _,name in ipairs(filenames) do if dry_run then - print("- " .. file) + print("- " .. name) else - local path,file = splitpath(file) + local path,file = splitpath(name) insert(installmap, {file = file, source = sourcepaths[file], dest = target .. "/" .. path}) end @@ -187,30 +189,30 @@ function install_files(target,full,dry_run) if errorlevel ~= 0 then return errorlevel end -- Creates a 'controlled' list of files - local function excludelist(dir,include,exclude) + local function create_file_list(dir,include,exclude) + dir = dir or currentdir include = include or { } exclude = exclude or { } - dir = dir or currentdir - local includelist = { } local excludelist = { } for _,glob_table in pairs(exclude) do for _,glob in pairs(glob_table) do - for file,_ in pairs(tree(dir,glob)) do - excludelist[file] = true + for _,p in ipairs(tree(dir,glob)) do + excludelist[p.src] = true end end end + local result = { } for _,glob in pairs(include) do - for file,_ in pairs(tree(dir,glob)) do - if not excludelist[file] then - insert(includelist, file) + for _,p in ipairs(tree(dir,glob)) do + if not excludelist[p.src] then + insert(result, p.src) end end end - return includelist + return result end - local installlist = excludelist(unpackdir,installfiles,{scriptfiles}) + local installlist = create_file_list(unpackdir,installfiles,{scriptfiles}) if full then errorlevel = doc() @@ -229,8 +231,8 @@ function install_files(target,full,dry_run) end -- Set up lists: global as they are also needed to do CTAN releases - typesetlist = excludelist(docfiledir,typesetfiles,{sourcefiles}) - sourcelist = excludelist(sourcefiledir,sourcefiles, + typesetlist = create_file_list(docfiledir,typesetfiles,{sourcefiles}) + sourcelist = create_file_list(sourcefiledir,sourcefiles, {bstfiles,installfiles,makeindexfiles,scriptfiles}) if dry_run then @@ -255,15 +257,15 @@ function install_files(target,full,dry_run) -- Any script man files need special handling local manfiles = { } for _,glob in pairs(scriptmanfiles) do - for file,_ in pairs(tree(docfiledir,glob)) do + for _,p in ipairs(tree(docfiledir,glob)) do if dry_run then - insert(manfiles,"man" .. match(file,".$") .. "/" .. - select(2,splitpath(file))) + insert(manfiles,"man" .. match(p.src,".$") .. "/" .. + select(2,splitpath(p.src))) else -- Man files should have a single-digit extension: the type - local installdir = target .. "/doc/man/man" .. match(file,".$") + local installdir = target .. "/doc/man/man" .. match(p.src,".$") errorlevel = errorlevel + mkdir(installdir) - errorlevel = errorlevel + cp(file,docfiledir,installdir) + errorlevel = errorlevel + cp(p.src,docfiledir,installdir) end end end diff --git a/Master/texmf-dist/scripts/l3build/l3build-manifest-setup.lua b/Master/texmf-dist/scripts/l3build/l3build-manifest-setup.lua index 26bf1ca008a..9fcf4937297 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-manifest-setup.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-manifest-setup.lua @@ -1,6 +1,6 @@ --[[ -File l3build-manifest-setup.lua Copyright (C) 2018,2020 The LaTeX3 Project +File l3build-manifest-setup.lua Copyright (C) 2018,2020 The LaTeX Project It may be distributed and/or modified under the conditions of the LaTeX Project Public License (LPPL), either version 1.3c of this @@ -54,7 +54,7 @@ for those people who are interested. --]] -manifest_setup = manifest_setup or function() +function manifest_setup() local groups = { { subheading = "Repository manifest", @@ -221,18 +221,16 @@ end --]] manifest_sort_within_match = manifest_sort_within_match or function(files) - local f = files - table.sort(f) - return f + table.sort(files) + return files end manifest_sort_within_group = manifest_sort_within_group or function(files) - local f = files --[[ -- no-op by default; make your own definition to customise. E.g.: - table.sort(f) + table.sort(files) --]] - return f + return files end --[[ @@ -240,7 +238,7 @@ end --------------- --]] -manifest_write_opening = manifest_write_opening or function(filehandle) +function manifest_write_opening(filehandle) filehandle:write("# Manifest for " .. module .. "\n\n") filehandle:write([[ @@ -250,7 +248,7 @@ It is automatically generated with `texlua build.lua manifest`. end -manifest_write_subheading = manifest_write_subheading or function(filehandle,heading,description) +function manifest_write_subheading(filehandle,heading,description) filehandle:write("\n\n## " .. heading .. "\n\n") @@ -260,7 +258,7 @@ manifest_write_subheading = manifest_write_subheading or function(filehandle,hea end -manifest_write_group_heading = manifest_write_group_heading or function (filehandle,heading,description) +function manifest_write_group_heading(filehandle,heading,description) filehandle:write("\n### " .. heading .. "\n\n") @@ -270,7 +268,7 @@ manifest_write_group_heading = manifest_write_group_heading or function (filehan end -manifest_write_group_file = manifest_write_group_file or function(filehandle,filename,param) +function manifest_write_group_file(filehandle,filename,param) --[[ filehandle : write file object filename : the count of the filename to be written @@ -296,7 +294,7 @@ manifest_write_group_file = manifest_write_group_file or function(filehandle,fil end -manifest_write_group_file_descr = manifest_write_group_file_descr or function(filehandle,filename,descr,param) +function manifest_write_group_file_descr(filehandle,filename,descr,param) --[[ filehandle : write file object filename : the name of the file to write @@ -326,7 +324,7 @@ end ------------------------------------------- --]] -manifest_extract_filedesc = manifest_extract_filedesc or function(filehandle) +function manifest_extract_filedesc(filehandle) -- no-op by default; two examples below diff --git a/Master/texmf-dist/scripts/l3build/l3build-manifest.lua b/Master/texmf-dist/scripts/l3build/l3build-manifest.lua index dc5138952c8..f9b1c4e82e9 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-manifest.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-manifest.lua @@ -1,6 +1,6 @@ --[[ -File l3build-manifest.lua Copyright (C) 2018,2020 The LaTeX3 Project +File l3build-manifest.lua Copyright (C) 2018,2020 The LaTeX Project It may be distributed and/or modified under the conditions of the LaTeX Project Public License (LPPL), either version 1.3c of this @@ -33,7 +33,7 @@ for those people who are interested. `l3build-manifest-setup.lua`. --]] -manifest = manifest or function() +function manifest() -- build list of ctan files ctanfiles = {} @@ -277,4 +277,3 @@ manifest_write_group = function(f,entry) end end - diff --git a/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua b/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua index 10303b71f71..80b5cc84be1 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua @@ -1,6 +1,6 @@ --[[ -File l3build-stdmain.lua Copyright (C) 2018-2020 The LaTeX3 Project +File l3build-stdmain.lua Copyright (C) 2018-2020 The LaTeX Project It may be distributed and/or modified under the conditions of the LaTeX Project Public License (LPPL), either version 1.3c of this @@ -22,6 +22,8 @@ for those people who are interested. --]] +local lfs = require("lfs") + local exit = os.exit local insert = table.insert @@ -49,7 +51,7 @@ target_list = bundlecheck = { func = check, - pre = function() + pre = function(names) if names then print("Bundle checks should not list test names") help() @@ -65,7 +67,7 @@ target_list = bundleunpack = { func = bundleunpack, - pre = function() return(depinstall(unpackdeps)) end + pre = function() return(dep_install(unpackdeps)) end }, -- Public targets check = @@ -149,7 +151,7 @@ target_list = -- The overall main function -- -function stdmain(target,names) +function main(target,names) -- Deal with unknown targets up-front if not target_list[target] then help() diff --git a/Master/texmf-dist/scripts/l3build/l3build-tagging.lua b/Master/texmf-dist/scripts/l3build/l3build-tagging.lua index a00180b34cb..e255aba5cdd 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-tagging.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-tagging.lua @@ -1,6 +1,6 @@ --[[ -File l3build-tagging.lua Copyright (C) 2018-2020 The LaTeX3 Project +File l3build-tagging.lua Copyright (C) 2018-2020 The LaTeX Project It may be distributed and/or modified under the conditions of the LaTeX Project Public License (LPPL), either version 1.3c of this @@ -28,7 +28,7 @@ local os_date = os.date local match = string.match local gsub = string.gsub -update_tag = update_tag or function(filename,content,tagname,tagdate) +function update_tag(filename,content,tagname,tagdate) return content end @@ -51,7 +51,7 @@ local function update_file_tag(file,tagname,tagdate) else local path = dirname(file) ren(path,filename,filename .. ".bak") - local f = assert(open(file,"w")) + f = assert(open(file,"w")) -- Convert line ends back if required during write -- Watch for the second return value! f:write((gsub(updated_content,"\n",os_newline))) @@ -71,8 +71,8 @@ function tag(tagnames) local errorlevel = 0 for _,dir in pairs(dirs) do for _,filetype in pairs(tagfiles) do - for file,_ in pairs(tree(dir,filetype)) do - errorlevel = update_file_tag(dir .. "/" .. file,tagname,tagdate) + for _,p in ipairs(tree(dir,filetype)) do + errorlevel = update_file_tag(dir .. "/" .. p.src,tagname,tagdate) if errorlevel ~= 0 then return errorlevel end @@ -81,4 +81,3 @@ function tag(tagnames) end return tag_hook(tagname,tagdate) end - diff --git a/Master/texmf-dist/scripts/l3build/l3build-typesetting.lua b/Master/texmf-dist/scripts/l3build/l3build-typesetting.lua index 540ac3f384a..716648f7fd4 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-typesetting.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-typesetting.lua @@ -1,6 +1,6 @@ --[[ -File l3build-typesetting.lua Copyright (C) 2018-2020 The LaTeX3 Project +File l3build-typesetting.lua Copyright (C) 2018-2020 The LaTeX Project It may be distributed and/or modified under the conditions of the LaTeX Project Public License (LPPL), either version 1.3c of this @@ -38,20 +38,20 @@ local os_type = os.type function dvitopdf(name, dir, engine, hide) run( dir, - (forcecheckepoch and setepoch() or "") .. + set_epoch_cmd(epoch, forcecheckepoch) .. "dvips " .. name .. dviext .. (hide and (" > " .. os_null) or "") .. os_concat .. - "ps2pdf " .. ps2pdfopt .. name .. psext + "ps2pdf " .. ps2pdfopt .. name .. psext .. (hide and (" > " .. os_null) or "") ) end -- An auxiliary used to set up the environmental variables function runcmd(cmd,dir,vars) - local dir = dir or "." - local dir = abspath(dir) - local vars = vars or {} + dir = dir or "." + dir = abspath(dir) + vars = vars or {} -- Allow for local texmf files local env = os_setenv .. " TEXMFCNF=." .. os_pathsep local localtexmf = "" @@ -68,7 +68,7 @@ function runcmd(cmd,dir,vars) for _,var in pairs(vars) do env = env .. os_concat .. os_setenv .. " " .. var .. "=" .. envpaths end - return run(dir,(forcedocepoch and setepoch() or "") .. env .. os_concat .. cmd) + return run(dir,set_epoch_cmd(epoch, forcedocepoch) .. env .. os_concat .. cmd) end function biber(name,dir) @@ -80,7 +80,7 @@ function biber(name,dir) end function bibtex(name,dir) - local dir = dir or "." + dir = dir or "." if fileexists(dir .. "/" .. name .. ".aux") then -- LaTeX always generates an .aux file, so there is a need to -- look inside it for a \citation line @@ -105,7 +105,7 @@ function bibtex(name,dir) end function makeindex(name,dir,inext,outext,logext,style) - local dir = dir or "." + dir = dir or "." if fileexists(dir .. "/" .. name .. inext) then if style == "" then style = nil end return runcmd(makeindexexe .. " " .. makeindexopts @@ -119,15 +119,15 @@ function makeindex(name,dir,inext,outext,logext,style) end function tex(file,dir,cmd) - local dir = dir or "." - local cmd = cmd or typesetexe .. typesetopts + dir = dir or "." + cmd = cmd or typesetexe .. " " .. typesetopts return runcmd(cmd .. " \"" .. typesetcmds .. "\\input " .. file .. "\"", dir,{"TEXINPUTS","LUAINPUTS"}) end local function typesetpdf(file,dir) - local dir = dir or "." + dir = dir or "." local name = jobname(file) print("Typesetting " .. name) local fn = typeset @@ -141,12 +141,12 @@ local function typesetpdf(file,dir) print(" ! Compilation failed") return errorlevel end - pdfname = name .. pdfext + local pdfname = name .. pdfext rm(docfiledir,pdfname) return cp(pdfname,dir,docfiledir) end -typeset = typeset or function(file,dir,exe) +function typeset(file,dir,exe) dir = dir or "." local errorlevel = tex(file,dir,exe) if errorlevel ~= 0 then @@ -168,7 +168,7 @@ typeset = typeset or function(file,dir,exe) end -- A hook to allow additional typesetting of demos -typeset_demo_tasks = typeset_demo_tasks or function() +function typeset_demo_tasks() return 0 end @@ -188,7 +188,7 @@ local function docinit() for _,file in pairs(typesetsuppfiles) do cp(file, supportdir, typesetdir) end - depinstall(typesetdeps) + dep_install(typesetdeps) unpack({sourcefiles, typesetsourcefiles}, {sourcefiledir, docfiledir}) -- Main loop for doc creation local errorlevel = typeset_demo_tasks() @@ -198,7 +198,7 @@ local function docinit() return docinit_hook() end -docinit_hook = docinit_hook or function() return 0 end +function docinit_hook() return 0 end -- Typeset all required documents -- Uses a set of dedicated auxiliaries that need to be available to others @@ -209,8 +209,8 @@ function doc(files) for _,typesetfiles in ipairs({typesetdemofiles,typesetfiles}) do for _,glob in pairs(typesetfiles) do for _,dir in ipairs({typesetdir,unpackdir}) do - for _,file in pairs(tree(dir,glob)) do - local path,srcname = splitpath(file) + for _,p in ipairs(tree(dir,glob)) do + local path,srcname = splitpath(p.cwd) local name = jobname(srcname) if not done[name] then local typeset = true @@ -226,7 +226,7 @@ function doc(files) end -- Now know if we should typeset this source if typeset then - local errorlevel = typesetpdf(srcname,path) + errorlevel = typesetpdf(srcname,path) if errorlevel ~= 0 then return errorlevel else @@ -240,4 +240,3 @@ function doc(files) end return 0 end - diff --git a/Master/texmf-dist/scripts/l3build/l3build-unpack.lua b/Master/texmf-dist/scripts/l3build/l3build-unpack.lua index 412b5e84da7..7ab21ee09d4 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-unpack.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-unpack.lua @@ -1,6 +1,6 @@ --[[ -File l3build-unpack.lua Copyright (C) 2018-2020 The LaTeX3 Project +File l3build-unpack.lua Copyright (C) 2018-2020 The LaTeX Project It may be distributed and/or modified under the conditions of the LaTeX Project Public License (LPPL), either version 1.3c of this @@ -22,12 +22,10 @@ for those people who are interested. --]] -local execute = os.execute - -- Unpack the package files using an 'isolated' system: this requires -- a copy of the 'basic' DocStrip program, which is used then removed function unpack(sources, sourcedirs) - local errorlevel = depinstall(unpackdeps) + local errorlevel = dep_install(unpackdeps) if errorlevel ~= 0 then return errorlevel end @@ -46,7 +44,7 @@ end -- Split off from the main unpack so it can be used on a bundle and not -- leave only one modules files -bundleunpack = bundleunpack or function(sourcedirs, sources) +function bundleunpack(sourcedirs, sources) local errorlevel = mkdir(localdir) if errorlevel ~=0 then return errorlevel @@ -72,8 +70,8 @@ bundleunpack = bundleunpack or function(sourcedirs, sources) end end for _,i in ipairs(unpackfiles) do - for j,_ in pairs(tree(unpackdir, i)) do - local path, name = splitpath(j) + for _,p in ipairs(tree(unpackdir, i)) do + local path, name = splitpath(p.src) local localdir = abspath(localdir) local success = io.popen( "cd " .. unpackdir .. "/" .. path .. os_concat .. diff --git a/Master/texmf-dist/scripts/l3build/l3build-upload.lua b/Master/texmf-dist/scripts/l3build/l3build-upload.lua index ba743094924..629d2ee7c77 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-upload.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-upload.lua @@ -1,6 +1,6 @@ --[[ -File l3build-upload.lua Copyright (C) 2018-2020 The LaTeX3 Project +File l3build-upload.lua Copyright (C) 2018-2020 The LaTeX Project It may be distributed and/or modified under the conditions of the LaTeX Project Public License (LPPL), either version 1.3c of this @@ -78,6 +78,12 @@ end -- if upload is anything else, the user will be prompted whether to upload. -- For now, this is undocumented. I think I would prefer to keep it always set to ask for the time being. +local ctan_post -- this is private to the module + +-- TODO: next is a public global method, +-- but following functions are semantically local +-- despite they are declared globally. + function upload(tagnames) local uploadfile = ctanzip..".zip" @@ -100,7 +106,7 @@ function upload(tagnames) uploadconfig.note = uploadconfig.note or file_contents(uploadconfig.note_file) - local tagnames = tagnames or { } + tagnames = tagnames or { } uploadconfig.version = tagnames[1] or uploadconfig.version local override_update_check = false @@ -214,7 +220,7 @@ end function shell(s) local h = assert(popen(s, 'r')) - t = assert(h:read('*a')) + local t = assert(h:read('*a')) h:close() return t end @@ -248,12 +254,6 @@ function construct_ctan_post(uploadfile,debug) ctan_field("uploader", uploadconfig.uploader, 255, "Name of uploader", true, false ) ctan_field("version", uploadconfig.version, 32, "Package version", true, false ) - -- finish constructing the curl command: - local qq = '"' - if os_type == "windows" then - qq = '\"' - end --- commandline ctan_post = ctan_post .. ' --form ' .. qq .. 'file=@' .. tostring(uploadfile) .. ';filename=' .. tostring(uploadfile) .. qq ctan_post = ctan_post .. '\nform="file=@' .. tostring(uploadfile) .. ';filename=' .. tostring(uploadfile) .. '"' return ctan_post @@ -352,7 +352,7 @@ function file_contents (filename) local f= open(filename,"r") if f==nil then return nil - else + else local s = f:read("*all") close(f) return s diff --git a/Master/texmf-dist/scripts/l3build/l3build-variables.lua b/Master/texmf-dist/scripts/l3build/l3build-variables.lua index 87ce01d611f..e3ae64e13be 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-variables.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-variables.lua @@ -1,6 +1,6 @@ --[[ -File l3build-variables.lua Copyright (C) 2018-2020 The LaTeX3 Project +File l3build-variables.lua Copyright (C) 2018-2020 The LaTeX Project It may be distributed and/or modified under the conditions of the LaTeX Project Public License (LPPL), either version 1.3c of this @@ -22,8 +22,6 @@ for those people who are interested. --]] -local exit = os.exit - -- "module" is a deprecated function in Lua 5.2: as we want the name -- for other purposes, and it should eventually be 'free', simply -- remove the built-in @@ -209,6 +207,24 @@ pvtext = pvtext or ".pvt" tlgext = tlgext or ".tlg" tpfext = tpfext or ".tpf" +test_types = test_types or { } +test_types.log = test_types.log or { + test = lvtext, + generated = logext, + reference = tlgext, + expectation = lveext, + compare = compare_tlg, + rewrite = rewrite_log, +} +test_types.pdf = test_types.pdf or { + test = pvtext, + generated = pdfext, + reference = tpfext, + rewrite = rewrite_pdf, +} + +test_order = test_order or {"log", "pdf"} + -- Manifest options manifestfile = manifestfile or "MANIFEST.md" diff --git a/Master/texmf-dist/scripts/l3build/l3build.lua b/Master/texmf-dist/scripts/l3build/l3build.lua index e7acd47fb3b..0b68ee1617d 100755 --- a/Master/texmf-dist/scripts/l3build/l3build.lua +++ b/Master/texmf-dist/scripts/l3build/l3build.lua @@ -2,7 +2,7 @@ --[[ -File l3build.lua Copyright (C) 2014-2020 The LaTeX3 Project +File l3build.lua Copyright (C) 2014-2020 The LaTeX Project It may be distributed and/or modified under the conditions of the LaTeX Project Public License (LPPL), either version 1.3c of this @@ -25,7 +25,7 @@ for those people who are interested. --]] -- Version information -release_date = "2020-06-04" +release_date = "2021-05-05" -- File operations are aided by the LuaFileSystem module local lfs = require("lfs") @@ -79,18 +79,12 @@ elseif options["target"] == "version" then exit(0) end --- Allow main function to be disabled 'higher up' -main = main or stdmain - --- Load configuration file if running as a script -if match(arg[0], "l3build$") or match(arg[0], "l3build%.lua$") then - -- Look for some configuration details - if fileexists("build.lua") then - dofile("build.lua") - else - print("Error: Cannot find configuration build.lua") - exit(1) - end +-- Look for some configuration details +if fileexists("build.lua") then + dofile("build.lua") +else + print("Error: Cannot find configuration build.lua") + exit(1) end -- Load standard settings for variables: @@ -120,7 +114,7 @@ if options["epoch"] then forcecheckepoch = true forcedocepoch = true end -normalise_epoch() +epoch = normalise_epoch(epoch) -- Sanity check check_engines() @@ -159,7 +153,6 @@ if options["target"] == "check" then print("\n Check failed with difference files") local testdir = testdir if config ~= "build" then - resultdir = resultdir .. "-" .. config testdir = testdir .. "-" .. config end for _,i in ipairs(filelist(testdir,"*" .. os_diffext)) do |