From a7064da16b2ffde620c83efcea151fe0a619ff22 Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Tue, 12 Sep 2017 22:28:01 +0000 Subject: l3build (13sep17) git-svn-id: svn://tug.org/texlive/trunk@45283 c570f23f-e606-0410-a88d-b1316a301751 --- Master/texmf-dist/tex/latex/l3build/l3build.lua | 392 +++++++++++++++--------- 1 file changed, 255 insertions(+), 137 deletions(-) (limited to 'Master/texmf-dist/tex/latex/l3build') diff --git a/Master/texmf-dist/tex/latex/l3build/l3build.lua b/Master/texmf-dist/tex/latex/l3build/l3build.lua index f5b15f4088f..bb4bf12df5f 100644 --- a/Master/texmf-dist/tex/latex/l3build/l3build.lua +++ b/Master/texmf-dist/tex/latex/l3build/l3build.lua @@ -23,7 +23,7 @@ for those people who are interested. --]] -- Version information -release_date = "2017/07/01" +release_date = "2017/09/12" -- "module" is a deprecated function in Lua 5.2: as we want the name -- for other purposes, and it should eventually be 'free', simply @@ -108,6 +108,7 @@ sourcefiles = sourcefiles or {"*.dtx", "*.ins"} textfiles = textfiles or {"*.md", "*.txt"} typesetdemofiles = typesetdemofiles or { } typesetfiles = typesetfiles or {"*.dtx"} +docfiledir = docfiledir or maindir typesetsuppfiles = typesetsuppfiles or { } typesetsourcefiles = typesetsourcefiles or { } unpackfiles = unpackfiles or {"*.ins"} @@ -159,6 +160,14 @@ bibtexopts = bibtexopts or "-W" makeindexexe = makeindexexe or "makeindex" makeindexopts = makeindexopts or "" +-- Forcing epoch +if forcecheckepoch == nil then + forcecheckepoch = true +end +if forcedocepoch == nil then + forcedocepoch = true +end + -- Other required settings asciiengines = asciiengines or {"pdftex"} checkruns = checkruns or 1 @@ -168,6 +177,7 @@ packtdszip = packtdszip or false scriptname = scriptname or "build.lua" typesetcmds = typesetcmds or "" versionform = versionform or "" +recordstatus = recordstatus or false -- Extensions for various file types: used to abstract out stuff a bit bakext = bakext or ".bak" @@ -204,7 +214,9 @@ local execute = os.execute local exit = os.exit local getenv = os.getenv local os_remove = os.remove +local os_time = os.time local os_type = os.type +local len = string.len local luatex_revision = status.luatex_revision local luatex_version = status.luatex_version local char = string.char @@ -214,57 +226,96 @@ local gmatch = string.gmatch local gsub = string.gsub local len = string.len local match = string.match +local rep = string.rep +local sort = table.sort local sub = string.sub local concat = table.concat local insert = table.insert local utf8_char = unicode.utf8.char -- Parse command line options + +local option_list = + { + date = + { + desc = "Sets the date to insert into sources", + short = "d", + type = "string" + }, + engine = + { + desc = "Sets the engine to use for running test", + short = "e", + type = "table" + }, + epoch = + { + desc = "Sets the epoch for tests and typesetting", + short = "E", + type = "string" + }, + force = + { + desc = "Force tests to run if engine is not set up", + short = "f", + type = "boolean" + }, + ["halt-on-error"] = + { + desc = "Stops running tests after the first failure", + short = "H", + type = "boolean" + }, + help = + { + short = "h", + type = "boolean" + }, + pdf = + { + desc = "Check/save PDF files", + short = "p", + type = "boolean" + }, + quiet = + { + desc = "Suppresses TeX output when unpacking", + short = "q", + type = "boolean" + }, + rerun = + { + desc = "Suppresses TeX output when unpacking", + short = "r", + type = "boolean" + }, + testfiledir = + { + desc = "Selects the specified testfile location", + short = "t", + type = "table" + }, + version = + { + desc = "Sets the version to insert into sources", + short = "v", + type = "string" + }, + } + -- This is done as a function (rather than do ... end) as it allows early -- termination (break) local function argparse() local result = { } local files = { } - local long_options = - { - date = "date" , - engine = "engine" , - force = "force" , - ["halt-on-error"] = "halt" , - ["halt-on-failure"] = "halt" , - help = "help" , - pdf = "pdf" , - quiet = "quiet" , - rerun = "rerun" , - testfiledir = "testfiledir", - version = "version" - } - local short_options = - { - d = "date" , - e = "engine" , - f = "force" , - h = "help" , - H = "halt" , - p = "pdf" , - q = "quiet" , - r = "rerun" , - t = "testfiledir", - v = "version" - } - local option_args = - { - date = true , - engine = true , - force = false, - halt = false, - help = false, - pdf = false, - quiet = false, - rerun = false, - testfiledir = true , - version = true - } + local long_options = { } + local short_options = { } + -- Turn long/short options into two lookup tables + for k,v in pairs(option_list) do + short_options[v["short"]] = k + long_options[k] = k + end local args = args -- arg[1] is a special case: must be a command or "-h"/"--help" -- Deal with this by assuming help and storing only apparently-valid @@ -327,19 +378,22 @@ local function argparse() -- if required local optname = opts[opt] if optname then - local reqarg = option_args[optname] -- Tidy up arguments - if reqarg and not optarg then + if option_list[optname]["type"] == "boolean" then + if optarg then + local opt = "-" .. (match(a, "^%-%-") and "-" or "") .. opt + stderr:write("Value not allowed for option " .. opt .."\n") + return {"help"} + end + else + if not optarg then optarg = arg[i + 1] if not optarg then stderr:write("Missing value for option " .. a .."\n") return {"help"} end i = i + 1 - end - if not reqarg and optarg then - stderr:write("Value not allowed for option " .. a .."\n") - return {"help"} + end end else stderr:write("Unknown option " .. a .."\n") @@ -347,11 +401,15 @@ local function argparse() end -- Store the result if optarg then - local opts = result[optname] or { } - for hit in gmatch(optarg, "([^,%s]+)") do - insert(opts, hit) + if option_list[optname]["type"] == "string" then + result[optname] = optarg + else + local opts = result[optname] or { } + for hit in gmatch(optarg, "([^,%s]+)") do + insert(opts, hit) + end + result[optname] = opts end - result[optname] = opts else result[optname] = true end @@ -362,29 +420,22 @@ local function argparse() break end end - result["files"] = files + if next(files) then + result["files"] = files + end return result end options = argparse() -local optdate = options["date"] -local optengines = options["engine"] -local optforce = options["force"] -local opthalt = options["halt"] -local optpdf = options["pdf"] -local optquiet = options["quiet"] -local optrerun = options["rerun"] -local optversion = options["version"] - -- Sanity check -if optengines and not optforce then +if options["engine"] and not options["force"] then -- Make a lookup table local t = { } for _, engine in pairs(checkengines) do t[engine] = true end - for _, engine in pairs(optengines) do + for _, engine in pairs(options["engine"]) do if not t[engine] then print("\n! Error: Engine \"" .. engine .. "\" not set up for testing!") print("\n Valid values are:") @@ -397,6 +448,27 @@ if optengines and not optforce then end end +-- Tidy up the epoch setting +-- Force an epoch if set at the command line +if options["epoch"] then + epoch = options["epoch"] + forcecheckepoch = true + forcedocepoch = true +end +-- If given as an ISO date, turn into an epoch number +do + local y, m, d = match(epoch, "^(%d%d%d%d)-(%d%d)-(%d%d)$") + if y then + epoch = + os_time({year = y, month = m, day = d, hour = 0, sec = 0, isdst = nil}) - + os_time({year = 1970, month = 1, day = 1, hour = 0, sec = 0, isdst = nil}) + elseif match(epoch, "^%d+$") then + epoch = tonumber(epoch) + else + epoch = 0 + end +end + -- Convert a file glob into a pattern for use by e.g. string.gub -- Based on https://github.com/davidm/lua-glob-pattern -- Simplified substantially: "[...]" syntax not supported as is not @@ -530,17 +602,17 @@ function cp(glob, source, dest) if os_type == "windows" then if lfs_attributes(source)["mode"] == "directory" then errorlevel = execute( - "xcopy /y /e /i " .. unix_to_win(source) .. " " - .. unix_to_win(dest .. "/" .. i) .. " > nul" + 'xcopy /y /e /i "' .. unix_to_win(source) .. '" "' + .. unix_to_win(dest .. '/' .. i) .. '" > nul' ) else errorlevel = execute( - "xcopy /y " .. unix_to_win(source) .. " " - .. unix_to_win(dest) .. " > nul" + 'xcopy /y "' .. unix_to_win(source) .. '" "' + .. unix_to_win(dest) .. '" > nul' ) end else - errorlevel = execute("cp -rf " .. source .. " " .. dest) + errorlevel = execute("cp -RLf '" .. source .. "' '" .. dest .. "'") end if errorlevel ~=0 then return errorlevel @@ -708,10 +780,33 @@ end -- -- Do some subtarget for all modules in a bundle -function call(dirs, target) +function call(dirs, target, opts) + -- Turn the option table into a string + local opts = opts or options local s = "" - for i = 2, #arg do - s = s .. " " .. arg[i] + for k,v in pairs(opts) do + if k ~= "files" and k ~= "target" then -- Special cases + local t = option_list[k] or { } + local arg = "" + if t["type"] == "string" then + arg = arg .. "=" .. v + end + if t["type"] == "table" then + for _,a in pairs(v) do + if arg == "" then + arg = "=" .. a -- Add the initial "=" here + else + arg = arg .. "," .. a + end + end + end + s = s .. " --" .. k .. arg + end + end + if opts["files"] then + for _,v in pairs(opts["files"]) do + s = s .. " " .. v + end end for _,i in ipairs(dirs) do print( @@ -840,7 +935,7 @@ end -- Convert the raw log file into one for comparison/storage: keeps only -- the 'business' part from the tests and removes system-dependent stuff -local function formatlog(logfile, newfile, engine) +local function formatlog(logfile, newfile, engine, errlevels) local maxprintline = maxprintline if engine == "luatex" or engine == "luajittex" then maxprintline = maxprintline + 1 -- Deal with an out-by-one error @@ -972,6 +1067,12 @@ local function formatlog(logfile, newfile, engine) local newfile = open(newfile, "w") output(newfile) write(newlog) + if recordstatus then + write('***************\n') + for i = 1, checkruns do + write('Compilation ' .. i .. ' of test file completed with exit status ' .. errlevels[i] ) + end + end close(newfile) end @@ -1192,12 +1293,22 @@ function listmodules() return modules end +local function setepoch() + return + os_setenv .. " SOURCE_DATE_EPOCH=" .. epoch + .. os_concat .. + os_setenv .. " SOURCE_DATE_EPOCH_TEX_PRIMITIVES=1" + .. os_concat .. + os_setenv .. " FORCE_SOURCE_DATE=1" + .. os_concat +end + -- Run one test which may have multiple engine-dependent comparisons -- Should create a difference file for each failed test function runcheck(name, hide) local checkengines = checkengines - if optengines then - checkengines = optengines + if options["engine"] then + checkengines = options["engine"] end local errorlevel = 0 for _,i in ipairs(checkengines) do @@ -1215,7 +1326,7 @@ function runcheck(name, hide) else errlevel = compare_tlg(name, engine) end - if errlevel ~= 0 and opthalt then + if errlevel ~= 0 and options["halt-on-error"] then showfaileddiff() if errlevel ~= 0 then return 1 @@ -1396,8 +1507,9 @@ function runtest(name, engine, hide, ext, makepdf) break end end + local errlevels = {} for i = 1, checkruns do - run( + errlevels[i] = run( testdir, -- No use of localdir here as the files get copied to testdir: -- avoids any paths in the logs @@ -1406,13 +1518,7 @@ function runtest(name, engine, hide, ext, makepdf) -- Avoid spurious output from (u)pTeX os_setenv .. " GUESS_INPUT_KANJI_ENCODING=0" .. os_concat .. - -- Fix the time of the run - os_setenv .. " SOURCE_DATE_EPOCH=" .. epoch - .. os_concat .. - os_setenv .. " SOURCE_DATE_EPOCH_TEX_PRIMITIVES=1" - .. os_concat .. - os_setenv .. " FORCE_SOURCE_DATE=1" - .. os_concat .. + (forcecheckepoch and setepoch() or "") .. -- Ensure lines are of a known length os_setenv .. " max_print_line=" .. maxprintline .. os_concat .. @@ -1426,7 +1532,7 @@ function runtest(name, engine, hide, ext, makepdf) if makepdf and fileexists(testdir .. "/" .. name .. dviext) then dvitopdf(name, testdir, engine, hide) end - formatlog(logfile, newfile, engine) + formatlog(logfile, newfile, engine, errlevels) -- Store secondary files for this engine for _,i in ipairs(filelist(testdir, name .. ".???")) do local ext = match(i, "%....") @@ -1451,16 +1557,14 @@ function dvitopdf(name, dir, engine, hide) if match(engine, "^u?ptex$") then run( dir, - os_setenv .. " SOURCE_DATE_EPOCH=" .. epoch - .. os_concat .. + (forcecheckepoch and setepoch() or "") .. "dvipdfmx " .. name .. dviext .. (hide and (" > " .. os_null) or "") ) else run( dir, - os_setenv .. " SOURCE_DATE_EPOCH=" .. epoch - .. os_concat .. + (forcecheckepoch and setepoch() or "") .. "dvips " .. name .. dviext .. (hide and (" > " .. os_null) or "") .. os_concat .. @@ -1491,7 +1595,7 @@ end -- Strip the extension from a file name (if present) function jobname(file) - local name = match(select(2, splitpath(file)), "^(.*)%.") + local name = match(basename(file), "^(.*)%.") return name or file end @@ -1510,11 +1614,12 @@ function runtool(subdir, dir, envvar, command) return( run( typesetdir .. "/" .. subdir, + (forcedocepoch and setepoch() or "") .. os_setenv .. " " .. envvar .. "=." .. os_pathsep .. abspath(localdir) .. os_pathsep .. abspath(dir .. "/" .. subdir) - .. (typesetsearch and os_pathsep or "") .. - os_concat .. + .. (typesetsearch and os_pathsep or "") + .. os_concat .. command ) ) @@ -1596,12 +1701,13 @@ function tex(file, dir) end function typesetpdf(file, dir) - local name = jobname(file) + local name = gsub(file, "%.[^.]+$", "") print("Typesetting " .. name) local errorlevel = typeset(file, dir) if errorlevel == 0 then - os_remove(name .. ".pdf") - cp(name .. ".pdf", typesetdir, ".") + name = name .. ".pdf" + os_remove(jobname(name)) + cp(name, typesetdir, ".") else print(" ! Compilation failed") end @@ -1658,15 +1764,27 @@ function help() print(" setversion Update version information in sources") print("") print("Valid options are:") - print(" --date|-d Sets the date to insert into sources") - print(" --engine|-e Sets the engine to use for running test") - print(" --force|-f Force tests to run if engine is not set up") - print(" --halt-on-error|-H Stops running tests after the first failure") - print(" --pdf|-p Check/save PDF files") - print(" --quiet|-q Suppresses TeX output when unpacking") - print(" --rerun|-r Runs tests without any unpacking, etc.") - print(" --testfiledir|-t Selects the specified testfile location") - print(" --version|-v Sets the version to insert into sources") + local longest = 0 + for k,v in pairs(option_list) do + if len(k) > longest then + longest = len(k) + end + end + -- Sort the options + local t = { } + for k,_ in pairs(option_list) do + insert(t, k) + end + sort(t) + for _,k in ipairs(t) do + local opt = option_list[k] + local filler = rep(" ", longest - len(k)) + if opt["desc"] then -- Skip --help as it has no desc + print( + " --" .. k .. "|-" .. opt["short"] .. filler .. opt["desc"] + ) + end + end print("") print("See l3build.pdf for further details.") end @@ -1674,7 +1792,7 @@ end function check(names) local errorlevel = 0 if testfiledir ~= "" and direxists(testfiledir) then - if not optrerun then + if not options["rerun"] then checkinit() end local hide = true @@ -1703,7 +1821,7 @@ function check(names) local errlevel = runcheck(name, hide) -- Return value must be 1 not errlevel if errlevel ~= 0 then - if opthalt then + if options["halt-on-error"] then return 1 else errorlevel = 1 @@ -1817,7 +1935,7 @@ end function ctan(standalone) -- Always run tests for all engines - optengines = nil + options["engine"] = nil local function dirzip(dir, name) local zipname = name .. ".zip" local function tab_to_str(table) @@ -1954,27 +2072,33 @@ function doc(files) cp(i, supportdir, typesetdir) end depinstall(typesetdeps) - unpack({sourcefiles, typesetsourcefiles}) + unpack({sourcefiles, typesetsourcefiles}, {".", docfiledir}) -- Main loop for doc creation + local done = {} for _, typesetfiles in ipairs({typesetdemofiles, typesetfiles}) do for _,i in ipairs(typesetfiles) do for _, dir in ipairs({unpackdir, typesetdir}) do for j,_ in pairs(tree(dir, i)) do - -- Allow for command line selection of files - local typeset = true - if files and next(files) then - typeset = false - for _,k in ipairs(files) do - if k == jobname(j) then - typeset = true - break + if not done[j] then + j = gsub(j, "^%./", "") + -- Allow for command line selection of files + local typeset = true + if files and next(files) then + typeset = false + for _,k in ipairs(files) do + if k == gsub(j, "%.[^.]+$", "") then + typeset = true + break + end end end - end - if typeset then - local errorlevel = typesetpdf(j, dir) - if errorlevel ~= 0 then - return errorlevel + if typeset then + local errorlevel = typesetpdf(j, dir) + if errorlevel ~= 0 then + return errorlevel + else + done[j] = true + end end end end @@ -2008,7 +2132,7 @@ end function save(names) checkinit() - local engines = optengines or {stdengine} + local engines = options["engine"] or {stdengine} for _,name in pairs(names) do local engine for _,engine in pairs(engines) do @@ -2017,11 +2141,11 @@ function save(names) local spdffile = name .. tlgengine .. pdfext local newfile = name .. "." .. engine .. logext local pdffile = name .. "." .. engine .. pdfext - local refext = ((optpdf and pdfext) or tlgext) + local refext = ((options["pdf"] and pdfext) or tlgext) if testexists(name) then print("Creating and copying " .. refext) - runtest(name, engine, false, lvtext, optpdf) - if optpdf then + runtest(name, engine, false, lvtext, options["pdf"]) + if options["pdf"] then ren(testdir, pdffile, spdffile) cp(spdffile, testdir, testfiledir) else @@ -2144,14 +2268,8 @@ function setversion(dir) rm(dir, file .. bakext) end end - local date = os_date("%Y-%m-%d") - if optdate then - date = optdate[1] or date - end - local version = -1 - if optversion then - version = optversion[1] or version - end + local date = options["date"] or os_date("%Y-%m-%d") + local version = options["version"] or -1 local dir = dir or "." for _,i in pairs(versionfiles) do for _,j in pairs(filelist(dir, i)) do @@ -2163,12 +2281,12 @@ end -- Unpack the package files using an 'isolated' system: this requires -- a copy of the 'basic' DocStrip program, which is used then removed -function unpack(sources) +function unpack(sources, sourcedirs) local errorlevel = depinstall(unpackdeps) if errorlevel ~= 0 then return errorlevel end - errorlevel = bundleunpack({"."}, sources) + errorlevel = bundleunpack(sourcedirs, sources) if errorlevel ~= 0 then return errorlevel end @@ -2183,7 +2301,7 @@ end -- Split off from the main unpack so it can be used on a bundle and not -- leave only one modules files -bundleunpack = bundleunpack or function(sourcedir, sources) +bundleunpack = bundleunpack or function(sourcedirs, sources) local errorlevel = mkdir(localdir) if errorlevel ~=0 then return errorlevel @@ -2192,7 +2310,7 @@ bundleunpack = bundleunpack or function(sourcedir, sources) if errorlevel ~=0 then return errorlevel end - for _,i in ipairs(sourcedir or {"."}) do + for _,i in ipairs(sourcedirs or {"."}) do for _,j in ipairs(sources or {sourcefiles}) do for _,k in ipairs(j) do errorlevel = cp(k, i, unpackdir) @@ -2225,7 +2343,7 @@ bundleunpack = bundleunpack or function(sourcedir, sources) os_concat .. unpackexe .. " " .. unpackopts .. " " .. name .. " < " .. localdir .. "/yes" - .. (optquiet and (" > " .. os_null) or "") + .. (options["quiet"] and (" > " .. os_null) or "") ) if errorlevel ~=0 then return errorlevel -- cgit v1.2.3