From 66f6e60b6dc60cc08482030b92df223b6a51b15f Mon Sep 17 00:00:00 2001 From: Norbert Preining Date: Sun, 29 Aug 2021 03:02:07 +0000 Subject: CTAN sync 202108290302 --- macros/latex/contrib/l3build/CHANGELOG.md | 21 +++- macros/latex/contrib/l3build/README.md | 2 +- macros/latex/contrib/l3build/l3build-arguments.lua | 6 ++ macros/latex/contrib/l3build/l3build-check.lua | 76 +++++++++++-- macros/latex/contrib/l3build/l3build-ctan.lua | 50 +++++---- .../contrib/l3build/l3build-file-functions.lua | 46 +++----- macros/latex/contrib/l3build/l3build-upload.lua | 40 +++++-- macros/latex/contrib/l3build/l3build-variables.lua | 3 +- macros/latex/contrib/l3build/l3build-zip.lua | 119 +++++++++++++++++++++ macros/latex/contrib/l3build/l3build.1 | 2 +- macros/latex/contrib/l3build/l3build.dtx | 39 ++++++- macros/latex/contrib/l3build/l3build.lua | 33 +++++- macros/latex/contrib/l3build/l3build.pdf | Bin 708832 -> 740399 bytes 13 files changed, 359 insertions(+), 78 deletions(-) create mode 100644 macros/latex/contrib/l3build/l3build-zip.lua mode change 100644 => 100755 macros/latex/contrib/l3build/l3build.lua (limited to 'macros/latex/contrib/l3build') diff --git a/macros/latex/contrib/l3build/CHANGELOG.md b/macros/latex/contrib/l3build/CHANGELOG.md index e46a935c3f..f5be81a937 100644 --- a/macros/latex/contrib/l3build/CHANGELOG.md +++ b/macros/latex/contrib/l3build/CHANGELOG.md @@ -7,6 +7,23 @@ this project uses date-based 'snapshot' version identifiers. ## [Unreleased] +## [2021-08-28] + +### Fixed +- Creation of zip files on Windows +- Only match filename and not full path for `exefiles` + +## [2021-08-27] + +### Added +- Add the `--show-saves` flag for `l3build check` to generate a list of + `l3build save` commands to regenerate all failing tests + +### Changed +- No longer call an external program to generate `zip` files and generate + them directly instead. This disables the options `zipexe` and `zipopts`. +- Copy TDS-style zip files to main dir + ## [2021-05-06] ### Fixed @@ -492,7 +509,9 @@ this project uses date-based 'snapshot' version identifiers. - Rationalise short option names: removed `-d`, `-E`, `-r` - Target `cmdcheck`: specific to LaTeX kernel work -[Unreleased]: https://github.com/latex3/l3build/compare/2021-05-06...HEAD +[Unreleased]: https://github.com/latex3/l3build/compare/2021-08-28...HEAD +[2021-08-28]: https://github.com/latex3/l3build/compare/2021-08-27...2021-08-28 +[2021-08-27]: https://github.com/latex3/l3build/compare/2021-05-06...2021-08-27 [2021-05-06]: https://github.com/latex3/l3build/compare/2021-05-05...2021-05-06 [2021-05-05]: https://github.com/latex3/l3build/compare/2020-06-04...2021-05-05 [2020-06-04]: https://github.com/latex3/l3build/compare/2020-03-25...2020-06-04 diff --git a/macros/latex/contrib/l3build/README.md b/macros/latex/contrib/l3build/README.md index 219b1ad36c..635693ae9f 100644 --- a/macros/latex/contrib/l3build/README.md +++ b/macros/latex/contrib/l3build/README.md @@ -1,7 +1,7 @@ l3build: a testing and building system for LaTeX ================================================= -Release 2021-05-06 +Release 2021-08-28 Overview -------- diff --git a/macros/latex/contrib/l3build/l3build-arguments.lua b/macros/latex/contrib/l3build/l3build-arguments.lua index 80f19d66ad..8f62e8b433 100644 --- a/macros/latex/contrib/l3build/l3build-arguments.lua +++ b/macros/latex/contrib/l3build/l3build-arguments.lua @@ -139,6 +139,12 @@ option_list = desc = "If 'halt-on-error' stops, show the full log of the failure", type = "boolean" }, + ["show-saves"] = + { + desc = "If tests fail, show the l3build save invocation to regenerate the tests", + short = "S", + type = "boolean" + }, shuffle = { desc = "Shuffle order of tests", diff --git a/macros/latex/contrib/l3build/l3build-check.lua b/macros/latex/contrib/l3build/l3build-check.lua index 47ab19ced6..f83dee4e44 100644 --- a/macros/latex/contrib/l3build/l3build-check.lua +++ b/macros/latex/contrib/l3build/l3build-check.lua @@ -574,14 +574,16 @@ function runcheck(name, hide) if options["engine"] then checkengines = options["engine"] end + local failedengines = {} -- Used for both .lvt and .pvt tests local test_type = test_types[kind] local function check_and_diff(engine) - runtest(name, engine, hide, test_type.test, test_type, true) + runtest(name, engine, hide, test_type.test, test_type, not forcecheckruns) local errorlevel = base_compare(test_type,name,engine) if errorlevel == 0 then return errorlevel end + failedengines[#failedengines + 1] = engine if options["show-log-on-error"] then showfailedlog(name) end @@ -595,14 +597,20 @@ function runcheck(name, hide) setup_check(name,engine) local errlevel = check_and_diff(engine) if errlevel ~= 0 and options["halt-on-error"] then - return 1 + return 1, failedengines end if errlevel > errorlevel then errorlevel = errlevel end end + for i=1, #failedengines do + if failedengines[i] == stdengine then + failedengines = {stdengine} + break + end + end -- Return everything - return errorlevel + return errorlevel, failedengines end function setup_check(name, engine) @@ -845,11 +853,57 @@ function testexists(test) end end +-- A short auxiliary to print the list of differences for check +local function showsavecommands(failurelist) + local savecmds = {} + local checkcmd = "l3build check --show-saves" + local prefix = "l3build save" + if options.config and options.config[1] ~= 'build' then + local config = " -c " .. options.config[1] + prefix = prefix .. config + checkcmd = checkcmd .. config + end + for name, engines in pairs(failurelist) do + for i = 1, #engines do + local engine = engines[i] + local cmd = savecmds[engine] + if not cmd then + if engine == stdengine then + cmd = prefix + else + cmd = prefix .. " -e " .. engine + end + end + savecmds[engine] = cmd .. " " .. name + if engine == stdengine then + checkcmd = checkcmd .. " " .. name + end + end + end + print(" To regenerate the test files, run\n") + local f = open(testdir .. "/.savecommands", "w") + for _, cmds in pairs(savecmds) do + print(" " .. cmds) + f:write(cmds, "\n") + end + f:write"\n" + if savecmds[stdengine] then + print("\n Afterwards test for engine specific changes using\n") + print(" " .. checkcmd) + f:write(checkcmd) + end + f:close() + print("") +end + function check(names) local errorlevel = 0 if testfiledir ~= "" and direxists(testfiledir) then if not options["rerun"] then - checkinit() + errorlevel = checkinit() + if errorlevel ~= 0 then + return errorlevel + end end local hide = true if names and next(names) then @@ -933,11 +987,13 @@ function check(names) end -- Actually run the tests print("Running checks on") + local failurelist = {} for i, name in ipairs(names) do print(" " .. name .. " (" .. i .. "/" .. #names ..")") - local errlevel = runcheck(name, hide) + local errlevel, failedengines = runcheck(name, hide) -- Return value must be 1 not errlevel if errlevel ~= 0 then + failurelist[name] = failedengines if options["halt-on-error"] then return 1 else @@ -949,6 +1005,9 @@ function check(names) end if errorlevel ~= 0 then checkdiff() + if options["show-saves"] then + showsavecommands(failurelist) + end else print("\n All checks passed\n") end @@ -994,7 +1053,12 @@ function showfaileddiff() end function save(names) - checkinit() + do + local errorlevel = checkinit() + if errorlevel ~= 0 then + return errorlevel + end + end local engines = options["engine"] or {stdengine} if names == nil then print("Arguments are required for the save command") diff --git a/macros/latex/contrib/l3build/l3build-ctan.lua b/macros/latex/contrib/l3build/l3build-ctan.lua index fd1491dd58..e5b3b7e8f8 100644 --- a/macros/latex/contrib/l3build/l3build-ctan.lua +++ b/macros/latex/contrib/l3build/l3build-ctan.lua @@ -25,9 +25,12 @@ for those people who are interested. local pairs = pairs local print = print +local attributes = lfs.attributes local lower = string.lower local match = string.match +local newzip = require"l3build-zip" + -- Copy files to the main CTAN release directory function copyctan() mkdir(ctandir .. "/" .. ctanpkg) @@ -68,33 +71,33 @@ end function ctan() -- Always run tests for all engines options["engine"] = nil - local function dirzip(dir, name) - local zipname = name .. ".zip" - local function tab_to_str(table) - local string = "" - for _,i in ipairs(table) do - string = string .. " " .. "\"" .. i .. "\"" + local function dirzip(dir, zipname) + zipname = zipname .. ".zip" + local zip = newzip(dir .. '/' .. zipname) + local function tab_to_check(table) + local patterns = {} + for n,i in ipairs(table) do + patterns[n] = glob_to_pattern(i) + end + return function(name) + for n, patt in ipairs(patterns) do + if name:match"([^/]*)$":match(patt) then return true end + end + return false end - return string end -- Convert the tables of files to quoted strings - local binfiles = tab_to_str(binaryfiles) - local exclude = tab_to_str(excludefiles) + local binfile = tab_to_check(binaryfiles) + local exclude = tab_to_check(excludefiles) + local exefile = tab_to_check(exefiles) -- First, zip up all of the text files - run( - dir, - zipexe .. " " .. zipopts .. " -ll ".. zipname .. " " .. "." - .. ( - (binfiles or exclude) and (" -x" .. binfiles .. " " .. exclude) - or "" - ) - ) - -- Then add the binary ones - run( - dir, - zipexe .. " " .. zipopts .. " -g ".. zipname .. " " .. ". -i" .. - binfiles .. (exclude and (" -x" .. exclude) or "") - ) + for _, p in ipairs(tree(dir, "**")) do + local src = p.src:sub(3) -- Strip ./ + if not (attributes(p.cwd, "mode") == "directory" or exclude(src) or src == zipname) then + zip:add(p.cwd, src, binfile(src), exefile(src)) + end + end + return zip:close() end local errorlevel local standalone = false @@ -146,6 +149,7 @@ function ctan() dirzip(tdsdir, ctanpkg .. ".tds") if packtdszip then cp(ctanpkg .. ".tds.zip", tdsdir, ctandir) + cp(ctanpkg .. ".tds.zip", tdsdir, currentdir) end dirzip(ctandir, ctanzip) cp(ctanzip .. ".zip", ctandir, currentdir) diff --git a/macros/latex/contrib/l3build/l3build-file-functions.lua b/macros/latex/contrib/l3build/l3build-file-functions.lua index 6ca1cc6011..53364bd301 100644 --- a/macros/latex/contrib/l3build/l3build-file-functions.lua +++ b/macros/latex/contrib/l3build/l3build-file-functions.lua @@ -211,6 +211,20 @@ function cleandir(dir) return rm(dir, "**") end +function direxists(dir) + return attributes(dir, "mode") == "directory" +end + +function fileexists(file) + local f = open(file, "r") + if f ~= nil then + f:close() + return true + else + return false -- also file exits and is not readable + end +end + -- Copy files 'quietly' function cp(glob, source, dest) local errorlevel @@ -218,7 +232,7 @@ function cp(glob, source, dest) -- p_src is a path relative to `source` whereas -- p_cwd is the counterpart relative to the current working directory if os_type == "windows" then - if attributes(p.cwd, "mode") == "directory" then + if direxists(p.cwd) then errorlevel = execute( 'xcopy /y /e /i "' .. unix_to_win(p.cwd) .. '" "' .. unix_to_win(dest .. '/' .. p.src) .. '" > nul' @@ -241,31 +255,6 @@ function cp(glob, source, dest) return 0 end --- OS-dependent test for a directory -function direxists(dir) - local errorlevel - if os_type == "windows" then - errorlevel = - execute("if not exist \"" .. unix_to_win(dir) .. "\" exit 1") - else - errorlevel = execute("[ -d '" .. dir .. "' ]") - end - if errorlevel ~= 0 then - return false - end - return true -end - -function fileexists(file) - local f = open(file, "r") - if f ~= nil then - f:close() - return true - else - return false -- also file exits and is not readable - end -end - -- Generate a table containing all file names of the given glob or all files -- if absent function filelist(path, glob) @@ -309,16 +298,13 @@ function tree(src_path, glob) local function always_true() return true end - local function is_dir(file) - return attributes(file, "mode") == "directory" - end ---@type table local result = { { src = ".", cwd = src_path, } } for glob_part, sep in glob:gmatch("([^/]+)(/?)/*") do - local accept = sep == "/" and is_dir or always_true + local accept = sep == "/" and direxists or always_true ---Feeds the given table according to `glob_part` ---@param p tree_entry_t path counterpart relative to the current working directory ---@param table table diff --git a/macros/latex/contrib/l3build/l3build-upload.lua b/macros/latex/contrib/l3build/l3build-upload.lua index 629d2ee7c7..47268f5b6a 100644 --- a/macros/latex/contrib/l3build/l3build-upload.lua +++ b/macros/latex/contrib/l3build/l3build-upload.lua @@ -116,11 +116,24 @@ function upload(tagnames) end -- avoid lower level error from post command if zip file missing - local zip=open(trim_space(tostring(uploadfile)),"r") - if zip~=nil then - close(zip) - else - error("Missing zip file '" .. tostring(uploadfile) .. "'") + local ziptime = lfs.attributes(trim_space(tostring(uploadfile)), 'modification') + if not ziptime then + error("Missing zip file '" .. tostring(uploadfile) .. "'. \z + Maybe you forgot to run 'l3build ctan' first?") + end + local age = os.time() - ziptime + if age >= 86400 then + print(string.format("------------------------------------------\n\z + | The local archive is older than %3i days. |\n\z + | Are you sure that you executed 'l3build ctan' first? |\n\z + --------------------------------------------------------", + age // 86400)) + print("Are you sure you want to continue? [y/n]" ) + io.stdout:write("> "):flush() + if lower(read(),1,1) ~= "y" then + print'Aborting' + return 1 + end end ctan_post = construct_ctan_post(uploadfile,options["debug"]) @@ -174,12 +187,23 @@ end end -- if upload requested and validation succeeded repost to the upload URL - if (exit_status==0 or exit_status==nil) then + if (exit_status==0 or exit_status==nil) then if (ctanupload ~=nil and ctanupload ~=false and ctanupload ~= true) then if (match(fp_return,"WARNING")) then - print("Warnings from CTAN package validation:" .. fp_return:gsub("%[","\n["):gsub("%]%]","]\n]")) + print("Warnings from CTAN package validation:" .. fp_return:gsub("%[","\n["):gsub("%]%]","]\n]")) else - print("Validation successful." ) + print("Validation successful." ) + end + print("" ) + if age < 86400 and age >= 60 then + if age >= 3600 then + print("----------------------------------------------------" ) + print(string.format("| The local archive is older than %2i hours. |", age//3600 )) + print("| Have you executed l3build ctan first? If so ... |" ) + print("----------------------------------------------------" ) + else + print(string.format("The local archive is %i minutes old.", age//60 )) + end end print("Do you want to upload to CTAN? [y/n]" ) local answer="" diff --git a/macros/latex/contrib/l3build/l3build-variables.lua b/macros/latex/contrib/l3build/l3build-variables.lua index e3ae64e13b..f1032004b6 100644 --- a/macros/latex/contrib/l3build/l3build-variables.lua +++ b/macros/latex/contrib/l3build/l3build-variables.lua @@ -84,6 +84,7 @@ demofiles = demofiles or { } docfiles = docfiles or { } dynamicfiles = dynamicfiles or { } excludefiles = excludefiles or {"*~"} +exefiles = exefiles or { } installfiles = installfiles or {"*.sty","*.cls"} makeindexfiles = makeindexfiles or {"*.ist"} scriptfiles = scriptfiles or { } @@ -106,12 +107,10 @@ unpackdeps = unpackdeps or { } -- Executable names plus following options typesetexe = typesetexe or "pdflatex" unpackexe = unpackexe or "pdftex" -zipexe = zipexe or "zip" checkopts = checkopts or "-interaction=nonstopmode" typesetopts = typesetopts or "-interaction=nonstopmode" unpackopts = unpackopts or "" -zipopts = zipopts or "-v -r -X" -- Engines for testing checkengines = checkengines or {"pdftex", "xetex", "luatex"} diff --git a/macros/latex/contrib/l3build/l3build-zip.lua b/macros/latex/contrib/l3build/l3build-zip.lua new file mode 100644 index 0000000000..06f37cd8b3 --- /dev/null +++ b/macros/latex/contrib/l3build/l3build-zip.lua @@ -0,0 +1,119 @@ +--[[ + +File l3build-zip.lua Copyright (C) 2021 The LaTeX Project + +It may be distributed and/or modified under the conditions of the +LaTeX Project Public License (LPPL), either version 1.3c of this +license or (at your option) any later version. The latest version +of this license is in the file + + http://www.latex-project.org/lppl.txt + +This file is part of the "l3build bundle" (The Work in LPPL) +and all files in that bundle must be distributed together. + +----------------------------------------------------------------------- + +The development version of the bundle can be found at + + https://github.com/latex3/l3build + +for those people who are interested. + +--]] + +local concat = table.concat +local open = io.open +local osdate = os.date +local pack = string.pack +local setmetatable = setmetatable +local iotype = io.type + +local compress = zlib.compress +local crc32 = zlib.crc32 + +local function encode_time(unix) + local t = osdate('*t', unix) + local date = t.day | (t.month << 5) | ((t.year-1980) << 9) + local time = (t.sec//2) | (t.min << 5) | (t.hour << 11) + return date, time +end + +local meta = {__index = { + add = function(z, filename, innername, binary, executable) + innername = innername or filename + + local offset = z.f:seek'cur' + + local content do + local f = iotype(filename) and filename or assert(open(filename, binary and 'rb' or 'r')) + content = f:read'*a' + f:close() + end + local crc32 = crc32(crc32(), content) + local compressed = compress(content, nil, nil, -15) + if #compressed >= #content then + compressed = nil + end + local date, time = encode_time(nil) + z.f:write(pack("