From e701677aa361258a69f40dad5187444352c6e14b Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Sat, 4 Aug 2018 20:54:38 +0000 Subject: l3build (4aug18) git-svn-id: svn://tug.org/texlive/trunk@48348 c570f23f-e606-0410-a88d-b1316a301751 --- .../texlive/linked_scripts/l3build/l3build.lua | 24 ++- Master/texmf-dist/doc/latex/l3build/CHANGELOG.md | 24 ++- Master/texmf-dist/doc/latex/l3build/README.md | 2 +- Master/texmf-dist/doc/latex/l3build/l3build.pdf | Bin 671522 -> 672071 bytes Master/texmf-dist/doc/man/man1/l3build.1 | 2 +- Master/texmf-dist/doc/man/man1/l3build.man1.pdf | Bin 15146 -> 15209 bytes .../scripts/l3build/l3build-arguments.lua | 5 + .../texmf-dist/scripts/l3build/l3build-check.lua | 184 +++++++++---------- Master/texmf-dist/scripts/l3build/l3build-ctan.lua | 119 +------------ .../scripts/l3build/l3build-file-functions.lua | 1 - .../texmf-dist/scripts/l3build/l3build-install.lua | 194 +++++++++++++++++---- .../texmf-dist/scripts/l3build/l3build-stdmain.lua | 2 +- .../texmf-dist/scripts/l3build/l3build-tagging.lua | 5 +- .../scripts/l3build/l3build-variables.lua | 1 + Master/texmf-dist/scripts/l3build/l3build.lua | 24 ++- Master/texmf-dist/source/latex/l3build/l3build.dtx | 50 ++++-- 16 files changed, 356 insertions(+), 281 deletions(-) diff --git a/Build/source/texk/texlive/linked_scripts/l3build/l3build.lua b/Build/source/texk/texlive/linked_scripts/l3build/l3build.lua index 5cefd6d5dc0..2107f024d3a 100644 --- a/Build/source/texk/texlive/linked_scripts/l3build/l3build.lua +++ b/Build/source/texk/texlive/linked_scripts/l3build/l3build.lua @@ -25,7 +25,7 @@ for those people who are interested. --]] -- Version information -release_date = "2018-08-02" +release_date = "2018-08-04" -- File operations are aided by the LuaFileSystem module local lfs = require("lfs") @@ -34,6 +34,7 @@ local lfs = require("lfs") local assert = assert local ipairs = ipairs +local insert = table.insert local lookup = kpse.lookup local match = string.match local next = next @@ -123,13 +124,28 @@ if options["target"] == "check" then if #checkconfigs > 1 then local errorlevel = 0 local opts = options + local failed = { } for i = 1, #checkconfigs do opts["config"] = {checkconfigs[i]} errorlevel = call({"."}, "check", opts) - if errorlevel ~= 0 then exit(1) end + if errorlevel ~= 0 then + if options["halt-on-error"] then + exit(1) + else + insert(failed,checkconfigs[i]) + end + end + end + if next(failed) then + print(" Failed tests for configs:") + for _,config in ipairs(failed) do + print(" - " .. config) + end + exit(1) + else + -- Avoid running the 'main' set of tests twice + exit(0) end - -- Avoid running the 'main' set of tests twice - exit(0) end end if #checkconfigs == 1 and diff --git a/Master/texmf-dist/doc/latex/l3build/CHANGELOG.md b/Master/texmf-dist/doc/latex/l3build/CHANGELOG.md index b9a434787b0..2d59804c698 100644 --- a/Master/texmf-dist/doc/latex/l3build/CHANGELOG.md +++ b/Master/texmf-dist/doc/latex/l3build/CHANGELOG.md @@ -7,6 +7,21 @@ this project uses date-based 'snapshot' version identifiers. ## [Unreleased] +## [2018-08-04] + +### Added +- `ctanzip` variable +- `--full` option + +### Changed +- Run PDF-based tests for all engines +- Tweaks to PDF-based normalisation: new `.tpf` files will be required +- The `--halt-on-error|-H` setting now applies to multiple configs + +### Fixed +- Testing using `.lve` files +- Tagging with new approach to top-level targets + ## [2018-08-02] ### Added @@ -19,10 +34,10 @@ this project uses date-based 'snapshot' version identifiers. ### Changed - PDF-based testing now uses 'digested' PDF file for comparison, - working from dedicated .pvt input files + working from dedicated `.pvt` input files ### Removed -- "--pdf" command line switch +- `--pdf|-p` command line switch ## [2018-05-10] @@ -127,8 +142,9 @@ this project uses date-based 'snapshot' version identifiers. - Rationalise short option names: removed `-d`, `-E`, `-r` - Target `cmdcheck`: specific to LaTeX3 kernel work -[Unreleased]: https://github.com/latex3/l3build/compare/2018-08-02...HEAD -[2018-08-02]: https://github.com/latex3/l3build/compare/2018-05-06...2018-08-02 +[Unreleased]: https://github.com/latex3/l3build/compare/2018-08-04...HEAD +[2018-08-04]: https://github.com/latex3/l3build/compare/2018-08-02...2018-08-04 +[2018-08-02]: https://github.com/latex3/l3build/compare/2018-05-10...2018-08-02 [2018-05-10]: https://github.com/latex3/l3build/compare/2018-05-06...2018-05-10 [2018-05-06]: https://github.com/latex3/l3build/compare/2018-03-26...2018-05-06 [2018-03-26]: https://github.com/latex3/l3build/compare/2018-03-24...2018-03-26 diff --git a/Master/texmf-dist/doc/latex/l3build/README.md b/Master/texmf-dist/doc/latex/l3build/README.md index 43ac56bf0e9..1014ef3cd4a 100644 --- a/Master/texmf-dist/doc/latex/l3build/README.md +++ b/Master/texmf-dist/doc/latex/l3build/README.md @@ -1,7 +1,7 @@ l3build: a testing and building system for LaTeX3 ================================================= -Release 2018-08-02 +Release 2018-08-04 Overview -------- diff --git a/Master/texmf-dist/doc/latex/l3build/l3build.pdf b/Master/texmf-dist/doc/latex/l3build/l3build.pdf index c5675f9d9a9..cdbb633e42f 100644 Binary files a/Master/texmf-dist/doc/latex/l3build/l3build.pdf and b/Master/texmf-dist/doc/latex/l3build/l3build.pdf differ diff --git a/Master/texmf-dist/doc/man/man1/l3build.1 b/Master/texmf-dist/doc/man/man1/l3build.1 index fb85523ad0b..c3ef55a52c8 100644 --- a/Master/texmf-dist/doc/man/man1/l3build.1 +++ b/Master/texmf-dist/doc/man/man1/l3build.1 @@ -1,4 +1,4 @@ -.TH l3build 1 "2018-08-02" +.TH l3build 1 "2018-08-04" .SH NAME l3build \- Checking and building packages .SH SYNOPSIS diff --git a/Master/texmf-dist/doc/man/man1/l3build.man1.pdf b/Master/texmf-dist/doc/man/man1/l3build.man1.pdf index 63568956eab..2c0b133a7be 100644 Binary files a/Master/texmf-dist/doc/man/man1/l3build.man1.pdf and b/Master/texmf-dist/doc/man/man1/l3build.man1.pdf differ diff --git a/Master/texmf-dist/scripts/l3build/l3build-arguments.lua b/Master/texmf-dist/scripts/l3build/l3build-arguments.lua index 4416ab69cda..e693ab929df 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-arguments.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-arguments.lua @@ -79,6 +79,11 @@ option_list = short = "f", type = "boolean" }, + full = + { + desc = "Install all files", + type = "boolean" + }, ["halt-on-error"] = { desc = "Stops running tests after the first failure", diff --git a/Master/texmf-dist/scripts/l3build/l3build-check.lua b/Master/texmf-dist/scripts/l3build/l3build-check.lua index 7cac25a7d93..f5d1716de9e 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-check.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-check.lua @@ -83,9 +83,20 @@ function checkinit() execute(os_ascii .. ">" .. testdir .. "/ascii.tcx") end +local function rewrite(source,result,processor,...) + local file = assert(open(source,"rb")) + local content = gsub(file:read("*all") .. "\n","\r\n","\n") + close(file) + local new_content = processor(content,...) + local newfile = open(result,"w") + output(newfile) + write(new_content) + close(newfile) +end + -- Convert the raw log file into one for comparison/storage: keeps only -- the 'business' part from the tests and removes system-dependent stuff -local function formatlog(logfile, newfile, engine, errlevels) +local function normalize_log(content,engine,errlevels) local maxprintline = maxprintline if engine == "luatex" or engine == "luajittex" then maxprintline = maxprintline + 1 -- Deal with an out-by-one error @@ -239,14 +250,10 @@ local function formatlog(logfile, newfile, engine, errlevels) return line, lastline end local lastline = "" - local newlog = "" + local new_content = "" local prestart = true local skipping = false - -- Read the entire log file as a binary: deals with ^@/^[, etc. - local file = assert(open(logfile, "rb")) - local contents = gsub(file:read("*all") .. "\n", "\r\n", "\n") - close(file) - for line in gmatch(contents, "([^\n]*)\n") do + for line in gmatch(content, "([^\n]*)\n") do if line == "START-TEST-LOG" then prestart = false elseif line == "END-TEST-LOG" or @@ -259,24 +266,23 @@ local function formatlog(logfile, newfile, engine, errlevels) elseif not prestart and not skipping then line, lastline = normalize(line, lastline) if not match(line, "^ *$") and not killcheck(line) then - newlog = newlog .. line .. os_newline + new_content = new_content .. line .. os_newline end end end - local newfile = open(newfile, "w") - output(newfile) - write(newlog) if recordstatus then - write('***************\n') + new_content = new_content .. '***************' .. os_newline for i = 1, checkruns do - write('Compilation ' .. i .. ' of test file completed with exit status ' .. errlevels[i] .. '\n') + new_content = new_content .. + 'Compilation ' .. i .. ' of test file completed with exit status ' .. + errlevels[i] .. os_newline end end - close(newfile) + return new_content end -- Additional normalization for LuaTeX -local function formatlualog(logfile, newfile, luatex) +local function normalize_lua_log(content,luatex) local function normalize(line, lastline, dropping) -- Find \discretionary or \whatsit lines: -- These may come back later @@ -468,34 +474,24 @@ local function formatlualog(logfile, newfile, luatex) -- uses a different number to the other engines return gsub(line, "^%s+", ""), "" end - local newlog = "" + local new_content = "" local lastline = "" local dropping = false - -- Read the entire log file as a binary: deals with ^@/^[, etc. - local file = assert(open(logfile, "rb")) - local contents = gsub(file:read("*all") .. "\n", "\r\n", "\n") - close(file) - for line in gmatch(contents, "([^\n]*)\n") do + for line in gmatch(content, "([^\n]*)\n") do line, lastline, dropping = normalize(line, lastline, dropping) if not match(line, "^ *$") then - newlog = newlog .. line .. os_newline + new_content = new_content .. line .. os_newline end end - local newfile = open(newfile, "w") - output(newfile) - write(newlog) - close(newfile) + return new_content end -local function normalise_pdf(pdffile,npdffile) - local file = assert(open(pdffile, "rb")) - local contents = gsub(file:read("*all") .. "\n", "\r\n", "\n") - close(file) +local function normalize_pdf(content) local new_content = "" local stream_content = "" local binary = false local stream = false - for line in gmatch(contents, "([^\n]*)\n") do + for line in gmatch(content,"([^\n]*)\n") do if stream then if match(line,"endstream") then stream = false @@ -524,15 +520,13 @@ local function normalise_pdf(pdffile,npdffile) new_content = new_content .. line .. os_newline end end - local newfile = open(npdffile, "w") - output(newfile) - write(new_content) - close(newfile) + return new_content end -- Run one test which may have multiple engine-dependent comparisons -- Should create a difference file for each failed test function runcheck(name, hide) + if not testexists(name) then return 1 end local checkengines = checkengines if options["engine"] then checkengines = options["engine"] @@ -548,30 +542,25 @@ function runcheck(name, hide) return errorlevel end local errorlevel = 0 - -- First check by log - if fileexists(testfiledir .. "/" .. name .. lvtext) then - for _,i in pairs(checkengines) do - -- Allow for luatex == luajittex for .tlg purposes - local engine = i - if i == "luajittex" then - engine = "luatex" - end - setup_check(name, engine) - local errlevel = 0 - errlevel = check_and_diff(lvtext,i,engine,compare_tlg) - if errlevel ~= 0 and options["halt-on-error"] then - return 1 - end - if errlevel > errorlevel then - errorlevel = errlevel - end + for _,engine in pairs(checkengines) do + local enginename = engine + -- Allow for luatex == luajittex for .tlg/.tpf purposes + if engine == "luajittex" then + engine = "luatex" + end + setup_check(name,engine) + local errlevel = 0 + if fileexists(testfiledir .. "/" .. name .. pvtext) then + errlevel = check_and_diff(pvtext,engine,enginename,compare_pdf,true) + else + errlevel = check_and_diff(lvtext,engine,enginename,compare_tlg) + end + if errlevel ~= 0 and options["halt-on-error"] then + return 1 + end + if errlevel > errorlevel then + errorlevel = errlevel end - end - -- Then check by PDF - if fileexists(testfiledir .. "/" .. name .. pvtext) then - setup_check(name, stdengine) - errorlevel = check_and_diff(pvtext,stdengine,stdengine,compare_pdf,true) - + errorlevel end -- Return everything return errorlevel @@ -596,7 +585,7 @@ function setup_check(name, engine) ) exit(1) end - runtest(name, engine, true, lveext, true, false) + runtest(name, engine, true, lveext) ren(testdir, testname .. logext, testname .. tlgext) else -- Install comparison files found @@ -613,15 +602,14 @@ function setup_check(name, engine) end function compare_pdf(name,engine,cleanup) - local errorlevel local testname = name .. "." .. engine - local difffile = testdir .. "/" .. name .. pdfext .. os_diffext + local difffile = testdir .. "/" .. testname .. pdfext .. os_diffext local pdffile = testdir .. "/" .. testname .. pdfext - local tpffile = testdir .. "/" .. name .. tpfext + local tpffile = locate({testdir}, {testname .. tpfext, name .. tpfext}) if not tpffile then return 1 end - errorlevel = execute(os_diffexe .. " " + local errorlevel = execute(os_diffexe .. " " .. normalize_path(tpffile .. " " .. pdffile .. " > " .. difffile)) if errorlevel == 0 or cleanup then remove(difffile) @@ -650,8 +638,8 @@ function compare_tlg(name, engine,cleanup) lualogfile = testdir .. "/" .. testname .. ".tmp" .. logext end local luatlgfile = testdir .. "/" .. testname .. tlgext - formatlualog(tlgfile, luatlgfile, false) - formatlualog(logfile, lualogfile, true) + rewrite(tlgfile,luatlgfile,normalize_lua_log) + rewrite(logfile,lualogfile,normalize_lua_log,true) errorlevel = execute(os_diffexe .. " " .. normalize_path(luatlgfile .. " " .. lualogfile .. " > " .. difffile)) if cleanup then @@ -669,7 +657,7 @@ function compare_tlg(name, engine,cleanup) end -- Run one of the test files: doesn't check the result so suitable for --- both creating and verifying .tlg files +-- both creating and verifying function runtest(name, engine, hide, ext, pdfmode, breakout) local lvtfile = name .. (ext or lvtext) cp(lvtfile, fileexists(testfiledir .. "/" .. lvtfile) @@ -677,13 +665,11 @@ function runtest(name, engine, hide, ext, pdfmode, breakout) local engine = engine or stdengine -- Set up the format file name if it's one ending "...tex" local realengine = engine - local format + local format = "" if match(checkformat, "tex$") and not match(engine, checkformat) then format = " -fmt=" .. gsub(engine, "(.*)tex$", "%1") .. checkformat - else - format = "" end -- Special casing for e-LaTeX format if @@ -720,10 +706,11 @@ function runtest(name, engine, hide, ext, pdfmode, breakout) exit(1) end end - local logfile = testdir .. "/" .. name .. logext - local newfile = testdir .. "/" .. name .. "." .. engine .. logext - local pdffile = testdir .. "/" .. name .. pdfext - local npffile = testdir .. "/" .. name .. "." .. engine .. pdfext + local basename = testdir .. "/" .. name + local logfile = basename .. logext + local newfile = basename .. "." .. engine .. logext + local pdffile = basename .. pdfext + local npffile = basename .. "." .. engine .. pdfext local asciiopt = "" for _,i in ipairs(asciiengines) do if realengine == i then @@ -760,12 +747,12 @@ function runtest(name, engine, hide, ext, pdfmode, breakout) -- Break the loop if the result is stable if breakout and i < checkruns then if pdfmode then - normalise_pdf(pdffile,npffile) + rewrite(pdffile,npffile,normalize_pdf) if compare_pdf(name,engine,true) == 0 then break end else - formatlog(logfile, newfile, engine, errlevels) + rewrite(logfile,newfile,normalize_log,engine,errlevels) if compare_tlg(name,engine,true) == 0 then break end @@ -776,9 +763,9 @@ function runtest(name, engine, hide, ext, pdfmode, breakout) dvitopdf(name, testdir, engine, hide) end if pdfmode then - normalise_pdf(pdffile,npffile) + rewrite(pdffile,npffile,normalize_pdf) else - formatlog(logfile, newfile, engine, errlevels) + rewrite(logfile,newfile,normalize_log,engine,errlevels) end -- Store secondary files for this engine for _,filetype in pairs(auxfiles) do @@ -794,6 +781,7 @@ function runtest(name, engine, hide, ext, pdfmode, breakout) end end end + return 0 end -- A hook to allow additional tasks to run for the tests @@ -807,8 +795,6 @@ function testexists(test) {test .. lvtext, test .. pvtext})) end --- Standard versions of the main targets for building modules - function check(names) local errorlevel = 0 if testfiledir ~= "" and direxists(testfiledir) then @@ -950,37 +936,37 @@ function save(names) for _,name in pairs(names) do if testexists(name) then for _,engine in pairs(engines) do - local testengine = ((engine == stdengine and "") or "." .. engine) - if fileexists(testfiledir .. "/" .. name .. lvtext) then - -- Create one or more .tlg files - print("Creating and copying " .. tlgext) - local tlgfile = name .. testengine .. tlgext - local newfile = name .. "." .. engine .. logext - runtest(name,engine,false,lvtext) - ren(testdir,newfile,tlgfile) - cp(tlgfile,testdir,testfiledir) - if fileexists(unpackdir .. "/" .. tlgfile) then - print("Saved " .. tlgext + local testengine = ((engine == stdengine and "") or "." .. engine) + local function save_test(test_ext,gen_ext,out_ext,pdfmode) + local out_file = name .. testengine .. out_ext + local gen_file = name .. "." .. engine .. gen_ext + print("Creating and copying " .. out_file) + runtest(name,engine,false,test_ext,pdfmode) + ren(testdir,gen_file,out_file) + cp(out_file,testdir,testfiledir) + if fileexists(unpackdir .. "/" .. out_file) then + print("Saved " .. out_ext .. " file overrides unpacked version of the same name") + return 1 end + return 0 + end + local errorlevel + if fileexists(testfiledir .. "/" .. name .. lvtext) then + errorlevel = save_test(lvtext,logext,tlgext) else - -- Create one .tpf file - print("Creating and copying " .. tpfext) - local tpffile = name .. tpfext - local newfile = name .. "." .. engine .. pdfext - runtest(name,engine,false,pvtext,true) - ren(testdir,newfile,tpffile) - cp(tpffile,testdir,testfiledir) + errorlevel = save_test(pvtext,pdfext,tpfext,true) end - return 0 + if errorlevel ~=0 then return errorlevel end end elseif locate({unpackdir, testfiledir}, {name .. lveext}) then print("Saved " .. tlgext .. " file overrides a " .. lveext .. " file of the same name") + return 1 else print('Test "'.. name .. '"not found') return 1 end end + return 0 end - diff --git a/Master/texmf-dist/scripts/l3build/l3build-ctan.lua b/Master/texmf-dist/scripts/l3build/l3build-ctan.lua index 875f50dc4fa..9d29f63f1c7 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-ctan.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-ctan.lua @@ -22,9 +22,8 @@ for those people who are interested. --]] -local gsub = string.gsub -local match = string.match -local insert = table.insert +local pairs = pairs +local print = print -- Copy files to the main CTAN release directory function copyctan() @@ -56,62 +55,6 @@ function copyctan() end end --- Copy files to the correct places in the TDS tree -function copytds() - local function install(source, dest, files, tool) - local moduledir = moduledir - -- For material associated with secondary tools (BibTeX, MakeIndex) - -- the structure needed is slightly different from those items going - -- into the tex/doc/source trees - if tool then - -- "base" is reserved for the tools themselves: make the assumption - -- in this case that the tdsroot name is the right place for stuff to - -- go (really just for the team) - if module == "base" then - moduledir = tdsroot - else - moduledir = module - end - end - -- Convert the file table(s) to a list of individual files - local filenames = { } - for _,i in ipairs(files) do - for _,j in ipairs(i) do - for file,_ in pairs(tree(source, j)) do - insert(filenames, file) - end - end - end - -- The target is only created if there are actual files to install - if next(filenames) ~= nil then - local installdir = tdsdir .. "/" .. dest .. "/" .. moduledir - mkdir(installdir) - for _,i in ipairs(filenames) do - cp(i, source, installdir) - end - end - end - install( - docfiledir, - "doc", - {bibfiles, demofiles, docfiles, pdffiles, textfiles, typesetlist} - ) - install(unpackdir, "makeindex", {makeindexfiles}, true) - install(unpackdir, "bibtex/bst", {bstfiles}, true) - install(sourcefiledir, "source", {sourcelist}) - install(unpackdir, "tex", {installfiles}) - install(unpackdir, "scripts", {scriptfiles}, true) - -- Any script man files need special handling - for _,glob in pairs(scriptmanfiles) do - for file,_ in pairs(tree(docfiledir,glob)) do - -- Man files should have a single-digit extension: the type - local installdir = tdsdir .. "/doc/man/man" .. match(file,".$") - mkdir(installdir) - cp(file,docfiledir,installdir) - end - end -end - -- Standard versions of the main targets for building modules function ctan() @@ -162,7 +105,9 @@ function ctan() rmdir(tdsdir) mkdir(tdsdir) if standalone then - errorlevel = bundlectan() + errorlevel = install_files(tdsdir,true) + if errorlevel ~=0 then return errorlevel end + copyctan() else errorlevel = call(modules, "bundlectan") end @@ -183,8 +128,8 @@ function ctan() if packtdszip then cp(ctanpkg .. ".tds.zip", tdsdir, ctandir) end - dirzip(ctandir, ctanpkg) - cp(ctanpkg .. ".zip", ctandir, currentdir) + dirzip(ctandir, ctanzip) + cp(ctanzip .. ".zip", ctandir, currentdir) else print("\n====================") print("Typesetting failed, zip stage skipped!") @@ -193,53 +138,3 @@ function ctan() return errorlevel end -function bundlectan() - -- Generate a list of individual file names excluding those in the second - -- argument: the latter is a table - local function excludelist(include, exclude, dir) - local include = include or { } - local exclude = exclude or { } - local dir = dir or currentdir - local includelist = { } - local excludelist = { } - for _,i in ipairs(exclude) do - for _,j in ipairs(i) do - for file,_ in pairs(tree(dir, j)) do - excludelist[file] = true - end - end - end - for _,i in ipairs(include) do - for file,_ in pairs(tree(dir, i)) do - if not excludelist[file] then - insert(includelist, file) - end - end - end - return includelist - end - unpack() - local errorlevel = doc() - if errorlevel == 0 then - -- Work out what PDF files are available - pdffiles = { } - for _,i in ipairs(typesetfiles) do - insert(pdffiles, (gsub(i, "%.%w+$", ".pdf"))) - end - -- For the purposes here, any typesetting demo files need to be - -- part of the main typesetting list - local typesetfiles = typesetfiles - for _,v in pairs(typesetdemofiles) do - insert(typesetfiles, v) - end - typesetlist = excludelist(typesetfiles, {sourcefiles}, docfiledir) - sourcelist = excludelist( - sourcefiles, {bstfiles, installfiles, makeindexfiles, scriptfiles}, - sourcefiledir - ) - copyctan() - copytds() - end - return errorlevel -end - diff --git a/Master/texmf-dist/scripts/l3build/l3build-file-functions.lua b/Master/texmf-dist/scripts/l3build/l3build-file-functions.lua index 510567e5342..bc462d3ff33 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-file-functions.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-file-functions.lua @@ -217,7 +217,6 @@ if os_type == "windows" then os_diffext = getenv("diffext") or ".fc" os_diffexe = getenv("diffexe") or "fc /n" os_grepexe = "findstr /r" - os_newline = "\n" if tonumber(luatex_version) < 100 or (tonumber(luatex_version) == 100 and tonumber(luatex_revision) < 4) then diff --git a/Master/texmf-dist/scripts/l3build/l3build-install.lua b/Master/texmf-dist/scripts/l3build/l3build-install.lua index c18677e4271..2c4274454db 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-install.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-install.lua @@ -28,71 +28,191 @@ local print = print local set_program = kpse.set_program_name local var_value = kpse.var_value +local gsub = string.gsub +local match = string.match + +local insert = table.insert + local function gethome() set_program("latex") return options["texmfhome"] or var_value("TEXMFHOME") end function uninstall() - local function uninstall_files(target) - local installdir = gethome() .. target + local function uninstall_files(dir,subdir) + subdir = subdir or moduledir + dir = dir .. "/" .. subdir + local installdir = gethome() .. "/" .. dir if options["dry-run"] then - print("\n" .. "Installation root: " .. installdir) local files = filelist(installdir) - -- Deal with an empty directory if next(files) then - print("\n" .. "Files for removal:") + print("\n" .. "For removal from " .. installdir .. ":") for _,file in pairs(filelist(installdir)) do print("- " .. file) end - else - print("No files present") end return 0 else - return rmdir(installdir) + if direxists(installdir) then + return rmdir(installdir) + end + end + return 0 + end + local errorlevel = 0 + -- Any script man files need special handling + local manfiles = { } + for _,glob in pairs(scriptmanfiles) do + for file,_ in pairs(tree(docfiledir,glob)) do + -- Man files should have a single-digit extension: the type + local installdir = gethome() .. "/doc/man/man" .. match(file,".$") + if fileexists(installdir .. "/" .. file) then + if options["dry-run"] then + insert(manfiles,"man" .. match(file,".$") .. "/" .. + select(2,splitpath(file))) + else + errorlevel = errorlevel + rm(installdir,file) + end + end + end + end + if next(manfiles) then + print("\n" .. "For removal from " .. gethome() .. "/doc/man:") + for _,v in ipairs(manfiles) do + print("- " .. v) end end - return uninstall_files("/tex/" .. moduledir) - + uninstall_files("/scripts/" .. module) + return uninstall_files("doc") + + uninstall_files("source") + + uninstall_files("tex") + + uninstall_files("bibtex/bst",module) + + uninstall_files("makeindex",module) + + uninstall_files("scripts",module) + + errorlevel end - --- Locally install files: only deals with those extracted, not docs etc. -function install() - local function install_files(files,target) - if not next(files) then - return 0 +function install_files(target,full,dry_run) + local function install_files(source,dir,files,subdir,tool) + subdir = subdir or moduledir + -- For material associated with secondary tools (BibTeX, MakeIndex) + -- the structure needed is slightly different from those items going + -- into the tex/doc/source trees + if tool and module == "base" then + subdir = nil end - local installdir = gethome() .. target - if options["dry-run"] then - print("\n" .. "Installation root: " .. installdir - .. "\n" .. "Installation files:" - ) - for _,filetype in pairs(files) do - for _,file in pairs(filelist(unpackdir,filetype)) do - print("- " .. file) + dir = dir .. (subdir and "/" or "") .. subdir + local filenames = { } + for _,glob_table in pairs(files) do + for _,glob in pairs(glob_table) do + for file,_ in pairs(tree(source,glob)) do + insert(filenames,file) end end - return 0 - else - errorlevel = cleandir(installdir) - if errorlevel ~= 0 then - return errorlevel + end + local errorlevel = 0 + -- The target is only created if there are actual files to install + if next(filenames) then + local installdir = target .. "/" .. dir + if dry_run then + print("\n" .. "For installation in " .. installdir .. ":") + else + errorlevel = cleandir(installdir) + if errorlevel ~= 0 then return errorlevel end end - for _,filetype in pairs(files) do - errorlevel = cp(filetype, unpackdir, installdir) - if errorlevel ~= 0 then - return errorlevel + for _,file in ipairs(filenames) do + if dry_run then + print("- " .. select(2,splitpath(file))) + else + errorlevel = cp(file,source,installdir) + if errorlevel ~= 0 then return errorlevel end end end end return 0 end local errorlevel = unpack() - if errorlevel ~= 0 then - return errorlevel + if errorlevel ~= 0 then return errorlevel end + errorlevel = install_files(unpackdir,"tex",{installfiles}) + + install_files(unpackdir,"bibtex/bst",{bstfiles},module,true) + + install_files(unpackdir,"makeindex",{makeindexfiles},module,true) + + install_files(unpackdir,"scripts",{scriptfiles},module) + if errorlevel ~= 0 then return errorlevel end + if full then + errorlevel = doc() + if errorlevel ~= 0 then return errorlevel end + + -- Creates a 'controlled' list of files + local function excludelist(dir,include,exclude) + include = include or { } + exclude = exclude or { } + dir = dir or currentdir + local includelist = { } + local excludelist = { } + for _,glob_table in pairs(exclude) do + for _,glob in pairs(glob_table) do + for file,_ in pairs(tree(dir,glob)) do + excludelist[file] = true + end + end + end + for _,glob in pairs(include) do + for file,_ in pairs(tree(dir,glob)) do + if not excludelist[file] then + insert(includelist, file) + end + end + end + return includelist + end + + -- For the purposes here, any typesetting demo files need to be + -- part of the main typesetting list + local typesetfiles = typesetfiles + for _,glob in pairs(typesetdemofiles) do + insert(typesetfiles,glob) + end + + -- Find PDF files + pdffiles = { } + for _,glob in pairs(typesetfiles) do + insert(pdffiles,(gsub(glob,"%.%w+$",".pdf"))) + end + + -- Set up lists: global as they are also needed to do CTAN releases + typesetlist = excludelist(docfiledir,typesetfiles,{sourcefiles}) + sourcelist = excludelist(sourcefiledir,sourcefiles, + {bstfiles,installfiles,makeindexfiles,scriptfiles}) + + errorlevel = install_files(sourcefiledir,"source",{sourcelist}) + + install_files(docfiledir,"doc", + {bibfiles,demofiles,docfiles,pdffiles,textfiles,typesetlist}) + if errorlevel ~= 0 then return errorlevel end + + -- Any script man files need special handling + local manfiles = { } + for _,glob in pairs(scriptmanfiles) do + for file,_ in pairs(tree(docfiledir,glob)) do + if dry_run then + insert(manfiles,"man" .. match(file,".$") .. "/" .. + select(2,splitpath(file))) + else + -- Man files should have a single-digit extension: the type + local installdir = target .. "/doc/man/man" .. match(file,".$") + errorlevel = errorlevel + mkdir(installdir) + errorlevel = errorlevel + cp(file,docfiledir,installdir) + end + end + end + if next(manfiles) then + print("\n" .. "For installation in " .. target .. "/doc/man:") + for _,v in ipairs(manfiles) do + print("- " .. v) + end + end end - return install_files(installfiles, "/tex/" .. moduledir) - + install_files(scriptfiles, "/scripts/" .. module) + return errorlevel +end + +function install() + return install_files(gethome(),options["full"],options["dry-run"]) end diff --git a/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua b/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua index 8c39da91600..6398288a0a2 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua @@ -109,7 +109,7 @@ target_list = local errorlevel = call(modules,"tag") -- Deal with any files in the bundle dir itself if errorlevel == 0 then - errorlevel = tag(names[1]) + errorlevel = tag(names) end return errorlevel end, diff --git a/Master/texmf-dist/scripts/l3build/l3build-tagging.lua b/Master/texmf-dist/scripts/l3build/l3build-tagging.lua index 9a6f8cf5014..6ad11e68943 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-tagging.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-tagging.lua @@ -28,7 +28,7 @@ local os_date = os.date local match = string.match local gsub = string.gsub -function update_tag(filename,content,tagname,tagdate) +update_tag = update_tag or function(filename,content,tagname,tagdate) return content end @@ -61,8 +61,9 @@ local function update_file_tag(file,tagname,tagdate) return 0 end -function tag(tagname) +function tag(tagnames) local tagdate = options["date"] or os_date("%Y-%m-%d") + local tagname = tagnames[1] or tagdate local dirs = remove_duplicates({currentdir, sourcefiledir, docfiledir}) local errorlevel = 0 for _,dir in pairs(dirs) do diff --git a/Master/texmf-dist/scripts/l3build/l3build-variables.lua b/Master/texmf-dist/scripts/l3build/l3build-variables.lua index 98ce7fd6c0e..f24b11acb3a 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-variables.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-variables.lua @@ -159,6 +159,7 @@ end -- Other required settings asciiengines = asciiengines or {"pdftex"} checkruns = checkruns or 1 +ctanzip = ctanzip or ctanpkg .. "-ctan" epoch = epoch or 1463734800 if flatten == nil then flatten = true diff --git a/Master/texmf-dist/scripts/l3build/l3build.lua b/Master/texmf-dist/scripts/l3build/l3build.lua index 5cefd6d5dc0..2107f024d3a 100755 --- a/Master/texmf-dist/scripts/l3build/l3build.lua +++ b/Master/texmf-dist/scripts/l3build/l3build.lua @@ -25,7 +25,7 @@ for those people who are interested. --]] -- Version information -release_date = "2018-08-02" +release_date = "2018-08-04" -- File operations are aided by the LuaFileSystem module local lfs = require("lfs") @@ -34,6 +34,7 @@ local lfs = require("lfs") local assert = assert local ipairs = ipairs +local insert = table.insert local lookup = kpse.lookup local match = string.match local next = next @@ -123,13 +124,28 @@ if options["target"] == "check" then if #checkconfigs > 1 then local errorlevel = 0 local opts = options + local failed = { } for i = 1, #checkconfigs do opts["config"] = {checkconfigs[i]} errorlevel = call({"."}, "check", opts) - if errorlevel ~= 0 then exit(1) end + if errorlevel ~= 0 then + if options["halt-on-error"] then + exit(1) + else + insert(failed,checkconfigs[i]) + end + end + end + if next(failed) then + print(" Failed tests for configs:") + for _,config in ipairs(failed) do + print(" - " .. config) + end + exit(1) + else + -- Avoid running the 'main' set of tests twice + exit(0) end - -- Avoid running the 'main' set of tests twice - exit(0) end end if #checkconfigs == 1 and diff --git a/Master/texmf-dist/source/latex/l3build/l3build.dtx b/Master/texmf-dist/source/latex/l3build/l3build.dtx index f15e1b970a9..5032e78ce5c 100644 --- a/Master/texmf-dist/source/latex/l3build/l3build.dtx +++ b/Master/texmf-dist/source/latex/l3build/l3build.dtx @@ -152,6 +152,7 @@ \luavarseparator \luavarset{asciiengines}{\{"pdftex"\}}{Engines which should log as sure ASCII} \luavarset{checkruns} {1} {Number of runs to complete for a test before comparing the log} +\luavarset{ctanzip}{ctanpkg ... "-ctan"}{Name of the zip file created for upload to CTAN} \luavarset{epoch} {1463734800} {Epoch (Unix date) to set for test runs} \luavarset{flatten} {true} {Switch to flatten any source structure when sending to CTAN} \luavarset{maxprintline}{79} {Length of line to use in log files} @@ -221,7 +222,7 @@ % }^^A % } % -% \date{Released 2018-08-02} +% \date{Released 2018-08-04} % % \maketitle % \tableofcontents @@ -329,6 +330,8 @@ % \item |--force| (|-f|) Force checks to run even if sanity % checks fail, \emph{e.g.}~when |--engine| is not given in % \luavar{checkengines} +% \item |--full| Instructs the \texttt{install} target to include the +% \texttt{doc} and \texttt{source} trees % \item |--halt-on-error| (|-H|) Specifies that checks % should stop as soon as possible, rather than running all requested % tests; the difference file is printed in the terminal directly in the case of failure @@ -1122,10 +1125,10 @@ % \item Removal of blank lines % \end{itemize} % -% Testing on the result of typesetting \texttt{.pvt} files is carried out using -% only the standard engine, as aspects of PDF files are not identical between -% engines. To allow platform-independence, PDF-based tests must use only -% Type1 or OpenType fonts: Type3 fonts are system-dependent. +% To allow platform-independence, PDF-based tests must use only Type~1 or +% OpenType fonts: Type3 fonts are system-dependent. PDF files are +% engine-specific, thus one |.tpf| file should be stored per engine to be +% tested. % % \section{Release-focussed features} % @@ -1251,16 +1254,22 @@ % \toprule % Entry & Type \\ % \midrule -% \var{date} & String \\ -% \var{engine} & Table \\ -% \var{force} & Boolean \\ -% \var{halt} & Boolean \\ -% \var{help} & Boolean \\ -% \var{names} & Table \\ -% \var{quiet} & Boolean \\ -% \var{rerun} & Boolean \\ -% \var{testfiledir} & Table \\ -% \var{version} & String \\ +% \var{config} & Table \\ +% \var{date} & String \\ +% \var{dirty} & Boolean \\ +% \var{dry-run} & Boolean \\ +% \var{engine} & Table \\ +% \var{epoch} & String \\ +% \var{first} & Boolean \\ +% \var{force} & Boolean \\ +% \var{full} & Boolean \\ +% \var{halt-on-error} & Boolean \\ +% \var{help} & Boolean \\ +% \var{names} & Table \\ +% \var{quiet} & Boolean \\ +% \var{rerun} & Boolean \\ +% \var{shuffle} & Boolean \\ +% \var{texmfhome} & String \\ % \bottomrule % \end{tabular} % \end{center} @@ -1452,6 +1461,17 @@ % |target| in this table is ignored. % \end{function} % +% \begin{function}{install_files()} +% \begin{syntax} +% |install_files(|\meta{target},\meta{full},\meta{dry-run}|)| +% \end{syntax} +% Installs the files from the module into the TDS root \meta{target}. +% If \meta{full} is \texttt{true}, all files are copied: if it is +% \texttt{false}, the \texttt{doc} and \texttt{source} trees are skipped. +% If \meta{dry-run} is \texttt{true}, no files are copied, but instead the +% files which would be copied are reported. +% \end{function} +% % \subsection{Customising the target list} % % The targets known to \pkg{l3build} are stored in the global table -- cgit v1.2.3