diff options
17 files changed, 679 insertions, 293 deletions
diff --git a/Build/source/texk/texlive/linked_scripts/l3build/l3build.lua b/Build/source/texk/texlive/linked_scripts/l3build/l3build.lua index 1ded83cc5b6..5cefd6d5dc0 100644 --- a/Build/source/texk/texlive/linked_scripts/l3build/l3build.lua +++ b/Build/source/texk/texlive/linked_scripts/l3build/l3build.lua @@ -2,7 +2,7 @@ --[[ -File l3build.lua Copyright (C) 2014-2017 The LaTeX3 Project +File l3build.lua Copyright (C) 2014-2018 The LaTeX3 Project It may be distributed and/or modified under the conditions of the LaTeX Project Public License (LPPL), either version 1.3c of this @@ -25,7 +25,7 @@ for those people who are interested. --]] -- Version information -release_date = "2018-05-10" +release_date = "2018-08-02" -- File operations are aided by the LuaFileSystem module local lfs = require("lfs") @@ -44,7 +44,7 @@ local exit = os.exit -- l3build setup and functions kpse.set_program_name("kpsewhich") -build_kpse_path = string.match(lookup("l3build.lua"),"(.*[/])") +build_kpse_path = match(lookup("l3build.lua"),"(.*[/])") local function build_require(s) require(lookup("l3build-"..s..".lua", { path = build_kpse_path } ) ) end @@ -53,15 +53,6 @@ end build_require("arguments") build_require("help") --- Filter out special cases early -if options["target"] == "help" then - help() - exit(0) -elseif options["target"] == "version" then - version() - exit(0) -end - build_require("file-functions") build_require("typesetting") build_require("aux") @@ -75,6 +66,16 @@ build_require("manifest-setup") build_require("tagging") build_require("stdmain") +-- This has to come after stdmain(), +-- and that has to come after the functions are defined +if options["target"] == "help" then + help() + exit(0) +elseif options["target"] == "version" then + version() + exit(0) +end + -- Allow main function to be disabled 'higher up' main = main or stdmain diff --git a/Master/texmf-dist/doc/latex/l3build/CHANGELOG.md b/Master/texmf-dist/doc/latex/l3build/CHANGELOG.md new file mode 100644 index 00000000000..b9a434787b0 --- /dev/null +++ b/Master/texmf-dist/doc/latex/l3build/CHANGELOG.md @@ -0,0 +1,142 @@ +# Changelog +All notable changes to the `l3build` bundle since the start of 2018 +will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +this project uses date-based 'snapshot' version identifiers. + +## [Unreleased] + +## [2018-08-02] + +### Added +- `CHANGELOG.md` +- `--dirty` option +- `includetests` and `excludetests` variables for controlling which tests + run +- `target_list` table to allow control of targets without redefining + `main()` + +### Changed +- PDF-based testing now uses 'digested' PDF file for comparison, + working from dedicated .pvt input files + +### Removed +- "--pdf" command line switch + +## [2018-05-10] + +### Changed +- Revert appearance of date lines in `.tlg` files: + this is on balance problematic + +## [2018-05-06] + +### Added +- Variable `dynamicfiles` to be cleaned between each test run + +### Changed +- Normalise dates to placeholder "YYYY-MM-DD": may require `.tlg` updates + +### Fixed +- Include dot files in `tree()` (fixes #30) + +## [2018-03-26] + +### Changed +- Omit ISO date lines in `.tlg` files + +## [2018-03-24] + +### Changed +- Allow 'short cut' of check runs +- Support for upcoming LaTeX kernel release functions + +## [2018-03-10] + +### Changed +- Add `#!` line for POSIX users +- Set POSIX u+x on `l3build.lua` + +### Fixed +- Handling of script name with or without extension + +## [2018-03-09] + +### Fixed +- Pass through script name correctly with new set up + +## [2018-03-08] + +### Added +- Target `tag`, variable `tagfiles` and function `update_tag()` +- Variables `scriptfiles` and `scriptmanfiles` to support installation + of scripts + +### Changed +- `l3build` can now be run as a top-level script rather than using + `texlua build.lua ...` +- Normalisation of LuaTeX-derived `tlg` files, in preparation for + TeX Live 2018 + +### Deprecated +- Use of wrapper `build.lua` script to call `l3build`: the new + top-level script approach is preferred + +### Removed +- Target `setversion` and variable `versionfiles` + +## [2018-02-20] + +### Changed +- Allow for `checkopts` adding code/files + +### Fixed +- Creation of 'structured' CTAN releases +- Quote test names correctly + +## [2018-01-27] + +### Added +- Target `uninstall` +- Options `--first` and `--last` + +## Changed +- Normalisation for upcoming LuaTeX 1.07 release + +### Fixed +- Behaviour of check on Windows when using standard `fc` tool + +## [2018-01-10] + +## Added +- Target `manifest` for construction of file manifests automatically +- Variable `auxfiles` +- Option `--dry-run` for installation/cleaning +- Option `--texmfhome` to allow customisation of installation +- Option `--shuffle` to run tests in a random order + +### Changed +- Sort list of tests to avoid system-dependent ordering +- Split `l3build` into multiple files for improved maintenance + +### Fixed +- Issue with `recordstatus` + +### Removed +- Rationalise short option names: removed `-d`, `-E`, `-r` +- Target `cmdcheck`: specific to LaTeX3 kernel work + +[Unreleased]: https://github.com/latex3/l3build/compare/2018-08-02...HEAD +[2018-08-02]: https://github.com/latex3/l3build/compare/2018-05-06...2018-08-02 +[2018-05-10]: https://github.com/latex3/l3build/compare/2018-05-06...2018-05-10 +[2018-05-06]: https://github.com/latex3/l3build/compare/2018-03-26...2018-05-06 +[2018-03-26]: https://github.com/latex3/l3build/compare/2018-03-24...2018-03-26 +[2018-03-24]: https://github.com/latex3/l3build/compare/2018-03-10...2018-03-24 +[2018-03-10]: https://github.com/latex3/l3build/compare/2018-03-09...2018-03-10 +[2018-03-09]: https://github.com/latex3/l3build/compare/2018-03-08...2018-03-09 +[2018-03-08]: https://github.com/latex3/l3build/compare/2018-02-20...2018-03-08 +[2018-02-20]: https://github.com/latex3/l3build/compare/2018-01-27...2018-02-20 +[2018-01-27]: https://github.com/latex3/l3build/compare/2018-01-10...2018-01-27 +[2018-01-10]: https://github.com/latex3/l3build/compare/2017-12-12...2018-01-10 + diff --git a/Master/texmf-dist/doc/latex/l3build/README.md b/Master/texmf-dist/doc/latex/l3build/README.md index 756ce85a42f..43ac56bf0e9 100644 --- a/Master/texmf-dist/doc/latex/l3build/README.md +++ b/Master/texmf-dist/doc/latex/l3build/README.md @@ -1,7 +1,7 @@ l3build: a testing and building system for LaTeX3 ================================================= -Release 2018-05-10 +Release 2018-08-02 Overview -------- @@ -55,6 +55,6 @@ by e-mail: <latex-team@latex-project.org>. ----- -<p>Copyright (C) 2014-2017 The LaTeX3 Project <br /> +<p>Copyright (C) 2014-2018 The LaTeX3 Project <br /> <a href="http://latex-project.org/">http://latex-project.org/</a> <br /> All rights reserved.</p> diff --git a/Master/texmf-dist/doc/latex/l3build/l3build.pdf b/Master/texmf-dist/doc/latex/l3build/l3build.pdf Binary files differindex fb2f936ae56..c5675f9d9a9 100644 --- a/Master/texmf-dist/doc/latex/l3build/l3build.pdf +++ b/Master/texmf-dist/doc/latex/l3build/l3build.pdf diff --git a/Master/texmf-dist/doc/man/man1/l3build.1 b/Master/texmf-dist/doc/man/man1/l3build.1 index e3cc9f1e933..fb85523ad0b 100644 --- a/Master/texmf-dist/doc/man/man1/l3build.1 +++ b/Master/texmf-dist/doc/man/man1/l3build.1 @@ -1,4 +1,4 @@ -.TH l3build 1 "2018-05-10" +.TH l3build 1 "2018-08-02" .SH NAME l3build \- Checking and building packages .SH SYNOPSIS diff --git a/Master/texmf-dist/doc/man/man1/l3build.man1.pdf b/Master/texmf-dist/doc/man/man1/l3build.man1.pdf Binary files differindex 419d5a2b93e..63568956eab 100644 --- a/Master/texmf-dist/doc/man/man1/l3build.man1.pdf +++ b/Master/texmf-dist/doc/man/man1/l3build.man1.pdf diff --git a/Master/texmf-dist/scripts/l3build/l3build-arguments.lua b/Master/texmf-dist/scripts/l3build/l3build-arguments.lua index 499e2c4031e..4416ab69cda 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-arguments.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-arguments.lua @@ -47,6 +47,11 @@ option_list = desc = "Sets the date to insert into sources", type = "string" }, + dirty = + { + desc = "Skip cleaning up the test area", + type = "boolean" + }, ["dry-run"] = { desc = "Dry run for install", @@ -85,12 +90,6 @@ option_list = short = "h", type = "boolean" }, - pdf = - { - desc = "Check/save PDF files", - short = "p", - type = "boolean" - }, quiet = { desc = "Suppresses TeX output when unpacking", diff --git a/Master/texmf-dist/scripts/l3build/l3build-aux.lua b/Master/texmf-dist/scripts/l3build/l3build-aux.lua index f081054d575..d93c00c6745 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-aux.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-aux.lua @@ -100,10 +100,7 @@ function call(dirs, target, opts) if i == "." then text = " with configuration " .. opts["config"][1] end - print( - "Running script " .. scriptname .. " with target \"" .. target - .. "\"" .. text - ) + print("Running l3build with target \"" .. target .. "\"" .. text ) local errorlevel = run( i, "texlua " .. scriptname .. " " .. target .. s diff --git a/Master/texmf-dist/scripts/l3build/l3build-check.lua b/Master/texmf-dist/scripts/l3build/l3build-check.lua index 442bd916814..7cac25a7d93 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-check.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-check.lua @@ -55,7 +55,9 @@ local remove = os.remove -- Set up the check system files: needed for checking one or more tests and -- for saving the test files function checkinit() - cleandir(testdir) + if not options["dirty"] then + cleandir(testdir) + end depinstall(checkdeps) -- Copy dependencies to the test directory itself: this makes the paths -- a lot easier to manage, and is important for dealing with the log and @@ -207,6 +209,9 @@ local function formatlog(logfile, newfile, engine, errlevels) if match(line, "^%.*\\special%{pdf: docinfo << /Creator") then return "" end + if match(line, "^%.*\\special%{dvipdfmx:config") then + return "" + end -- Remove the \special line possibly present in DVI mode for paper size if match(line, "^%.*\\special%{papersize") then return "" @@ -482,6 +487,49 @@ local function formatlualog(logfile, newfile, luatex) close(newfile) end +local function normalise_pdf(pdffile,npdffile) + local file = assert(open(pdffile, "rb")) + local contents = gsub(file:read("*all") .. "\n", "\r\n", "\n") + close(file) + local new_content = "" + local stream_content = "" + local binary = false + local stream = false + for line in gmatch(contents, "([^\n]*)\n") do + if stream then + if match(line,"endstream") then + stream = false + if binary then + new_content = new_content .. "[BINARY STREAM]" .. os_newline + else + new_content = new_content .. stream_content .. line .. os_newline + end + binary = false + else + for i = 0, 31 do + if match(line,char(i)) then + binary = true + break + end + end + if not binary and not match(line, "^ *$") then + stream_content = stream_content .. line .. os_newline + end + end + elseif match(line,"^stream$") then + binary = false + stream = true + stream_content = "stream" .. os_newline + elseif not match(line, "^ *$") then + new_content = new_content .. line .. os_newline + end + end + local newfile = open(npdffile, "w") + output(newfile) + write(new_content) + close(newfile) +end + -- Run one test which may have multiple engine-dependent comparisons -- Should create a difference file for each failed test function runcheck(name, hide) @@ -489,64 +537,70 @@ function runcheck(name, hide) if options["engine"] then checkengines = options["engine"] end + -- Used for both .lvt and .pvt tests + local function check_and_diff(ext,engine,enginename,comp,pdftest) + runtest(name,engine,hide,ext,pdftest,true) + local errorlevel = comp(name,enginename) + if errorlevel ~= 0 and options["halt-on-error"] then + showfaileddiff() + return 1 + end + return errorlevel + end local errorlevel = 0 - for _,i in ipairs(checkengines) do + -- First check by log + if fileexists(testfiledir .. "/" .. name .. lvtext) then + for _,i in pairs(checkengines) do -- Allow for luatex == luajittex for .tlg purposes - local engine = i - if i == "luajittex" then - engine = "luatex" - end - checkpdf = setup_check(name, engine) - runtest(name, i, hide, lvtext, checkpdf, true) - -- Generation of results depends on test type - local errlevel - if checkpdf then - errlevel = compare_pdf(name, engine) - else - errlevel = compare_tlg(name, engine) - end - if errlevel ~= 0 and options["halt-on-error"] then - showfaileddiff() - if errlevel ~= 0 then + local engine = i + if i == "luajittex" then + engine = "luatex" + end + setup_check(name, engine) + local errlevel = 0 + errlevel = check_and_diff(lvtext,i,engine,compare_tlg) + if errlevel ~= 0 and options["halt-on-error"] then return 1 end - end - if errlevel > errorlevel then - errorlevel = errlevel + if errlevel > errorlevel then + errorlevel = errlevel + end end end + -- Then check by PDF + if fileexists(testfiledir .. "/" .. name .. pvtext) then + setup_check(name, stdengine) + errorlevel = check_and_diff(pvtext,stdengine,stdengine,compare_pdf,true) + + errorlevel + end + -- Return everything return errorlevel end function setup_check(name, engine) local testname = name .. "." .. engine - local pdffile = locate( - {testfiledir, unpackdir}, - {testname .. pdfext, name .. pdfext} - ) local tlgfile = locate( {testfiledir, unpackdir}, {testname .. tlgext, name .. tlgext} ) + local tpffile = locate( + {testfiledir, unpackdir}, + {testname .. tpfext, name .. tpfext} + ) -- Attempt to generate missing reference file from expectation - if not (pdffile or tlgfile) then + if not (tlgfile or tpffile) then if not locate({unpackdir, testfiledir}, {name .. lveext}) then print( - "Error: failed to find " .. pdfext .. ", " .. tlgext .. " or " + "Error: failed to find " .. tlgext .. ", " .. tlpext .. " or " .. lveext .. " file for " .. name .. "!" ) exit(1) end runtest(name, engine, true, lveext, true, false) - pdffile = testdir .. "/" .. testname .. pdfext - -- If a PDF is generated use it for comparisons - if not fileexists(pdffile) then - pdffile = nil - ren(testdir, testname .. logext, testname .. tlgext) - end + ren(testdir, testname .. logext, testname .. tlgext) else -- Install comparison files found - for _,v in pairs({pdffile, tlgfile}) do + for _,v in pairs({tlgfile, tpffile}) do if v then cp( match(v, ".*/(.*)"), @@ -556,36 +610,21 @@ function setup_check(name, engine) end end end - if pdffile then - local pdffile = match(pdffile, ".*/(.*)") - ren( - testdir, - pdffile, - gsub(pdffile, pdfext .. "$", ".ref" .. pdfext) - ) - return true - else - return false - end end -function compare_pdf(name, engine) +function compare_pdf(name,engine,cleanup) local errorlevel local testname = name .. "." .. engine - local cmpfile = testdir .. "/" .. testname .. os_cmpext - local pdffile = testdir .. "/" .. testname .. pdfext - local refpdffile = locate( - {testdir}, {testname .. ".ref" .. pdfext, name .. ".ref" .. pdfext} - ) - if not refpdffile then - return + local difffile = testdir .. "/" .. name .. pdfext .. os_diffext + local pdffile = testdir .. "/" .. testname .. pdfext + local tpffile = testdir .. "/" .. name .. tpfext + if not tpffile then + return 1 end - errorlevel = execute( - os_cmpexe .. " " .. normalize_path(refpdffile) - .. " " .. pdffile .. " > " .. cmpfile - ) - if errorlevel == 0 then - remove(cmpfile) + errorlevel = execute(os_diffexe .. " " + .. normalize_path(tpffile .. " " .. pdffile .. " > " .. difffile)) + if errorlevel == 0 or cleanup then + remove(difffile) end return errorlevel end @@ -631,7 +670,7 @@ end -- Run one of the test files: doesn't check the result so suitable for -- both creating and verifying .tlg files -function runtest(name, engine, hide, ext, makepdf, breakout) +function runtest(name, engine, hide, ext, pdfmode, breakout) local lvtfile = name .. (ext or lvtext) cp(lvtfile, fileexists(testfiledir .. "/" .. lvtfile) and testfiledir or unpackdir, testdir) @@ -660,7 +699,7 @@ function runtest(name, engine, hide, ext, makepdf, breakout) end -- Special casing for XeTeX engine local checkopts = checkopts - if match(engine, "xetex") and not makepdf then + if match(engine, "xetex") and not pdfmode then checkopts = checkopts .. " -no-pdf" end -- Special casing for ConTeXt @@ -683,6 +722,8 @@ function runtest(name, engine, hide, ext, makepdf, breakout) end local logfile = testdir .. "/" .. name .. logext local newfile = testdir .. "/" .. name .. "." .. engine .. logext + local pdffile = testdir .. "/" .. name .. pdfext + local npffile = testdir .. "/" .. name .. "." .. engine .. pdfext local asciiopt = "" for _,i in ipairs(asciiengines) do if realengine == i then @@ -718,16 +759,27 @@ function runtest(name, engine, hide, ext, makepdf, breakout) ) -- Break the loop if the result is stable if breakout and i < checkruns then - formatlog(logfile, newfile, engine, errlevels) - if compare_tlg(name,engine,true) == 0 then - break + if pdfmode then + normalise_pdf(pdffile,npffile) + if compare_pdf(name,engine,true) == 0 then + break + end + else + formatlog(logfile, newfile, engine, errlevels) + if compare_tlg(name,engine,true) == 0 then + break + end end end end - if makepdf and fileexists(testdir .. "/" .. name .. dviext) then + if pdfmode and fileexists(testdir .. "/" .. name .. dviext) then dvitopdf(name, testdir, engine, hide) end - formatlog(logfile, newfile, engine, errlevels) + if pdfmode then + normalise_pdf(pdffile,npffile) + else + formatlog(logfile, newfile, engine, errlevels) + end -- Store secondary files for this engine for _,filetype in pairs(auxfiles) do for _,file in pairs(filelist(testdir, filetype)) do @@ -751,7 +803,8 @@ end -- Look for a test: could be in the testfiledir or the unpackdir function testexists(test) - return(locate({testfiledir, unpackdir}, {test .. lvtext})) + return(locate({testfiledir, unpackdir}, + {test .. lvtext, test .. pvtext})) end -- Standard versions of the main targets for building modules @@ -769,15 +822,36 @@ function check(names) names = names or { } -- No names passed: find all test files if not next(names) then - for _,i in pairs(filelist(testfiledir, "*" .. lvtext)) do - insert(names, jobname(i)) + local excludenames = { } + for _,glob in pairs(excludetests) do + for _,name in pairs(filelist(testfiledir, glob .. lvtext)) do + excludenames[jobname(name)] = true + end + for _,name in pairs(filelist(unpackdir, glob .. lvtext)) do + excludenames[jobname(name)] = true + end + for _,name in pairs(filelist(testfiledir, glob .. pvtext)) do + excludenames[jobname(name)] = true + end end - for _,i in ipairs(filelist(unpackdir, "*" .. lvtext)) do - if fileexists(testfiledir .. "/" .. i) then - print("Duplicate test file: " .. i) - return 1 - else - insert(names, jobname(i)) + local function addname(name) + if not excludenames[jobname(name)] then + insert(names,jobname(name)) + end + end + for _,glob in pairs(includetests) do + for _,name in pairs(filelist(testfiledir, glob .. lvtext)) do + addname(name) + end + for _,name in pairs(filelist(testfiledir, glob .. pvtext)) do + addname(name) + end + for _,name in pairs(filelist(unpackdir, glob .. lvtext)) do + if fileexists(testfiledir .. "/" .. name) then + print("Duplicate test file: " .. i) + return 1 + end + addname(name) end end sort(names) @@ -853,9 +927,6 @@ function checkdiff() for _,i in ipairs(filelist(testdir, "*" .. os_diffext)) do print(" - " .. testdir .. "/" .. i) end - for _,i in ipairs(filelist(testdir, "*" .. os_cmpext)) do - print(" - " .. testdir .. "/" .. i) - end print("") end @@ -871,50 +942,44 @@ function showfaileddiff() print(content) print("-----------------------------------------------------------------------------------") end - for _,i in ipairs(filelist(testdir, "*" .. os_cmpext)) do - print(" - " .. testdir .. "/" .. i) - end end function save(names) checkinit() local engines = options["engine"] or {stdengine} for _,name in pairs(names) do - local engine - for _,engine in pairs(engines) do - local tlgengine = ((engine == stdengine and "") or "." .. engine) - local tlgfile = name .. tlgengine .. tlgext - local spdffile = name .. tlgengine .. pdfext - local newfile = name .. "." .. engine .. logext - local pdffile = name .. "." .. engine .. pdfext - local refext = ((options["pdf"] and pdfext) or tlgext) - if testexists(name) then - print("Creating and copying " .. refext) - runtest(name, engine, false, lvtext, options["pdf"]) - if options["pdf"] then - ren(testdir, pdffile, spdffile) - cp(spdffile, testdir, testfiledir) + if testexists(name) then + for _,engine in pairs(engines) do + local testengine = ((engine == stdengine and "") or "." .. engine) + if fileexists(testfiledir .. "/" .. name .. lvtext) then + -- Create one or more .tlg files + print("Creating and copying " .. tlgext) + local tlgfile = name .. testengine .. tlgext + local newfile = name .. "." .. engine .. logext + runtest(name,engine,false,lvtext) + ren(testdir,newfile,tlgfile) + cp(tlgfile,testdir,testfiledir) + if fileexists(unpackdir .. "/" .. tlgfile) then + print("Saved " .. tlgext + .. " file overrides unpacked version of the same name") + end else - ren(testdir, newfile, tlgfile) - cp(tlgfile, testdir, testfiledir) - end - if fileexists(unpackdir .. "/" .. tlgfile) then - print( - "Saved " .. tlgext - .. " file overrides unpacked version of the same name" - ) + -- Create one .tpf file + print("Creating and copying " .. tpfext) + local tpffile = name .. tpfext + local newfile = name .. "." .. engine .. pdfext + runtest(name,engine,false,pvtext,true) + ren(testdir,newfile,tpffile) + cp(tpffile,testdir,testfiledir) end - elseif locate({unpackdir, testfiledir}, {name .. lveext}) then - print( - "Saved " .. tlgext .. " file overrides a " - .. lveext .. " file of the same name" - ) - else - print( - "Test input \"" .. testfiledir .. "/" .. name .. lvtext - .. "\" not found" - ) + return 0 end + elseif locate({unpackdir, testfiledir}, {name .. lveext}) then + print("Saved " .. tlgext .. " file overrides a " + .. lveext .. " file of the same name") + else + print('Test "'.. name .. '"not found') + return 1 end end end diff --git a/Master/texmf-dist/scripts/l3build/l3build-file-functions.lua b/Master/texmf-dist/scripts/l3build/l3build-file-functions.lua index 83b4bfb6539..510567e5342 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-file-functions.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-file-functions.lua @@ -90,6 +90,67 @@ local function glob_to_pattern(glob) return match(char, "^%w$") and char or "%" .. char end + -- remove a leading "\" if one is present: + -- this can simple be discarded + local function unescape() + if char == "\\" then + i = i + 1 + char = sub(glob,i,i) + if char == "" then + pattern = "[^]" + return false + end + end + return true + end + + -- look for the body of a set: + -- char is already the first token + local function collect_set() + while true do + if char == "" then + pattern = "[^]" + return false + elseif char == "]" then + pattern = pattern .. "]" + break + else + if not unescape(char) then break end + local char_one = char + i = i + 1 + char = sub(glob,i,i) + if char == "" then + pattern = "[^]" + return false + elseif char == "-" then + i = i + 1 + char = sub(glob,i,i) + if char == "" then + pattern = "[^]" + return false + elseif char == "]" then + -- Not a range + pattern = pattern .. escape(char_one) .. "%-]" + return true + else + if not unescape(char) then break end + pattern = pattern .. escape(char_one) .. "-" .. escape(char) + end + elseif char == "]" then + -- Successfully done + pattern = pattern .. escape(char_one) .. "]" + return true + else + pattern = pattern .. escape(char_one) + i = i - 1 -- Back up one + end + end + i = i + 1 + char = sub(glob,i,i) + end + return true + end + -- Convert tokens. while true do i = i + 1 @@ -102,9 +163,24 @@ local function glob_to_pattern(glob) elseif char == "*" then pattern = pattern .. ".*" elseif char == "[" then - -- Ignored - print("[...] syntax not supported in globs!") - elseif char == "\\" then + -- Search inside a charset + i = i + 1 + char = sub(glob,i,i) + if char == "" or char == "]" then -- Matches nothing + pattern = "[^]" + break + elseif char == "^" or char == "!" then + i = i + 1 + char = sub(glob,i,i) + if char ~= "]" then -- Matches anything so ignore + pattern = pattern .. "[^" + if not collect_set() then break end + end + else + pattern = pattern .. "[" + if not collect_set() then break end + end + elseif char == "\\" then -- Discarded i = i + 1 char = sub(glob, i, i) if char == "" then @@ -130,8 +206,6 @@ os_setenv = "export" os_yes = "printf 'y\\n%.0s' {1..200}" os_ascii = "echo \"\"" -os_cmpexe = getenv("cmpexe") or "cmp" -os_cmpext = getenv("cmpext") or ".cmp" os_diffext = getenv("diffext") or ".diff" os_diffexe = getenv("diffexe") or "diff -c --strip-trailing-cr" os_grepexe = "grep" @@ -139,8 +213,6 @@ os_newline = "\n" if os_type == "windows" then os_ascii = "@echo." - os_cmpexe = getenv("cmpexe") or "fc /b" - os_cmpext = getenv("cmpext") or ".cmp" os_concat = "&" os_diffext = getenv("diffext") or ".fc" os_diffexe = getenv("diffexe") or "fc /n" diff --git a/Master/texmf-dist/scripts/l3build/l3build-help.lua b/Master/texmf-dist/scripts/l3build/l3build-help.lua index c7050c44210..18caf87daae 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-help.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-help.lua @@ -22,6 +22,11 @@ for those people who are interested. --]] +local insert = table.insert +local match = string.match +local rep = string.rep +local sort = table.sort + function version() print( "\n" .. @@ -31,46 +36,44 @@ function version() end function help() + local function setup_list(list) + local longest = 0 + for k,v in pairs(list) do + if k:len() > longest then + longest = k:len() + end + end + -- Sort the options + local t = { } + for k,_ in pairs(list) do + insert(t, k) + end + sort(t) + return longest,t + end + local scriptname = "l3build" - if not string.match(arg[0], "l3build(%.lua)$") then + if not match(arg[0], "l3build(%.lua)$") then scriptname = arg[0] end print("usage: " .. scriptname .. " <command> [<options>] [<names>]") print("") print("The most commonly used l3build commands are:") - if testfiledir ~= "" then - print(" check Run all automated tests") - end - print(" clean Clean out directory tree") - if module == "" or bundle == "" then - print(" ctan Create CTAN-ready archive") - end - print(" doc Typesets all documentation files") - print(" install Installs files into the local texmf tree") - if module ~= "" and testfiledir ~= "" then - print(" save Saves test validation log") + local longest,t = setup_list(target_list) + for _,k in ipairs(t) do + local target = target_list[k] + local filler = rep(" ", longest - k:len() + 1) + if target["desc"] then + print(" " .. k .. filler .. target["desc"]) + end end - print(" tag Update release tags in files") - print(" uninstall Uninstalls files from the local texmf tree") - print(" unpack Unpacks the source files into the build tree") print("") print("Valid options are:") - local longest = 0 - for k,v in pairs(option_list) do - if k:len() > longest then - longest = k:len() - end - end - -- Sort the options - local t = { } - for k,_ in pairs(option_list) do - table.insert(t, k) - end - table.sort(t) + local longest,t = setup_list(option_list) for _,k in ipairs(t) do local opt = option_list[k] - local filler = string.rep(" ", longest - k:len() + 1) - if opt["desc"] then -- Skip --help as it has no desc + local filler = rep(" ", longest - k:len() + 1) + if opt["desc"] then if opt["short"] then print(" --" .. k .. "|-" .. opt["short"] .. filler .. opt["desc"]) else diff --git a/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua b/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua index 7dac27aa983..8c39da91600 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua @@ -22,7 +22,8 @@ for those people who are interested. --]] -local exit = os.exit +local exit = os.exit +local insert = table.insert -- List all modules function listmodules() @@ -34,7 +35,7 @@ function listmodules() assert(type(attr) == "table") if attr.mode == "directory" then if not exclmodules[entry] then - table.insert(modules, entry) + insert(modules, entry) end end end @@ -42,91 +43,132 @@ function listmodules() return modules end +target_list = + { + -- Some hidden targets + bundlecheck = + { + func = check, + pre = function() + if names then + print("Bundle checks should not list test names") + help() + exit(1) + end + return 0 + end + }, + bundleunpack = + { + func = bundleunpack, + pre = function() return(depinstall(unpackdeps)) end + }, + -- Public targets + check = + { + bundle_target = true, + desc = "Run all automated tests", + func = check, + }, + clean = + { + bundle_func = bundleclean, + desc = "Clean out directory tree", + func = clean + }, + ctan = + { + bundle_func = ctan, + desc = "Create CTAN-ready archive", + func = ctan + }, + doc = + { + desc = "Typesets all documentation files", + func = doc + }, + install = + { + desc = "Installs files into the local textmf tree", + func = install + }, + manifest = + { + desc = "Creates a manifest file", + func = manifest + }, + save = + { + desc = "Saves test validation log", + func = save + }, + tag = + { + bundle_func = function(names) + local modules = modules or listmodules() + local errorlevel = call(modules,"tag") + -- Deal with any files in the bundle dir itself + if errorlevel == 0 then + errorlevel = tag(names[1]) + end + return errorlevel + end, + desc = "Updates release tags in files", + func = tag, + pre = function(names) + if not names or #names ~=1 then + print("Tag name required") + help() + exit(1) + end + return 0 + end + }, + uninstall = + { + desc = "Uninstalls files from the local textmf tree", + func = uninstall + }, + unpack= + { + bundle_target = true, + desc = "Unpacks the source files into the build tree", + func = unpack + } + } + -- -- The overall main function -- -function stdmain(target, names) - local errorlevel - -- If the module name is empty, the script is running in a bundle: - -- apart from ctan all of the targets are then just mappings +function stdmain(target,names) + -- Deal with unknown targets up-front + if not target_list[target] then + help() + exit(1) + end + local errorlevel = 0 if module == "" then - -- Detect all of the modules modules = modules or listmodules() - if target == "doc" then - errorlevel = call(modules, "doc") - elseif target == "check" then - errorlevel = call(modules, "bundlecheck") - if errorlevel ~=0 then - print("There were errors: checks halted!\n") - end - elseif target == "clean" then - errorlevel = bundleclean() - elseif target == "ctan" then - errorlevel = ctan() - elseif target == "install" then - errorlevel = call(modules, "install") - elseif target == "tag" then - if options["names"] and #options["names"] == 1 then - errorlevel = call(modules,"tag") - -- Deal with any files in the bundle dir itself - if errorlevel == 0 then - errorlevel = tag(options["names"][1]) - end - else - print("Tag name required") - help() - exit(1) - end - elseif target == "uninstall" then - errorlevel = call(modules, "uninstall") - elseif target == "unpack" then - errorlevel = call(modules, "bundleunpack") + if target_list[target].bundle_func then + errorlevel = target_list[target].bundle_func(names) else - help() + -- Detect all of the modules + if target_list[target].bundle_target then + target = "bundle" .. target + end + errorlevel = call(modules,target) end else - if target == "bundleunpack" then -- 'Hidden' as only needed 'higher up' - depinstall(unpackdeps) - errorlevel = bundleunpack() - elseif target == "bundlecheck" then - errorlevel = check() - elseif target == "bundlectan" then - errorlevel = bundlectan() - elseif target == "doc" then - errorlevel = doc(names) - elseif target == "check" then - errorlevel = check(names) - elseif target == "clean" then - errorlevel = clean() - elseif target == "ctan" then - errorlevel = ctan() - elseif target == "install" then - errorlevel = install() - elseif target == "manifest" then - errorlevel = manifest() - elseif target == "save" then - if next(names) then - errorlevel = save(names) - else - help() - end - elseif target == "tag" then - if options["names"] and #options["names"] == 1 then - errorlevel = tag(options["names"][1]) - else - print("Tag name required") - help() - exit(1) - end - elseif target == "uninstall" then - errorlevel = uninstall() - elseif target == "unpack" then - errorlevel = unpack() - else - help() + if target_list[target].pre then + errorlevel = target_list[target].pre(names) + if errorlevel ~= 0 then + exit(1) + end end + errorlevel = target_list[target].func(names) end + -- All done, finish up if errorlevel ~= 0 then exit(1) else diff --git a/Master/texmf-dist/scripts/l3build/l3build-typesetting.lua b/Master/texmf-dist/scripts/l3build/l3build-typesetting.lua index 229745dafc6..71fe0ea6c4b 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-typesetting.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-typesetting.lua @@ -125,7 +125,7 @@ function makeindex(name, dir, inext, outext, logext, style) makeindexexe .. " " .. makeindexopts .. " -o " .. name .. outext .. (style and (" -s " .. style) or "") - .. " -t " .. name .. " " .. name .. inext + .. " -t " .. name .. logext .. " " .. name .. inext ) ) end diff --git a/Master/texmf-dist/scripts/l3build/l3build-variables.lua b/Master/texmf-dist/scripts/l3build/l3build-variables.lua index 38be1c7c5ea..98ce7fd6c0e 100644 --- a/Master/texmf-dist/scripts/l3build/l3build-variables.lua +++ b/Master/texmf-dist/scripts/l3build/l3build-variables.lua @@ -117,6 +117,10 @@ checkengines = checkengines or {"pdftex", "xetex", "luatex"} checkformat = checkformat or "latex" stdengine = stdengine or "pdftex" +-- The tests themselves +includetests = includetests or {"*"} +excludetests = excludetests or { } + -- Configs for testing checkconfigs = checkconfigs or {"build"} @@ -163,7 +167,6 @@ maxprintline = maxprintline or 79 packtdszip = packtdszip or false typesetcmds = typesetcmds or "" typesetruns = typesetruns or 2 -versionform = versionform or "" recordstatus = recordstatus or false -- Extensions for various file types: used to abstract out stuff a bit @@ -174,7 +177,9 @@ lveext = lveext or ".lve" lvtext = lvtext or ".lvt" pdfext = pdfext or ".pdf" psext = psext or ".ps" +pvtext = pvtext or ".pvt" tlgext = tlgext or ".tlg" +tpfext = tpfext or ".tpf" -- Manifest options manifestfile = manifestfile or "MANIFEST.md" diff --git a/Master/texmf-dist/scripts/l3build/l3build.lua b/Master/texmf-dist/scripts/l3build/l3build.lua index 1ded83cc5b6..5cefd6d5dc0 100755 --- a/Master/texmf-dist/scripts/l3build/l3build.lua +++ b/Master/texmf-dist/scripts/l3build/l3build.lua @@ -2,7 +2,7 @@ --[[ -File l3build.lua Copyright (C) 2014-2017 The LaTeX3 Project +File l3build.lua Copyright (C) 2014-2018 The LaTeX3 Project It may be distributed and/or modified under the conditions of the LaTeX Project Public License (LPPL), either version 1.3c of this @@ -25,7 +25,7 @@ for those people who are interested. --]] -- Version information -release_date = "2018-05-10" +release_date = "2018-08-02" -- File operations are aided by the LuaFileSystem module local lfs = require("lfs") @@ -44,7 +44,7 @@ local exit = os.exit -- l3build setup and functions kpse.set_program_name("kpsewhich") -build_kpse_path = string.match(lookup("l3build.lua"),"(.*[/])") +build_kpse_path = match(lookup("l3build.lua"),"(.*[/])") local function build_require(s) require(lookup("l3build-"..s..".lua", { path = build_kpse_path } ) ) end @@ -53,15 +53,6 @@ end build_require("arguments") build_require("help") --- Filter out special cases early -if options["target"] == "help" then - help() - exit(0) -elseif options["target"] == "version" then - version() - exit(0) -end - build_require("file-functions") build_require("typesetting") build_require("aux") @@ -75,6 +66,16 @@ build_require("manifest-setup") build_require("tagging") build_require("stdmain") +-- This has to come after stdmain(), +-- and that has to come after the functions are defined +if options["target"] == "help" then + help() + exit(0) +elseif options["target"] == "version" then + version() + exit(0) +end + -- Allow main function to be disabled 'higher up' main = main or stdmain diff --git a/Master/texmf-dist/source/latex/l3build/l3build.dtx b/Master/texmf-dist/source/latex/l3build/l3build.dtx index fd2e652347b..f15e1b970a9 100644 --- a/Master/texmf-dist/source/latex/l3build/l3build.dtx +++ b/Master/texmf-dist/source/latex/l3build/l3build.dtx @@ -102,13 +102,18 @@ \luavarseparator \luavarset{bakext}{".bak"}{Extension of backup files} \luavarset{dviext}{".dvi"}{Extension of DVI files} -\luavarset{lvtext}{".lvt"}{Extension of test files} +\luavarset{lvtext}{".lvt"}{Extension of log-based test files} \luavarset{tlgext}{".tlg"}{Extension of test file output} +\luavarset{tpfext}{".tpf"}{Extension of PDF-based test output} \luavarset{lveext}{".lve"}{Extension of auto-generating test file output} \luavarset{logext}{".log"}{Extension of checking output, before processing it into a \texttt{.tlg}} +\luavarset{pvtext}{".pvt"}{Extension of PDF-based test files} \luavarset{pdfext}{".pdf"}{Extension of PDF file for checking and saving} \luavarset{psext} {".ps"} {Extension of PostScript files} \luavarseparator +\luavarset{includetests}{\{"*"\}}{Test names to include when checking} +\luavarset{excludetests}{\{\}} {Test names to exclude when checking} +\luavarseparator \luavarset{checkdeps} {\{\}}{List of dependencies for running checks} \luavarset{typesetdeps}{\{\}}{List of dependencies for typesetting docs} \luavarset{unpackdeps} {\{\}}{List of dependencies for unpacking} @@ -135,7 +140,6 @@ \luavarset{glossarystyle}{"gglo.ist"}{MakeIndex style file for glossary/changes creation} \luavarset{indexstyle} {"gind.ist"}{MakeIndex style for index creation} \luavarseparator -\luavarseparator \luavarset{biberexe} {"biber"} {Biber executable} \luavarset{biberopts} {""} {Biber options} \luavarset{bibtexexe} {"bibtex8"} {\BibTeX{} executable} @@ -217,7 +221,7 @@ % }^^A % } % -% \date{Released 2018-05-10} +% \date{Released 2018-08-02} % % \maketitle % \tableofcontents @@ -315,6 +319,7 @@ % \item |--config| (|-c|) Configuration(s) to use for testing % \item |--date| Date to use when tagging % data +% \item |--dirty| Skip cleaning up of the test area % \item |--dry-run| Runs the \texttt{install} target but does not copy % any files: simply lists those that would be installed % \item |--engine| (|-e|) Sets the engine to use for @@ -328,11 +333,9 @@ % should stop as soon as possible, rather than running all requested % tests; the difference file is printed in the terminal directly in the case of failure % \item |--last| Name of the last test to run -% \item |--pdf| (|-p|) Test PDF file against a reference -% version rather than using a log comparison % \item |--quiet| (|-q|) Suppresses output from unpacking % \item |--rerun| Run tests without unpacking/set up -% \item |--shuffle| Shuffle the order in whichs tests run +% \item |--shuffle| Shuffle the order in which tests run % \item |--texmfhome| Sets the location of the user tree for installing % \end{itemize} % @@ -382,16 +385,6 @@ % test directory: this may be skipped using the |-s| option. % \end{buildcmd} % -% \begin{buildcmd}{check -p} -% Rather than the log-based checking carried out by the standard -% |check| target, running with the |-p| option carries out a binary -% comparison of the PDF files produced by typesetting against those -% saved in \var{testfiledir}. -% -% This functionality requires \TeX{} Live 2016 or later as it needs support -% from the engines not available in earlier releases. -% \end{buildcmd} -% % \begin{buildcmd}{clean} % This command removes all temporary files used for package bundling and regression testing. % In the standard layout, these are all files within the directories defined by \var{localdir}, \var{testdir}, \var{typesetdir} and \var{unpackdir}, as well as all files defined in the \var{cleanfiles} variable in the same directory as the script. @@ -496,15 +489,6 @@ % the \enquote{exit status} is zero, else non-zero. % \end{buildcmd} % -% \begin{buildcmd}{save -p \meta{name(s)}} -% This version of |save| will store the PDF files produced from -% \texttt{\meta{name(s)}.lvt} in addition to the |.tlg| file, and thus allows -% binary comparison of the result of typesetting. -% -% This functionality requires \TeX{} Live 2016 or later as it needs support -% from the engines not available in earlier releases. -% \end{buildcmd} -% % \begin{buildcmd}{manifest} % Generates a `manifest' file which lists the files of the package as known to \pkg{l3build}. % The filename of this file (by default \luavar{manifestfile}) can be set with the variable \var{manifestfile}. @@ -650,6 +634,15 @@ % % \luavartypeset % +% \subsection{Selective running of tests} +% +% The variables |includetests| and |excludetests| may be used to select which +% tests are run: these variables take test \emph{names} not full file names. +% The list of tests in |excludetests| overrides any matches in |includetests|, +% meaning that tests can be disabled selectively. It also makes it possible +% to disable test on for example a platform basis: the Lua core variable +% |os.type| may be used to set |excludetests| only on some systems. +% % \subsection{Multiple sets of tests} % % In most cases, a single set of tests will be appropriate for the module, with @@ -1116,6 +1109,24 @@ % \label{fig:expect-ins} % \end{figure} % +% \subsection{PDF-based tests} +% +% In most cases, testing is best handled by using the text-based methods +% outlined above. However, there are cases where the detail of output structure +% is important. This can only be fully tested by comparing PDF structure. +% To support this, \pkg{l3build} can be instructed to build and compare +% PDF files by setting up tests in \texttt{.pvt} files. The following +% normalization takes place: +% \begin{itemize} +% \item Replacement of binary streams by the marker |[BINARY STREAM]| +% \item Removal of blank lines +% \end{itemize} +% +% Testing on the result of typesetting \texttt{.pvt} files is carried out using +% only the standard engine, as aspects of PDF files are not identical between +% engines. To allow platform-independence, PDF-based tests must use only +% Type1 or OpenType fonts: Type3 fonts are system-dependent. +% % \section{Release-focussed features} % % \subsection{Automatic tagging} @@ -1222,7 +1233,6 @@ % \label{fig:PDF} % \end{figure} % -% % \section{Lua interfaces} % % Whilst for the majority of users the simple variable-based control methods @@ -1247,7 +1257,6 @@ % \var{halt} & Boolean \\ % \var{help} & Boolean \\ % \var{names} & Table \\ -% \var{pdf} & Boolean \\ % \var{quiet} & Boolean \\ % \var{rerun} & Boolean \\ % \var{testfiledir} & Table \\ @@ -1443,6 +1452,24 @@ % |target| in this table is ignored. % \end{function} % +% \subsection{Customising the target list} +% +% The targets known to \pkg{l3build} are stored in the global table +% |target_list|. Each entry should have at least a |func|, pointing to the +% function used to implement the target. This function will receive the +% list of names given at the command line as a table argument. +% In most cases, targets will also have a |desc|, used to construct |help()| +% automatically. In addition, the following may also be used: +% \begin{itemize} +% \item |bundle_func| A variant of |func| used when at the top level of +% a bundle +% \item |bundle_target| A boolean to specify that when passing the target +% name in a bundle, it should have |bundle| prepended. +% \item |pre| A function executed before the main function, and receiving +% the |names| as an argument; this allows checking of the |name| data +% without impact on the main |func|. +% \end{itemize} +% % \subsection{Customising the manifest file} % \label{sec:manifest} % @@ -1815,7 +1842,6 @@ \SEPARATOR \begingroup \let\TYPE\LONGTYPEOUT - } \protected\long\def\ENDTEST{% \endgroup @@ -1899,6 +1925,25 @@ \fi % \end{macrocode} % +% Disable compression in PDF output. +% \begin{macrocode} +\ifnum 0% + \ifx\pdfoutput\@undefined\else\ifnum\pdfoutput>0 1\fi\fi + \ifx\outputmode\@undefined\else\ifnum\outputmode>0 1\fi\fi + >0 % + \ifx\pdfvariable\@undefined + \pdfcompresslevel=0 % + \pdfobjcompresslevel=0 % + \else + \pdfvariable compresslevel=0 % + \pdfvariable objcompresslevel=0 % + \fi +\else + \special{dvipdfmx:config z 0}% Compress level + \special{dvipdfmx:config C 0x40}% Object compression +\fi +% \end{macrocode} +% % Finish up. % \begin{macrocode} \reset@catcodes diff --git a/Master/texmf-dist/tex/latex/l3build/regression-test.tex b/Master/texmf-dist/tex/latex/l3build/regression-test.tex index 4c12a1d5444..8f559a7e3c7 100644 --- a/Master/texmf-dist/tex/latex/l3build/regression-test.tex +++ b/Master/texmf-dist/tex/latex/l3build/regression-test.tex @@ -108,7 +108,6 @@ \SEPARATOR \begingroup \let\TYPE\LONGTYPEOUT - } \protected\long\def\ENDTEST{% \endgroup @@ -173,6 +172,21 @@ >> } \fi +\ifnum 0% + \ifx\pdfoutput\@undefined\else\ifnum\pdfoutput>0 1\fi\fi + \ifx\outputmode\@undefined\else\ifnum\outputmode>0 1\fi\fi + >0 % + \ifx\pdfvariable\@undefined + \pdfcompresslevel=0 % + \pdfobjcompresslevel=0 % + \else + \pdfvariable compresslevel=0 % + \pdfvariable objcompresslevel=0 % + \fi +\else + \special{dvipdfmx:config z 0}% Compress level + \special{dvipdfmx:config C 0x40}% Object compression +\fi \reset@catcodes %% %% |