diff options
author | Norbert Preining <norbert@preining.info> | 2021-05-06 03:00:58 +0000 |
---|---|---|
committer | Norbert Preining <norbert@preining.info> | 2021-05-06 03:00:58 +0000 |
commit | 25f20b6d8ccd4edcdfec23c1a3c908516076ee06 (patch) | |
tree | f889e071118d7d2db80066d14d14b020c50a2238 /macros/latex/contrib/l3build/l3build-check.lua | |
parent | 75db33dc347ec9cfbaa2f3ec538a2697351f571b (diff) |
CTAN sync 202105060300
Diffstat (limited to 'macros/latex/contrib/l3build/l3build-check.lua')
-rw-r--r-- | macros/latex/contrib/l3build/l3build-check.lua | 423 |
1 files changed, 214 insertions, 209 deletions
diff --git a/macros/latex/contrib/l3build/l3build-check.lua b/macros/latex/contrib/l3build/l3build-check.lua index ff022c6079..dce9e68c6a 100644 --- a/macros/latex/contrib/l3build/l3build-check.lua +++ b/macros/latex/contrib/l3build/l3build-check.lua @@ -1,6 +1,6 @@ --[[ -File l3build-check.lua Copyright (C) 2018-2020 The LaTeX3 Project +File l3build-check.lua Copyright (C) 2018-2020 The LaTeX Project It may be distributed and/or modified under the conditions of the LaTeX Project Public License (LPPL), either version 1.3c of this @@ -34,7 +34,7 @@ local luatex_version = status.luatex_version local len = string.len local char = string.char -local format = string.format +local str_format = string.format local gmatch = string.gmatch local gsub = string.gsub local match = string.match @@ -59,7 +59,7 @@ function checkinit() cleandir(testdir) cleandir(resultdir) end - depinstall(checkdeps) + dep_install(checkdeps) -- Copy dependencies to the test directory itself: this makes the paths -- a lot easier to manage, and is important for dealing with the log and -- with file input/output tests @@ -85,7 +85,7 @@ function checkinit() return checkinit_hook() end -checkinit_hook = checkinit_hook or function() return 0 end +function checkinit_hook() return 0 end local function rewrite(source,result,processor,...) local file = assert(open(source,"rb")) @@ -102,7 +102,7 @@ end -- the 'business' part from the tests and removes system-dependent stuff local function normalize_log(content,engine,errlevels) local maxprintline = maxprintline - if match(engine,"^lua") or match(engine,"^harf") then + if (match(engine,"^lua") or match(engine,"^harf")) and luatex_version < 113 then maxprintline = maxprintline + 1 -- Deal with an out-by-one error end local function killcheck(line) @@ -134,7 +134,7 @@ local function normalize_log(content,engine,errlevels) not match(line, "%.%.%.$") then return "", (lastline or "") .. line end - local line = (lastline or "") .. line + line = (lastline or "") .. line lastline = "" -- Zap ./ at begin of filename line = gsub(line, "%(%.%/", "(") @@ -223,15 +223,19 @@ local function normalize_log(content,engine,errlevels) -- Remove '\displace 0.0' lines in (u)pTeX if match(line,"^%.*\\displace 0%.0$") then return "" - end - end - -- Remove the \special line that in DVI mode keeps PDFs comparable + end + end + -- Deal with Lua function calls + if match(line, "^Lua function") then + line = gsub(line,"= %d+$","= ...") + end + -- Remove the \special line that in DVI mode keeps PDFs comparable if match(line, "^%.*\\special%{pdf: docinfo << /Creator") or match(line, "^%.*\\special%{ps: /setdistillerparams") or match(line, "^%.*\\special%{! <</........UUID") then return "" end - -- Remove \special lines for DVI .pro files + -- Remove \special lines for DVI .pro files if match(line, "^%.*\\special%{header=") then return "" end @@ -251,7 +255,7 @@ local function normalize_log(content,engine,errlevels) match(line, "^used file >") or match(line, "^used option >") or match(line, "^used structure >") then - return "" + return "" end -- The first time a new font is used by LuaTeX, it shows up -- as being cached: make it appear loaded every time @@ -265,7 +269,7 @@ local function normalize_log(content,engine,errlevels) -- tidy up to match pdfTeX if an ASCII engine is in use if next(asciiengines) then for i = 128, 255 do - line = gsub(line, utf8_char(i), "^^" .. format("%02x", i)) + line = gsub(line, utf8_char(i), "^^" .. str_format("%02x", i)) end end return line, lastline @@ -341,7 +345,7 @@ local function normalize_lua_log(content,luatex) l, m .. " (%-?)%d+%.%d+", m .. " %1" - .. format( + .. str_format( "%.3f", match(line, m .. " %-?(%d+%.%d+)") or 0 ) @@ -370,20 +374,20 @@ local function normalize_lua_log(content,luatex) -- This block only applies to the output of LuaTeX itself, -- hence needing a flag to skip the case of the reference log if luatex and - tonumber(luatex_version) >= 107 and - match(line, "^%.*\\kern") then - -- Re-insert the space in explicit kerns - if match(line, "kern%-?%d+%.%d+ *$") then - line = gsub(line, "kern", "kern ") - elseif match(line, "%(accent%)$") then - line = gsub(line, "kern", "kern ") - line = gsub(line, "%(accent%)$", "(for accent)") - elseif match(line, "%(italic%)$") then - line = gsub(line, "kern", "kern ") - line = gsub(line, " %(italic%)$", "") - else - line = gsub(line, " %(font%)$", "") - end + tonumber(luatex_version) >= 107 and + match(line, "^%.*\\kern") then + -- Re-insert the space in explicit kerns + if match(line, "kern%-?%d+%.%d+ *$") then + line = gsub(line, "kern", "kern ") + elseif match(line, "%(accent%)$") then + line = gsub(line, "kern", "kern ") + line = gsub(line, "%(accent%)$", "(for accent)") + elseif match(line, "%(italic%)$") then + line = gsub(line, "kern", "kern ") + line = gsub(line, " %(italic%)$", "") + else + line = gsub(line, " %(font%)$", "") + end end -- Changes in PDF specials line = gsub(line, "\\pdfliteral origin", "\\pdfliteral") @@ -394,8 +398,8 @@ local function normalize_lua_log(content,luatex) -- 'Recover' some discretionary data if match(lastline, "^%.+\\discretionary %(penalty 50%)$") and match(line, boxprefix(lastline) .. "%.= ") then - line = gsub(line," %(font%)$","") - return gsub(line, "%.= ", ""),"" + line = gsub(line," %(font%)$","") + return gsub(line, "%.= ", ""),"" end -- Where the last line was a discretionary, looks for the -- info one level in about what it represents @@ -406,13 +410,13 @@ local function normalize_lua_log(content,luatex) local prefix = boxprefix(lastline) if match(line, prefix .. "%.") or match(line, prefix .. "%|") then - if match(lastline, " replacing $") and - not dropping then - -- Modify the return line - return gsub(line, "^%.", ""), lastline, true - else - return "", lastline, true - end + if match(lastline, " replacing $") and + not dropping then + -- Modify the return line + return gsub(line, "^%.", ""), lastline, true + else + return "", lastline, true + end else if dropping then -- End of a \discretionary block @@ -431,7 +435,7 @@ local function normalize_lua_log(content,luatex) end end -- Look for another form of \discretionary, replacing a "-" - pattern = "^%.+\\discretionary replacing *$" + local pattern = "^%.+\\discretionary replacing *$" if match(line, pattern) then return "", line else @@ -461,7 +465,7 @@ local function normalize_lua_log(content,luatex) match(line, "^%.+\\localbrokenpenalty=0$") or match(line, "^%.+\\localleftbox=null$") or match(line, "^%.+\\localrightbox=null$") then - return "", "" + return "", "" end -- Older LuaTeX versions set the above up as a whatsit -- (at some stage this can therefore go) @@ -522,7 +526,7 @@ local function normalize_pdf(content) if binary then new_content = new_content .. "[BINARY STREAM]" .. os_newline else - new_content = new_content .. stream_content .. line .. os_newline + new_content = new_content .. stream_content .. line .. os_newline end binary = false else @@ -541,7 +545,7 @@ local function normalize_pdf(content) stream = true stream_content = "stream" .. os_newline elseif not match(line, "^ *$") and - not match(line,"^%%%%Invocation") and + not match(line,"^%%%%Invocation") and not match(line,"^%%%%%+") then line = gsub(line,"%/ID( ?)%[<[^>]+><[^>]+>]","/ID%1[<ID-STRING><ID-STRING>]") new_content = new_content .. line .. os_newline @@ -550,10 +554,19 @@ local function normalize_pdf(content) return new_content end +function rewrite_log(source, result, engine, errlevels) + return rewrite(source, result, normalize_log, engine, errlevels) +end + +function rewrite_pdf(source, result, engine, errlevels) + return rewrite(source, result, normalize_pdf, engine, errlevels) +end + -- Run one test which may have multiple engine-dependent comparisons -- Should create a difference file for each failed test function runcheck(name, hide) - if not testexists(name) then + local test_filename, kind = testexists(name) + if not test_filename then print("Failed to find input for test " .. name) return 1 end @@ -562,9 +575,10 @@ function runcheck(name, hide) checkengines = options["engine"] end -- Used for both .lvt and .pvt tests - local function check_and_diff(ext,engine,comp,pdftest) - runtest(name,engine,hide,ext,pdftest,true) - local errorlevel = comp(name,engine) + local test_type = test_types[kind] + local function check_and_diff(engine) + runtest(name, engine, hide, test_type.test, test_type, true) + local errorlevel = base_compare(test_type,name,engine) if errorlevel == 0 then return errorlevel end @@ -579,12 +593,7 @@ function runcheck(name, hide) local errorlevel = 0 for _,engine in pairs(checkengines) do setup_check(name,engine) - local errlevel = 0 - if fileexists(testfiledir .. "/" .. name .. pvtext) then - errlevel = check_and_diff(pvtext,engine,compare_pdf,true) - else - errlevel = check_and_diff(lvtext,engine,compare_tlg) - end + local errlevel = check_and_diff(engine) if errlevel ~= 0 and options["halt-on-error"] then return 1 end @@ -598,64 +607,73 @@ end function setup_check(name, engine) local testname = name .. "." .. engine - local tlgfile = locate( - {testfiledir, unpackdir}, - {testname .. tlgext, name .. tlgext} - ) - local tpffile = locate( - {testfiledir, unpackdir}, - {testname .. tpfext, name .. tpfext} - ) - -- Attempt to generate missing reference file from expectation - if not (tlgfile or tpffile) then - if not locate({unpackdir, testfiledir}, {name .. lveext}) then - print( - "Error: failed to find " .. tlgext .. ", " .. tpfext .. " or " - .. lveext .. " file for " .. name .. "!" + local found + for _, kind in ipairs(test_order) do + local reference_ext = test_types[kind].reference + local reference_file = locate( + {testfiledir, unpackdir}, + {testname .. reference_ext, name .. reference_ext} + ) + if reference_file then + found = true + -- Install comparison file found + cp( + match(reference_file, ".*/(.*)"), + match(reference_file, "(.*)/.*"), + testdir ) - exit(1) end - runtest(name, engine, true, lveext) - ren(testdir, testname .. logext, testname .. tlgext) - else - -- Install comparison files found - for _,v in pairs({tlgfile, tpffile}) do - if v then - cp( - match(v, ".*/(.*)"), - match(v, "(.*)/.*"), - testdir - ) - end + end + if found then + return + end + -- Attempt to generate missing reference file from expectation + for _, kind in ipairs(test_order) do + local test_type = test_types[kind] + local exp_ext = test_type.expectation + local expectation_file = exp_ext and locate( + {testfiledir, unpackdir}, + {name .. exp_ext} + ) + if expectation_file then + found = true + runtest(name, engine, true, exp_ext, test_type) + ren(testdir, testname .. test_type.generated, testname .. test_type.reference) end end + if found then + return + end + print( + "Error: failed to find any reference or expectation file for " + .. name .. "!" + ) + exit(1) end -function compare_pdf(name,engine,cleanup) +function base_compare(test_type,name,engine,cleanup) local testname = name .. "." .. engine - local difffile = testdir .. "/" .. testname .. pdfext .. os_diffext - local pdffile = testdir .. "/" .. testname .. pdfext - local tpffile = locate({testdir}, {testname .. tpfext, name .. tpfext}) - if not tpffile then + local difffile = testdir .. "/" .. testname.. os_diffext + local genfile = testdir .. "/" .. testname .. test_type.generated + local reffile = locate({testdir}, {testname .. test_type.reference, name .. test_type.reference}) + if not reffile then return 1 end + local compare = test_type.compare + if compare then + return compare(difffile, reffile, genfile, cleanup, name, engine) + end local errorlevel = execute(os_diffexe .. " " - .. normalize_path(tpffile .. " " .. pdffile .. " > " .. difffile)) + .. normalize_path(reffile .. " " .. genfile .. " > " .. difffile)) if errorlevel == 0 or cleanup then remove(difffile) end return errorlevel end -function compare_tlg(name,engine,cleanup) +function compare_tlg(difffile, tlgfile, logfile, cleanup, name, engine) local errorlevel local testname = name .. "." .. engine - local difffile = testdir .. "/" .. testname .. os_diffext - local logfile = testdir .. "/" .. testname .. logext - local tlgfile = locate({testdir}, {testname .. tlgext, name .. tlgext}) - if not tlgfile then - return 1 - end -- Do additional log formatting if the engine is LuaTeX, there is no -- LuaTeX-specific .tlg file and the default engine is not LuaTeX if (match(engine,"^lua") or match(engine,"^harf")) @@ -687,22 +705,22 @@ end -- Run one of the test files: doesn't check the result so suitable for -- both creating and verifying -function runtest(name, engine, hide, ext, pdfmode, breakout) +function runtest(name, engine, hide, ext, test_type, breakout) local lvtfile = name .. (ext or lvtext) cp(lvtfile, fileexists(testfiledir .. "/" .. lvtfile) and testfiledir or unpackdir, testdir) local checkopts = checkopts - local engine = engine or stdengine + engine = engine or stdengine local binary = engine local format = gsub(engine,"tex$",checkformat) -- Special binary/format combos - if specialformats[checkformat] and next(specialformats[checkformat]) then - local t = specialformats[checkformat] - if t[engine] and next(t[engine]) then - local t = t[engine] - binary = t.binary or binary - checkopts = t.options or checkopts - format = t.format or format + local special_check = specialformats[checkformat] + if special_check and next(special_check) then + local engine_info = special_check[engine] + if engine_info then + binary = engine_info.binary or binary + format = engine_info.format or format + checkopts = engine_info.options or checkopts end end -- Finalise format string @@ -710,7 +728,7 @@ function runtest(name, engine, hide, ext, pdfmode, breakout) format = " --fmt=" .. format end -- Special casing for XeTeX engine - if match(engine, "xetex") and not pdfmode then + if match(engine, "xetex") and test_type.generated ~= pdfext then checkopts = checkopts .. " -no-pdf" end -- Special casing for ConTeXt @@ -721,10 +739,8 @@ function runtest(name, engine, hide, ext, pdfmode, breakout) function setup(file) return ' "' .. file .. '" ' end end local basename = testdir .. "/" .. name - local logfile = basename .. logext - local newfile = basename .. "." .. engine .. logext - local pdffile = basename .. pdfext - local npffile = basename .. "." .. engine .. pdfext + local gen_file = basename .. test_type.generated + local new_file = basename .. "." .. engine .. test_type.generated local asciiopt = "" for _,i in ipairs(asciiengines) do if binary == i then @@ -737,7 +753,7 @@ function runtest(name, engine, hide, ext, pdfmode, breakout) rm(testdir,filetype) end -- Ensure there is no stray .log file - rm(testdir,name .. logext) + rmfile(testdir,name .. logext) local errlevels = {} local localtexmf = "" if texmfdir and texmfdir ~= "" and direxists(texmfdir) then @@ -760,7 +776,7 @@ function runtest(name, engine, hide, ext, pdfmode, breakout) -- Allow for local texmf files os_setenv .. " TEXMFCNF=." .. os_pathsep .. os_concat .. - (forcecheckepoch and setepoch() or "") .. + set_epoch_cmd(epoch, forcecheckepoch) .. -- Ensure lines are of a known length os_setenv .. " max_print_line=" .. maxprintline .. os_concat .. @@ -773,47 +789,34 @@ function runtest(name, engine, hide, ext, pdfmode, breakout) ) -- Break the loop if the result is stable if breakout and i < checkruns then - if pdfmode then + if test_type.generated == pdfext then if fileexists(testdir .. "/" .. name .. dviext) then dvitopdf(name, testdir, engine, hide) end - rewrite(pdffile,npffile,normalize_pdf) - if compare_pdf(name,engine,true) == 0 then - break - end - else - rewrite(logfile,newfile,normalize_log,engine,errlevels) - if compare_tlg(name,engine,true) == 0 then - break - end + end + test_type.rewrite(gen_file,new_file,engine,errlevels) + if base_compare(test_type,name,engine,true) == 0 then + break end end end - if pdfmode and fileexists(testdir .. "/" .. name .. dviext) then - dvitopdf(name, testdir, engine, hide) - end - if pdfmode then + if test_type.generated == pdfext then + if fileexists(testdir .. "/" .. name .. dviext) then + dvitopdf(name, testdir, engine, hide) + end cp(name .. pdfext,testdir,resultdir) ren(resultdir,name .. pdfext,name .. "." .. engine .. pdfext) - rewrite(pdffile,npffile,normalize_pdf) - else - rewrite(logfile,newfile,normalize_log,engine,errlevels) end + test_type.rewrite(gen_file,new_file,engine,errlevels) -- Store secondary files for this engine for _,filetype in pairs(auxfiles) do for _,file in pairs(filelist(testdir, filetype)) do - if match(file,"^" .. name .. ".[^.]+$") then - local ext = match(file, "%.[^.]+$") - if ext ~= lvtext and - ext ~= tlgext and - ext ~= lveext and - ext ~= logext then - local newname = gsub(file,"(%.[^.]+)$","." .. engine .. "%1") - if fileexists(testdir,newname) then - rm(testdir,newname) - end - ren(testdir,file,newname) + if match(file,"^" .. name .. "%.[^.]+$") then + local newname = gsub(file,"(%.[^.]+)$","." .. engine .. "%1") + if fileexists(testdir .. "/" .. newname) then + rmfile(testdir,newname) end + ren(testdir,file,newname) end end end @@ -821,14 +824,25 @@ function runtest(name, engine, hide, ext, pdfmode, breakout) end -- A hook to allow additional tasks to run for the tests -runtest_tasks = runtest_tasks or function(name,run) +function runtest_tasks(name,run) return "" end -- Look for a test: could be in the testfiledir or the unpackdir function testexists(test) - return(locate({testfiledir, unpackdir}, - {test .. lvtext, test .. pvtext})) + local filenames = {} + for i, kind in ipairs(test_order) do + filenames[i] = test .. test_types[kind].test + end + local found = locate({testfiledir, unpackdir}, filenames) + if found then + for i, kind in ipairs(test_order) do + local filename = filenames[i] + if found:sub(-#filename) == filename then + return found, kind + end + end + end end function check(names) @@ -844,36 +858,42 @@ function check(names) names = names or { } -- No names passed: find all test files if not next(names) then - local excludenames = { } - for _,glob in pairs(excludetests) do - for _,name in pairs(filelist(testfiledir, glob .. lvtext)) do - excludenames[jobname(name)] = true - end - for _,name in pairs(filelist(unpackdir, glob .. lvtext)) do - excludenames[jobname(name)] = true - end - for _,name in pairs(filelist(testfiledir, glob .. pvtext)) do - excludenames[jobname(name)] = true - end - end - local function addname(name) - if not excludenames[jobname(name)] then - insert(names,jobname(name)) - end - end - for _,glob in pairs(includetests) do - for _,name in pairs(filelist(testfiledir, glob .. lvtext)) do - addname(name) - end - for _,name in pairs(filelist(testfiledir, glob .. pvtext)) do - addname(name) + for _, kind in ipairs(test_order) do + local ext = test_types[kind].test + local excludepatterns = { } + local num_exclude = 0 + for _,glob in pairs(excludetests) do + num_exclude = num_exclude+1 + excludepatterns[num_exclude] = glob_to_pattern(glob .. ext) end - for _,name in pairs(filelist(unpackdir, glob .. lvtext)) do - if fileexists(testfiledir .. "/" .. name) then - print("Duplicate test file: " .. i) - return 1 + for _,glob in pairs(includetests) do + for _,name in pairs(filelist(testfiledir, glob .. ext)) do + local exclude + for i=1, num_exclude do + if match(name, excludepatterns[i]) then + exclude = true + break + end + end + if not exclude then + insert(names,jobname(name)) + end + end + for _,name in pairs(filelist(unpackdir, glob .. ext)) do + local exclude + for i=1, num_exclude do + if not match(name, excludepatterns[i]) then + exclude = true + break + end + end + if not exclude then + if fileexists(testfiledir .. "/" .. name) then + return 1 + end + insert(names,jobname(name)) + end end - addname(name) end end sort(names) @@ -904,24 +924,17 @@ function check(names) end end end - -- https://stackoverflow.com/a/32167188 - local function shuffle(tbl) - local len, random = #tbl, rnd - for i = len, 2, -1 do - local j = random(1, i) - tbl[i], tbl[j] = tbl[j], tbl[i] - end - return tbl - end if options["shuffle"] then - names = shuffle(names) + -- https://stackoverflow.com/a/32167188 + for i = #names, 2, -1 do + local j = rnd(1, i) + names[i], names[j] = names[j], names[i] + end end -- Actually run the tests print("Running checks on") - local i = 0 - for _,name in ipairs(names) do - i = i + 1 - print(" " .. name .. " (" .. i.. "/" .. #names ..")") + for i, name in ipairs(names) do + print(" " .. name .. " (" .. i .. "/" .. #names ..")") local errlevel = runcheck(name, hide) -- Return value must be 1 not errlevel if errlevel ~= 0 then @@ -988,39 +1001,31 @@ function save(names) return 1 end for _,name in pairs(names) do - if testexists(name) then - for _,engine in pairs(engines) do - local testengine = ((engine == stdengine and "") or "." .. engine) - local function save_test(test_ext,gen_ext,out_ext,pdfmode) - local out_file = name .. testengine .. out_ext - local gen_file = name .. "." .. engine .. gen_ext - print("Creating and copying " .. out_file) - runtest(name,engine,false,test_ext,pdfmode) - ren(testdir,gen_file,out_file) - cp(out_file,testdir,testfiledir) - if fileexists(unpackdir .. "/" .. out_file) then - print("Saved " .. out_ext - .. " file overrides unpacked version of the same name") - return 1 - end - return 0 - end - local errorlevel - if fileexists(testfiledir .. "/" .. name .. lvtext) then - errorlevel = save_test(lvtext,logext,tlgext) - else - errorlevel = save_test(pvtext,pdfext,tpfext,true) - end - if errorlevel ~=0 then return errorlevel end - end - elseif locate({unpackdir, testfiledir}, {name .. lveext}) then - print("Saved " .. tlgext .. " file overrides a " - .. lveext .. " file of the same name") - return 1 - else + local test_filename, kind = testexists(name) + if not test_filename then print('Test "' .. name .. '" not found') return 1 end + local test_type = test_types[kind] + if locate({unpackdir, testfiledir}, {name .. test_type.expectation}) then + print("Saved " .. test_type.test .. " file would override a " + .. test_type.expectation .. " file of the same name") + return 1 + end + for _,engine in pairs(engines) do + local testengine = engine == stdengine and "" or ("." .. engine) + local out_file = name .. testengine .. test_type.reference + local gen_file = name .. "." .. engine .. test_type.generated + print("Creating and copying " .. out_file) + runtest(name, engine, false, test_type.test, test_type) + ren(testdir, gen_file, out_file) + cp(out_file, testdir, testfiledir) + if fileexists(unpackdir .. "/" .. test_type.reference) then + print("Saved " .. test_type.reference + .. " file overrides unpacked version of the same name") + return 1 + end + end end return 0 end |