From ee9860b8c1122fcf2d566d52c867c223f948977c Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Wed, 10 Jan 2018 21:47:40 +0000 Subject: l3build (10jan18) git-svn-id: svn://tug.org/texlive/trunk@46283 c570f23f-e606-0410-a88d-b1316a301751 --- .../tex/latex/l3build/l3build-arguments.lua | 276 +++ .../texmf-dist/tex/latex/l3build/l3build-aux.lua | 96 + .../texmf-dist/tex/latex/l3build/l3build-check.lua | 842 +++++++ .../texmf-dist/tex/latex/l3build/l3build-clean.lua | 56 + .../texmf-dist/tex/latex/l3build/l3build-ctan.lua | 238 ++ .../tex/latex/l3build/l3build-file-functions.lua | 406 ++++ .../texmf-dist/tex/latex/l3build/l3build-help.lua | 78 + .../tex/latex/l3build/l3build-install.lua | 57 + .../tex/latex/l3build/l3build-manifest-setup.lua | 364 +++ .../tex/latex/l3build/l3build-manifest.lua | 280 +++ .../tex/latex/l3build/l3build-setversion.lua | 140 ++ .../tex/latex/l3build/l3build-stdmain.lua | 123 + .../tex/latex/l3build/l3build-typesetting.lua | 247 ++ .../tex/latex/l3build/l3build-unpack.lua | 99 + .../tex/latex/l3build/l3build-variables.lua | 199 ++ Master/texmf-dist/tex/latex/l3build/l3build.lua | 2517 +------------------- 16 files changed, 3526 insertions(+), 2492 deletions(-) create mode 100644 Master/texmf-dist/tex/latex/l3build/l3build-arguments.lua create mode 100644 Master/texmf-dist/tex/latex/l3build/l3build-aux.lua create mode 100644 Master/texmf-dist/tex/latex/l3build/l3build-check.lua create mode 100644 Master/texmf-dist/tex/latex/l3build/l3build-clean.lua create mode 100644 Master/texmf-dist/tex/latex/l3build/l3build-ctan.lua create mode 100644 Master/texmf-dist/tex/latex/l3build/l3build-file-functions.lua create mode 100644 Master/texmf-dist/tex/latex/l3build/l3build-help.lua create mode 100644 Master/texmf-dist/tex/latex/l3build/l3build-install.lua create mode 100644 Master/texmf-dist/tex/latex/l3build/l3build-manifest-setup.lua create mode 100644 Master/texmf-dist/tex/latex/l3build/l3build-manifest.lua create mode 100644 Master/texmf-dist/tex/latex/l3build/l3build-setversion.lua create mode 100644 Master/texmf-dist/tex/latex/l3build/l3build-stdmain.lua create mode 100644 Master/texmf-dist/tex/latex/l3build/l3build-typesetting.lua create mode 100644 Master/texmf-dist/tex/latex/l3build/l3build-unpack.lua create mode 100644 Master/texmf-dist/tex/latex/l3build/l3build-variables.lua (limited to 'Master/texmf-dist/tex/latex/l3build') diff --git a/Master/texmf-dist/tex/latex/l3build/l3build-arguments.lua b/Master/texmf-dist/tex/latex/l3build/l3build-arguments.lua new file mode 100644 index 00000000000..83129a071f4 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3build/l3build-arguments.lua @@ -0,0 +1,276 @@ +--[[ + +File l3build-arguments.lua Copyright (C) 2018 The LaTeX3 Project + +It may be distributed and/or modified under the conditions of the +LaTeX Project Public License (LPPL), either version 1.3c of this +license or (at your option) any later version. The latest version +of this license is in the file + + http://www.latex-project.org/lppl.txt + +This file is part of the "l3build bundle" (The Work in LPPL) +and all files in that bundle must be distributed together. + +----------------------------------------------------------------------- + +The development version of the bundle can be found at + + https://github.com/latex3/l3build + +for those people who are interested. + +--]] + +local exit = os.exit +local stderr = io.stderr + +-- Parse command line options + +option_list = + { + config = + { + desc = "Sets the config(s) used for running tests", + short = "c", + type = "table" + }, + date = + { + desc = "Sets the date to insert into sources", + type = "string" + }, + ["dry-run"] = + { + desc = "Dry run for install", + type = "boolean" + }, + engine = + { + desc = "Sets the engine(s) to use for running test", + short = "e", + type = "table" + }, + epoch = + { + desc = "Sets the epoch for tests and typesetting", + type = "string" + }, + force = + { + desc = "Force tests to run if engine is not set up", + short = "f", + type = "boolean" + }, + ["halt-on-error"] = + { + desc = "Stops running tests after the first failure", + short = "H", + type = "boolean" + }, + help = + { + short = "h", + type = "boolean" + }, + pdf = + { + desc = "Check/save PDF files", + short = "p", + type = "boolean" + }, + quiet = + { + desc = "Suppresses TeX output when unpacking", + short = "q", + type = "boolean" + }, + rerun = + { + desc = "Skip setup: simply rerun tests", + type = "boolean" + }, + shuffle = + { + desc = "Shuffle order of tests", + type = "boolean" + }, + texmfhome = + { + desc = "Location of user texmf tree", + type = "string" + }, + version = + { + desc = "Sets the version to insert into sources", + short = "v", + type = "string" + }, + } + +-- This is done as a function (rather than do ... end) as it allows early +-- termination (break) +local function argparse() + local result = { } + local files = { } + local long_options = { } + local short_options = { } + -- Turn long/short options into two lookup tables + for k,v in pairs(option_list) do + if v["short"] then + short_options[v["short"]] = k + end + long_options[k] = k + end + local args = args + -- arg[1] is a special case: must be a command or "-h"/"--help" + -- Deal with this by assuming help and storing only apparently-valid + -- input + local a = arg[1] + result["target"] = "help" + if a then + -- No options are allowed in position 1, so filter those out + if not string.match(a, "^%-") then + result["target"] = a + end + end + -- Stop here if help is required + if result["target"] == "help" then + return result + end + -- An auxiliary to grab all file names into a table + local function remainder(num) + local files = { } + for i = num, #arg do + table.insert(files, arg[i]) + end + return files + end + -- Examine all other arguments + -- Use a while loop rather than for as this makes it easier + -- to grab arg for optionals where appropriate + local i = 2 + while i <= #arg do + local a = arg[i] + -- Terminate search for options + if a == "--" then + files = remainder(i + 1) + break + end + -- Look for optionals + local opt + local optarg + local opts + -- Look for and option and get it into a variable + if string.match(a, "^%-") then + if string.match(a, "^%-%-") then + opts = long_options + local pos = string.find(a, "=", 1, true) + if pos then + opt = string.sub(a, 3, pos - 1) + optarg = string.sub(a, pos + 1) + else + opt = string.sub(a, 3) + end + else + opts = short_options + opt = string.sub(a, 2, 2) + -- Only set optarg if it is there + if #a > 2 then + optarg = string.sub(a, 3) + end + end + -- Now check that the option is valid and sort out the argument + -- if required + local optname = opts[opt] + if optname then + -- Tidy up arguments + if option_list[optname]["type"] == "boolean" then + if optarg then + local opt = "-" .. (string.match(a, "^%-%-") and "-" or "") .. opt + stderr:write("Value not allowed for option " .. opt .."\n") + return {"help"} + end + else + if not optarg then + optarg = arg[i + 1] + if not optarg then + stderr:write("Missing value for option " .. a .."\n") + return {"help"} + end + i = i + 1 + end + end + else + stderr:write("Unknown option " .. a .."\n") + return {"help"} + end + -- Store the result + if optarg then + if option_list[optname]["type"] == "string" then + result[optname] = optarg + else + local opts = result[optname] or { } + for hit in string.gmatch(optarg, "([^,%s]+)") do + table.insert(opts, hit) + end + result[optname] = opts + end + else + result[optname] = true + end + i = i + 1 + end + if not opt then + files = remainder(i) + break + end + end + if next(files) then + result["files"] = files + end + return result +end + +options = argparse() + +-- Sanity check +if options["engine"] and not options["force"] then + -- Make a lookup table + local t = { } + for _, engine in pairs(checkengines) do + t[engine] = true + end + for _, engine in pairs(options["engine"]) do + if not t[engine] then + print("\n! Error: Engine \"" .. engine .. "\" not set up for testing!") + print("\n Valid values are:") + for _, engine in ipairs(checkengines) do + print(" - " .. engine) + end + print("") + exit(1) + end + end +end + +-- Tidy up the epoch setting +-- Force an epoch if set at the command line +if options["epoch"] then + epoch = options["epoch"] + forcecheckepoch = true + forcedocepoch = true +end +-- If given as an ISO date, turn into an epoch number +do + local y, m, d = string.match(epoch, "^(%d%d%d%d)-(%d%d)-(%d%d)$") + if y then + epoch = + os_time({year = y, month = m, day = d, hour = 0, sec = 0, isdst = nil}) - + os_time({year = 1970, month = 1, day = 1, hour = 0, sec = 0, isdst = nil}) + elseif string.match(epoch, "^%d+$") then + epoch = tonumber(epoch) + else + epoch = 0 + end +end diff --git a/Master/texmf-dist/tex/latex/l3build/l3build-aux.lua b/Master/texmf-dist/tex/latex/l3build/l3build-aux.lua new file mode 100644 index 00000000000..102f9ba83be --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3build/l3build-aux.lua @@ -0,0 +1,96 @@ +--[[ + +File l3build-aux.lua Copyright (C) 2018 The LaTeX3 Project + +It may be distributed and/or modified under the conditions of the +LaTeX Project Public License (LPPL), either version 1.3c of this +license or (at your option) any later version. The latest version +of this license is in the file + + http://www.latex-project.org/lppl.txt + +This file is part of the "l3build bundle" (The Work in LPPL) +and all files in that bundle must be distributed together. + +----------------------------------------------------------------------- + +The development version of the bundle can be found at + + https://github.com/latex3/l3build + +for those people who are interested. + +--]] + +-- +-- Auxiliary functions which are used by more than one main function +-- + +function setepoch() + return + os_setenv .. " SOURCE_DATE_EPOCH=" .. epoch + .. os_concat .. + os_setenv .. " SOURCE_DATE_EPOCH_TEX_PRIMITIVES=1" + .. os_concat .. + os_setenv .. " FORCE_SOURCE_DATE=1" + .. os_concat +end + +-- Do some subtarget for all modules in a bundle +function call(dirs, target, opts) + -- Turn the option table into a string + local opts = opts or options + local s = "" + for k,v in pairs(opts) do + if k ~= "files" and k ~= "target" then -- Special cases + local t = option_list[k] or { } + local arg = "" + if t["type"] == "string" then + arg = arg .. "=" .. v + end + if t["type"] == "table" then + for _,a in pairs(v) do + if arg == "" then + arg = "=" .. a -- Add the initial "=" here + else + arg = arg .. "," .. a + end + end + end + s = s .. " --" .. k .. arg + end + end + if opts["files"] then + for _,v in pairs(opts["files"]) do + s = s .. " " .. v + end + end + for _,i in ipairs(dirs) do + print( + "Running script " .. scriptname .. " with target \"" .. target + .. "\" for module " + .. i + ) + local errorlevel = run( + i, + "texlua " .. scriptname .. " " .. target .. s + ) + if errorlevel ~= 0 then + return errorlevel + end + end + return 0 +end + +-- Unpack files needed to support testing/typesetting/unpacking +function depinstall(deps) + local errorlevel + for _,i in ipairs(deps) do + print("Installing dependency: " .. i) + errorlevel = run(i, "texlua " .. scriptname .. " unpack -q") + if errorlevel ~= 0 then + return errorlevel + end + end + return 0 +end diff --git a/Master/texmf-dist/tex/latex/l3build/l3build-check.lua b/Master/texmf-dist/tex/latex/l3build/l3build-check.lua new file mode 100644 index 00000000000..617fa56a0c6 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3build/l3build-check.lua @@ -0,0 +1,842 @@ +--[[ + +File l3build-check.lua Copyright (C) 2018 The LaTeX3 Project + +It may be distributed and/or modified under the conditions of the +LaTeX Project Public License (LPPL), either version 1.3c of this +license or (at your option) any later version. The latest version +of this license is in the file + + http://www.latex-project.org/lppl.txt + +This file is part of the "l3build bundle" (The Work in LPPL) +and all files in that bundle must be distributed together. + +----------------------------------------------------------------------- + +The development version of the bundle can be found at + + https://github.com/latex3/l3build + +for those people who are interested. + +--]] + +-- Local access to functions +local open = io.open +local close = io.close +local write = io.write +local output = io.output + +local rnd = math.random + +local len = string.len +local char = string.char +local format = string.format +local gmatch = string.gmatch +local gsub = string.gsub +local match = string.match + +local sort = table.sort + +local utf8_char = unicode.utf8.char + +local exit = os.exit +local execute = os.execute + +-- +-- Auxiliary functions which are used by more than one main function +-- + +-- Set up the check system files: needed for checking one or more tests and +-- for saving the test files +function checkinit() + cleandir(testdir) + depinstall(checkdeps) + -- Copy dependencies to the test directory itself: this makes the paths + -- a lot easier to manage, and is important for dealing with the log and + -- with file input/output tests + for _,i in ipairs(filelist(localdir)) do + cp(i, localdir, testdir) + end + bundleunpack({sourcefiledir, testfiledir}) + for _,i in ipairs(installfiles) do + cp(i, unpackdir, testdir) + end + for _,i in ipairs(checkfiles) do + cp(i, unpackdir, testdir) + end + if direxists(testsuppdir) then + for _,i in ipairs(filelist(testsuppdir)) do + cp(i, testsuppdir, testdir) + end + end + for _,i in ipairs(checksuppfiles) do + cp(i, supportdir, testdir) + end + execute(os_ascii .. ">" .. testdir .. "/ascii.tcx") +end + +-- Convert the raw log file into one for comparison/storage: keeps only +-- the 'business' part from the tests and removes system-dependent stuff +local function formatlog(logfile, newfile, engine, errlevels) + local maxprintline = maxprintline + if engine == "luatex" or engine == "luajittex" then + maxprintline = maxprintline + 1 -- Deal with an out-by-one error + end + local function killcheck(line) + -- Skip lines containing file dates + if match(line, "[^<]%d%d%d%d/%d%d/%d%d") then + return true + elseif + -- Skip \openin/\openout lines in web2c 7.x + -- As Lua doesn't allow "(in|out)", a slightly complex approach: + -- do a substitution to check the line is exactly what is required! + match( + gsub(line, "^\\openin", "\\openout"), "^\\openout%d%d? = " + ) then + return true + end + return false + end + -- Substitutions to remove some non-useful changes + local function normalize(line, lastline) + -- Zap line numbers from \show, \showbox, \box_show and the like: + -- do this before wrapping lines + line = gsub(line, "^l%.%d+ ", "l. ...") + -- Also from lua stack traces. + line = gsub(line, "lua:%d+: in function", "lua:...: in function") + -- Allow for wrapped lines: preserve the content and wrap + -- Skip lines that have an explicit marker for truncation + if len(line) == maxprintline and + not match(line, "%.%.%.$") then + return "", (lastline or "") .. line + end + local line = (lastline or "") .. line + lastline = "" + -- Zap ./ at begin of filename + line = gsub(line, "%(%.%/", "(") + -- Zap paths if places other than 'here' are accessible + if checksearch then + -- The pattern excludes < and > as the image part can have + -- several entries on one line + local pattern = "%w?:?/[^ %<%>]*/([^/%(%)]*%.%w*)" + -- Files loaded from TeX: all start ( -- ) + line = gsub(line, "%(" .. pattern, "(../%1") + -- Images + line = gsub(line, "<" .. pattern .. ">", "<../%1>") + -- luaotfload files start with keywords + line = gsub(line, "from " .. pattern .. "%(", "from. ./%1(") + line = gsub(line, ": " .. pattern .. "%)", ": ../%1)") + -- Deal with XeTeX specials + if match(line, "^%.+\\XeTeX.?.?.?file") then + line = gsub(line, pattern, "../%1") + end + end + -- Deal with the fact that "(.aux)" may have still a leading space + line = gsub(line, "^ %(%.aux%)", "(.aux)") + -- Merge all of .fd data into one line so will be removed later + if match(line, "^ *%([%.%/%w]+%.fd[^%)]*$") then + lastline = (lastline or "") .. line + return "", (lastline or "") .. line + end + -- TeX90/XeTeX knows only the smaller set of dimension units + line = gsub( + line, + "cm, mm, dd, cc, bp, or sp", "cm, mm, dd, cc, nd, nc, bp, or sp" + ) + -- Normalise a case where fixing a TeX bug changes the message text + line = gsub(line, "\\csname\\endcsname ", "\\csname\\endcsname") + -- Zap "on line " and replace with "on line ..." + -- Two similar cases, Lua patterns mean we need to do them separately + line = gsub(line, "on line %d*", "on line ...") + line = gsub(line, "on input line %d*", "on input line ...") + -- Tidy up to ^^ notation + for i = 0, 31 do + line = gsub(line, char(i), "^^" .. char(64 + i)) + end + -- Normalise register allocation to hard-coded numbers + -- No regex, so use a pattern plus lookup approach + local register_types = { + attribute = true, + box = true, + bytecode = true, + catcodetable = true, + count = true, + dimen = true, + insert = true, + language = true, + luabytecode = true, + luachunk = true, + luafunction = true, + marks = true, + muskip = true, + read = true, + skip = true, + toks = true, + whatsit = true, + write = true, + XeTeXcharclass = true + } + if register_types[match(line, "^\\[^%]]+=\\([a-z]+)%d+$")] then + line = gsub(line, "%d+$", "...") + end + -- Also deal with showing boxes + if match(line, "^> \\box%d+=$") or match(line, "^> \\box%d+=(void)$") then + line = gsub(line, "%d+=", "...=") + end + -- Remove 'normal' direction information on boxes with (u)pTeX + line = gsub(line, ",? yoko direction,?", "") + line = gsub(line, ",? yoko%(math%) direction,?", "") + -- Remove the \special line that in DVI mode keeps PDFs comparable + if match(line, "^%.*\\special%{pdf: docinfo << /Creator") then + return "" + end + -- Remove the \special line possibly present in DVI mode for paper size + if match(line, "^%.*\\special%{papersize") then + return "" + end + -- Remove ConTeXt stuff + if match(line, "^backend >") or + match(line, "^close source >") or + match(line, "^mkiv lua stats >") or + match(line, "^pages >") or + match(line, "^system >") or + match(line, "^used file >") or + match(line, "^used option >") or + match(line, "^used structure >") then + return "" + end + -- A tidy-up to keep LuaTeX and other engines in sync + line = gsub(line, utf8_char(127), "^^?") + -- Unicode engines display chars in the upper half of the 8-bit range: + -- tidy up to match pdfTeX if an ASCII engine is in use + if next(asciiengines) then + for i = 128, 255 do + line = gsub(line, utf8_char(i), "^^" .. format("%02x", i)) + end + end + return line, lastline + end + local lastline = "" + local newlog = "" + local prestart = true + local skipping = false + -- Read the entire log file as a binary: deals with ^@/^[, etc. + local file = assert(open(logfile, "rb")) + local contents = gsub(file:read("*all") .. "\n", "\r\n", "\n") + close(file) + for line in gmatch(contents, "([^\n]*)\n") do + if line == "START-TEST-LOG" then + prestart = false + elseif line == "END-TEST-LOG" or + match(line, "^Here is how much of .?.?.?TeX\'s memory you used:") then + break + elseif line == "OMIT" then + skipping = true + elseif match(line, "^%)?TIMO$") then + skipping = false + elseif not prestart and not skipping then + line, lastline = normalize(line, lastline) + if not match(line, "^ *$") and not killcheck(line) then + newlog = newlog .. line .. os_newline + end + end + end + local newfile = open(newfile, "w") + output(newfile) + write(newlog) + if recordstatus then + write('***************\n') + for i = 1, checkruns do + write('Compilation ' .. i .. ' of test file completed with exit status ' .. errlevels[i] .. '\n') + end + end + close(newfile) +end + +-- Additional normalization for LuaTeX +local function formatlualog(logfile, newfile) + local function normalize(line, lastline, dropping) + -- Find \discretionary or \whatsit lines: + -- These may come back later + if match(line, "^%.+\\discretionary$") or + match(line, "^%.+\\discretionary %(penalty 50%)$") or + match(line, "^%.+\\discretionary50%|$") or + match(line, "^%.+\\discretionary50%| replacing $") or + match(line, "^%.+\\whatsit$") then + return "", line + end + -- For \mathon, we always need this line but the next + -- may be affected + if match(line, "^%.+\\mathon$") then + return line, line + end + -- LuaTeX has a flexible output box + line = gsub(line,"\\box\\outputbox", "\\box255") + -- LuaTeX identifies spaceskip glue + line = gsub(line,"%(\\spaceskip%) ", " ") + -- Remove 'display' at end of display math boxes: + -- LuaTeX omits this as it includes direction in all cases + line = gsub(line, "(\\hbox%(.*), display$", "%1") + -- Remove 'normal' direction information on boxes: + -- any bidi/vertical stuff will still show + line = gsub(line, ", direction TLT", "") + -- Find glue setting and round out the last place + local function round_digits(l, m) + return gsub( + l, + m .. " (%-?)%d+%.%d+", + m .. " %1" + .. format( + "%.3f", + match(line, m .. " %-?(%d+%.%d+)") or 0 + ) + ) + end + if match(line, "glue set %-?%d+%.%d+") then + line = round_digits(line, "glue set") + end + if match( + line, "glue %-?%d+%.%d+ plus %-?%d+%.%d+ minus %-?%d+%.%d+$" + ) + then + line = round_digits(line, "glue") + line = round_digits(line, "plus") + line = round_digits(line, "minus") + end + -- LuaTeX writes ^^M as a new line, which we lose + line = gsub(line, "%^%^M", "") + -- Remove U+ notation in the "Missing character" message + line = gsub( + line, + "Missing character: There is no (%^%^..) %(U%+(....)%)", + "Missing character: There is no %1" + ) + -- The first time a new font is used, it shows up + -- as being cached + line = gsub(line, "(save cache:", "(load cache:") + -- A function to handle the box prefix part + local function boxprefix(s) + return gsub(match(s, "^(%.+)"), "%.", "%%.") + end + -- 'Recover' some discretionary data + if match(lastline, "^%.+\\discretionary %(penalty 50%)$") and + match(line, boxprefix(lastline) .. "%.= ") then + return gsub(line, "%.= ", ""),"" + end + -- Where the last line was a discretionary, looks for the + -- info one level in about what it represents + if match(lastline, "^%.+\\discretionary$") or + match(lastline, "^%.+\\discretionary %(penalty 50%)$") or + match(lastline, "^%.+\\discretionary50%|$") or + match(lastline, "^%.+\\discretionary50%| replacing $") then + local prefix = boxprefix(lastline) + if match(line, prefix .. "%.") or + match(line, prefix .. "%|") then + if match(lastline, " replacing $") and + not dropping then + -- Modify the return line + return gsub(line, "^%.", ""), lastline, true + else + return "", lastline, true + end + else + if dropping then + -- End of a \discretionary block + return line, "" + else + -- Not quite a normal discretionary + if match(lastline, "^%.+\\discretionary50%|$") then + lastline = gsub(lastline, "50%|$", "") + end + -- Remove some info that TeX90 lacks + lastline = gsub(lastline, " %(penalty 50%)$", "") + -- A normal (TeX90) discretionary: + -- add with the line break reintroduced + return lastline .. os_newline .. line, "" + end + end + end + -- Look for another form of \discretionary, replacing a "-" + pattern = "^%.+\\discretionary replacing *$" + if match(line, pattern) then + return "", line + else + if match(lastline, pattern) then + local prefix = boxprefix(lastline) + if match(line, prefix .. "%.\\kern") then + return gsub(line, "^%.", ""), lastline, true + elseif dropping then + return "", "" + else + return lastline .. os_newline .. line, "" + end + end + end + -- For \mathon, if the current line is an empty \hbox then + -- drop it + if match(lastline, "^%.+\\mathon$") then + local prefix = boxprefix(lastline) + if match(line, prefix .. "\\hbox%(0%.0%+0%.0%)x0%.0$") then + return "", "" + end + end + -- Various \local... things that other engines do not do: + -- Only remove the no-op versions + if match(line, "^%.+\\localpar$") or + match(line, "^%.+\\localinterlinepenalty=0$") or + match(line, "^%.+\\localbrokenpenalty=0$") or + match(line, "^%.+\\localleftbox=null$") or + match(line, "^%.+\\localrightbox=null$") then + return "", "" + end + -- Older LuaTeX versions set the above up as a whatsit + -- (at some stage this can therefore go) + if match(lastline, "^%.+\\whatsit$") then + local prefix = boxprefix(lastline) + if match(line, prefix .. "%.") then + return "", lastline, true + else + -- End of a \whatsit block + return line, "" + end + end + -- Wrap some cases that can be picked out + -- In some places LuaTeX does use max_print_line, then we + -- get into issues with different wrapping approaches + if len(line) == maxprintline then + return "", lastline .. line + elseif len(lastline) == maxprintline then + if match(line, "\\ETC%.%}$") then + -- If the line wrapped at \ETC we might have lost a space + return lastline + .. ((match(line, "^\\ETC%.%}$") and " ") or "") + .. line, "" + elseif match(line, "^%}%}%}$") then + return lastline .. line, "" + else + return lastline .. os_newline .. line, "" + end + -- Return all of the text for a wrapped (multi)line + elseif len(lastline) > maxprintline then + return lastline .. line, "" + end + -- Remove spaces at the start of lines: deals with the fact that LuaTeX + -- uses a different number to the other engines + return gsub(line, "^%s+", ""), "" + end + local newlog = "" + local lastline = "" + local dropping = false + -- Read the entire log file as a binary: deals with ^@/^[, etc. + local file = assert(open(logfile, "rb")) + local contents = gsub(file:read("*all") .. "\n", "\r\n", "\n") + close(file) + for line in gmatch(contents, "([^\n]*)\n") do + line, lastline, dropping = normalize(line, lastline, dropping) + if not match(line, "^ *$") then + newlog = newlog .. line .. os_newline + end + end + local newfile = open(newfile, "w") + output(newfile) + write(newlog) + close(newfile) +end + +-- Run one test which may have multiple engine-dependent comparisons +-- Should create a difference file for each failed test +function runcheck(name, hide) + local checkengines = checkengines + if options["engine"] then + checkengines = options["engine"] + end + local errorlevel = 0 + for _,i in ipairs(checkengines) do + -- Allow for luatex == luajittex for .tlg purposes + local engine = i + if i == "luajittex" then + engine = "luatex" + end + checkpdf = setup_check(name, engine) + runtest(name, i, hide, lvtext, checkpdf) + -- Generation of results depends on test type + local errlevel + if checkpdf then + errlevel = compare_pdf(name, engine) + else + errlevel = compare_tlg(name, engine) + end + if errlevel ~= 0 and options["halt-on-error"] then + showfaileddiff() + if errlevel ~= 0 then + return 1 + end + end + if errlevel > errorlevel then + errorlevel = errlevel + end + end + return errorlevel +end + +function setup_check(name, engine) + local testname = name .. "." .. engine + local pdffile = locate( + {testfiledir, unpackdir}, + {testname .. pdfext, name .. pdfext} + ) + local tlgfile = locate( + {testfiledir, unpackdir}, + {testname .. tlgext, name .. tlgext} + ) + -- Attempt to generate missing reference file from expectation + if not (pdffile or tlgfile) then + if not locate({unpackdir, testfiledir}, {name .. lveext}) then + print( + "Error: failed to find " .. pdfext .. ", " .. tlgext .. " or " + .. lveext .. " file for " .. name .. "!" + ) + exit(1) + end + runtest(name, engine, true, lveext, true) + pdffile = testdir .. "/" .. testname .. pdfext + -- If a PDF is generated use it for comparisons + if not fileexists(pdffile) then + pdffile = nil + ren(testdir, testname .. logext, testname .. tlgext) + end + else + -- Install comparison files found + for _,v in pairs({pdffile, tlgfile}) do + if v then + cp( + match(v, ".*/(.*)"), + match(v, "(.*)/.*"), + testdir + ) + end + end + end + if pdffile then + local pdffile = match(pdffile, ".*/(.*)") + ren( + testdir, + pdffile, + gsub(pdffile, pdfext .. "$", ".ref" .. pdfext) + ) + return true + else + return false + end +end + +function compare_pdf(name, engine) + local errorlevel + local testname = name .. "." .. engine + local cmpfile = testdir .. "/" .. testname .. os_cmpext + local pdffile = testdir .. "/" .. testname .. pdfext + local refpdffile = locate( + {testdir}, {testname .. ".ref" .. pdfext, name .. ".ref" .. pdfext} + ) + if not refpdffile then + return + end + if os_type == "windows" then + refpdffile = unix_to_win(refpdffile) + end + errorlevel = execute( + os_cmpexe .. " " .. refpdffile .. " " .. pdffile .. " > " .. cmpfile + ) + if errorlevel == 0 then + os.remove(cmpfile) + end + return errorlevel +end + +function compare_tlg(name, engine) + local errorlevel + local testname = name .. "." .. engine + local difffile = testdir .. "/" .. testname .. os_diffext + local logfile = testdir .. "/" .. testname .. logext + local tlgfile = locate({testdir}, {testname .. tlgext, name .. tlgext}) + if not tlgfile then + return + end + if os_type == "windows" then + tlgfile = unix_to_win(tlgfile) + end + -- Do additional log formatting if the engine is LuaTeX, there is no + -- LuaTeX-specific .tlg file and the default engine is not LuaTeX + if engine == "luatex" + and not match(tlgfile, "%.luatex" .. "%" .. tlgext) + and stdengine ~= "luatex" + and stdengine ~= "luajittex" + then + local luatlgfile = testdir .. "/" .. name .. ".luatex" .. tlgext + if os_type == "windows" then + luatlgfile = unix_to_win(luatlgfile) + end + formatlualog(tlgfile, luatlgfile) + formatlualog(logfile, logfile) + -- This allows code sharing below: we only need the .tlg name in one place + tlgfile = luatlgfile + end + errorlevel = execute( + os_diffexe .. " " .. tlgfile .. " " .. logfile .. " > " .. difffile + ) + if errorlevel == 0 then + os.remove(difffile) + end + return errorlevel +end + +-- Run one of the test files: doesn't check the result so suitable for +-- both creating and verifying .tlg files +function runtest(name, engine, hide, ext, makepdf) + local lvtfile = name .. (ext or lvtext) + cp(lvtfile, fileexists(testfiledir .. "/" .. lvtfile) + and testfiledir or unpackdir, testdir) + local engine = engine or stdengine + -- Set up the format file name if it's one ending "...tex" + local realengine = engine + local format + if + match(checkformat, "tex$") and + not match(engine, checkformat) then + format = " -fmt=" .. gsub(engine, "(.*)tex$", "%1") .. checkformat + else + format = "" + end + -- Special casing for e-LaTeX format + if + match(checkformat, "^latex$") and + match(engine, "^etex$") then + format = " -fmt=latex" + end + -- Special casing for (u)pTeX LaTeX formats + if + match(checkformat, "^latex$") and + match(engine, "^u?ptex$") then + realengine = "e" .. engine + end + -- Special casing for XeTeX engine + local checkopts = checkopts + if match(engine, "xetex") and not makepdf then + checkopts = checkopts .. " -no-pdf" + end + -- Special casing for ConTeXt + if match(checkformat, "^context$") then + format = "" + if engine == "luatex" or engine == "luajittex" then + realengine = "context" + elseif engine == "pdftex" then + realengine = "texexec" + elseif engine == "xetex" then + realengine = "texexec --xetex" + else + print("Engine incompatible with format") + exit(1) + end + end + local logfile = testdir .. "/" .. name .. logext + local newfile = testdir .. "/" .. name .. "." .. engine .. logext + local asciiopt = "" + for _,i in ipairs(asciiengines) do + if realengine == i then + asciiopt = "-translate-file ./ascii.tcx " + break + end + end + local errlevels = {} + for i = 1, checkruns do + errlevels[i] = run( + testdir, + -- No use of localdir here as the files get copied to testdir: + -- avoids any paths in the logs + os_setenv .. " TEXINPUTS=." .. (checksearch and os_pathsep or "") + .. os_concat .. + -- Avoid spurious output from (u)pTeX + os_setenv .. " GUESS_INPUT_KANJI_ENCODING=0" + .. os_concat .. + (forcecheckepoch and setepoch() or "") .. + -- Ensure lines are of a known length + os_setenv .. " max_print_line=" .. maxprintline + .. os_concat .. + realengine .. format .. " " + .. checkopts .. " " .. asciiopt .. lvtfile + .. (hide and (" > " .. os_null) or "") + .. os_concat .. + runtest_tasks(jobname(lvtfile)) + ) + end + if makepdf and fileexists(testdir .. "/" .. name .. dviext) then + dvitopdf(name, testdir, engine, hide) + end + formatlog(logfile, newfile, engine, errlevels) + -- Store secondary files for this engine + for _,filetype in pairs(auxfiles) do + for _,file in pairs(filelist(testdir, filetype)) do + if match(file,"^" .. name) then + local ext = match(file, "%.[^.]+$") + if ext ~= lvtext and + ext ~= tlgext and + ext ~= lveext and + ext ~= logext then + ren(testdir, file, gsub(file, "(%.[^.]+)$", "." .. engine .. "%1")) + end + end + end + end +end + +-- A hook to allow additional tasks to run for the tests +runtest_tasks = runtest_tasks or function(name) + return "" +end + +-- Look for a test: could be in the testfiledir or the unpackdir +function testexists(test) + return(locate({testfiledir, unpackdir}, {test .. lvtext})) +end + +-- Standard versions of the main targets for building modules + +function check(names) + local errorlevel = 0 + if testfiledir ~= "" and direxists(testfiledir) then + if not options["rerun"] then + checkinit() + end + local hide = true + if names and next(names) then + hide = false + end + names = names or { } + -- No names passed: find all test files + if not next(names) then + for _,i in pairs(filelist(testfiledir, "*" .. lvtext)) do + table.insert(names, jobname(i)) + end + for _,i in ipairs(filelist(unpackdir, "*" .. lvtext)) do + if fileexists(testfiledir .. "/" .. i) then + print("Duplicate test file: " .. i) + return 1 + else + table.insert(names, jobname(i)) + end + end + sort(names) + end + -- https://stackoverflow.com/a/32167188 + local function shuffle(tbl) + local len, random = #tbl, rnd + for i = len, 2, -1 do + local j = random(1, i) + tbl[i], tbl[j] = tbl[j], tbl[i] + end + return tbl + end + if options["shuffle"] then + names = shuffle(names) + end + -- Actually run the tests + print("Running checks on") + local i = 0 + for _,name in ipairs(names) do + i = i + 1 + print(" " .. name .. " (" .. i.. "/" .. #names ..")") + local errlevel = runcheck(name, hide) + -- Return value must be 1 not errlevel + if errlevel ~= 0 then + if options["halt-on-error"] then + return 1 + else + errorlevel = 1 + -- visually show that something has failed + print(" --> failed\n") + end + end + end + if errorlevel ~= 0 then + checkdiff() + else + print("\n All checks passed\n") + end + end + return errorlevel +end + +-- A short auxiliary to print the list of differences for check +function checkdiff() + print("\n Check failed with difference files") + for _,i in ipairs(filelist(testdir, "*" .. os_diffext)) do + print(" - " .. testdir .. "/" .. i) + end + for _,i in ipairs(filelist(testdir, "*" .. os_cmpext)) do + print(" - " .. testdir .. "/" .. i) + end + print("") +end + +function showfaileddiff() + print("\nCheck failed with difference file") + for _,i in ipairs(filelist(testdir, "*" .. os_diffext)) do + print(" - " .. testdir .. "/" .. i) + print("") + local f = open(testdir .. "/" .. i,"r") + local content = f:read("*all") + close(f) + print("-----------------------------------------------------------------------------------") + print(content) + print("-----------------------------------------------------------------------------------") + end + for _,i in ipairs(filelist(testdir, "*" .. os_cmpext)) do + print(" - " .. testdir .. "/" .. i) + end +end + +function save(names) + checkinit() + local engines = options["engine"] or {stdengine} + for _,name in pairs(names) do + local engine + for _,engine in pairs(engines) do + local tlgengine = ((engine == stdengine and "") or "." .. engine) + local tlgfile = name .. tlgengine .. tlgext + local spdffile = name .. tlgengine .. pdfext + local newfile = name .. "." .. engine .. logext + local pdffile = name .. "." .. engine .. pdfext + local refext = ((options["pdf"] and pdfext) or tlgext) + if testexists(name) then + print("Creating and copying " .. refext) + runtest(name, engine, false, lvtext, options["pdf"]) + if options["pdf"] then + ren(testdir, pdffile, spdffile) + cp(spdffile, testdir, testfiledir) + else + ren(testdir, newfile, tlgfile) + cp(tlgfile, testdir, testfiledir) + end + if fileexists(unpackdir .. "/" .. tlgfile) then + print( + "Saved " .. tlgext + .. " file overrides unpacked version of the same name" + ) + end + elseif locate({unpackdir, testfiledir}, {name .. lveext}) then + print( + "Saved " .. tlgext .. " file overrides a " + .. lveext .. " file of the same name" + ) + else + print( + "Test input \"" .. testfiledir .. "/" .. name .. lvtext + .. "\" not found" + ) + end + end + end +end + diff --git a/Master/texmf-dist/tex/latex/l3build/l3build-clean.lua b/Master/texmf-dist/tex/latex/l3build/l3build-clean.lua new file mode 100644 index 00000000000..3fd8396c9ce --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3build/l3build-clean.lua @@ -0,0 +1,56 @@ +--[[ + +File l3build-clean.lua Copyright (C) 2018 The LaTeX3 Project + +It may be distributed and/or modified under the conditions of the +LaTeX Project Public License (LPPL), either version 1.3c of this +license or (at your option) any later version. The latest version +of this license is in the file + + http://www.latex-project.org/lppl.txt + +This file is part of the "l3build bundle" (The Work in LPPL) +and all files in that bundle must be distributed together. + +----------------------------------------------------------------------- + +The development version of the bundle can be found at + + https://github.com/latex3/l3build + +for those people who are interested. + +--]] + +-- Remove all generated files +function clean() + -- To make sure that distribdir never contains any stray subdirs, + -- it is entirely removed then recreated rather than simply deleting + -- all of the files + local errorlevel = + rmdir(distribdir) + + mkdir(distribdir) + + cleandir(localdir) + + cleandir(testdir) + + cleandir(typesetdir) + + cleandir(unpackdir) + for _,i in ipairs(cleanfiles) do + for _,dir in pairs(remove_duplicates({maindir, sourcefiledir, docfiledir})) do + errorlevel = rm(dir, i) + errorlevel + end + end + return errorlevel +end + +function bundleclean() + local errorlevel = call(modules, "clean") + for _,i in ipairs(cleanfiles) do + errorlevel = rm(maindir, i) + errorlevel + end + return ( + errorlevel + + rmdir(ctandir) + + rmdir(tdsdir) + ) +end + diff --git a/Master/texmf-dist/tex/latex/l3build/l3build-ctan.lua b/Master/texmf-dist/tex/latex/l3build/l3build-ctan.lua new file mode 100644 index 00000000000..47688349d8e --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3build/l3build-ctan.lua @@ -0,0 +1,238 @@ +--[[ + +File l3build-ctan.lua Copyright (C) 2018 The LaTeX3 Project + +It may be distributed and/or modified under the conditions of the +LaTeX Project Public License (LPPL), either version 1.3c of this +license or (at your option) any later version. The latest version +of this license is in the file + + http://www.latex-project.org/lppl.txt + +This file is part of the "l3build bundle" (The Work in LPPL) +and all files in that bundle must be distributed together. + +----------------------------------------------------------------------- + +The development version of the bundle can be found at + + https://github.com/latex3/l3build + +for those people who are interested. + +--]] + +local gsub = string.gsub +local insert = table.insert + +-- Copy files to the main CTAN release directory +function copyctan() + local ctantarget = ctanpkg + if docfiledir ~= currentdir then + ctantarget = ctanpkg .. "/" .. gsub(docfiledir, "^%.*/", "") + end + mkdir(ctandir .. "/" .. ctantarget) + for _,filetype in pairs( + { + bibfiles, + demofiles, + docfiles, + pdffiles, + typesetlist + } + ) do + for _,file in pairs(filetype) do + cp(file, docfiledir, ctandir .. "/" .. ctantarget) + end + end + ctantarget = ctanpkg + if sourcefiledir ~= currentdir then + ctantarget = ctanpkg .. "/" .. gsub(sourcefiledir, "^%.*/", "") + end + mkdir(ctandir .. "/" .. ctantarget) + for _,file in pairs(sourcefiles) do + if sourcedir ~= currentdir then + end + cp(file, sourcefiledir, ctandir .. "/" .. ctantarget) + end + for _,file in pairs(textfiles) do + cp(file, currentdir, ctandir .. "/" .. ctanpkg) + end +end + +-- Copy files to the correct places in the TDS tree +function copytds() + local function install(source, dest, files, tool) + local moduledir = moduledir + -- For material associated with secondary tools (BibTeX, MakeIndex) + -- the structure needed is slightly different from those items going + -- into the tex/doc/source trees + if tool then + -- "base" is reserved for the tools themselves: make the assumption + -- in this case that the tdsroot name is the right place for stuff to + -- go (really just for the team) + if module == "base" then + moduledir = tdsroot + else + moduledir = module + end + end + -- Convert the file table(s) to a list of individual files + local filenames = { } + for _,i in ipairs(files) do + for _,j in ipairs(i) do + for file,_ in pairs(tree(source, j)) do + insert(filenames, file) + end + end + end + -- The target is only created if there are actual files to install + if next(filenames) ~= nil then + local installdir = tdsdir .. "/" .. dest .. "/" .. moduledir + mkdir(installdir) + for _,i in ipairs(filenames) do + cp(i, source, installdir) + end + end + end + install( + docfiledir, + "doc", + {bibfiles, demofiles, docfiles, pdffiles, textfiles, typesetlist} + ) + install(unpackdir, "makeindex", {makeindexfiles}, true) + install(unpackdir, "bibtex/bst", {bstfiles}, true) + install(sourcefiledir, "source", {sourcelist}) + install(unpackdir, "tex", {installfiles}) +end + +-- Standard versions of the main targets for building modules + +function ctan() + -- Always run tests for all engines + options["engine"] = nil + local function dirzip(dir, name) + local zipname = name .. ".zip" + local function tab_to_str(table) + local string = "" + for _,i in ipairs(table) do + string = string .. " " .. "\"" .. i .. "\"" + end + return string + end + -- Convert the tables of files to quoted strings + local binfiles = tab_to_str(binaryfiles) + local exclude = tab_to_str(excludefiles) + -- First, zip up all of the text files + run( + dir, + zipexe .. " " .. zipopts .. " -ll ".. zipname .. " " .. "." + .. ( + (binfiles or exclude) and (" -x" .. binfiles .. " " .. exclude) + or "" + ) + ) + -- Then add the binary ones + run( + dir, + zipexe .. " " .. zipopts .. " -g ".. zipname .. " " .. ". -i" .. + binfiles .. (exclude and (" -x" .. exclude) or "") + ) + end + local errorlevel + local standalone = false + if bundle == "" then + standalone = true + end + if standalone then + errorlevel = check() + bundle = module + else + errorlevel = call(modules, "bundlecheck") + end + if errorlevel == 0 then + rmdir(ctandir) + mkdir(ctandir .. "/" .. ctanpkg) + rmdir(tdsdir) + mkdir(tdsdir) + if standalone then + errorlevel = bundlectan() + else + errorlevel = call(modules, "bundlectan") + end + else + print("\n====================") + print("Tests failed, zip stage skipped!") + print("====================\n") + return errorlevel + end + if errorlevel == 0 then + for _,i in ipairs(textfiles) do + for _,j in pairs({unpackdir, currentdir}) do + cp(i, j, ctandir .. "/" .. ctanpkg) + cp(i, j, tdsdir .. "/doc/" .. tdsroot .. "/" .. bundle) + end + end + dirzip(tdsdir, ctanpkg .. ".tds") + if packtdszip then + cp(ctanpkg .. ".tds.zip", tdsdir, ctandir) + end + dirzip(ctandir, ctanpkg) + cp(ctanpkg .. ".zip", ctandir, currentdir) + else + print("\n====================") + print("Typesetting failed, zip stage skipped!") + print("====================\n") + end + return errorlevel +end + +function bundlectan() + -- Generate a list of individual file names excluding those in the second + -- argument: the latter is a table + local function excludelist(include, exclude, dir) + local include = include or { } + local exclude = exclude or { } + local dir = dir or currentdir + local includelist = { } + local excludelist = { } + for _,i in ipairs(exclude) do + for _,j in ipairs(i) do + for file,_ in pairs(tree(dir, j)) do + excludelist[file] = true + end + end + end + for _,i in ipairs(include) do + for file,_ in pairs(tree(dir, i)) do + if not excludelist[file] then + insert(includelist, file) + end + end + end + return includelist + end + unpack() + local errorlevel = doc() + if errorlevel == 0 then + -- Work out what PDF files are available + pdffiles = { } + for _,i in ipairs(typesetfiles) do + insert(pdffiles, (gsub(i, "%.%w+$", ".pdf"))) + end + -- For the purposes here, any typesetting demo files need to be + -- part of the main typesetting list + local typesetfiles = typesetfiles + for _,v in pairs(typesetdemofiles) do + insert(typesetfiles, v) + end + typesetlist = excludelist(typesetfiles, {sourcefiles}, docfiledir) + sourcelist = excludelist( + sourcefiles, {bstfiles, installfiles, makeindexfiles}, sourcefiledir + ) + copyctan() + copytds() + end + return errorlevel +end + diff --git a/Master/texmf-dist/tex/latex/l3build/l3build-file-functions.lua b/Master/texmf-dist/tex/latex/l3build/l3build-file-functions.lua new file mode 100644 index 00000000000..d85a3d203d2 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3build/l3build-file-functions.lua @@ -0,0 +1,406 @@ +--[[ + +File l3build-file-functions.lua Copyright (C) 2018 The LaTeX3 Project + +It may be distributed and/or modified under the conditions of the +LaTeX Project Public License (LPPL), either version 1.3c of this +license or (at your option) any later version. The latest version +of this license is in the file + + http://www.latex-project.org/lppl.txt + +This file is part of the "l3build bundle" (The Work in LPPL) +and all files in that bundle must be distributed together. + +----------------------------------------------------------------------- + +The development version of the bundle can be found at + + https://github.com/latex3/l3build + +for those people who are interested. + +--]] + + +local execute = os.execute +local getenv = os.getenv +local os_time = os.time +local os_type = os.type + +local match = string.match +local sub = string.sub +local gsub = string.gsub + + + +-- Convert a file glob into a pattern for use by e.g. string.gub +-- Based on https://github.com/davidm/lua-glob-pattern +-- Simplified substantially: "[...]" syntax not supported as is not +-- required by the file patterns used by the team. Also note style +-- changes to match coding approach in rest of this file. +-- +-- License for original globtopattern +--[[ + + (c) 2008-2011 David Manura. Licensed under the same terms as Lua (MIT). + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + (end license) + +--]] +local function glob_to_pattern(glob) + + local pattern = "^" -- pattern being built + local i = 0 -- index in glob + local char -- char at index i in glob + + -- escape pattern char + local function escape(char) + return match(char, "^%w$") and char or "%" .. char + end + + -- Convert tokens. + while true do + i = i + 1 + char = sub(glob, i, i) + if char == "" then + pattern = pattern .. "$" + break + elseif char == "?" then + pattern = pattern .. "." + elseif char == "*" then + pattern = pattern .. ".*" + elseif char == "[" then + -- Ignored + print("[...] syntax not supported in globs!") + elseif char == "\\" then + i = i + 1 + char = sub(glob, i, i) + if char == "" then + pattern = pattern .. "\\$" + break + end + pattern = pattern .. escape(char) + else + pattern = pattern .. escape(char) + end + end + return pattern +end + +-- Detect the operating system in use +-- Support items are defined here for cases where a single string can cover +-- both Windows and Unix cases: more complex situations are handled inside +-- the support functions +os_concat = ";" +os_null = "/dev/null" +os_pathsep = ":" +os_setenv = "export" +os_yes = "printf 'y\\n%.0s' {1..200}" + +os_ascii = "echo \"\"" +os_cmpexe = getenv("cmpexe") or "cmp" +os_cmpext = getenv("cmpext") or ".cmp" +os_diffext = getenv("diffext") or ".diff" +os_diffexe = getenv("diffexe") or "diff -c --strip-trailing-cr" +os_grepexe = "grep" +os_newline = "\n" + +if os_type == "windows" then + os_ascii = "@echo." + os_cmpexe = getenv("cmpexe") or "fc /b" + os_cmpext = getenv("cmpext") or ".cmp" + os_concat = "&" + os_diffext = getenv("diffext") or ".fc" + os_diffexe = getenv("diffexe") or "fc /n" + os_grepexe = "findstr /r" + os_newline = "\n" + if tonumber(status.luatex_version) < 100 or + (tonumber(status.luatex_version) == 100 + and tonumber(status.luatex_revision) < 4) then + os_newline = "\r\n" + end + os_null = "nul" + os_pathsep = ";" + os_setenv = "set" + os_yes = "for /l %I in (1,1,200) do @echo y" +end + +-- Return an absolute path from a relative one +function abspath(path) + local oldpwd = lfs.currentdir() + lfs.chdir(path) + local result = lfs.currentdir() + lfs.chdir(oldpwd) + return gsub(result, "\\", "/") +end + +-- For cleaning out a directory, which also ensures that it exists +function cleandir(dir) + local errorlevel = mkdir(dir) + if errorlevel ~= 0 then + return errorlevel + end + return rm(dir, "*") +end + +-- Copy files 'quietly' +function cp(glob, source, dest) + local errorlevel + for i,_ in pairs(tree(source, glob)) do + local source = source .. "/" .. i + if os_type == "windows" then + if lfs.attributes(source)["mode"] == "directory" then + errorlevel = execute( + 'xcopy /y /e /i "' .. unix_to_win(source) .. '" "' + .. unix_to_win(dest .. '/' .. i) .. '" > nul' + ) + else + errorlevel = execute( + 'xcopy /y "' .. unix_to_win(source) .. '" "' + .. unix_to_win(dest .. '/') .. '" > nul' + ) + end + else + errorlevel = execute("cp -RLf '" .. source .. "' '" .. dest .. "'") + end + if errorlevel ~=0 then + return errorlevel + end + end + return 0 +end + +-- OS-dependent test for a directory +function direxists(dir) + local errorlevel + if os_type == "windows" then + errorlevel = + execute("if not exist \"" .. unix_to_win(dir) .. "\" exit 1") + else + errorlevel = execute("[ -d " .. dir .. " ]") + end + if errorlevel ~= 0 then + return false + end + return true +end + +function fileexists(file) + local f = io.open(file, "r") + if f ~= nil then + f:close() + return true + else + return false + end +end + +-- Generate a table containing all file names of the given glob or all files +-- if absent +function filelist(path, glob) + local files = { } + local pattern + if glob then + pattern = glob_to_pattern(glob) + end + if direxists(path) then + for entry in lfs.dir(path) do + if pattern then + if match(entry, pattern) then + table.insert(files, entry) + end + else + if entry ~= "." and entry ~= ".." then + table.insert(files, entry) + end + end + end + end + return files +end + +-- Does what filelist does, but can also glob subdirectories. In the returned +-- table, the keys are paths relative to the given starting path, the values +-- are their counterparts relative to the current working directory. +function tree(path, glob) + local function cropdots(path) + return gsub(gsub(path, "^%./", ""), "/%./", "/") + end + local function always_true() + return true + end + local function is_dir(file) + return lfs.attributes(file)["mode"] == "directory" + end + local dirs = {["."] = cropdots(path)} + for pattern, criterion in string.gmatch(cropdots(glob), "([^/]+)(/?)") do + local criterion = criterion == "/" and is_dir or always_true + function fill(path, dir, table) + for _, file in ipairs(filelist(dir, pattern)) do + local fullpath = path .. "/" .. file + if file ~= "." and file ~= ".." and + fullpath ~= builddir and + (sub(pattern, 1, 1) == "." + or sub(file, 1, 1) ~= ".") + then + local fulldir = dir .. "/" .. file + if criterion(fulldir) then + table[fullpath] = fulldir + end + end + end + end + local newdirs = {} + if pattern == "**" then + while true do + path, dir = next(dirs) + if not path then + break + end + dirs[path] = nil + newdirs[path] = dir + fill(path, dir, dirs) + end + else + for path, dir in pairs(dirs) do + fill(path, dir, newdirs) + end + end + dirs = newdirs + end + return dirs +end + +function remove_duplicates(a) + -- Return array with duplicate entries removed from input array `a`. + + local uniq = {} + local hash = {} + + for _,v in ipairs(a) do + if (not hash[v]) then + hash[v] = true + uniq[#uniq+1] = v + end + end + + return uniq +end + +function mkdir(dir) + if os_type == "windows" then + -- Windows (with the extensions) will automatically make directory trees + -- but issues a warning if the dir already exists: avoid by including a test + local dir = unix_to_win(dir) + return execute( + "if not exist " .. dir .. "\\nul " .. "mkdir " .. dir + ) + else + return execute("mkdir -p " .. dir) + end +end + +-- Rename +function ren(dir, source, dest) + local dir = dir .. "/" + if os_type == "windows" then + local source = gsub(source, "^%.+/", "") + local dest = gsub(dest, "^%.+/", "") + return execute("ren " .. unix_to_win(dir) .. source .. " " .. dest) + else + return execute("mv " .. dir .. source .. " " .. dir .. dest) + end +end + +-- Remove file(s) based on a glob +function rm(source, glob) + for _,i in ipairs(filelist(source, glob)) do + rmfile(source,i) + end + -- os.remove doesn't give a sensible errorlevel + return 0 +end + +-- Remove file +function rmfile(source, file) + os.remove(source .. "/" .. file) + -- os.remove doesn't give a sensible errorlevel + return 0 +end + +-- Remove a directory tree +function rmdir(dir) + -- First, make sure it exists to avoid any errors + mkdir(dir) + if os_type == "windows" then + return execute("rmdir /s /q " .. unix_to_win(dir)) + else + return execute("rm -r " .. dir) + end +end + +-- Run a command in a given directory +function run(dir, cmd) + return execute("cd " .. dir .. os_concat .. cmd) +end + +-- Deal with the fact that Windows and Unix use different path separators +function unix_to_win(path) + return gsub(path, "/", "\\") +end + + +-- Split a path into file and directory component +function splitpath(file) + local path, name = match(file, "^(.*)/([^/]*)$") + if path then + return path, name + else + return ".", file + end +end + +-- Arguably clearer names +function basename(file) + return(select(2, splitpath(file))) +end + +function dirname(file) + return(select(1, splitpath(file))) +end + +-- Strip the extension from a file name (if present) +function jobname(file) + local name = match(basename(file), "^(.*)%.") + return name or file +end + +-- Look for files, directory by directory, and return the first existing +function locate(dirs, names) + for _,i in ipairs(dirs) do + for _,j in ipairs(names) do + local path = i .. "/" .. j + if fileexists(path) then + return path + end + end + end +end diff --git a/Master/texmf-dist/tex/latex/l3build/l3build-help.lua b/Master/texmf-dist/tex/latex/l3build/l3build-help.lua new file mode 100644 index 00000000000..656f82999ab --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3build/l3build-help.lua @@ -0,0 +1,78 @@ +--[[ + +File l3build-help.lua Copyright (C) 2018 The LaTeX3 Project + +It may be distributed and/or modified under the conditions of the +LaTeX Project Public License (LPPL), either version 1.3c of this +license or (at your option) any later version. The latest version +of this license is in the file + + http://www.latex-project.org/lppl.txt + +This file is part of the "l3build bundle" (The Work in LPPL) +and all files in that bundle must be distributed together. + +----------------------------------------------------------------------- + +The development version of the bundle can be found at + + https://github.com/latex3/l3build + +for those people who are interested. + +--]] + +function version() + print( + "\n" .. + "l3build: A testing and building system for LaTeX\n\n" .. + "Release " .. release_date + ) +end + +function help() + print("usage: " .. arg[0] .. " [] []") + print("") + print("The most commonly used l3build commands are:") + if testfiledir ~= "" then + print(" check Run all automated tests") + end + print(" clean Clean out directory tree") + if module == "" or bundle == "" then + print(" ctan Create CTAN-ready archive") + end + print(" doc Typesets all documentation files") + print(" install Installs files into the local texmf tree") + if module ~= "" and testfiledir ~= "" then + print(" save Saves test validation log") + end + print(" setversion Update version information in sources") + print(" unpack Unpacks the source files into the build tree") + print("") + print("Valid options are:") + local longest = 0 + for k,v in pairs(option_list) do + if k:len() > longest then + longest = k:len() + end + end + -- Sort the options + local t = { } + for k,_ in pairs(option_list) do + table.insert(t, k) + end + table.sort(t) + for _,k in ipairs(t) do + local opt = option_list[k] + local filler = string.rep(" ", longest - k:len() + 1) + if opt["desc"] then -- Skip --help as it has no desc + if opt["short"] then + print(" --" .. k .. "|-" .. opt["short"] .. filler .. opt["desc"]) + else + print(" --" .. k .. " " .. filler .. opt["desc"]) + end + end + end + print("") + print("See l3build.pdf for further details.") +end diff --git a/Master/texmf-dist/tex/latex/l3build/l3build-install.lua b/Master/texmf-dist/tex/latex/l3build/l3build-install.lua new file mode 100644 index 00000000000..dd1526a0473 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3build/l3build-install.lua @@ -0,0 +1,57 @@ +--[[ + +File l3build-install.lua Copyright (C) 2018 The LaTeX3 Project + +It may be distributed and/or modified under the conditions of the +LaTeX Project Public License (LPPL), either version 1.3c of this +license or (at your option) any later version. The latest version +of this license is in the file + + http://www.latex-project.org/lppl.txt + +This file is part of the "l3build bundle" (The Work in LPPL) +and all files in that bundle must be distributed together. + +----------------------------------------------------------------------- + +The development version of the bundle can be found at + + https://github.com/latex3/l3build + +for those people who are interested. + +--]] + + +-- Locally install files: only deals with those extracted, not docs etc. +function install() + local errorlevel = unpack() + if errorlevel ~= 0 then + return errorlevel + end + kpse.set_program_name("latex") + local texmfhome = options["texmfhome"] or kpse.var_value("TEXMFHOME") + local installdir = texmfhome .. "/tex/" .. moduledir + if options["dry-run"] then + print("\n" .. "Installation root: " .. installdir + .. "\n" .. "Intallation files:" + ) + for _,filetype in ipairs(installfiles) do + for _,file in pairs(filelist(unpackdir,filetype)) do + print("- " .. file) + end + end + else + errorlevel = cleandir(installdir) + if errorlevel ~= 0 then + return errorlevel + end + for _,filetype in ipairs(installfiles) do + errorlevel = cp(filetype, unpackdir, installdir) + if errorlevel ~= 0 then + return errorlevel + end + end + end + return 0 +end diff --git a/Master/texmf-dist/tex/latex/l3build/l3build-manifest-setup.lua b/Master/texmf-dist/tex/latex/l3build/l3build-manifest-setup.lua new file mode 100644 index 00000000000..fb46eb1d369 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3build/l3build-manifest-setup.lua @@ -0,0 +1,364 @@ +--[[ + +File l3build-manifest-setup.lua Copyright (C) 2017-2018 The LaTeX3 Project + +It may be distributed and/or modified under the conditions of the +LaTeX Project Public License (LPPL), either version 1.3c of this +license or (at your option) any later version. The latest version +of this license is in the file + + http://www.latex-project.org/lppl.txt + +This file is part of the "l3build bundle" (The Work in LPPL) +and all files in that bundle must be distributed together. + +----------------------------------------------------------------------- + +The development version of the bundle can be found at + + https://github.com/latex3/l3build + +for those people who are interested. + +--]] + + +--[[ + L3BUILD MANIFEST SETUP + ====================== + This file contains all of the code that is easily replaceable by the user. + Either create a copy of this file, rename, and include alongside your `build.lua` + script and load it with `dofile()`, or simply copy/paste the definitions below + into your `build.lua` script directly. +--]] + + +--[[ + Setup of manifest "groups" + -------------------------- + + The grouping of manifest files is broken into three subheadings: + + * The development repository + * The TDS structure from `ctan` + * The CTAN structure from `ctan` + + The latter two will only be produced if the `manifest` target is run *after* + the `ctan` target. Contrarily, if you run `clean` before `manifest` then + only the first grouping will be printed. + + If you want to omit the files in the development repository, essentially + producing a minimalist manifest with only the files included for distribution, + make a copy of the `manifest_setup` function and delete the groups under + the ‘Repository manifest’ subheading below. +--]] + + +manifest_setup = manifest_setup or function() + local groups = { + { + subheading = "Repository manifest", + description = [[ +The following groups list the files included in the development repository of the package. +Files listed with a ‘†’ marker are included in the TDS but not CTAN files, and files listed +with ‘‡’ are included in both. +]], + }, + { + name = "Source files", + description = [[ +These are source files for a number of purposes, including the `unpack` process which +generates the installation files of the package. Additional files included here will also +be installed for processing such as testing. +]], + files = {sourcefiles}, + dir = sourcefiledir or maindir, -- TODO: remove "or maindir" after rebasing onto master + }, + { + name = "Typeset documentation source files", + description = [[ +These files are typeset using LaTeX to produce the PDF documentation for the package. +]], + files = {typesetfiles,typesetsourcefiles,typesetdemofiles}, + }, + { + name = "Documentation files", + description = [[ +These files form part of the documentation but are not typeset. Generally they will be +additional input files for the typeset documentation files listed above. +]], + files = {docfiles}, + dir = docfiledir or maindir, -- TODO: remove "or maindir" after rebasing onto master + }, + { + name = "Text files", + description = [[ +Plain text files included as documentation or metadata. +]], + files = {textfiles}, + skipfiledescription = true, + }, + { + name = "Demo files", + description = [[ +Files included to demonstrate package functionality. These files are *not* +typeset or compiled in any way. +]], + files = {demofiles}, + }, + { + name = "Bibliography and index files", + description = [[ +Supplementary files used for compiling package documentation. +]], + files = {bibfiles,bstfiles,makeindexfiles}, + }, + { + name = "Derived files", + description = [[ +The files created by ‘unpacking’ the package sources. This typically includes +`.sty` and `.cls` files created from DocStrip `.dtx` files. +]], + files = {installfiles}, + exclude = {excludefiles,sourcefiles}, + dir = unpackdir, + skipfiledescription = true, + }, + { + name = "Typeset documents", + description = [[ +The output files (PDF, essentially) from typesetting the various source, demo, +etc., package files. +]], + files = {typesetfiles,typesetsourcefiles,typesetdemofiles}, + rename = {"%.%w+$", ".pdf"}, + skipfiledescription = true, + }, + { + name = "Support files", + description = [[ +These files are used for unpacking, typesetting, or checking purposes. +]], + files = {unpacksuppfiles,typesetsuppfiles,checksuppfiles}, + dir = supportdir, + }, + { + name = "Checking-specific support files", + description = [[ +Support files for checking the test suite. +]], + files = {"*.*"}, + exclude = {{".",".."},excludefiles}, + dir = testsuppdir, + }, + { + name = "Test files", + description = [[ +These files form the test suite for the package. `.lvt` or `.lte` files are the individual +unit tests, and `.tlg` are the stored output for ensuring changes to the package produce +the same output. These output files are sometimes shared and sometime specific for +different engines (pdfTeX, XeTeX, LuaTeX, etc.). +]], + files = {"*"..lvtext,"*"..lveext,"*"..tlgext}, + dir = testfiledir, + skipfiledescription = true, + }, + { + subheading = "TDS manifest", + description = [[ +The following groups list the files included in the TeX Directory Structure used to install +the package into a TeX distribution. +]], + }, + { + name = "Source files (TDS)", + description = "All files included in the `"..module.."/source` directory.\n", + dir = tdsdir.."/source/"..moduledir, + files = {"*.*"}, + exclude = {".",".."}, + flag = false, + skipfiledescription = true, + }, + { + name = "TeX files (TDS)", + description = "All files included in the `"..module.."/tex` directory.\n", + dir = tdsdir.."/tex/"..moduledir, + files = {"*.*"}, + exclude = {".",".."}, + flag = false, + skipfiledescription = true, + }, + { + name = "Doc files (TDS)", + description = "All files included in the `"..module.."/doc` directory.\n", + dir = tdsdir.."/doc/"..moduledir, + files = {"*.*"}, + exclude = {".",".."}, + flag = false, + skipfiledescription = true, + }, + { + subheading = "CTAN manifest", + description = [[ +The following group lists the files included in the CTAN package. +]], + }, + { + name = "CTAN files", + dir = ctandir.."/"..module, + files = {"*.*"}, + exclude = {".",".."}, + flag = false, + skipfiledescription = true, + }, + } + return groups +end + +--[[ + Sorting within groups + --------------------- +--]] + +manifest_sort_within_match = manifest_sort_within_match or function(files) + local f = files + table.sort(f) + return f +end + +manifest_sort_within_group = manifest_sort_within_group or function(files) + local f = files + --[[ + -- no-op by default; make your own definition to customise. E.g.: + table.sort(f) + --]] + return f +end + +--[[ + Writing to file + --------------- +--]] + +manifest_write_opening = manifest_write_opening or function(filehandle) + + filehandle:write("# Manifest for " .. module .. "\n\n") + filehandle:write([[ +This file is a listing of all files considered to be part of this package. +It is automatically generated with `texlua build.lua manifest`. +]]) + +end + +manifest_write_subheading = manifest_write_subheading or function(filehandle,heading,description) + + filehandle:write("\n\n## " .. heading .. "\n\n") + + if description then + filehandle:write(description) + end + +end + +manifest_write_group_heading = manifest_write_group_heading or function (filehandle,heading,description) + + filehandle:write("\n### " .. heading .. "\n\n") + + if description then + filehandle:write(description .. "\n") + end + +end + +manifest_write_group_file = manifest_write_group_file or function(filehandle,filename,param) + --[[ + filehandle : write file object + filename : the count of the filename to be written + + param.dir : the directory of the file + param.count : the name of the file to write + param.filemaxchar : the maximum number of chars of all filenames in this group + param.flag : false OR string for indicating CTAN/TDS location + param.ctanfile : (boolean) if file is in CTAN dir + param.tdsfile : (boolean) if file is in TDS dir + --]] + + -- no file description: plain bullet list item: + + flagstr = param.flag or "" + filehandle:write("* " .. filename .. " " .. flagstr .. "\n") + + --[[ + -- or if you prefer an enumerated list: + filehandle:write(param.count..". " .. filename .. "\n") + --]] + + +end + +manifest_write_group_file_descr = manifest_write_group_file_descr or function(filehandle,filename,descr,param) + --[[ + filehandle : write file object + filename : the name of the file to write + descr : description of the file to write + + param.dir : the directory of the file + param.count : the count of the filename to be written + param.filemaxchar : the maximum number of chars of all filenames in this group + param.descmaxchar : the maximum number of chars of all descriptions in this group + param.flag : false OR string for indicating CTAN/TDS location + param.ctanfile : (boolean) if file is in CTAN dir + param.tdsfile : (boolean) if file is in TDS dir + --]] + + -- filename+description: Github-flavoured Markdown table + + filestr = string.format(" | %-"..param.filemaxchar.."s",filename) + flagstr = param.flag and string.format(" | %s",param.flag) or "" + descstr = string.format(" | %-"..param.descmaxchar.."s",descr) + + filehandle:write(filestr..flagstr..descstr.." |\n") + +end + +--[[ + Extracting ‘descriptions’ from source files + ------------------------------------------- +--]] + +manifest_extract_filedesc = manifest_extract_filedesc or function(filehandle) + + -- no-op by default; two examples below + +end + +--[[ + +-- From the first match of a pattern in a file: +manifest_extract_filedesc = function(filehandle) + + local all_file = filehandle:read("*all") + local matchstr = "\\section{(.-)}" + + filedesc = string.match(all_file,matchstr) + + return filedesc +end + +-- From the match of the 2nd line (say) of a file: +manifest_extract_filedesc = function(filehandle) + + local end_read_loop = 2 + local matchstr = "%%%S%s+(.*)" + local this_line = "" + + for ii = 1, end_read_loop do + this_line = filehandle:read("*line") + end + + filedesc = string.match(this_line,matchstr) + + return filedesc +end + +]]-- diff --git a/Master/texmf-dist/tex/latex/l3build/l3build-manifest.lua b/Master/texmf-dist/tex/latex/l3build/l3build-manifest.lua new file mode 100644 index 00000000000..3b899d55fd8 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3build/l3build-manifest.lua @@ -0,0 +1,280 @@ +--[[ + +File l3build-manifest.lua Copyright (C) 2017-2018 The LaTeX3 Project + +It may be distributed and/or modified under the conditions of the +LaTeX Project Public License (LPPL), either version 1.3c of this +license or (at your option) any later version. The latest version +of this license is in the file + + http://www.latex-project.org/lppl.txt + +This file is part of the "l3build bundle" (The Work in LPPL) +and all files in that bundle must be distributed together. + +----------------------------------------------------------------------- + +The development version of the bundle can be found at + + https://github.com/latex3/l3build + +for those people who are interested. + +--]] + + +--[[ + L3BUILD MANIFEST + ================ + If desired this entire function can be replaced; if not, it uses a number of + auxiliary functions which are included in this file. + + Additional setup can be performed by replacing the functions lists in the file + `l3build-manifest-setup.lua`. +--]] + +manifest = manifest or function() + + -- build list of ctan files + ctanfiles = {} + for _,f in ipairs(filelist(ctandir.."/"..ctanpkg,"*.*")) do + ctanfiles[f] = true + end + tdsfiles = {} + for _,subdir in ipairs({"/doc/","/source/","/tex/"}) do + for _,f in ipairs(filelist(tdsdir..subdir..moduledir,"*.*")) do + tdsfiles[f] = true + end + end + + local manifest_entries = manifest_setup() + + for ii,_ in ipairs(manifest_entries) do + manifest_entries[ii] = manifest_build_list(manifest_entries[ii]) + end + + manifest_write(manifest_entries) + + printline = "Manifest written to " .. manifestfile + print((printline:gsub(".","*"))) print(printline) print((printline:gsub(".","*"))) + +end + +--[[ + Internal Manifest functions: build_list + --------------------------------------- +--]] + +manifest_build_list = function(entry) + + if not(entry.subheading) then + + entry = manifest_build_init(entry) + + -- build list of excluded files + for _,glob_list in ipairs(entry.exclude) do + for _,this_glob in ipairs(glob_list) do + for _,this_file in ipairs(filelist(maindir,this_glob)) do + entry.excludes[this_file] = true + end + end + end + + -- build list of matched files + for _,glob_list in ipairs(entry.files) do + for _,this_glob in ipairs(glob_list) do + + local these_files = filelist(entry.dir,this_glob) + these_files = manifest_sort_within_match(these_files) + + for _,this_file in ipairs(these_files) do + entry = manifest_build_file(entry,this_file) + end + + entry.files_ordered = manifest_sort_within_group(entry.files_ordered) + + end + end + + end + + return entry + +end + + +manifest_build_init = function(entry) + + -- currently these aren't customisable; I guess they could be? + local manifest_group_defaults = { + skipfiledescription = false , + rename = false , + dir = maindir , + exclude = {excludefiles} , + flag = true , + } + + -- internal data added to each group in the table that needs to be initialised + local manifest_group_init = { + N = 0 , -- # matched files + ND = 0 , -- # descriptions + matches = {} , + excludes = {} , + files_ordered = {} , + descr = {} , + Nchar_file = 4 , -- TODO: generalise + Nchar_descr = 11 , -- TODO: generalise + } + + -- copy default options to each group if necessary + for kk,ll in pairs(manifest_group_defaults) do + if entry[kk] == nil then + entry[kk] = ll + end + -- can't use "entry[kk] = entry[kk] or ll" because false/nil are indistinguishable! + end + + -- initialisation for internal data + for kk,ll in pairs(manifest_group_init) do + entry[kk] = ll + end + + -- allow nested tables by requiring two levels of nesting + if type(entry.files[1])=="string" then + entry.files = {entry.files} + end + if type(entry.exclude[1])=="string" then + entry.exclude = {entry.exclude} + end + + return entry + +end + + +manifest_build_file = function(entry,this_file) + + if entry.rename then + this_file = this_file:gsub(entry.rename[1],entry.rename[2]) + end + + if not entry.excludes[this_file] then + + entry.N = entry.N+1 + if not(entry.matches[this_file]) then + + entry.matches[this_file] = true -- store the file name + entry.files_ordered[entry.N] = this_file -- store the file order + entry.Nchar_file = math.max(entry.Nchar_file,this_file:len()) + + end + + if not(entry.skipfiledescription) then + + local ff = assert(io.open(entry.dir .. "/" .. this_file, "r")) + this_descr = manifest_extract_filedesc(ff,this_file) + ff:close() + + if this_descr and this_descr ~= "" then + entry.descr[this_file] = this_descr + entry.ND = entry.ND+1 + entry.Nchar_descr = math.max(entry.Nchar_descr,this_descr:len()) + end + + end + end + + return entry + +end + +--[[ + Internal Manifest functions: write + ---------------------------------- +--]] + +manifest_write = function(manifest_entries) + + local f = assert(io.open(manifestfile, "w")) + manifest_write_opening(f) + + for ii,vv in ipairs(manifest_entries) do + if manifest_entries[ii].subheading then + manifest_write_subheading(f,manifest_entries[ii].subheading,manifest_entries[ii].description) + elseif manifest_entries[ii].N > 0 then + manifest_write_group(f,manifest_entries[ii]) + end + end + + f:close() + +end + + +manifest_write_group = function(f,entry) + + manifest_write_group_heading(f,entry.name,entry.description) + + if entry.ND > 0 then + + for ii,file in ipairs(entry.files_ordered) do + local descr = entry.descr[file] or "" + local param = { + dir = entry.dir , + count = ii , + filemaxchar = entry.Nchar_file , + descmaxchar = entry.Nchar_descr , + ctanfile = ctanfiles[file] , + tdsfile = tdsfiles[file] , + flag = false , + } + + if entry.flag then + param.flag = " " + if tdsfiles[file] and not(ctanfiles[file]) then + param.flag = "† " + elseif ctanfiles[file] then + param.flag = "‡ " + end + end + + if ii == 1 then + -- header of table + -- TODO: generalise + local p = {} + for k,v in pairs(param) do p[k] = v end + p.count = -1 + p.flag = p.flag and "Flag" + manifest_write_group_file_descr(f,"File","Description",p) + p.flag = p.flag and "--- " + manifest_write_group_file_descr(f,"---","---",p) + end + + manifest_write_group_file_descr(f,file,descr,param) + end + + else + + for ii,file in ipairs(entry.files_ordered) do + local param = { + dir = entry.dir , + count = ii , + filemaxchar = entry.Nchar_file , + ctanfile = ctanfiles[file] , + tdsfile = tdsfiles[file] , + } + if entry.flag then + param.flag = "" + if tdsfiles[file] and not(ctanfiles[file]) then + param.flag = "†" + elseif ctanfiles[file] then + param.flag = "‡" + end + end + manifest_write_group_file(f,file,param) + end + + end + +end + diff --git a/Master/texmf-dist/tex/latex/l3build/l3build-setversion.lua b/Master/texmf-dist/tex/latex/l3build/l3build-setversion.lua new file mode 100644 index 00000000000..178bb29489a --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3build/l3build-setversion.lua @@ -0,0 +1,140 @@ +--[[ + +File l3build-setversion.lua Copyright (C) 2018 The LaTeX3 Project + +It may be distributed and/or modified under the conditions of the +LaTeX Project Public License (LPPL), either version 1.3c of this +license or (at your option) any later version. The latest version +of this license is in the file + + http://www.latex-project.org/lppl.txt + +This file is part of the "l3build bundle" (The Work in LPPL) +and all files in that bundle must be distributed together. + +----------------------------------------------------------------------- + +The development version of the bundle can be found at + + https://github.com/latex3/l3build + +for those people who are interested. + +--]] + + +--[[ + L3BUILD SETVERSION + ================== +--]] + +local gsub = string.gsub +local match = string.match + +-- Provide some standard search-and-replace functions +if versionform ~= "" and not setversion_update_line then + if versionform == "ProvidesPackage" then + function setversion_update_line(line, date, version) + -- No real regex so do it one type at a time + for _,i in pairs({"Class", "File", "Package"}) do + if match( + line, + "^\\Provides" .. i .. "{[a-zA-Z0-9%-%.]+}%[[^%]]*%]$" + ) then + line = gsub(line, "%[%d%d%d%d/%d%d/%d%d", "[" + .. gsub(date, "%-", "/")) + line = gsub( + line, "(%[%d%d%d%d/%d%d/%d%d) [^ ]*", "%1 " .. version + ) + break + end + end + return line + end + elseif versionform == "ProvidesExplPackage" then + function setversion_update_line(line, date, version) + -- No real regex so do it one type at a time + for _,i in pairs({"Class", "File", "Package"}) do + if match( + line, + "^\\ProvidesExpl" .. i .. " *{[a-zA-Z0-9%-%.]+}" + ) then + line = gsub( + line, + "{%d%d%d%d/%d%d/%d%d}( *){[^}]*}", + "{" .. gsub(date, "%-", "/") .. "}%1{" .. version .. "}" + ) + break + end + end + return line + end + elseif versionform == "filename" then + function setversion_update_line(line, date, version) + if match(line, "^\\def\\filedate{%d%d%d%d/%d%d/%d%d}$") then + line = "\\def\\filedate{" .. gsub(date, "%-", "/") .. "}" + end + if match(line, "^\\def\\fileversion{[^}]+}$") then + line = "\\def\\fileversion{" .. version .. "}" + end + return line + end + elseif versionform == "ExplFileDate" then + function setversion_update_line(line, date, version) + if match(line, "^\\def\\ExplFileDate{%d%d%d%d/%d%d/%d%d}$") then + line = "\\def\\ExplFileDate{" .. gsub(date, "%-", "/") .. "}" + end + if match(line, "^\\def\\ExplFileVersion{[^}]+}$") then + line = "\\def\\ExplFileVersion{" .. version .. "}" + end + return line + end + end +end + +-- Used to actually carry out search-and-replace +setversion_update_line = setversion_update_line or function(line, date, version) + return line +end + +function setversion() + local function rewrite(dir, file, date, version) + local changed = false + local result = "" + for line in io.lines(dir .. "/" .. file) do + local newline = setversion_update_line(line, date, version) + if newline ~= line then + line = newline + changed = true + end + result = result .. line .. os_newline + end + if changed then + -- Avoid adding/removing end-of-file newline + local f = io.open(dir .. "/" .. file, "rb") + local content = f:read("*all") + f:close() + if not match(content, os_newline .. "$") then + gsub(result, os_newline .. "$", "") + end + -- Write the new file + ren(dir, file, file .. bakext) + local f = io.open(dir .. "/" .. file, "w") + io.output(f) + io.write(result) + f:close() + rmfile(dir, file .. bakext) + end + end + local date = options["date"] or os.date("%Y-%m-%d") + local version = options["version"] or -1 + for _,dir in pairs(remove_duplicates({currentdir, sourcefiledir, docfiledir})) do + for _,i in pairs(versionfiles) do + for file,_ in pairs(tree(dir, i)) do + rewrite(dir, file, date, version) + end + end + end + return 0 +end + diff --git a/Master/texmf-dist/tex/latex/l3build/l3build-stdmain.lua b/Master/texmf-dist/tex/latex/l3build/l3build-stdmain.lua new file mode 100644 index 00000000000..d8cd8da6278 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3build/l3build-stdmain.lua @@ -0,0 +1,123 @@ +--[[ + +File l3build-stdmain.lua Copyright (C) 2018 The LaTeX3 Project + +It may be distributed and/or modified under the conditions of the +LaTeX Project Public License (LPPL), either version 1.3c of this +license or (at your option) any later version. The latest version +of this license is in the file + + http://www.latex-project.org/lppl.txt + +This file is part of the "l3build bundle" (The Work in LPPL) +and all files in that bundle must be distributed together. + +----------------------------------------------------------------------- + +The development version of the bundle can be found at + + https://github.com/latex3/l3build + +for those people who are interested. + +--]] + +local exit = os.exit + +-- List all modules +function listmodules() + local modules = { } + local exclmodules = exclmodules or { } + for entry in lfs.dir(".") do + if entry ~= "." and entry ~= ".." then + local attr = lfs.attributes(entry) + assert(type(attr) == "table") + if attr.mode == "directory" then + if not exclmodules[entry] then + table.insert(modules, entry) + end + end + end + end + return modules +end + +-- +-- The overall main function +-- + +function stdmain(target, files) + local errorlevel + -- If the module name is empty, the script is running in a bundle: + -- apart from ctan all of the targets are then just mappings + if module == "" then + -- Detect all of the modules + modules = modules or listmodules() + if target == "doc" then + errorlevel = call(modules, "doc") + elseif target == "check" then + errorlevel = call(modules, "bundlecheck") + if errorlevel ~=0 then + print("There were errors: checks halted!\n") + end + elseif target == "clean" then + errorlevel = bundleclean() + elseif target == "ctan" then + errorlevel = ctan() + elseif target == "install" then + errorlevel = call(modules, "install") + elseif target == "setversion" then + errorlevel = call(modules, "setversion") + -- Deal with any files in the bundle dir itself + if errorlevel == 0 then + errorlevel = setversion() + end + elseif target == "unpack" then + errorlevel = call(modules, "bundleunpack") + elseif target == "version" then + version() + else + help() + end + else + if target == "bundleunpack" then -- 'Hidden' as only needed 'higher up' + depinstall(unpackdeps) + errorlevel = bundleunpack() + elseif target == "bundlecheck" then + errorlevel = check() + elseif target == "bundlectan" then + errorlevel = bundlectan() + elseif target == "doc" then + errorlevel = doc(files) + elseif target == "check" then + errorlevel = check(files) + elseif target == "clean" then + errorlevel = clean() + elseif target == "ctan" then + errorlevel = ctan() + elseif target == "install" then + errorlevel = install() + elseif target == "manifest" then + errorlevel = manifest() + elseif target == "save" then + if next(files) then + errorlevel = save(files) + else + help() + end + elseif target == "setversion" then + errorlevel = setversion() + elseif target == "unpack" then + errorlevel = unpack() + elseif target == "version" then + version() + else + help() + end + end + if errorlevel ~= 0 then + exit(1) + else + exit(0) + end +end diff --git a/Master/texmf-dist/tex/latex/l3build/l3build-typesetting.lua b/Master/texmf-dist/tex/latex/l3build/l3build-typesetting.lua new file mode 100644 index 00000000000..46feb0375d7 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3build/l3build-typesetting.lua @@ -0,0 +1,247 @@ +--[[ + +File l3build-typesetting.lua Copyright (C) 2018 The LaTeX3 Project + +It may be distributed and/or modified under the conditions of the +LaTeX Project Public License (LPPL), either version 1.3c of this +license or (at your option) any later version. The latest version +of this license is in the file + + http://www.latex-project.org/lppl.txt + +This file is part of the "l3build bundle" (The Work in LPPL) +and all files in that bundle must be distributed together. + +----------------------------------------------------------------------- + +The development version of the bundle can be found at + + https://github.com/latex3/l3build + +for those people who are interested. + +--]] + +-- +-- Auxiliary functions for typesetting: need to be generally available +-- + +local gsub = string.gsub +local match = string.match + +function dvitopdf(name, dir, engine, hide) + if match(engine, "^u?ptex$") then + run( + dir, + (forcecheckepoch and setepoch() or "") .. + "dvipdfmx " .. name .. dviext + .. (hide and (" > " .. os_null) or "") + ) + else + run( + dir, + (forcecheckepoch and setepoch() or "") .. + "dvips " .. name .. dviext + .. (hide and (" > " .. os_null) or "") + .. os_concat .. + "ps2pdf " .. name .. psext + .. (hide and (" > " .. os_null) or "") + ) + end +end + +-- An auxiliary used to set up the environmental variables +function runtool(subdir, dir, envvar, command) + dir = dir or "." + return( + run( + typesetdir .. "/" .. subdir, + (forcedocepoch and setepoch() or "") .. + os_setenv .. " " .. envvar .. "=." .. os_pathsep + .. abspath(localdir) .. os_pathsep + .. abspath(dir .. "/" .. subdir) + .. (typesetsearch and os_pathsep or "") + .. os_concat .. + command + ) + ) +end + +function biber(name, dir) + if fileexists(typesetdir .. "/" .. name .. ".bcf") then + local path, name = splitpath(name) + return( + runtool(path, dir, "BIBINPUTS", biberexe .. " " .. biberopts .. " " .. name) + ) + end + return 0 +end + +function bibtex(name, dir) + if fileexists(typesetdir .. "/" .. name .. ".aux") then + -- LaTeX always generates an .aux file, so there is a need to + -- look inside it for a \citation line + local grep + if os_type == "windows" then + grep = "\\\\" + else + grep = "\\\\\\\\" + end + local path, name = splitpath(name) + if run( + typesetdir, + os_grepexe .. " \"^" .. grep .. "citation{\" " .. name .. ".aux > " + .. os_null + ) + run( + typesetdir, + os_grepexe .. " \"^" .. grep .. "bibdata{\" " .. name .. ".aux > " + .. os_null + ) == 0 then + return( + -- Cheat slightly as we need to set two variables + runtool( + path, dir, + "BIBINPUTS", + os_setenv .. " BSTINPUTS=." .. os_pathsep + .. abspath(localdir) + .. (typesetsearch and os_pathsep or "") .. + os_concat .. + bibtexexe .. " " .. bibtexopts .. " " .. name + ) + ) + end + end + return 0 +end + +function makeindex(name, dir, inext, outext, logext, style) + if fileexists(typesetdir .. "/" .. name .. inext) then + local path, name = splitpath(name) + return( + runtool( + path, dir, + "INDEXSTYLE", + makeindexexe .. " " .. makeindexopts .. " " + .. " -s " .. style .. " -o " .. name .. outext + .. " -t " .. name .. " " .. name .. inext + ) + ) + end + return 0 +end + +function tex(file, dir) + local path, name = splitpath(file) + return( + runtool( + path, dir, + "TEXINPUTS", + typesetexe .. " " .. typesetopts .. " \"" .. typesetcmds + .. "\\input " .. name .. "\"" + ) + ) +end + +function typesetpdf(file, dir) + local name = gsub(file, "%.[^.]+$", "") + print("Typesetting " .. name) + local errorlevel = typeset(file, dir) + if errorlevel == 0 then + name = name .. ".pdf" + os.remove(jobname(name)) + cp(name, typesetdir, docfiledir) + else + print(" ! Compilation failed") + end + return errorlevel +end + +typeset = typeset or function(file, dir) + dir = dir or "." + local errorlevel = tex(file, dir) + if errorlevel ~= 0 then + return errorlevel + else + local name = jobname(file) + errorlevel = biber(name, dir) + bibtex(name, dir) + if errorlevel == 0 then + local function cycle(name, dir) + return( + makeindex(name, dir, ".glo", ".gls", ".glg", glossarystyle) + + makeindex(name, dir, ".idx", ".ind", ".ilg", indexstyle) + + tex(file, dir) + ) + end + for i = 1, typesetruns do + errorlevel = cycle(name, dir) + if errorlevel ~= 0 then break end + end + end + return errorlevel + end +end + + +-- A hook to allow additional typesetting of demos +typeset_demo_tasks = typeset_demo_tasks or function() + return 0 +end + +-- Typeset all required documents +-- Uses a set of dedicated auxiliaries that need to be available to others +function doc(files) + -- Set up + cleandir(typesetdir) + for _,filetype in pairs( + {bibfiles, docfiles, typesetfiles, typesetdemofiles} + ) do + for _,file in pairs(filetype) do + cp(file, docfiledir, typesetdir) + end + end + for _,file in pairs(sourcefiles) do + cp(file, sourcefiledir, typesetdir) + end + for _,file in pairs(typesetsuppfiles) do + cp(file, supportdir, typesetdir) + end + depinstall(typesetdeps) + unpack({sourcefiles, typesetsourcefiles}, {sourcefiledir, docfiledir}) + -- Main loop for doc creation + local done = {} + local errorlevel = typeset_demo_tasks() + if errorlevel ~= 0 then + return errorlevel + end + for _, typesetfiles in ipairs({typesetdemofiles, typesetfiles}) do + for _,i in ipairs(typesetfiles) do + for _, dir in ipairs({unpackdir, typesetdir}) do + for j,_ in pairs(tree(dir, i)) do + if not done[j] then + j = gsub(j, "^%./", "") + -- Allow for command line selection of files + local typeset = true + if files and next(files) then + typeset = false + for _,k in ipairs(files) do + if k == gsub(j, "%.[^.]+$", "") then + typeset = true + break + end + end + end + if typeset then + local errorlevel = typesetpdf(j, dir) + if errorlevel ~= 0 then + return errorlevel + else + done[j] = true + end + end + end + end + end + end + end + return 0 +end diff --git a/Master/texmf-dist/tex/latex/l3build/l3build-unpack.lua b/Master/texmf-dist/tex/latex/l3build/l3build-unpack.lua new file mode 100644 index 00000000000..9779502c2fc --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3build/l3build-unpack.lua @@ -0,0 +1,99 @@ +--[[ + +File l3build-unpack.lua Copyright (C) 2018 The LaTeX3 Project + +It may be distributed and/or modified under the conditions of the +LaTeX Project Public License (LPPL), either version 1.3c of this +license or (at your option) any later version. The latest version +of this license is in the file + + http://www.latex-project.org/lppl.txt + +This file is part of the "l3build bundle" (The Work in LPPL) +and all files in that bundle must be distributed together. + +----------------------------------------------------------------------- + +The development version of the bundle can be found at + + https://github.com/latex3/l3build + +for those people who are interested. + +--]] + +local execute = os.execute + +-- Unpack the package files using an 'isolated' system: this requires +-- a copy of the 'basic' DocStrip program, which is used then removed +function unpack(sources, sourcedirs) + local errorlevel = depinstall(unpackdeps) + if errorlevel ~= 0 then + return errorlevel + end + errorlevel = bundleunpack(sourcedirs, sources) + if errorlevel ~= 0 then + return errorlevel + end + for _,i in ipairs(installfiles) do + errorlevel = cp(i, unpackdir, localdir) + if errorlevel ~= 0 then + return errorlevel + end + end + return 0 +end + +-- Split off from the main unpack so it can be used on a bundle and not +-- leave only one modules files +bundleunpack = bundleunpack or function(sourcedirs, sources) + local errorlevel = mkdir(localdir) + if errorlevel ~=0 then + return errorlevel + end + errorlevel = cleandir(unpackdir) + if errorlevel ~=0 then + return errorlevel + end + for _,i in ipairs(sourcedirs or {sourcefiledir}) do + for _,j in ipairs(sources or {sourcefiles}) do + for _,k in ipairs(j) do + errorlevel = cp(k, i, unpackdir) + if errorlevel ~=0 then + return errorlevel + end + end + end + end + for _,i in ipairs(unpacksuppfiles) do + errorlevel = cp(i, supportdir, localdir) + if errorlevel ~=0 then + return errorlevel + end + end + for _,i in ipairs(unpackfiles) do + for j,_ in pairs(tree(unpackdir, i)) do + -- This 'yes' business is needed to pass a series of "y\n" to + -- TeX if \askforoverwrite is true + -- That is all done using a file as it's the only way on Windows and + -- on Unix the "yes" command can't be used inside execute (it never + -- stops, which confuses Lua) + execute(os_yes .. ">>" .. localdir .. "/yes") + local path, name = splitpath(j) + local localdir = abspath(localdir) + errorlevel = run( + unpackdir .. "/" .. path, + os_setenv .. " TEXINPUTS=." .. os_pathsep + .. localdir .. (unpacksearch and os_pathsep or "") .. + os_concat .. + unpackexe .. " " .. unpackopts .. " " .. name .. " < " + .. localdir .. "/yes" + .. (options["quiet"] and (" > " .. os_null) or "") + ) + if errorlevel ~=0 then + return errorlevel + end + end + end + return 0 +end diff --git a/Master/texmf-dist/tex/latex/l3build/l3build-variables.lua b/Master/texmf-dist/tex/latex/l3build/l3build-variables.lua new file mode 100644 index 00000000000..7115d1f86ae --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3build/l3build-variables.lua @@ -0,0 +1,199 @@ +--[[ + +File l3build-variables.lua Copyright (C) 2018 The LaTeX3 Project + +It may be distributed and/or modified under the conditions of the +LaTeX Project Public License (LPPL), either version 1.3c of this +license or (at your option) any later version. The latest version +of this license is in the file + + http://www.latex-project.org/lppl.txt + +This file is part of the "l3build bundle" (The Work in LPPL) +and all files in that bundle must be distributed together. + +----------------------------------------------------------------------- + +The development version of the bundle can be found at + + https://github.com/latex3/l3build + +for those people who are interested. + +--]] + +local exit = os.exit + +-- "module" is a deprecated function in Lua 5.2: as we want the name +-- for other purposes, and it should eventually be 'free', simply +-- remove the built-in +if type(module) == "function" then + module = nil +end + +-- Ensure the module and bundle exist +module = module or "" +bundle = bundle or "" + +-- Sanity check +if module == "" and bundle == "" then + if string.match(arg[0], "l3build%.lua$") then + print( + "\n" .. + "l3build: A testing and building system for LaTeX\n\n" .. + "Release " .. release_date + ) + if arg[1] and not string.match(arg[1], "version") then + print( + "\n" + .. "Error: Call l3build using a configuration file, not directly.\n" + ) + end + else + print( + "\n" + .. "Error: Specify either bundle or module in configuration script.\n" + ) + end + exit(1) +end + +-- Directory structure for the build system +-- Use Unix-style path separators +currentdir = "." +maindir = maindir or currentdir + +-- Substructure for file locations +docfiledir = docfiledir or currentdir +sourcefiledir = sourcefiledir or currentdir +supportdir = supportdir or maindir .. "/support" +testfiledir = testfiledir or currentdir .. "/testfiles" +testsuppdir = testsuppdir or testfiledir .. "/support" + +-- Structure within a development area +builddir = builddir or maindir .. "/build" +distribdir = distribdir or builddir .. "/distrib" +localdir = localdir or builddir .. "/local" +testdir = testdir or builddir .. "/test" +typesetdir = typesetdir or builddir .. "/doc" +unpackdir = unpackdir or builddir .. "/unpacked" + +-- Substructure for CTAN release material +ctandir = ctandir or distribdir .. "/ctan" +tdsdir = tdsdir or distribdir .. "/tds" +tdsroot = tdsroot or "latex" + +-- Location for installation on CTAN or in TEXMFHOME +if bundle == "" then + moduledir = tdsroot .. "/" .. module + ctanpkg = ctanpkg or module +else + moduledir = tdsroot .. "/" .. bundle .. "/" .. module + ctanpkg = ctanpkg or bundle +end + +-- File types for various operations +-- Use Unix-style globs +-- All of these may be set earlier, so a initialised conditionally +auxfiles = auxfiles or {"*.aux", "*.lof", "*.lot", "*.toc"} +bibfiles = bibfiles or {"*.bib"} +binaryfiles = binaryfiles or {"*.pdf", "*.zip"} +bstfiles = bstfiles or {"*.bst"} +checkfiles = checkfiles or { } +checksuppfiles = checksuppfiles or { } +cleanfiles = cleanfiles or {"*.log", "*.pdf", "*.zip"} +demofiles = demofiles or { } +docfiles = docfiles or { } +excludefiles = excludefiles or {"*~"} +installfiles = installfiles or {"*.sty","*.cls"} +makeindexfiles = makeindexfiles or {"*.ist"} +sourcefiles = sourcefiles or {"*.dtx", "*.ins"} +textfiles = textfiles or {"*.md", "*.txt"} +typesetdemofiles = typesetdemofiles or { } +typesetfiles = typesetfiles or {"*.dtx"} +typesetsuppfiles = typesetsuppfiles or { } +typesetsourcefiles = typesetsourcefiles or { } +unpackfiles = unpackfiles or {"*.ins"} +unpacksuppfiles = unpacksuppfiles or { } +versionfiles = versionfiles or {"*.dtx"} + +-- Roots which should be unpacked to support unpacking/testing/typesetting +checkdeps = checkdeps or { } +typesetdeps = typesetdeps or { } +unpackdeps = unpackdeps or { } + +-- Executable names plus following options +typesetexe = typesetexe or "pdflatex" +unpackexe = unpackexe or "tex" +zipexe = zipexe or "zip" + +checkopts = checkopts or "-interaction=nonstopmode" +typesetopts = typesetopts or "-interaction=nonstopmode" +unpackopts = unpackopts or "" +zipopts = zipopts or "-v -r -X" + +-- Engines for testing +checkengines = checkengines or {"pdftex", "xetex", "luatex"} +checkformat = checkformat or "latex" +stdengine = stdengine or "pdftex" + +-- Configs for testing +stdconfig = stdconfig or string.gsub(arg[0], "%.lua$", "") +checkconfigs = checkconfigs or {stdconfig} + +-- Enable access to trees outside of the repo +-- As these may be set false, a more elaborate test than normal is needed +if checksearch == nil then + checksearch = true +end +if typesetsearch == nil then + typesetsearch = true +end +if unpacksearch == nil then + unpacksearch = true +end + +-- Additional settings to fine-tune typesetting +glossarystyle = glossarystyle or "gglo.ist" +indexstyle = indexstyle or "gind.ist" + +-- Supporting binaries and options +biberexe = biberexe or "biber" +biberopts = biberopts or "" +bibtexexe = bibtexexe or "bibtex8" +bibtexopts = bibtexopts or "-W" +makeindexexe = makeindexexe or "makeindex" +makeindexopts = makeindexopts or "" + +-- Forcing epoch +if forcecheckepoch == nil then + forcecheckepoch = true +end +if forcedocepoch == nil then + forcedocepoch = true +end + +-- Other required settings +asciiengines = asciiengines or {"pdftex"} +checkruns = checkruns or 1 +epoch = epoch or 1463734800 +maxprintline = maxprintline or 79 +packtdszip = packtdszip or false +scriptname = scriptname or "build.lua" +typesetcmds = typesetcmds or "" +typesetruns = typesetruns or 2 +versionform = versionform or "" +recordstatus = recordstatus or false + +-- Extensions for various file types: used to abstract out stuff a bit +bakext = bakext or ".bak" +dviext = dviext or ".dvi" +logext = logext or ".log" +lveext = lveext or ".lve" +lvtext = lvtext or ".lvt" +pdfext = pdfext or ".pdf" +psext = psext or ".ps" +tlgext = tlgext or ".tlg" + +-- Manifest options +manifestfile = "MANIFEST.md" diff --git a/Master/texmf-dist/tex/latex/l3build/l3build.lua b/Master/texmf-dist/tex/latex/l3build/l3build.lua index c4c76c7a1bb..b18c2cbea87 100644 --- a/Master/texmf-dist/tex/latex/l3build/l3build.lua +++ b/Master/texmf-dist/tex/latex/l3build/l3build.lua @@ -23,2511 +23,44 @@ for those people who are interested. --]] -- Version information -release_date = "2017/12/12" - --- "module" is a deprecated function in Lua 5.2: as we want the name --- for other purposes, and it should eventually be 'free', simply --- remove the built-in -if type(module) == "function" then - module = nil -end - --- Ensure the module and bundle exist -module = module or "" -bundle = bundle or "" - --- Sanity check -if module == "" and bundle == "" then - if string.match(arg[0], "l3build%.lua$") then - print( - "\n" .. - "l3build: A testing and building system for LaTeX\n\n" .. - "Release " .. release_date - ) - if arg[1] and not string.match(arg[1], "version") then - print( - "\n" - .. "Error: Call l3build using a configuration file, not directly.\n" - ) - end - else - print( - "\n" - .. "Error: Specify either bundle or module in configuration script.\n" - ) - end - os.exit(1) -end - --- Directory structure for the build system --- Use Unix-style path separators -currentdir = "." -maindir = maindir or currentdir - --- Substructure for file locations -docfiledir = docfiledir or currentdir -sourcefiledir = sourcefiledir or currentdir -supportdir = supportdir or maindir .. "/support" -testfiledir = testfiledir or currentdir .. "/testfiles" -testsuppdir = testsuppdir or testfiledir .. "/support" - --- Structure within a development area -builddir = builddir or maindir .. "/build" -distribdir = distribdir or builddir .. "/distrib" -localdir = localdir or builddir .. "/local" -testdir = testdir or builddir .. "/test" -typesetdir = typesetdir or builddir .. "/doc" -unpackdir = unpackdir or builddir .. "/unpacked" - --- Substructure for CTAN release material -ctandir = ctandir or distribdir .. "/ctan" -tdsdir = tdsdir or distribdir .. "/tds" -tdsroot = tdsroot or "latex" - --- Location for installation on CTAN or in TEXMFHOME -if bundle == "" then - moduledir = tdsroot .. "/" .. module - ctanpkg = ctanpkg or module -else - moduledir = tdsroot .. "/" .. bundle .. "/" .. module - ctanpkg = ctanpkg or bundle -end - --- File types for various operations --- Use Unix-style globs --- All of these may be set earlier, so a initialised conditionally -bibfiles = bibfiles or {"*.bib"} -binaryfiles = binaryfiles or {"*.pdf", "*.zip"} -bstfiles = bstfiles or {"*.bst"} -checkfiles = checkfiles or { } -checksuppfiles = checksuppfiles or { } -cmdchkfiles = cmdchkfiles or { } -cleanfiles = cleanfiles or {"*.log", "*.pdf", "*.zip"} -demofiles = demofiles or { } -docfiles = docfiles or { } -excludefiles = excludefiles or {"*~"} -installfiles = installfiles or {"*.sty","*.cls"} -makeindexfiles = makeindexfiles or {"*.ist"} -sourcefiles = sourcefiles or {"*.dtx", "*.ins"} -textfiles = textfiles or {"*.md", "*.txt"} -typesetdemofiles = typesetdemofiles or { } -typesetfiles = typesetfiles or {"*.dtx"} -typesetsuppfiles = typesetsuppfiles or { } -typesetsourcefiles = typesetsourcefiles or { } -unpackfiles = unpackfiles or {"*.ins"} -unpacksuppfiles = unpacksuppfiles or { } -versionfiles = versionfiles or {"*.dtx"} - --- Roots which should be unpacked to support unpacking/testing/typesetting -checkdeps = checkdeps or { } -typesetdeps = typesetdeps or { } -unpackdeps = unpackdeps or { } - --- Executable names plus following options -typesetexe = typesetexe or "pdflatex" -unpackexe = unpackexe or "tex" -zipexe = zipexe or "zip" - -checkopts = checkopts or "-interaction=nonstopmode" -cmdchkopts = cmdchkopts or "-interaction=batchmode" -typesetopts = typesetopts or "-interaction=nonstopmode" -unpackopts = unpackopts or "" -zipopts = zipopts or "-v -r -X" - --- Engines for testing -checkengines = checkengines or {"pdftex", "xetex", "luatex"} -checkformat = checkformat or "latex" -stdengine = stdengine or "pdftex" - --- Configs for testing -stdconfig = stdconfig or string.gsub(arg[0], "%.lua$", "") -checkconfigs = checkconfigs or {stdconfig} - --- Enable access to trees outside of the repo --- As these may be set false, a more elaborate test than normal is needed -if checksearch == nil then - checksearch = true -end -if typesetsearch == nil then - typesetsearch = true -end -if unpacksearch == nil then - unpacksearch = true -end - --- Additional settings to fine-tune typesetting -glossarystyle = glossarystyle or "gglo.ist" -indexstyle = indexstyle or "gind.ist" - --- Supporting binaries and options -biberexe = biberexe or "biber" -biberopts = biberopts or "" -bibtexexe = bibtexexe or "bibtex8" -bibtexopts = bibtexopts or "-W" -makeindexexe = makeindexexe or "makeindex" -makeindexopts = makeindexopts or "" - --- Forcing epoch -if forcecheckepoch == nil then - forcecheckepoch = true -end -if forcedocepoch == nil then - forcedocepoch = true -end - --- Other required settings -asciiengines = asciiengines or {"pdftex"} -checkruns = checkruns or 1 -epoch = epoch or 1463734800 -maxprintline = maxprintline or 79 -packtdszip = packtdszip or false -scriptname = scriptname or "build.lua" -typesetcmds = typesetcmds or "" -typesetruns = typesetruns or 2 -versionform = versionform or "" -recordstatus = recordstatus or false - --- Extensions for various file types: used to abstract out stuff a bit -bakext = bakext or ".bak" -dviext = dviext or ".dvi" -logext = logext or ".log" -lveext = lveext or ".lve" -lvtext = lvtext or ".lvt" -pdfext = pdfext or ".pdf" -psext = psext or ".ps" -tlgext = tlgext or ".tlg" +release_date = "2018/01/10" -- File operations are aided by the LuaFileSystem module local lfs = require("lfs") -- Local access to functions + local assert = assert local ipairs = ipairs local next = next local print = print local select = select local tonumber = tonumber -local close = io.close -local lines = io.lines -local open = io.open -local output = io.output -local stderr = io.stderr -local write = io.write -local set_program_name = kpse.set_program_name -local var_value = kpse.var_value -local lfs_attributes = lfs.attributes -local lfs_dir = lfs.dir -local os_date = os.date -local execute = os.execute local exit = os.exit -local getenv = os.getenv -local os_remove = os.remove -local os_time = os.time -local os_type = os.type -local len = string.len -local luatex_revision = status.luatex_revision -local luatex_version = status.luatex_version -local char = string.char -local find = string.find -local format = string.format -local gmatch = string.gmatch -local gsub = string.gsub -local len = string.len -local match = string.match -local rep = string.rep -local sort = table.sort -local sub = string.sub -local concat = table.concat -local insert = table.insert -local utf8_char = unicode.utf8.char - --- Parse command line options - -local option_list = - { - config = - { - desc = "Sets the config(s) used for running tests", - short = "c", - type = "table" - }, - date = - { - desc = "Sets the date to insert into sources", - short = "d", - type = "string" - }, - engine = - { - desc = "Sets the engine(s) to use for running test", - short = "e", - type = "table" - }, - epoch = - { - desc = "Sets the epoch for tests and typesetting", - short = "E", - type = "string" - }, - force = - { - desc = "Force tests to run if engine is not set up", - short = "f", - type = "boolean" - }, - ["halt-on-error"] = - { - desc = "Stops running tests after the first failure", - short = "H", - type = "boolean" - }, - help = - { - short = "h", - type = "boolean" - }, - pdf = - { - desc = "Check/save PDF files", - short = "p", - type = "boolean" - }, - quiet = - { - desc = "Suppresses TeX output when unpacking", - short = "q", - type = "boolean" - }, - rerun = - { - desc = "Skip setup: simply rerun tests", - short = "r", - type = "boolean" - }, - version = - { - desc = "Sets the version to insert into sources", - short = "v", - type = "string" - }, - } - --- This is done as a function (rather than do ... end) as it allows early --- termination (break) -local function argparse() - local result = { } - local files = { } - local long_options = { } - local short_options = { } - -- Turn long/short options into two lookup tables - for k,v in pairs(option_list) do - short_options[v["short"]] = k - long_options[k] = k - end - local args = args - -- arg[1] is a special case: must be a command or "-h"/"--help" - -- Deal with this by assuming help and storing only apparently-valid - -- input - local a = arg[1] - result["target"] = "help" - if a then - -- No options are allowed in position 1, so filter those out - if not match(a, "^%-") then - result["target"] = a - end - end - -- Stop here if help is required - if result["target"] == "help" then - return result - end - -- An auxiliary to grab all file names into a table - local function remainder(num) - local files = { } - for i = num, #arg do - insert(files, arg[i]) - end - return files - end - -- Examine all other arguments - -- Use a while loop rather than for as this makes it easier - -- to grab arg for optionals where appropriate - local i = 2 - while i <= #arg do - local a = arg[i] - -- Terminate search for options - if a == "--" then - files = remainder(i + 1) - break - end - -- Look for optionals - local opt - local optarg - local opts - -- Look for and option and get it into a variable - if match(a, "^%-") then - if match(a, "^%-%-") then - opts = long_options - local pos = find(a, "=", 1, true) - if pos then - opt = sub(a, 3, pos - 1) - optarg = sub(a, pos + 1) - else - opt = sub(a, 3) - end - else - opts = short_options - opt = sub(a, 2, 2) - -- Only set optarg if it is there - if #a > 2 then - optarg = sub(a, 3) - end - end - -- Now check that the option is valid and sort out the argument - -- if required - local optname = opts[opt] - if optname then - -- Tidy up arguments - if option_list[optname]["type"] == "boolean" then - if optarg then - local opt = "-" .. (match(a, "^%-%-") and "-" or "") .. opt - stderr:write("Value not allowed for option " .. opt .."\n") - return {"help"} - end - else - if not optarg then - optarg = arg[i + 1] - if not optarg then - stderr:write("Missing value for option " .. a .."\n") - return {"help"} - end - i = i + 1 - end - end - else - stderr:write("Unknown option " .. a .."\n") - return {"help"} - end - -- Store the result - if optarg then - if option_list[optname]["type"] == "string" then - result[optname] = optarg - else - local opts = result[optname] or { } - for hit in gmatch(optarg, "([^,%s]+)") do - insert(opts, hit) - end - result[optname] = opts - end - else - result[optname] = true - end - i = i + 1 - end - if not opt then - files = remainder(i) - break - end - end - if next(files) then - result["files"] = files - end - return result -end - -options = argparse() - --- Sanity check -if options["engine"] and not options["force"] then - -- Make a lookup table - local t = { } - for _, engine in pairs(checkengines) do - t[engine] = true - end - for _, engine in pairs(options["engine"]) do - if not t[engine] then - print("\n! Error: Engine \"" .. engine .. "\" not set up for testing!") - print("\n Valid values are:") - for _, engine in ipairs(checkengines) do - print(" - " .. engine) - end - print("") - exit(1) - end - end -end - --- Tidy up the epoch setting --- Force an epoch if set at the command line -if options["epoch"] then - epoch = options["epoch"] - forcecheckepoch = true - forcedocepoch = true -end --- If given as an ISO date, turn into an epoch number -do - local y, m, d = match(epoch, "^(%d%d%d%d)-(%d%d)-(%d%d)$") - if y then - epoch = - os_time({year = y, month = m, day = d, hour = 0, sec = 0, isdst = nil}) - - os_time({year = 1970, month = 1, day = 1, hour = 0, sec = 0, isdst = nil}) - elseif match(epoch, "^%d+$") then - epoch = tonumber(epoch) - else - epoch = 0 - end -end - --- Convert a file glob into a pattern for use by e.g. string.gub --- Based on https://github.com/davidm/lua-glob-pattern --- Simplified substantially: "[...]" syntax not supported as is not --- required by the file patterns used by the team. Also note style --- changes to match coding approach in rest of this file. --- --- License for original globtopattern ---[[ - - (c) 2008-2011 David Manura. Licensed under the same terms as Lua (MIT). - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE. - (end license) - ---]] -local function glob_to_pattern(glob) - - local pattern = "^" -- pattern being built - local i = 0 -- index in glob - local char -- char at index i in glob - - -- escape pattern char - local function escape(char) - return match(char, "^%w$") and char or "%" .. char - end - - -- Convert tokens. - while true do - i = i + 1 - char = sub(glob, i, i) - if char == "" then - pattern = pattern .. "$" - break - elseif char == "?" then - pattern = pattern .. "." - elseif char == "*" then - pattern = pattern .. ".*" - elseif char == "[" then - -- Ignored - print("[...] syntax not supported in globs!") - elseif char == "\\" then - i = i + 1 - char = sub(glob, i, i) - if char == "" then - pattern = pattern .. "\\$" - break - end - pattern = pattern .. escape(char) - else - pattern = pattern .. escape(char) - end - end - return pattern -end - --- Detect the operating system in use --- Support items are defined here for cases where a single string can cover --- both Windows and Unix cases: more complex situations are handled inside --- the support functions -os_concat = ";" -os_null = "/dev/null" -os_pathsep = ":" -os_setenv = "export" -os_yes = "printf 'y\\n%.0s' {1..200}" -local os_ascii = "echo \"\"" -local os_cmpexe = getenv("cmpexe") or "cmp" -local os_cmpext = getenv("cmpext") or ".cmp" -local os_diffext = getenv("diffext") or ".diff" -local os_diffexe = getenv("diffexe") or "diff -c --strip-trailing-cr" -local os_grepexe = "grep" -local os_newline = "\n" -if os_type == "windows" then - os_ascii = "@echo." - os_cmpexe = getenv("cmpexe") or "fc /b" - os_cmpext = getenv("cmpext") or ".cmp" - os_concat = "&" - os_diffext = getenv("diffext") or ".fc" - os_diffexe = getenv("diffexe") or "fc /n" - os_grepexe = "findstr /r" - os_newline = "\n" - if tonumber(luatex_version) < 100 or - (tonumber(luatex_version) == 100 - and tonumber(luatex_revision) < 4) then - os_newline = "\r\n" - end - os_null = "nul" - os_pathsep = ";" - os_setenv = "set" - os_yes = "for /l %I in (1,1,200) do @echo y" -end - --- Return an absolute path from a relative one -function abspath(path) - local oldpwd = lfs.currentdir() - lfs.chdir(path) - local result = lfs.currentdir() - lfs.chdir(oldpwd) - return gsub(result, "\\", "/") -end - --- For cleaning out a directory, which also ensures that it exists -function cleandir(dir) - local errorlevel = mkdir(dir) - if errorlevel ~= 0 then - return errorlevel - end - return rm(dir, "*") -end - --- Copy files 'quietly' -function cp(glob, source, dest) - local errorlevel - for i,_ in pairs(tree(source, glob)) do - local source = source .. "/" .. i - if os_type == "windows" then - if lfs_attributes(source)["mode"] == "directory" then - errorlevel = execute( - 'xcopy /y /e /i "' .. unix_to_win(source) .. '" "' - .. unix_to_win(dest .. '/' .. i) .. '" > nul' - ) - else - errorlevel = execute( - 'xcopy /y "' .. unix_to_win(source) .. '" "' - .. unix_to_win(dest .. '/') .. '" > nul' - ) - end - else - errorlevel = execute("cp -RLf '" .. source .. "' '" .. dest .. "'") - end - if errorlevel ~=0 then - return errorlevel - end - end - return 0 -end - --- OS-dependent test for a directory -function direxists(dir) - local errorlevel - if os_type == "windows" then - errorlevel = - execute("if not exist \"" .. unix_to_win(dir) .. "\" exit 1") - else - errorlevel = execute("[ -d " .. dir .. " ]") - end - if errorlevel ~= 0 then - return false - end - return true -end - -function fileexists(file) - local f = open(file, "r") - if f ~= nil then - close(f) - return true - else - return false - end -end - --- Generate a table containing all file names of the given glob or all files --- if absent -function filelist(path, glob) - local files = { } - local pattern - if glob then - pattern = glob_to_pattern(glob) - end - if direxists(path) then - for entry in lfs_dir(path) do - if pattern then - if match(entry, pattern) then - insert(files, entry) - end - else - if entry ~= "." and entry ~= ".." then - insert(files, entry) - end - end - end - end - return files -end - --- Does what filelist does, but can also glob subdirectories. In the returned --- table, the keys are paths relative to the given starting path, the values --- are their counterparts relative to the current working directory. -function tree(path, glob) - local function cropdots(path) - return gsub(gsub(path, "^%./", ""), "/%./", "/") - end - local function always_true() - return true - end - local function is_dir(file) - return lfs_attributes(file)["mode"] == "directory" - end - local dirs = {["."] = cropdots(path)} - for pattern, criterion in gmatch(cropdots(glob), "([^/]+)(/?)") do - local criterion = criterion == "/" and is_dir or always_true - function fill(path, dir, table) - for _, file in ipairs(filelist(dir, pattern)) do - local fullpath = path .. "/" .. file - if file ~= "." and file ~= ".." and - fullpath ~= builddir and - (sub(pattern, 1, 1) == "." - or sub(file, 1, 1) ~= ".") - then - local fulldir = dir .. "/" .. file - if criterion(fulldir) then - table[fullpath] = fulldir - end - end - end - end - local newdirs = {} - if pattern == "**" then - while true do - path, dir = next(dirs) - if not path then - break - end - dirs[path] = nil - newdirs[path] = dir - fill(path, dir, dirs) - end - else - for path, dir in pairs(dirs) do - fill(path, dir, newdirs) - end - end - dirs = newdirs - end - return dirs -end - -function mkdir(dir) - if os_type == "windows" then - -- Windows (with the extensions) will automatically make directory trees - -- but issues a warning if the dir already exists: avoid by including a test - local dir = unix_to_win(dir) - return execute( - "if not exist " .. dir .. "\\nul " .. "mkdir " .. dir - ) - else - return execute("mkdir -p " .. dir) - end -end - --- Rename -function ren(dir, source, dest) - local dir = dir .. "/" - if os_type == "windows" then - local source = gsub(source, "^%.+/", "") - local dest = gsub(dest, "^%.+/", "") - return execute("ren " .. unix_to_win(dir) .. source .. " " .. dest) - else - return execute("mv " .. dir .. source .. " " .. dir .. dest) - end -end - --- Remove file(s) based on a glob -function rm(source, glob) - for _,i in ipairs(filelist(source, glob)) do - os_remove(source .. "/" .. i) - end - -- os_remove doesn't give a sensible errorlevel - return 0 -end - --- Remove a directory tree -function rmdir(dir) - -- First, make sure it exists to avoid any errors - mkdir(dir) - if os_type == "windows" then - return execute("rmdir /s /q " .. unix_to_win(dir)) - else - return execute("rm -r " .. dir) - end -end - --- Run a command in a given directory -function run(dir, cmd) - return execute("cd " .. dir .. os_concat .. cmd) -end - --- Deal with the fact that Windows and Unix use different path separators -function unix_to_win(path) - return gsub(path, "/", "\\") -end - --- --- Auxiliary functions which are used by more than one main function --- - --- Do some subtarget for all modules in a bundle -function call(dirs, target, opts) - -- Turn the option table into a string - local opts = opts or options - local s = "" - for k,v in pairs(opts) do - if k ~= "files" and k ~= "target" then -- Special cases - local t = option_list[k] or { } - local arg = "" - if t["type"] == "string" then - arg = arg .. "=" .. v - end - if t["type"] == "table" then - for _,a in pairs(v) do - if arg == "" then - arg = "=" .. a -- Add the initial "=" here - else - arg = arg .. "," .. a - end - end - end - s = s .. " --" .. k .. arg - end - end - if opts["files"] then - for _,v in pairs(opts["files"]) do - s = s .. " " .. v - end - end - for _,i in ipairs(dirs) do - print( - "Running script " .. scriptname .. " with target \"" .. target - .. "\" for module " - .. i - ) - local errorlevel = run( - i, - "texlua " .. scriptname .. " " .. target .. s - ) - if errorlevel ~= 0 then - return errorlevel - end - end - return 0 -end - --- Set up the check system files: needed for checking one or more tests and --- for saving the test files -function checkinit() - cleandir(testdir) - depinstall(checkdeps) - -- Copy dependencies to the test directory itself: this makes the paths - -- a lot easier to manage, and is important for dealing with the log and - -- with file input/output tests - for _,i in ipairs(filelist(localdir)) do - cp(i, localdir, testdir) - end - bundleunpack({sourcefiledir, testfiledir}) - for _,i in ipairs(installfiles) do - cp(i, unpackdir, testdir) - end - for _,i in ipairs(checkfiles) do - cp(i, unpackdir, testdir) - end - if direxists(testsuppdir) then - for _,i in ipairs(filelist(testsuppdir)) do - cp(i, testsuppdir, testdir) - end - end - for _,i in ipairs(checksuppfiles) do - cp(i, supportdir, testdir) - end - execute(os_ascii .. ">" .. testdir .. "/ascii.tcx") -end - --- Copy files to the main CTAN release directory -function copyctan() - local ctantarget = ctanpkg - if docfiledir ~= currentdir then - ctantarget = ctanpkg .. "/" .. gsub(docfiledir, "^%.*/", "") - end - mkdir(ctandir .. "/" .. ctantarget) - for _,filetype in pairs( - { - bibfiles, - demofiles, - docfiles, - pdffiles, - typesetlist - } - ) do - for _,file in pairs(filetype) do - cp(file, docfiledir, ctandir .. "/" .. ctantarget) - end - end - ctantarget = ctanpkg - if sourcefiledir ~= currentdir then - ctantarget = ctanpkg .. "/" .. gsub(sourcefiledir, "^%.*/", "") - end - mkdir(ctandir .. "/" .. ctantarget) - for _,file in pairs(sourcefiles) do - if sourcedir ~= currentdir then - end - cp(file, sourcefiledir, ctandir .. "/" .. ctantarget) - end - for _,file in pairs(textfiles) do - cp(file, currentdir, ctandir .. "/" .. ctanpkg) - end -end - --- Copy files to the correct places in the TDS tree -function copytds() - local function install(source, dest, files, tool) - local moduledir = moduledir - -- For material associated with secondary tools (BibTeX, MakeIndex) - -- the structure needed is slightly different from those items going - -- into the tex/doc/source trees - if tool then - -- "base" is reserved for the tools themselves: make the assumption - -- in this case that the tdsroot name is the right place for stuff to - -- go (really just for the team) - if module == "base" then - moduledir = tdsroot - else - moduledir = module - end - end - -- Convert the file table(s) to a list of individual files - local filenames = { } - for _,i in ipairs(files) do - for _,j in ipairs(i) do - for file,_ in pairs(tree(source, j)) do - insert(filenames, file) - end - end - end - -- The target is only created if there are actual files to install - if next(filenames) ~= nil then - local installdir = tdsdir .. "/" .. dest .. "/" .. moduledir - mkdir(installdir) - for _,i in ipairs(filenames) do - cp(i, source, installdir) - end - end - end - install( - docfiledir, - "doc", - {bibfiles, demofiles, docfiles, pdffiles, textfiles, typesetlist} - ) - install(unpackdir, "makeindex", {makeindexfiles}, true) - install(unpackdir, "bibtex/bst", {bstfiles}, true) - install(sourcefiledir, "source", {sourcelist}) - install(unpackdir, "tex", {installfiles}) -end --- Unpack files needed to support testing/typesetting/unpacking -function depinstall(deps) - local errorlevel - for _,i in ipairs(deps) do - print("Installing dependency: " .. i) - errorlevel = run(i, "texlua " .. scriptname .. " unpack -q") - if errorlevel ~= 0 then - return errorlevel - end - end - return 0 -end - --- Convert the raw log file into one for comparison/storage: keeps only --- the 'business' part from the tests and removes system-dependent stuff -local function formatlog(logfile, newfile, engine, errlevels) - local maxprintline = maxprintline - if engine == "luatex" or engine == "luajittex" then - maxprintline = maxprintline + 1 -- Deal with an out-by-one error - end - local function killcheck(line) - -- Skip lines containing file dates - if match(line, "[^<]%d%d%d%d/%d%d/%d%d") then - return true - elseif - -- Skip \openin/\openout lines in web2c 7.x - -- As Lua doesn't allow "(in|out)", a slightly complex approach: - -- do a substitution to check the line is exactly what is required! - match( - gsub(line, "^\\openin", "\\openout"), "^\\openout%d%d? = " - ) then - return true - end - return false - end - -- Substitutions to remove some non-useful changes - local function normalize(line, lastline) - -- Zap line numbers from \show, \showbox, \box_show and the like: - -- do this before wrapping lines - line = gsub(line, "^l%.%d+ ", "l. ...") - -- Also from lua stack traces. - line = gsub(line, "lua:%d+: in function", "lua:...: in function") - -- Allow for wrapped lines: preserve the content and wrap - -- Skip lines that have an explicit marker for truncation - if len(line) == maxprintline and - not match(line, "%.%.%.$") then - return "", (lastline or "") .. line - end - local line = (lastline or "") .. line - lastline = "" - -- Zap ./ at begin of filename - line = gsub(line, "%(%.%/", "(") - -- Zap paths if places other than 'here' are accessible - if checksearch then - -- The pattern excludes < and > as the image part can have - -- several entries on one line - local pattern = "%w?:?/[^ %<%>]*/([^/%(%)]*%.%w*)" - -- Files loaded from TeX: all start ( -- ) - line = gsub(line, "%(" .. pattern, "(../%1") - -- Images - line = gsub(line, "<" .. pattern .. ">", "<../%1>") - -- luaotfload files start with keywords - line = gsub(line, "from " .. pattern .. "%(", "from. ./%1(") - line = gsub(line, ": " .. pattern .. "%)", ": ../%1)") - -- Deal with XeTeX specials - if match(line, "^%.+\\XeTeX.?.?.?file") then - line = gsub(line, pattern, "../%1") - end - end - -- Deal with the fact that "(.aux)" may have still a leading space - line = gsub(line, "^ %(%.aux%)", "(.aux)") - -- Merge all of .fd data into one line so will be removed later - if match(line, "^ *%([%.%/%w]+%.fd[^%)]*$") then - lastline = (lastline or "") .. line - return "", (lastline or "") .. line - end - -- TeX90/XeTeX knows only the smaller set of dimension units - line = gsub( - line, - "cm, mm, dd, cc, bp, or sp", "cm, mm, dd, cc, nd, nc, bp, or sp" - ) - -- Normalise a case where fixing a TeX bug changes the message text - line = gsub(line, "\\csname\\endcsname ", "\\csname\\endcsname") - -- Zap "on line " and replace with "on line ..." - -- Two similar cases, Lua patterns mean we need to do them separately - line = gsub(line, "on line %d*", "on line ...") - line = gsub(line, "on input line %d*", "on input line ...") - -- Tidy up to ^^ notation - for i = 0, 31 do - line = gsub(line, char(i), "^^" .. char(64 + i)) - end - -- Normalise register allocation to hard-coded numbers - -- No regex, so use a pattern plus lookup approach - local register_types = { - attribute = true, - box = true, - bytecode = true, - catcodetable = true, - count = true, - dimen = true, - insert = true, - language = true, - luabytecode = true, - luachunk = true, - luafunction = true, - marks = true, - muskip = true, - read = true, - skip = true, - toks = true, - whatsit = true, - write = true, - XeTeXcharclass = true - } - if register_types[match(line, "^\\[^%]]+=\\([a-z]+)%d+$")] then - line = gsub(line, "%d+$", "...") - end - -- Also deal with showing boxes - if match(line, "^> \\box%d+=$") or match(line, "^> \\box%d+=(void)$") then - line = gsub(line, "%d+=", "...=") - end - -- Remove 'normal' direction information on boxes with (u)pTeX - line = gsub(line, ",? yoko direction,?", "") - line = gsub(line, ",? yoko%(math%) direction,?", "") - -- Remove the \special line that in DVI mode keeps PDFs comparable - if match(line, "^%.*\\special%{pdf: docinfo << /Creator") then - return "" - end - -- Remove the \special line possibly present in DVI mode for paper size - if match(line, "^%.*\\special%{papersize") then - return "" - end - -- Remove ConTeXt stuff - if match(line, "^backend >") or - match(line, "^close source >") or - match(line, "^mkiv lua stats >") or - match(line, "^pages >") or - match(line, "^system >") or - match(line, "^used file >") or - match(line, "^used option >") or - match(line, "^used structure >") then - return "" - end - -- A tidy-up to keep LuaTeX and other engines in sync - line = gsub(line, utf8_char(127), "^^?") - -- Unicode engines display chars in the upper half of the 8-bit range: - -- tidy up to match pdfTeX if an ASCII engine is in use - if next(asciiengines) then - for i = 128, 255 do - line = gsub(line, utf8_char(i), "^^" .. format("%02x", i)) - end - end - return line, lastline - end - local lastline = "" - local newlog = "" - local prestart = true - local skipping = false - -- Read the entire log file as a binary: deals with ^@/^[, etc. - local file = assert(open(logfile, "rb")) - local contents = gsub(file:read("*all") .. "\n", "\r\n", "\n") - close(file) - for line in gmatch(contents, "([^\n]*)\n") do - if line == "START-TEST-LOG" then - prestart = false - elseif line == "END-TEST-LOG" or - match(line, "^Here is how much of .?.?.?TeX\'s memory you used:") then - break - elseif line == "OMIT" then - skipping = true - elseif match(line, "^%)?TIMO$") then - skipping = false - elseif not prestart and not skipping then - line, lastline = normalize(line, lastline) - if not match(line, "^ *$") and not killcheck(line) then - newlog = newlog .. line .. os_newline - end - end - end - local newfile = open(newfile, "w") - output(newfile) - write(newlog) - if recordstatus then - write('***************\n') - for i = 1, checkruns do - write('Compilation ' .. i .. ' of test file completed with exit status ' .. errlevels[i] '\n') - end - end - close(newfile) -end - --- Additional normalization for LuaTeX -local function formatlualog(logfile, newfile) - local function normalize(line, lastline, dropping) - -- Find \discretionary or \whatsit lines: - -- These may come back later - if match(line, "^%.+\\discretionary$") or - match(line, "^%.+\\discretionary %(penalty 50%)$") or - match(line, "^%.+\\discretionary50%|$") or - match(line, "^%.+\\discretionary50%| replacing $") or - match(line, "^%.+\\whatsit$") then - return "", line - end - -- For \mathon, we always need this line but the next - -- may be affected - if match(line, "^%.+\\mathon$") then - return line, line - end - -- LuaTeX has a flexible output box - line = gsub(line,"\\box\\outputbox", "\\box255") - -- LuaTeX identifies spaceskip glue - line = gsub(line,"%(\\spaceskip%) ", " ") - -- Remove 'display' at end of display math boxes: - -- LuaTeX omits this as it includes direction in all cases - line = gsub(line, "(\\hbox%(.*), display$", "%1") - -- Remove 'normal' direction information on boxes: - -- any bidi/vertical stuff will still show - line = gsub(line, ", direction TLT", "") - -- Find glue setting and round out the last place - local function round_digits(l, m) - return gsub( - l, - m .. " (%-?)%d+%.%d+", - m .. " %1" - .. format( - "%.3f", - match(line, m .. " %-?(%d+%.%d+)") or 0 - ) - ) - end - if match(line, "glue set %-?%d+%.%d+") then - line = round_digits(line, "glue set") - end - if match( - line, "glue %-?%d+%.%d+ plus %-?%d+%.%d+ minus %-?%d+%.%d+$" - ) - then - line = round_digits(line, "glue") - line = round_digits(line, "plus") - line = round_digits(line, "minus") - end - -- LuaTeX writes ^^M as a new line, which we lose - line = gsub(line, "%^%^M", "") - -- Remove U+ notation in the "Missing character" message - line = gsub( - line, - "Missing character: There is no (%^%^..) %(U%+(....)%)", - "Missing character: There is no %1" - ) - -- The first time a new font is used, it shows up - -- as being cached - line = gsub(line, "(save cache:", "(load cache:") - -- A function to handle the box prefix part - local function boxprefix(s) - return gsub(match(s, "^(%.+)"), "%.", "%%.") - end - -- 'Recover' some discretionary data - if match(lastline, "^%.+\\discretionary %(penalty 50%)$") and - match(line, boxprefix(lastline) .. "%.= ") then - return gsub(line, "%.= ", ""),"" - end - -- Where the last line was a discretionary, looks for the - -- info one level in about what it represents - if match(lastline, "^%.+\\discretionary$") or - match(lastline, "^%.+\\discretionary %(penalty 50%)$") or - match(lastline, "^%.+\\discretionary50%|$") or - match(lastline, "^%.+\\discretionary50%| replacing $") then - local prefix = boxprefix(lastline) - if match(line, prefix .. "%.") or - match(line, prefix .. "%|") then - if match(lastline, " replacing $") and - not dropping then - -- Modify the return line - return gsub(line, "^%.", ""), lastline, true - else - return "", lastline, true - end - else - if dropping then - -- End of a \discretionary block - return line, "" - else - -- Not quite a normal discretionary - if match(lastline, "^%.+\\discretionary50%|$") then - lastline = gsub(lastline, "50%|$", "") - end - -- Remove some info that TeX90 lacks - lastline = gsub(lastline, " %(penalty 50%)$", "") - -- A normal (TeX90) discretionary: - -- add with the line break reintroduced - return lastline .. os_newline .. line, "" - end - end - end - -- Look for another form of \discretionary, replacing a "-" - pattern = "^%.+\\discretionary replacing *$" - if match(line, pattern) then - return "", line - else - if match(lastline, pattern) then - local prefix = boxprefix(lastline) - if match(line, prefix .. "%.\\kern") then - return gsub(line, "^%.", ""), lastline, true - elseif dropping then - return "", "" - else - return lastline .. os_newline .. line, "" - end - end - end - -- For \mathon, if the current line is an empty \hbox then - -- drop it - if match(lastline, "^%.+\\mathon$") then - local prefix = boxprefix(lastline) - if match(line, prefix .. "\\hbox%(0%.0%+0%.0%)x0%.0$") then - return "", "" - end - end - -- Various \local... things that other engines do not do: - -- Only remove the no-op versions - if match(line, "^%.+\\localpar$") or - match(line, "^%.+\\localinterlinepenalty=0$") or - match(line, "^%.+\\localbrokenpenalty=0$") or - match(line, "^%.+\\localleftbox=null$") or - match(line, "^%.+\\localrightbox=null$") then - return "", "" - end - -- Older LuaTeX versions set the above up as a whatsit - -- (at some stage this can therefore go) - if match(lastline, "^%.+\\whatsit$") then - local prefix = boxprefix(lastline) - if match(line, prefix .. "%.") then - return "", lastline, true - else - -- End of a \whatsit block - return line, "" - end - end - -- Wrap some cases that can be picked out - -- In some places LuaTeX does use max_print_line, then we - -- get into issues with different wrapping approaches - if len(line) == maxprintline then - return "", lastline .. line - elseif len(lastline) == maxprintline then - if match(line, "\\ETC%.%}$") then - -- If the line wrapped at \ETC we might have lost a space - return lastline - .. ((match(line, "^\\ETC%.%}$") and " ") or "") - .. line, "" - elseif match(line, "^%}%}%}$") then - return lastline .. line, "" - else - return lastline .. os_newline .. line, "" - end - -- Return all of the text for a wrapped (multi)line - elseif len(lastline) > maxprintline then - return lastline .. line, "" - end - -- Remove spaces at the start of lines: deals with the fact that LuaTeX - -- uses a different number to the other engines - return gsub(line, "^%s+", ""), "" - end - local newlog = "" - local lastline = "" - local dropping = false - -- Read the entire log file as a binary: deals with ^@/^[, etc. - local file = assert(open(logfile, "rb")) - local contents = gsub(file:read("*all") .. "\n", "\r\n", "\n") - close(file) - for line in gmatch(contents, "([^\n]*)\n") do - line, lastline, dropping = normalize(line, lastline, dropping) - if not match(line, "^ *$") then - newlog = newlog .. line .. os_newline - end - end - local newfile = open(newfile, "w") - output(newfile) - write(newlog) - close(newfile) -end - --- Look for files, directory by directory, and return the first existing -function locate(dirs, names) - for _,i in ipairs(dirs) do - for _,j in ipairs(names) do - local path = i .. "/" .. j - if fileexists(path) then - return path - end - end - end -end - --- List all modules -function listmodules() - local modules = { } - local exclmodules = exclmodules or { } - for entry in lfs_dir(".") do - if entry ~= "." and entry ~= ".." then - local attr = lfs_attributes(entry) - assert(type(attr) == "table") - if attr.mode == "directory" then - if not exclmodules[entry] then - insert(modules, entry) - end - end - end - end - return modules -end - -local function setepoch() - return - os_setenv .. " SOURCE_DATE_EPOCH=" .. epoch - .. os_concat .. - os_setenv .. " SOURCE_DATE_EPOCH_TEX_PRIMITIVES=1" - .. os_concat .. - os_setenv .. " FORCE_SOURCE_DATE=1" - .. os_concat -end - --- Run one test which may have multiple engine-dependent comparisons --- Should create a difference file for each failed test -function runcheck(name, hide) - local checkengines = checkengines - if options["engine"] then - checkengines = options["engine"] - end - local errorlevel = 0 - for _,i in ipairs(checkengines) do - -- Allow for luatex == luajittex for .tlg purposes - local engine = i - if i == "luajittex" then - engine = "luatex" - end - checkpdf = setup_check(name, engine) - runtest(name, i, hide, lvtext, checkpdf) - -- Generation of results depends on test type - local errlevel - if checkpdf then - errlevel = compare_pdf(name, engine) - else - errlevel = compare_tlg(name, engine) - end - if errlevel ~= 0 and options["halt-on-error"] then - showfaileddiff() - if errlevel ~= 0 then - return 1 - end - end - if errlevel > errorlevel then - errorlevel = errlevel - end - end - return errorlevel -end - -function setup_check(name, engine) - local testname = name .. "." .. engine - local pdffile = locate( - {testfiledir, unpackdir}, - {testname .. pdfext, name .. pdfext} - ) - local tlgfile = locate( - {testfiledir, unpackdir}, - {testname .. tlgext, name .. tlgext} - ) - -- Attempt to generate missing reference file from expectation - if not (pdffile or tlgfile) then - if not locate({unpackdir, testfiledir}, {name .. lveext}) then - print( - "Error: failed to find " .. pdfext .. ", " .. tlgext .. " or " - .. lveext .. " file for " .. name .. "!" - ) - exit(1) - end - runtest(name, engine, true, lveext, true) - pdffile = testdir .. "/" .. testname .. pdfext - -- If a PDF is generated use it for comparisons - if not fileexists(pdffile) then - pdffile = nil - ren(testdir, testname .. logext, testname .. tlgext) - end - else - -- Install comparison files found - for _,v in pairs({pdffile, tlgfile}) do - if v then - cp( - match(v, ".*/(.*)"), - match(v, "(.*)/.*"), - testdir - ) - end - end - end - if pdffile then - local pdffile = match(pdffile, ".*/(.*)") - ren( - testdir, - pdffile, - gsub(pdffile, pdfext .. "$", ".ref" .. pdfext) - ) - return true - else - return false - end -end - -function compare_pdf(name, engine) - local errorlevel - local testname = name .. "." .. engine - local cmpfile = testdir .. "/" .. testname .. os_cmpext - local pdffile = testdir .. "/" .. testname .. pdfext - local refpdffile = locate( - {testdir}, {testname .. ".ref" .. pdfext, name .. ".ref" .. pdfext} - ) - if not refpdffile then - return - end - if os_type == "windows" then - refpdffile = unix_to_win(refpdffile) - end - errorlevel = execute( - os_cmpexe .. " " .. refpdffile .. " " .. pdffile .. " > " .. cmpfile - ) - if errorlevel == 0 then - os_remove(cmpfile) - end - return errorlevel -end - -function compare_tlg(name, engine) - local errorlevel - local testname = name .. "." .. engine - local difffile = testdir .. "/" .. testname .. os_diffext - local logfile = testdir .. "/" .. testname .. logext - local tlgfile = locate({testdir}, {testname .. tlgext, name .. tlgext}) - if not tlgfile then - return - end - if os_type == "windows" then - tlgfile = unix_to_win(tlgfile) - end - -- Do additional log formatting if the engine is LuaTeX, there is no - -- LuaTeX-specific .tlg file and the default engine is not LuaTeX - if engine == "luatex" - and not match(tlgfile, "%.luatex" .. "%" .. tlgext) - and stdengine ~= "luatex" - and stdengine ~= "luajittex" - then - local luatlgfile = testdir .. "/" .. name .. ".luatex" .. tlgext - if os_type == "windows" then - luatlgfile = unix_to_win(luatlgfile) - end - formatlualog(tlgfile, luatlgfile) - formatlualog(logfile, logfile) - -- This allows code sharing below: we only need the .tlg name in one place - tlgfile = luatlgfile - end - errorlevel = execute( - os_diffexe .. " " .. tlgfile .. " " .. logfile .. " > " .. difffile - ) - if errorlevel == 0 then - os_remove(difffile) - end - return errorlevel -end - --- Run one of the test files: doesn't check the result so suitable for --- both creating and verifying .tlg files -function runtest(name, engine, hide, ext, makepdf) - local lvtfile = name .. (ext or lvtext) - cp(lvtfile, fileexists(testfiledir .. "/" .. lvtfile) - and testfiledir or unpackdir, testdir) - local engine = engine or stdengine - -- Set up the format file name if it's one ending "...tex" - local realengine = engine - local format - if - match(checkformat, "tex$") and - not match(engine, checkformat) then - format = " -fmt=" .. gsub(engine, "(.*)tex$", "%1") .. checkformat - else - format = "" - end - -- Special casing for e-LaTeX format - if - match(checkformat, "^latex$") and - match(engine, "^etex$") then - format = " -fmt=latex" - end - -- Special casing for (u)pTeX LaTeX formats - if - match(checkformat, "^latex$") and - match(engine, "^u?ptex$") then - realengine = "e" .. engine - end - -- Special casing for XeTeX engine - local checkopts = checkopts - if match(engine, "xetex") and not makepdf then - checkopts = checkopts .. " -no-pdf" - end - -- Special casing for ConTeXt - if match(checkformat, "^context$") then - format = "" - if engine == "luatex" or engine == "luajittex" then - realengine = "context" - elseif engine == "pdftex" then - realengine = "texexec" - elseif engine == "xetex" then - realengine = "texexec --xetex" - else - print("Engine incompatible with format") - exit(1) - end - end - local logfile = testdir .. "/" .. name .. logext - local newfile = testdir .. "/" .. name .. "." .. engine .. logext - local asciiopt = "" - for _,i in ipairs(asciiengines) do - if realengine == i then - asciiopt = "-translate-file ./ascii.tcx " - break - end - end - local errlevels = {} - for i = 1, checkruns do - errlevels[i] = run( - testdir, - -- No use of localdir here as the files get copied to testdir: - -- avoids any paths in the logs - os_setenv .. " TEXINPUTS=." .. (checksearch and os_pathsep or "") - .. os_concat .. - -- Avoid spurious output from (u)pTeX - os_setenv .. " GUESS_INPUT_KANJI_ENCODING=0" - .. os_concat .. - (forcecheckepoch and setepoch() or "") .. - -- Ensure lines are of a known length - os_setenv .. " max_print_line=" .. maxprintline - .. os_concat .. - realengine .. format .. " " - .. checkopts .. " " .. asciiopt .. lvtfile - .. (hide and (" > " .. os_null) or "") - .. os_concat .. - runtest_tasks(jobname(lvtfile)) - ) - end - if makepdf and fileexists(testdir .. "/" .. name .. dviext) then - dvitopdf(name, testdir, engine, hide) - end - formatlog(logfile, newfile, engine, errlevels) - -- Store secondary files for this engine - for _,i in ipairs(filelist(testdir, name .. ".???")) do - local ext = match(i, "%....") - if ext ~= lvtext and ext ~= tlgext and ext ~= lveext and ext ~= logext then - if not fileexists(testsuppdir .. "/" .. i) then - ren( - testdir, i, gsub( - i, gsub(name, "%-", "%%-"), name .. "." .. engine - ) - ) - end - end - end -end - --- A hook to allow additional tasks to run for the tests -runtest_tasks = runtest_tasks or function(name) - return "" -end - -function dvitopdf(name, dir, engine, hide) - if match(engine, "^u?ptex$") then - run( - dir, - (forcecheckepoch and setepoch() or "") .. - "dvipdfmx " .. name .. dviext - .. (hide and (" > " .. os_null) or "") - ) - else - run( - dir, - (forcecheckepoch and setepoch() or "") .. - "dvips " .. name .. dviext - .. (hide and (" > " .. os_null) or "") - .. os_concat .. - "ps2pdf " .. name .. psext - .. (hide and (" > " .. os_null) or "") - ) - end -end - --- Split a path into file and directory component -function splitpath(file) - local path, name = match(file, "^(.*)/([^/]*)$") - if path then - return path, name - else - return ".", file - end -end - --- Arguably clearer names -function basename(file) - return(select(2, splitpath(file))) -end - -function dirname(file) - return(select(1, splitpath(file))) -end - --- Strip the extension from a file name (if present) -function jobname(file) - local name = match(basename(file), "^(.*)%.") - return name or file -end - --- Look for a test: could be in the testfiledir or the unpackdir -function testexists(test) - return(locate({testfiledir, unpackdir}, {test .. lvtext})) -end - --- --- Auxiliary functions for typesetting: need to be generally available --- - --- An auxiliary used to set up the environmental variables -function runtool(subdir, dir, envvar, command) - dir = dir or "." - return( - run( - typesetdir .. "/" .. subdir, - (forcedocepoch and setepoch() or "") .. - os_setenv .. " " .. envvar .. "=." .. os_pathsep - .. abspath(localdir) .. os_pathsep - .. abspath(dir .. "/" .. subdir) - .. (typesetsearch and os_pathsep or "") - .. os_concat .. - command - ) - ) -end - -function biber(name, dir) - if fileexists(typesetdir .. "/" .. name .. ".bcf") then - local path, name = splitpath(name) - return( - runtool(path, dir, "BIBINPUTS", biberexe .. " " .. biberopts .. " " .. name) - ) - end - return 0 -end - -function bibtex(name, dir) - if fileexists(typesetdir .. "/" .. name .. ".aux") then - -- LaTeX always generates an .aux file, so there is a need to - -- look inside it for a \citation line - local grep - if os_type == "windows" then - grep = "\\\\" - else - grep = "\\\\\\\\" - end - local path, name = splitpath(name) - if run( - typesetdir, - os_grepexe .. " \"^" .. grep .. "citation{\" " .. name .. ".aux > " - .. os_null - ) + run( - typesetdir, - os_grepexe .. " \"^" .. grep .. "bibdata{\" " .. name .. ".aux > " - .. os_null - ) == 0 then - return( - -- Cheat slightly as we need to set two variables - runtool( - path, dir, - "BIBINPUTS", - os_setenv .. " BSTINPUTS=." .. os_pathsep - .. abspath(localdir) - .. (typesetsearch and os_pathsep or "") .. - os_concat .. - bibtexexe .. " " .. bibtexopts .. " " .. name - ) - ) - end - end - return 0 -end - -function makeindex(name, dir, inext, outext, logext, style) - if fileexists(typesetdir .. "/" .. name .. inext) then - local path, name = splitpath(name) - return( - runtool( - path, dir, - "INDEXSTYLE", - makeindexexe .. " " .. makeindexopts .. " " - .. " -s " .. style .. " -o " .. name .. outext - .. " -t " .. name .. " " .. name .. inext - ) - ) - end - return 0 -end - -function tex(file, dir) - local path, name = splitpath(file) - return( - runtool( - path, dir, - "TEXINPUTS", - typesetexe .. " " .. typesetopts .. " \"" .. typesetcmds - .. "\\input " .. name .. "\"" - ) - ) -end - -function typesetpdf(file, dir) - local name = gsub(file, "%.[^.]+$", "") - print("Typesetting " .. name) - local errorlevel = typeset(file, dir) - if errorlevel == 0 then - name = name .. ".pdf" - os_remove(jobname(name)) - cp(name, typesetdir, docfiledir) - else - print(" ! Compilation failed") - end - return errorlevel -end - -typeset = typeset or function(file, dir) - dir = dir or "." - local errorlevel = tex(file, dir) - if errorlevel ~= 0 then - return errorlevel - else - local name = jobname(file) - errorlevel = biber(name, dir) + bibtex(name, dir) - if errorlevel == 0 then - local function cycle(name, dir) - return( - makeindex(name, dir, ".glo", ".gls", ".glg", glossarystyle) + - makeindex(name, dir, ".idx", ".ind", ".ilg", indexstyle) + - tex(file, dir) - ) - end - for i = 1, typesetruns do - errorlevel = cycle(name, dir) - if errorlevel ~= 0 then break end - end - end - return errorlevel - end -end - --- Standard versions of the main targets for building modules - --- Simply print out how to use the build system -function help() - print("usage: " .. arg[0] .. " [] []") - print("") - print("The most commonly used l3build commands are:") - if testfiledir ~= "" then - print(" check Run all automated tests") - end - print(" clean Clean out directory tree") - if next(cmdchkfiles) ~= nil then - print(" cmdcheck Check commands documented are defined") - end - if module == "" or bundle == "" then - print(" ctan Create CTAN-ready archive") - end - print(" doc Typesets all documentation files") - print(" install Installs files into the local texmf tree") - if module ~= "" and testfiledir ~= "" then - print(" save Saves test validation log") - end - print(" setversion Update version information in sources") - print("") - print("Valid options are:") - local longest = 0 - for k,v in pairs(option_list) do - if len(k) > longest then - longest = len(k) - end - end - -- Sort the options - local t = { } - for k,_ in pairs(option_list) do - insert(t, k) - end - sort(t) - for _,k in ipairs(t) do - local opt = option_list[k] - local filler = rep(" ", longest - len(k)) - if opt["desc"] then -- Skip --help as it has no desc - print( - " --" .. k .. "|-" .. opt["short"] .. filler .. opt["desc"] - ) - end - end - print("") - print("See l3build.pdf for further details.") -end - -function check(names) - local errorlevel = 0 - if testfiledir ~= "" and direxists(testfiledir) then - if not options["rerun"] then - checkinit() - end - local hide = true - if names and next(names) then - hide = false - end - names = names or { } - -- No names passed: find all test files - if not next(names) then - for _,i in pairs(filelist(testfiledir, "*" .. lvtext)) do - insert(names, jobname(i)) - end - for _,i in ipairs(filelist(unpackdir, "*" .. lvtext)) do - if fileexists(testfiledir .. "/" .. i) then - print("Duplicate test file: " .. i) - return 1 - else - insert(names, jobname(i)) - end - end - end - -- Actually run the tests - print("Running checks on") - for _,name in ipairs(names) do - print(" " .. name) - local errlevel = runcheck(name, hide) - -- Return value must be 1 not errlevel - if errlevel ~= 0 then - if options["halt-on-error"] then - return 1 - else - errorlevel = 1 - -- visually show that something has failed - print(" --> failed\n") - end - end - end - if errorlevel ~= 0 then - checkdiff() - else - print("\n All checks passed\n") - end - end - return errorlevel -end - --- A short auxiliary to print the list of differences for check -function checkdiff() - print("\n Check failed with difference files") - for _,i in ipairs(filelist(testdir, "*" .. os_diffext)) do - print(" - " .. testdir .. "/" .. i) - end - for _,i in ipairs(filelist(testdir, "*" .. os_cmpext)) do - print(" - " .. testdir .. "/" .. i) - end - print("") -end - -function showfaileddiff() - print("\nCheck failed with difference file") - for _,i in ipairs(filelist(testdir, "*" .. os_diffext)) do - print(" - " .. testdir .. "/" .. i) - print("") - local f = open(testdir .. "/" .. i,"r") - local content = f:read("*all") - f:close() - print("-----------------------------------------------------------------------------------") - print(content) - print("-----------------------------------------------------------------------------------") - end - for _,i in ipairs(filelist(testdir, "*" .. os_cmpext)) do - print(" - " .. testdir .. "/" .. i) - end -end - --- Remove all generated files -function clean() - -- To make sure that distribdir never contains any stray subdirs, - -- it is entirely removed then recreated rather than simply deleting - -- all of the files - local errorlevel = - rmdir(distribdir) + - mkdir(distribdir) + - cleandir(localdir) + - cleandir(testdir) + - cleandir(typesetdir) + - cleandir(unpackdir) - for _,i in ipairs(cleanfiles) do - for _,dir in pairs({maindir, sourcefiledir, docfiledir}) do - errorlevel = rm(dir, i) + errorlevel - end - end - return errorlevel -end - -function bundleclean() - local errorlevel = call(modules, "clean") - for _,i in ipairs(cleanfiles) do - errorlevel = rm(maindir, i) + errorlevel - end - return ( - errorlevel + - rmdir(ctandir) + - rmdir(tdsdir) - ) -end - --- Check commands are defined -function cmdcheck() - mkdir(localdir) - cleandir(testdir) - depinstall(checkdeps) - for _,filetype in pairs( - {bibfiles, docfiles, typesetfiles, typesetdemofiles} - ) do - for _,file in pairs(filetype) do - cp(file, docfiledir, typesetdir) - end - end - for _,file in pairs(sourcefiles) do - cp(file, sourcefiledir, testdir) - end - for _,file in pairs(typesetsuppfiles) do - cp(file, supportdir, testdir) - end - local engine = gsub(stdengine, "tex$", "latex") - local localdir = abspath(localdir) - print("Checking source files") - for _,i in ipairs(cmdchkfiles) do - for _,j in ipairs(filelist(sourcefiledir, i)) do - print(" " .. jobname(j)) - run( - testdir, - os_setenv .. " TEXINPUTS=." .. os_pathsep .. localdir - .. os_pathsep .. - os_concat .. - engine .. " " .. cmdchkopts .. - " \"\\PassOptionsToClass{check}{l3doc} \\input " .. j .. "\"" - .. " > " .. os_null - ) - for line in lines(testdir .. "/" .. jobname(j) .. ".cmds") do - if match(line, "^%!") then - print(" - " .. match(line, "^%! (.*)")) - end - end - end - end -end - -function ctan(standalone) - -- Always run tests for all engines - options["engine"] = nil - local function dirzip(dir, name) - local zipname = name .. ".zip" - local function tab_to_str(table) - local string = "" - for _,i in ipairs(table) do - string = string .. " " .. "\"" .. i .. "\"" - end - return string - end - -- Convert the tables of files to quoted strings - local binfiles = tab_to_str(binaryfiles) - local exclude = tab_to_str(excludefiles) - -- First, zip up all of the text files - run( - dir, - zipexe .. " " .. zipopts .. " -ll ".. zipname .. " " .. "." - .. ( - (binfiles or exclude) and (" -x" .. binfiles .. " " .. exclude) - or "" - ) - ) - -- Then add the binary ones - run( - dir, - zipexe .. " " .. zipopts .. " -g ".. zipname .. " " .. ". -i" .. - binfiles .. (exclude and (" -x" .. exclude) or "") - ) - end - local errorlevel - if standalone then - errorlevel = check() - bundle = module - else - errorlevel = call(modules, "bundlecheck") - end - if errorlevel == 0 then - rmdir(ctandir) - mkdir(ctandir .. "/" .. ctanpkg) - rmdir(tdsdir) - mkdir(tdsdir) - if standalone then - errorlevel = bundlectan() - else - errorlevel = call(modules, "bundlectan") - end - else - print("\n====================") - print("Tests failed, zip stage skipped!") - print("====================\n") - return errorlevel - end - if errorlevel == 0 then - for _,i in ipairs(textfiles) do - for _,j in pairs({unpackdir, currentdir}) do - cp(i, j, ctandir .. "/" .. ctanpkg) - cp(i, j, tdsdir .. "/doc/" .. tdsroot .. "/" .. bundle) - end - end - dirzip(tdsdir, ctanpkg .. ".tds") - if packtdszip then - cp(ctanpkg .. ".tds.zip", tdsdir, ctandir) - end - dirzip(ctandir, ctanpkg) - cp(ctanpkg .. ".zip", ctandir, currentdir) - else - print("\n====================") - print("Typesetting failed, zip stage skipped!") - print("====================\n") - end - return errorlevel -end - -function bundlectan() - -- Generate a list of individual file names excluding those in the second - -- argument: the latter is a table - local function excludelist(include, exclude, dir) - local include = include or { } - local exclude = exclude or { } - local dir = dir or currentdir - local includelist = { } - local excludelist = { } - for _,i in ipairs(exclude) do - for _,j in ipairs(i) do - for file,_ in pairs(tree(dir, j)) do - excludelist[file] = true - end - end - end - for _,i in ipairs(include) do - for file,_ in pairs(tree(dir, i)) do - if not excludelist[file] then - insert(includelist, file) - end - end - end - return includelist - end - unpack() - local errorlevel = doc() - if errorlevel == 0 then - -- Work out what PDF files are available - pdffiles = { } - for _,i in ipairs(typesetfiles) do - insert(pdffiles, (gsub(i, "%.%w+$", ".pdf"))) - end - -- For the purposes here, any typesetting demo files need to be - -- part of the main typesetting list - local typesetfiles = typesetfiles - for _,v in pairs(typesetdemofiles) do - insert(typesetfiles, v) - end - typesetlist = excludelist(typesetfiles, {sourcefiles}, docfiledir) - sourcelist = excludelist( - sourcefiles, {bstfiles, installfiles, makeindexfiles}, sourcefiledir - ) - copyctan() - copytds() - end - return errorlevel -end - --- A hook to allow additional typesetting of demos -typeset_demo_tasks = typeset_demo_tasks or function() - return 0 -end - --- Typeset all required documents --- Uses a set of dedicated auxiliaries that need to be available to others -function doc(files) - -- Set up - cleandir(typesetdir) - for _,filetype in pairs( - {bibfiles, docfiles, typesetfiles, typesetdemofiles} - ) do - for _,file in pairs(filetype) do - cp(file, docfiledir, typesetdir) - end - end - for _,file in pairs(sourcefiles) do - cp(file, sourcefiledir, typesetdir) - end - for _,file in pairs(typesetsuppfiles) do - cp(file, supportdir, typesetdir) - end - depinstall(typesetdeps) - unpack({sourcefiles, typesetsourcefiles}, {sourcefiledir, docfiledir}) - -- Main loop for doc creation - local done = {} - local errorlevel = typeset_demo_tasks() - if errorlevel ~= 0 then - return errorlevel - end - for _, typesetfiles in ipairs({typesetdemofiles, typesetfiles}) do - for _,i in ipairs(typesetfiles) do - for _, dir in ipairs({unpackdir, typesetdir}) do - for j,_ in pairs(tree(dir, i)) do - if not done[j] then - j = gsub(j, "^%./", "") - -- Allow for command line selection of files - local typeset = true - if files and next(files) then - typeset = false - for _,k in ipairs(files) do - if k == gsub(j, "%.[^.]+$", "") then - typeset = true - break - end - end - end - if typeset then - local errorlevel = typesetpdf(j, dir) - if errorlevel ~= 0 then - return errorlevel - else - done[j] = true - end - end - end - end - end - end - end - return 0 -end - --- Locally install files: only deals with those extracted, not docs etc. -function install() - local errorlevel = unpack() - if errorlevel ~= 0 then - return errorlevel - end - set_program_name("latex") - local texmfhome = var_value("TEXMFHOME") - local installdir = texmfhome .. "/tex/" .. moduledir - errorlevel = cleandir(installdir) - if errorlevel ~= 0 then - return errorlevel - end - for _,i in ipairs(installfiles) do - errorlevel = cp(i, unpackdir, installdir) - if errorlevel ~= 0 then - return errorlevel - end - end - return 0 -end - -function save(names) - checkinit() - local engines = options["engine"] or {stdengine} - for _,name in pairs(names) do - local engine - for _,engine in pairs(engines) do - local tlgengine = ((engine == stdengine and "") or "." .. engine) - local tlgfile = name .. tlgengine .. tlgext - local spdffile = name .. tlgengine .. pdfext - local newfile = name .. "." .. engine .. logext - local pdffile = name .. "." .. engine .. pdfext - local refext = ((options["pdf"] and pdfext) or tlgext) - if testexists(name) then - print("Creating and copying " .. refext) - runtest(name, engine, false, lvtext, options["pdf"]) - if options["pdf"] then - ren(testdir, pdffile, spdffile) - cp(spdffile, testdir, testfiledir) - else - ren(testdir, newfile, tlgfile) - cp(tlgfile, testdir, testfiledir) - end - if fileexists(unpackdir .. "/" .. tlgfile) then - print( - "Saved " .. tlgext - .. " file overrides unpacked version of the same name" - ) - end - elseif locate({unpackdir, testfiledir}, {name .. lveext}) then - print( - "Saved " .. tlgext .. " file overrides a " - .. lveext .. " file of the same name" - ) - else - print( - "Test input \"" .. testfiledir .. "/" .. name .. lvtext - .. "\" not found" - ) - end - end - end -end - --- Provide some standard search-and-replace functions -if versionform ~= "" and not setversion_update_line then - if versionform == "ProvidesPackage" then - function setversion_update_line(line, date, version) - -- No real regex so do it one type at a time - for _,i in pairs({"Class", "File", "Package"}) do - if match( - line, - "^\\Provides" .. i .. "{[a-zA-Z0-9%-%.]+}%[[^%]]*%]$" - ) then - line = gsub(line, "%[%d%d%d%d/%d%d/%d%d", "[" - .. gsub(date, "%-", "/")) - line = gsub( - line, "(%[%d%d%d%d/%d%d/%d%d) [^ ]*", "%1 " .. version - ) - break - end - end - return line - end - elseif versionform == "ProvidesExplPackage" then - function setversion_update_line(line, date, version) - -- No real regex so do it one type at a time - for _,i in pairs({"Class", "File", "Package"}) do - if match( - line, - "^\\ProvidesExpl" .. i .. " *{[a-zA-Z0-9%-%.]+}" - ) then - line = gsub( - line, - "{%d%d%d%d/%d%d/%d%d}( *){[^}]*}", - "{" .. gsub(date, "%-", "/") .. "}%1{" .. version .. "}" - ) - break - end - end - return line - end - elseif versionform == "filename" then - function setversion_update_line(line, date, version) - if match(line, "^\\def\\filedate{%d%d%d%d/%d%d/%d%d}$") then - line = "\\def\\filedate{" .. gsub(date, "%-", "/") .. "}" - end - if match(line, "^\\def\\fileversion{[^}]+}$") then - line = "\\def\\fileversion{" .. version .. "}" - end - return line - end - elseif versionform == "ExplFileDate" then - function setversion_update_line(line, date, version) - if match(line, "^\\def\\ExplFileDate{%d%d%d%d/%d%d/%d%d}$") then - line = "\\def\\ExplFileDate{" .. gsub(date, "%-", "/") .. "}" - end - if match(line, "^\\def\\ExplFileVersion{[^}]+}$") then - line = "\\def\\ExplFileVersion{" .. version .. "}" - end - return line - end - end -end - --- Used to actually carry out search-and-replace -setversion_update_line = setversion_update_line or function(line, date, version) - return line -end - -function setversion() - local function rewrite(dir, file, date, version) - local changed = false - local result = "" - for line in lines(dir .. "/" .. file) do - local newline = setversion_update_line(line, date, version) - if newline ~= line then - line = newline - changed = true - end - result = result .. line .. os_newline - end - if changed then - -- Avoid adding/removing end-of-file newline - local f = open(dir .. "/" .. file, "rb") - local content = f:read("*all") - close(f) - if not match(content, os_newline .. "$") then - gsub(result, os_newline .. "$", "") - end - -- Write the new file - ren(dir, file, file .. bakext) - local f = open(dir .. "/" .. file, "w") - output(f) - write(result) - close(f) - rm(dir, file .. bakext) - end - end - local date = options["date"] or os_date("%Y-%m-%d") - local version = options["version"] or -1 - for _,dir in pairs({currentdir, sourcefiledir, docfiledir}) do - for _,i in pairs(versionfiles) do - for file,_ in pairs(tree(dir, i)) do - rewrite(dir, file, date, version) - end - end - end - return 0 -end - --- Unpack the package files using an 'isolated' system: this requires --- a copy of the 'basic' DocStrip program, which is used then removed -function unpack(sources, sourcedirs) - local errorlevel = depinstall(unpackdeps) - if errorlevel ~= 0 then - return errorlevel - end - errorlevel = bundleunpack(sourcedirs, sources) - if errorlevel ~= 0 then - return errorlevel - end - for _,i in ipairs(installfiles) do - errorlevel = cp(i, unpackdir, localdir) - if errorlevel ~= 0 then - return errorlevel - end - end - return 0 -end - --- Split off from the main unpack so it can be used on a bundle and not --- leave only one modules files -bundleunpack = bundleunpack or function(sourcedirs, sources) - local errorlevel = mkdir(localdir) - if errorlevel ~=0 then - return errorlevel - end - errorlevel = cleandir(unpackdir) - if errorlevel ~=0 then - return errorlevel - end - for _,i in ipairs(sourcedirs or {sourcefiledir}) do - for _,j in ipairs(sources or {sourcefiles}) do - for _,k in ipairs(j) do - errorlevel = cp(k, i, unpackdir) - if errorlevel ~=0 then - return errorlevel - end - end - end - end - for _,i in ipairs(unpacksuppfiles) do - errorlevel = cp(i, supportdir, localdir) - if errorlevel ~=0 then - return errorlevel - end - end - for _,i in ipairs(unpackfiles) do - for j,_ in pairs(tree(unpackdir, i)) do - -- This 'yes' business is needed to pass a series of "y\n" to - -- TeX if \askforoverwrite is true - -- That is all done using a file as it's the only way on Windows and - -- on Unix the "yes" command can't be used inside execute (it never - -- stops, which confuses Lua) - execute(os_yes .. ">>" .. localdir .. "/yes") - local path, name = splitpath(j) - local localdir = abspath(localdir) - errorlevel = run( - unpackdir .. "/" .. path, - os_setenv .. " TEXINPUTS=." .. os_pathsep - .. localdir .. (unpacksearch and os_pathsep or "") .. - os_concat .. - unpackexe .. " " .. unpackopts .. " " .. name .. " < " - .. localdir .. "/yes" - .. (options["quiet"] and (" > " .. os_null) or "") - ) - if errorlevel ~=0 then - return errorlevel - end - end - end - return 0 -end - -function version() - print( - "\n" .. - "l3build: A testing and building system for LaTeX\n\n" .. - "Release " .. release_date - ) -end - --- --- The overall main function --- - -function stdmain(target, files) - local errorlevel - -- If the module name is empty, the script is running in a bundle: - -- apart from ctan all of the targets are then just mappings - if module == "" then - -- Detect all of the modules - modules = modules or listmodules() - if target == "doc" then - errorlevel = call(modules, "doc") - elseif target == "check" then - errorlevel = call(modules, "bundlecheck") - if errorlevel ~=0 then - print("There were errors: checks halted!\n") - end - elseif target == "clean" then - errorlevel = bundleclean() - elseif target == "cmdcheck" and next(cmdchkfiles) ~= nil then - errorlevel = call(modules, "cmdcheck") - elseif target == "ctan" then - errorlevel = ctan() - elseif target == "install" then - errorlevel = call(modules, "install") - elseif target == "setversion" then - errorlevel = call(modules, "setversion") - -- Deal with any files in the bundle dir itself - if errorlevel == 0 then - errorlevel = setversion() - end - elseif target == "unpack" then - errorlevel = call(modules, "bundleunpack") - elseif target == "version" then - version() - else - help() - end - else - if target == "bundleunpack" then -- 'Hidden' as only needed 'higher up' - depinstall(unpackdeps) - errorlevel = bundleunpack() - elseif target == "bundlecheck" then - errorlevel = check() - elseif target == "bundlectan" then - errorlevel = bundlectan() - elseif target == "doc" then - errorlevel = doc(files) - elseif target == "check" then - errorlevel = check(files) - elseif target == "clean" then - errorlevel = clean() - elseif target == "cmdcheck" and next(cmdchkfiles) ~= nil then - errorlevel = cmdcheck() - elseif target == "ctan" and bundle == "" then -- Stand-alone module - errorlevel = ctan(true) - elseif target == "install" then - errorlevel = install() - elseif target == "save" then - if next(files) then - errorlevel = save(files) - else - help() - end - elseif target == "setversion" then - errorlevel = setversion() - elseif target == "unpack" then - errorlevel = unpack() - elseif target == "version" then - version() - else - help() - end - end - if errorlevel ~= 0 then - exit(1) - else - exit(0) - end -end +-- l3build setup and functions + +kpse.set_program_name("kpsewhich") +build_kpse_path = string.match(kpse.lookup("l3build.lua"),"(.*[/])") +local function build_require(s) + require( kpse.lookup("l3build-"..s..".lua", { path = build_kpse_path } ) ) +end + +build_require("variables") +build_require("arguments") +build_require("file-functions") +build_require("typesetting") +build_require("aux") +build_require("clean") +build_require("check") +build_require("ctan") +build_require("install") +build_require("unpack") +build_require("manifest") +build_require("manifest-setup") +build_require("setversion") +build_require("help") +build_require("stdmain") -- Allow main function to be disabled 'higher up' main = main or stdmain -- cgit v1.2.3