l3build (12dec17)

git-svn-id: svn://tug.org/texlive/trunk@46055 c570f23f-e606-0410-a88d-b1316a301751

1 files changed, 141 insertions, 82 deletions

diff --git a/Master/texmf-dist/tex/latex/l3build/l3build.lua b/Master/texmf-dist/tex/latex/l3build/l3build.lua
index 5c5469375b9..c4c76c7a1bb 100644
--- a/Master/texmf-dist/tex/latex/l3build/l3build.lua
+++ b/Master/texmf-dist/tex/latex/l3build/l3build.lua
@@ -23,7 +23,7 @@ for those people who are interested.
 --]]
 
 -- Version information
-release_date = "2017/12/06"
+release_date = "2017/12/12"
 
 -- "module" is a deprecated function in Lua 5.2: as we want the name
 -- for other purposes, and it should eventually be 'free', simply
@@ -61,20 +61,23 @@ end
 
 -- Directory structure for the build system
 -- Use Unix-style path separators
-maindir = maindir or "."
+currentdir = "."
+maindir    = maindir or currentdir
 
 -- Substructure for file locations
-docfiledir  = docfiledir  or maindir
-supportdir  = supportdir  or maindir .. "/support"
-testfiledir = testfiledir or "testfiles"
-testsuppdir = testsuppdir or testfiledir .. "/support"
+docfiledir    = docfiledir    or currentdir
+sourcefiledir = sourcefiledir or currentdir
+supportdir    = supportdir    or maindir .. "/support"
+testfiledir   = testfiledir   or currentdir .. "/testfiles"
+testsuppdir   = testsuppdir   or testfiledir .. "/support"
 
 -- Structure within a development area
-distribdir = distribdir or maindir .. "/build/distrib"
-localdir   = localdir   or maindir .. "/build/local"
-testdir    = testdir    or maindir .. "/build/test"
-typesetdir = typesetdir or maindir .. "/build/doc"
-unpackdir  = unpackdir  or maindir .. "/build/unpacked"
+builddir   = builddir   or maindir .. "/build"
+distribdir = distribdir or builddir .. "/distrib"
+localdir   = localdir   or builddir .. "/local"
+testdir    = testdir    or builddir .. "/test"
+typesetdir = typesetdir or builddir .. "/doc"
+unpackdir  = unpackdir  or builddir .. "/unpacked"
 
 -- Substructure for CTAN release material
 ctandir = ctandir or distribdir .. "/ctan"
@@ -136,6 +139,10 @@ checkengines = checkengines or {"pdftex", "xetex", "luatex"}
 checkformat  = checkformat  or "latex"
 stdengine    = stdengine    or "pdftex"
 
+-- Configs for testing
+stdconfig    = stdconfig    or string.gsub(arg[0], "%.lua$", "")
+checkconfigs = checkconfigs or {stdconfig}
+
 -- Enable access to trees outside of the repo
 -- As these may be set false, a more elaborate test than normal is needed
 if checksearch == nil then
@@ -238,6 +245,12 @@ local utf8_char        = unicode.utf8.char
 
 local option_list =
   {
+    config =
+      {
+        desc  = "Sets the config(s) used for running tests",
+        short = "c",
+        type  = "table"
+      },
     date =
       {
         desc  = "Sets the date to insert into sources",
@@ -246,7 +259,7 @@ local option_list =
       },
     engine =
       {
-        desc  = "Sets the engine to use for running test",
+        desc  = "Sets the engine(s) to use for running test",
         short = "e",
         type  = "table"
       },
@@ -291,12 +304,6 @@ local option_list =
         short = "r",
         type  = "boolean"
       },
-    testfiledir =
-      {
-        desc  = "Selects the specified testfile location",
-        short = "t",
-        type  = "table"
-      },
     version =
       {
         desc  = "Sets the version to insert into sources",
@@ -609,7 +616,7 @@ function cp(glob, source, dest)
       else
         errorlevel = execute(
           'xcopy /y "' .. unix_to_win(source) .. '" "'
-             .. unix_to_win(dest) .. '" > nul'
+             .. unix_to_win(dest .. '/') .. '" > nul'
         )
       end
     else
@@ -675,28 +682,28 @@ end
 -- table, the keys are paths relative to the given starting path, the values
 -- are their counterparts relative to the current working directory.
 function tree(path, glob)
-  function cropdots(path)
+  local function cropdots(path)
     return gsub(gsub(path, "^%./", ""), "/%./", "/")
   end
-  function always_true()
+  local function always_true()
     return true
   end
-  function is_dir(file)
+  local function is_dir(file)
     return lfs_attributes(file)["mode"] == "directory"
   end
   local dirs = {["."] = cropdots(path)}
-  for pattern, critereon in gmatch(cropdots(glob), "([^/]+)(/?)") do
-    local critereon = critereon == "/" and is_dir or always_true
+  for pattern, criterion in gmatch(cropdots(glob), "([^/]+)(/?)") do
+    local criterion = criterion == "/" and is_dir or always_true
     function fill(path, dir, table)
       for _, file in ipairs(filelist(dir, pattern)) do
         local fullpath = path .. "/" .. file
         if file ~= "." and file ~= ".." and
-          fullpath ~= maindir .. "/build" and
+          fullpath ~= builddir and
           (sub(pattern, 1, 1) == "."
             or sub(file, 1, 1) ~= ".")
         then
           local fulldir = dir .. "/" .. file
-          if critereon(fulldir) then
+          if criterion(fulldir) then
             table[fullpath] = fulldir
           end
         end
@@ -740,6 +747,8 @@ end
 function ren(dir, source, dest)
   local dir = dir .. "/"
   if os_type == "windows" then
+    local source = gsub(source, "^%.+/", "")
+    local dest = gsub(dest, "^%.+/", "")
     return execute("ren " .. unix_to_win(dir) .. source .. " " .. dest)
   else
     return execute("mv " .. dir .. source .. " " .. dir .. dest)
@@ -837,7 +846,7 @@ function checkinit()
   for _,i in ipairs(filelist(localdir)) do
     cp(i, localdir, testdir)
   end
-  bundleunpack({".", testfiledir})
+  bundleunpack({sourcefiledir, testfiledir})
   for _,i in ipairs(installfiles) do
     cp(i, unpackdir, testdir)
   end
@@ -857,22 +866,37 @@ end
 
 -- Copy files to the main CTAN release directory
 function copyctan()
-  -- Do all of the copying in one go
-  for _,i in ipairs(
+  local ctantarget = ctanpkg
+  if docfiledir ~= currentdir then
+    ctantarget = ctanpkg .. "/" .. gsub(docfiledir, "^%.*/", "")
+  end
+  mkdir(ctandir .. "/" .. ctantarget)
+  for _,filetype in pairs(
       {
         bibfiles,
         demofiles,
         docfiles,
         pdffiles,
-        sourcefiles,
-        textfiles,
         typesetlist
       }
     ) do
-    for _,j in ipairs(i) do
-      cp(j, ".", ctandir .. "/" .. ctanpkg)
+    for _,file in pairs(filetype) do
+      cp(file, docfiledir, ctandir .. "/" .. ctantarget)
     end
   end
+  ctantarget = ctanpkg
+  if sourcefiledir ~= currentdir then
+    ctantarget = ctanpkg .. "/" .. gsub(sourcefiledir, "^%.*/", "")
+  end
+  mkdir(ctandir .. "/" .. ctantarget)
+  for _,file in pairs(sourcefiles) do
+    if sourcedir ~= currentdir then
+    end
+    cp(file, sourcefiledir, ctandir .. "/" .. ctantarget)
+  end
+  for _,file in pairs(textfiles) do
+    cp(file, currentdir, ctandir .. "/" .. ctanpkg)
+  end
 end
 
 -- Copy files to the correct places in the TDS tree
@@ -896,8 +920,8 @@ function copytds()
     local filenames = { }
     for _,i in ipairs(files) do
       for _,j in ipairs(i) do
-        for _,k in ipairs(filelist(source, j)) do
-          insert(filenames, k)
+        for file,_ in pairs(tree(source, j)) do
+          insert(filenames, file)
         end
       end
     end
@@ -911,13 +935,13 @@ function copytds()
     end
   end
   install(
-    ".",
+    docfiledir,
     "doc",
     {bibfiles, demofiles, docfiles, pdffiles, textfiles, typesetlist}
   )
   install(unpackdir, "makeindex", {makeindexfiles}, true)
   install(unpackdir, "bibtex/bst", {bstfiles}, true)
-  install(".", "source", {sourcelist})
+  install(sourcefiledir, "source", {sourcelist})
   install(unpackdir, "tex", {installfiles})
 end
 
@@ -1746,7 +1770,7 @@ function typesetpdf(file, dir)
   if errorlevel == 0 then
     name = name .. ".pdf"
     os_remove(jobname(name))
-    cp(name, typesetdir, ".")
+    cp(name, typesetdir, docfiledir)
   else
     print(" ! Compilation failed")
   end
@@ -1920,7 +1944,9 @@ function clean()
     cleandir(typesetdir) +
     cleandir(unpackdir)
   for _,i in ipairs(cleanfiles) do
-    errorlevel = rm(".", i) + errorlevel
+    for _,dir in pairs({maindir, sourcefiledir, docfiledir}) do
+      errorlevel = rm(dir, i) + errorlevel
+    end
   end
   return errorlevel
 end
@@ -1928,7 +1954,7 @@ end
 function bundleclean()
   local errorlevel = call(modules, "clean")
   for _,i in ipairs(cleanfiles) do
-    errorlevel = rm(".", i) + errorlevel
+    errorlevel = rm(maindir, i) + errorlevel
   end
   return (
     errorlevel     +
@@ -1942,19 +1968,24 @@ function cmdcheck()
   mkdir(localdir)
   cleandir(testdir)
   depinstall(checkdeps)
-  for _,i in ipairs({bibfiles, docfiles, sourcefiles, typesetfiles}) do
-    for _,j in ipairs(i) do
-      cp(j, ".", testdir)
+  for _,filetype in pairs(
+      {bibfiles, docfiles, typesetfiles, typesetdemofiles}
+    ) do
+    for _,file in pairs(filetype) do
+      cp(file, docfiledir, typesetdir)
     end
   end
-  for _,i in ipairs(typesetsuppfiles) do
-    cp(i, supportdir, testdir)
+  for _,file in pairs(sourcefiles) do
+    cp(file, sourcefiledir, testdir)
+  end
+  for _,file in pairs(typesetsuppfiles) do
+    cp(file, supportdir, testdir)
   end
   local engine = gsub(stdengine, "tex$", "latex")
   local localdir = abspath(localdir)
   print("Checking source files")
   for _,i in ipairs(cmdchkfiles) do
-    for _,j in ipairs(filelist(".", i)) do
+    for _,j in ipairs(filelist(sourcefiledir, i)) do
       print("  " .. jobname(j))
       run(
         testdir,
@@ -2030,7 +2061,7 @@ function ctan(standalone)
   end
   if errorlevel == 0 then
     for _,i in ipairs(textfiles) do
-      for _,j in pairs({unpackdir, "."}) do
+      for _,j in pairs({unpackdir, currentdir}) do
         cp(i, j, ctandir .. "/" .. ctanpkg)
         cp(i, j, tdsdir .. "/doc/" .. tdsroot .. "/" .. bundle)
       end
@@ -2040,7 +2071,7 @@ function ctan(standalone)
       cp(ctanpkg .. ".tds.zip", tdsdir, ctandir)
     end
     dirzip(ctandir, ctanpkg)
-    cp(ctanpkg .. ".zip", ctandir, ".")
+    cp(ctanpkg .. ".zip", ctandir, currentdir)
   else
     print("\n====================")
     print("Typesetting failed, zip stage skipped!")
@@ -2052,22 +2083,23 @@ end
 function bundlectan()
   -- Generate a list of individual file names excluding those in the second
   -- argument: the latter is a table
-  local function excludelist(include, exclude)
+  local function excludelist(include, exclude, dir)
     local include = include or { }
     local exclude = exclude or { }
+    local dir = dir or currentdir
     local includelist = { }
     local excludelist = { }
     for _,i in ipairs(exclude) do
       for _,j in ipairs(i) do
-        for _,k in ipairs(filelist(".", j)) do
-          excludelist[k] = true
+        for file,_ in pairs(tree(dir, j)) do
+          excludelist[file] = true
         end
       end
     end
     for _,i in ipairs(include) do
-      for _,j in ipairs(filelist(".", i)) do
-        if not excludelist[j] then
-          insert(includelist, j)
+      for file,_ in pairs(tree(dir, i)) do
+        if not excludelist[file] then
+          insert(includelist, file)
         end
       end
     end
@@ -2087,9 +2119,9 @@ function bundlectan()
     for _,v in pairs(typesetdemofiles) do
       insert(typesetfiles, v)
     end
-    typesetlist = excludelist(typesetfiles, {sourcefiles})
+    typesetlist = excludelist(typesetfiles, {sourcefiles}, docfiledir)
     sourcelist = excludelist(
-      sourcefiles, {bstfiles, installfiles, makeindexfiles}
+      sourcefiles, {bstfiles, installfiles, makeindexfiles}, sourcefiledir
     )
     copyctan()
     copytds()
@@ -2097,25 +2129,37 @@ function bundlectan()
   return errorlevel
 end
 
+-- A hook to allow additional typesetting of demos
+typeset_demo_tasks = typeset_demo_tasks or function()
+  return 0
+end
+
 -- Typeset all required documents
 -- Uses a set of dedicated auxiliaries that need to be available to others
 function doc(files)
   -- Set up
   cleandir(typesetdir)
-  for _,i in ipairs(
-    {bibfiles, docfiles, sourcefiles, typesetfiles, typesetdemofiles}
-  ) do
-    for _,j in ipairs(i) do
-      cp(j, ".", typesetdir)
-    end
-  end
-  for _,i in ipairs(typesetsuppfiles) do
-    cp(i, supportdir, typesetdir)
+  for _,filetype in pairs( 
+      {bibfiles, docfiles, typesetfiles, typesetdemofiles} 
+    ) do 
+    for _,file in pairs(filetype) do 
+      cp(file, docfiledir, typesetdir) 
+    end 
+  end 
+  for _,file in pairs(sourcefiles) do 
+    cp(file, sourcefiledir, typesetdir) 
+  end 
+  for _,file in pairs(typesetsuppfiles) do
+    cp(file, supportdir, typesetdir)
   end
   depinstall(typesetdeps)
-  unpack({sourcefiles, typesetsourcefiles}, {".", docfiledir})
+  unpack({sourcefiles, typesetsourcefiles}, {sourcefiledir, docfiledir})
   -- Main loop for doc creation
   local done = {}
+  local errorlevel = typeset_demo_tasks()
+  if errorlevel ~= 0 then
+    return errorlevel
+  end
   for _, typesetfiles in ipairs({typesetdemofiles, typesetfiles}) do
     for _,i in ipairs(typesetfiles) do
       for _, dir in ipairs({unpackdir, typesetdir}) do
@@ -2280,7 +2324,7 @@ setversion_update_line = setversion_update_line or function(line, date, version)
   return line
 end
 
-function setversion(dir)
+function setversion()
   local function rewrite(dir, file, date, version)
     local changed = false
     local result = ""
@@ -2311,10 +2355,11 @@ function setversion(dir)
   end
   local date = options["date"] or os_date("%Y-%m-%d")
   local version = options["version"] or -1
-  local dir = dir or "."
-  for _,i in pairs(versionfiles) do
-    for _,j in pairs(filelist(dir, i)) do
-      rewrite(dir, j, date, version)
+  for _,dir in pairs({currentdir, sourcefiledir, docfiledir}) do
+    for _,i in pairs(versionfiles) do
+      for file,_ in pairs(tree(dir, i)) do
+        rewrite(dir, file, date, version)
+      end
     end
   end
   return 0
@@ -2351,7 +2396,7 @@ bundleunpack = bundleunpack or function(sourcedirs, sources)
   if errorlevel ~=0 then
     return errorlevel
   end
-  for _,i in ipairs(sourcedirs or {"."}) do
+  for _,i in ipairs(sourcedirs or {sourcefiledir}) do
     for _,j in ipairs(sources or {sourcefiles}) do
       for _,k in ipairs(j) do
         errorlevel = cp(k, i, unpackdir)
@@ -2487,18 +2532,32 @@ end
 -- Allow main function to be disabled 'higher up'
 main = main or stdmain
 
--- Pick up and read any per-run testfiledir
-if options["testfiledir"] then
-  if #options["testfiledir"] == 1 then
-    testfiledir = options["testfiledir"][1]
-    if fileexists(testfiledir .. "/config.lua") then
-      dofile(testfiledir .. "/config.lua")
-    end
-  else
-    print("Cannot use more than one testfile dir at a time!")
-    return 1
+-- Deal with multiple configs for tests
+checkconfigs = options["config"] or checkconfigs
+if options["target"] == "check" then
+  if #checkconfigs > 1 then
+    local errorlevel = 0
+    local opts = options
+    for i = 1, #checkconfigs do
+      opts["config"] = {checkconfigs[i]}
+      errorlevel = call({"."}, "check", opts)
+      if errorlevel ~= 0 then exit(1) end
+    end
+    -- Avoid running the 'main' set of tests twice
+    exit(0)
   end
 end
+if #checkconfigs == 1 and
+   checkconfigs[1] ~= stdconfig and
+   (options["target"] == "check" or options["target"] == "save") then
+   local config = "./" .. checkconfigs[1] .. ".lua"
+   if fileexists(config) then
+     dofile(config)
+   else
+     print("Error: Cannot find configuration " ..  checkconfigs[1])
+     exit(1)
+   end
+end
 
 -- Call the main function
 main(options["target"], options["files"])