l3build (13sep17)

git-svn-id: svn://tug.org/texlive/trunk@45283 c570f23f-e606-0410-a88d-b1316a301751

1 files changed, 255 insertions, 137 deletions

diff --git a/Master/texmf-dist/tex/latex/l3build/l3build.lua b/Master/texmf-dist/tex/latex/l3build/l3build.lua
index f5b15f4088f..bb4bf12df5f 100644
--- a/Master/texmf-dist/tex/latex/l3build/l3build.lua
+++ b/Master/texmf-dist/tex/latex/l3build/l3build.lua
@@ -23,7 +23,7 @@ for those people who are interested.
 --]]
 
 -- Version information
-release_date = "2017/07/01"
+release_date = "2017/09/12"
 
 -- "module" is a deprecated function in Lua 5.2: as we want the name
 -- for other purposes, and it should eventually be 'free', simply
@@ -108,6 +108,7 @@ sourcefiles        = sourcefiles        or {"*.dtx", "*.ins"}
 textfiles          = textfiles          or {"*.md", "*.txt"}
 typesetdemofiles   = typesetdemofiles   or { }
 typesetfiles       = typesetfiles       or {"*.dtx"}
+docfiledir         = docfiledir         or maindir
 typesetsuppfiles   = typesetsuppfiles   or { }
 typesetsourcefiles = typesetsourcefiles or { }
 unpackfiles        = unpackfiles        or {"*.ins"}
@@ -159,6 +160,14 @@ bibtexopts    = bibtexopts    or "-W"
 makeindexexe  = makeindexexe  or "makeindex"
 makeindexopts = makeindexopts or ""
 
+-- Forcing epoch
+if forcecheckepoch == nil then
+  forcecheckepoch = true
+end
+if forcedocepoch == nil then
+  forcedocepoch = true
+end
+
 -- Other required settings
 asciiengines = asciiengines or {"pdftex"}
 checkruns    = checkruns    or 1
@@ -168,6 +177,7 @@ packtdszip   = packtdszip   or false
 scriptname   = scriptname   or "build.lua"
 typesetcmds  = typesetcmds  or ""
 versionform  = versionform  or ""
+recordstatus = recordstatus or false
 
 -- Extensions for various file types: used to abstract out stuff a bit
 bakext = bakext or ".bak"
@@ -204,7 +214,9 @@ local execute          = os.execute
 local exit             = os.exit
 local getenv           = os.getenv
 local os_remove        = os.remove
+local os_time          = os.time
 local os_type          = os.type
+local len              = string.len
 local luatex_revision  = status.luatex_revision
 local luatex_version   = status.luatex_version
 local char             = string.char
@@ -214,57 +226,96 @@ local gmatch           = string.gmatch
 local gsub             = string.gsub
 local len              = string.len
 local match            = string.match
+local rep              = string.rep
+local sort             = table.sort
 local sub              = string.sub
 local concat           = table.concat
 local insert           = table.insert
 local utf8_char        = unicode.utf8.char
 
 -- Parse command line options
+
+local option_list =
+  {
+    date =
+      {
+        desc  = "Sets the date to insert into sources",
+        short = "d",
+        type  = "string"
+      },
+    engine =
+      {
+        desc  = "Sets the engine to use for running test",
+        short = "e",
+        type  = "table"
+      },
+    epoch =
+      {
+        desc  = "Sets the epoch for tests and typesetting",
+        short = "E",
+        type  = "string"
+      },
+    force =
+      {
+        desc  = "Force tests to run if engine is not set up",
+        short = "f",
+        type  = "boolean"
+      },
+    ["halt-on-error"] =
+      {
+        desc  = "Stops running tests after the first failure",
+        short = "H",
+        type  = "boolean"
+      },
+    help =
+      {
+        short = "h",
+        type  = "boolean"
+      },
+    pdf =
+      {
+        desc  = "Check/save PDF files",
+        short = "p",
+        type  = "boolean"
+      },
+    quiet =
+      {
+        desc  = "Suppresses TeX output when unpacking",
+        short = "q",
+        type  = "boolean"
+      },
+    rerun =
+      {
+        desc  = "Suppresses TeX output when unpacking",
+        short = "r",
+        type  = "boolean"
+      },
+    testfiledir =
+      {
+        desc  = "Selects the specified testfile location",
+        short = "t",
+        type  = "table"
+      },
+    version =
+      {
+        desc  = "Sets the version to insert into sources",
+        short = "v",
+        type  = "string"
+      },
+  }
+
 -- This is done as a function (rather than do ... end) as it allows early
 -- termination (break)
 local function argparse()
   local result = { }
   local files  = { }
-  local long_options =
-    {
-      date                = "date"       ,
-      engine              = "engine"     ,
-      force               = "force"      ,
-      ["halt-on-error"]   = "halt"       ,
-      ["halt-on-failure"] = "halt"       ,
-      help                = "help"       ,
-      pdf                 = "pdf"        ,
-      quiet               = "quiet"      ,
-      rerun               = "rerun"      ,
-      testfiledir         = "testfiledir",
-      version             = "version"
-    }
-  local short_options =
-    {
-      d = "date"       ,
-      e = "engine"     ,
-      f = "force"      ,
-      h = "help"       ,
-      H = "halt"       ,
-      p = "pdf"        ,
-      q = "quiet"      ,
-      r = "rerun"      ,
-      t = "testfiledir",
-      v = "version"
-    }
-  local option_args =
-    {
-      date        = true ,
-      engine      = true ,
-      force       = false,
-      halt        = false,
-      help        = false,
-      pdf         = false,
-      quiet       = false,
-      rerun       = false,
-      testfiledir = true ,
-      version     = true
-    }
+  local long_options =  { }
+  local short_options = { }
+  -- Turn long/short options into two lookup tables
+  for k,v in pairs(option_list) do
+    short_options[v["short"]] = k
+    long_options[k] = k
+  end
   local args = args
   -- arg[1] is a special case: must be a command or "-h"/"--help"
   -- Deal with this by assuming help and storing only apparently-valid
@@ -327,19 +378,22 @@ local function argparse()
       -- if required
       local optname = opts[opt]
       if optname then
-        local reqarg = option_args[optname]
         -- Tidy up arguments
-        if reqarg and not optarg then
+        if option_list[optname]["type"] == "boolean" then
+          if optarg then
+            local opt = "-" .. (match(a, "^%-%-") and "-" or "") .. opt
+            stderr:write("Value not allowed for option " .. opt .."\n")
+            return {"help"}
+          end
+        else
+         if not optarg then
           optarg = arg[i + 1]
           if not optarg then
             stderr:write("Missing value for option " .. a .."\n")
             return {"help"}
           end
           i = i + 1
-        end
-        if not reqarg and optarg then
-          stderr:write("Value not allowed for option " .. a .."\n")
-          return {"help"}
+         end
         end
       else
         stderr:write("Unknown option " .. a .."\n")
@@ -347,11 +401,15 @@ local function argparse()
       end
       -- Store the result
       if optarg then
-        local opts = result[optname] or { }
-        for hit in gmatch(optarg, "([^,%s]+)") do
-          insert(opts, hit)
+        if option_list[optname]["type"] == "string" then
+          result[optname] = optarg
+        else
+          local opts = result[optname] or { }
+          for hit in gmatch(optarg, "([^,%s]+)") do
+            insert(opts, hit)
+          end
+          result[optname] = opts
         end
-        result[optname] = opts
       else
         result[optname] = true
       end
@@ -362,29 +420,22 @@ local function argparse()
       break
     end
   end
-  result["files"] = files
+  if next(files) then
+   result["files"] = files
+  end
   return result
 end
 
 options = argparse()
 
-local optdate    = options["date"]
-local optengines = options["engine"]
-local optforce   = options["force"]
-local opthalt    = options["halt"]
-local optpdf     = options["pdf"]
-local optquiet   = options["quiet"]
-local optrerun   = options["rerun"]
-local optversion = options["version"]
-
 -- Sanity check
-if optengines and not optforce then
+if options["engine"] and not options["force"] then
    -- Make a lookup table
    local t = { }
   for _, engine in pairs(checkengines) do
     t[engine] = true
   end
-  for _, engine in pairs(optengines) do
+  for _, engine in pairs(options["engine"]) do
     if not t[engine] then
       print("\n! Error: Engine \"" .. engine .. "\" not set up for testing!")
       print("\n  Valid values are:")
@@ -397,6 +448,27 @@ if optengines and not optforce then
   end
 end
 
+-- Tidy up the epoch setting
+-- Force an epoch if set at the command line
+if options["epoch"] then
+  epoch           = options["epoch"]
+  forcecheckepoch = true
+  forcedocepoch   = true
+end
+-- If given as an ISO date, turn into an epoch number
+do
+  local y, m, d = match(epoch, "^(%d%d%d%d)-(%d%d)-(%d%d)$")
+  if y then
+    epoch =
+      os_time({year = y, month = m, day = d, hour = 0, sec = 0, isdst = nil}) -
+      os_time({year = 1970, month = 1, day = 1, hour = 0, sec = 0, isdst = nil})
+  elseif match(epoch, "^%d+$") then
+    epoch = tonumber(epoch)
+  else
+    epoch = 0
+  end
+end
+
 -- Convert a file glob into a pattern for use by e.g. string.gub
 -- Based on https://github.com/davidm/lua-glob-pattern
 -- Simplified substantially: "[...]" syntax not supported as is not
@@ -530,17 +602,17 @@ function cp(glob, source, dest)
     if os_type == "windows" then
       if lfs_attributes(source)["mode"] == "directory" then
         errorlevel = execute(
-          "xcopy /y /e /i " .. unix_to_win(source) .. " "
-             .. unix_to_win(dest .. "/" .. i) .. " > nul"
+          'xcopy /y /e /i "' .. unix_to_win(source) .. '" "'
+             .. unix_to_win(dest .. '/' .. i) .. '" > nul'
         )
       else
         errorlevel = execute(
-          "xcopy /y " .. unix_to_win(source) .. " "
-             .. unix_to_win(dest) .. " > nul"
+          'xcopy /y "' .. unix_to_win(source) .. '" "'
+             .. unix_to_win(dest) .. '" > nul'
         )
       end
     else
-      errorlevel = execute("cp -rf " .. source .. " " .. dest)
+      errorlevel = execute("cp -RLf '" .. source .. "' '" .. dest .. "'")
     end
     if errorlevel ~=0 then
       return errorlevel
@@ -708,10 +780,33 @@ end
 --
 
 -- Do some subtarget for all modules in a bundle
-function call(dirs, target)
+function call(dirs, target, opts)
+  -- Turn the option table into a string
+  local opts = opts or options
   local s = ""
-  for i = 2, #arg do
-    s = s .. " " .. arg[i]
+  for k,v in pairs(opts) do
+    if k ~= "files" and k ~= "target" then -- Special cases
+      local t = option_list[k] or { }
+      local arg = ""
+      if t["type"] == "string" then
+        arg = arg .. "=" .. v
+      end
+      if t["type"] == "table" then
+        for _,a in pairs(v) do
+          if arg == "" then
+            arg = "=" .. a -- Add the initial "=" here
+          else
+            arg = arg .. "," .. a
+          end
+        end
+      end
+      s = s .. " --" .. k .. arg
+    end
+  end
+  if opts["files"] then
+    for _,v in pairs(opts["files"]) do
+      s = s .. " " .. v
+    end
   end
   for _,i in ipairs(dirs) do
     print(
@@ -840,7 +935,7 @@ end
 
 -- Convert the raw log file into one for comparison/storage: keeps only
 -- the 'business' part from the tests and removes system-dependent stuff
-local function formatlog(logfile, newfile, engine)
+local function formatlog(logfile, newfile, engine, errlevels)
   local maxprintline = maxprintline
   if engine == "luatex" or engine == "luajittex" then
     maxprintline = maxprintline + 1 -- Deal with an out-by-one error
@@ -972,6 +1067,12 @@ local function formatlog(logfile, newfile, engine)
   local newfile = open(newfile, "w")
   output(newfile)
   write(newlog)
+  if recordstatus then
+    write('***************\n')
+    for i = 1, checkruns do
+      write('Compilation ' .. i .. ' of test file completed with exit status ' .. errlevels[i] )
+    end
+  end
   close(newfile)
 end
 
@@ -1192,12 +1293,22 @@ function listmodules()
   return modules
 end
 
+local function setepoch()
+  return
+    os_setenv .. " SOURCE_DATE_EPOCH=" .. epoch
+      .. os_concat ..
+    os_setenv .. " SOURCE_DATE_EPOCH_TEX_PRIMITIVES=1"
+      .. os_concat ..
+    os_setenv .. " FORCE_SOURCE_DATE=1"
+      .. os_concat
+end
+
 -- Run one test which may have multiple engine-dependent comparisons
 -- Should create a difference file for each failed test
 function runcheck(name, hide)
   local checkengines = checkengines
-  if optengines then
-    checkengines = optengines
+  if options["engine"] then
+    checkengines = options["engine"]
   end
   local errorlevel = 0
   for _,i in ipairs(checkengines) do
@@ -1215,7 +1326,7 @@ function runcheck(name, hide)
     else
       errlevel = compare_tlg(name, engine)
     end
-    if errlevel ~= 0 and opthalt then
+    if errlevel ~= 0 and options["halt-on-error"] then
       showfaileddiff()
       if errlevel ~= 0 then
         return 1
@@ -1396,8 +1507,9 @@ function runtest(name, engine, hide, ext, makepdf)
       break
     end
   end
+  local errlevels = {}
   for i = 1, checkruns do
-    run(
+    errlevels[i] = run(
       testdir,
       -- No use of localdir here as the files get copied to testdir:
       -- avoids any paths in the logs
@@ -1406,13 +1518,7 @@ function runtest(name, engine, hide, ext, makepdf)
       -- Avoid spurious output from (u)pTeX
       os_setenv .. " GUESS_INPUT_KANJI_ENCODING=0"
         .. os_concat ..
-      -- Fix the time of the run
-      os_setenv .. " SOURCE_DATE_EPOCH=" .. epoch
-        .. os_concat ..
-      os_setenv .. " SOURCE_DATE_EPOCH_TEX_PRIMITIVES=1"
-        .. os_concat ..
-      os_setenv .. " FORCE_SOURCE_DATE=1"
-        .. os_concat ..
+      (forcecheckepoch and setepoch() or "") ..
       -- Ensure lines are of a known length
       os_setenv .. " max_print_line=" .. maxprintline
         .. os_concat ..
@@ -1426,7 +1532,7 @@ function runtest(name, engine, hide, ext, makepdf)
   if makepdf and fileexists(testdir .. "/" .. name .. dviext) then
     dvitopdf(name, testdir, engine, hide)
   end
-  formatlog(logfile, newfile, engine)
+  formatlog(logfile, newfile, engine, errlevels)
   -- Store secondary files for this engine
   for _,i in ipairs(filelist(testdir, name .. ".???")) do
     local ext = match(i, "%....")
@@ -1451,16 +1557,14 @@ function dvitopdf(name, dir, engine, hide)
   if match(engine, "^u?ptex$") then
     run(
       dir,
-      os_setenv .. " SOURCE_DATE_EPOCH=" .. epoch
-        .. os_concat ..
+      (forcecheckepoch and setepoch() or "") ..
      "dvipdfmx  " .. name .. dviext
        .. (hide and (" > " .. os_null) or "")
     )
   else
     run(
       dir,
-      os_setenv .. " SOURCE_DATE_EPOCH=" .. epoch
-        .. os_concat ..
+      (forcecheckepoch and setepoch() or "") ..
      "dvips " .. name .. dviext
        .. (hide and (" > " .. os_null) or "")
        .. os_concat ..
@@ -1491,7 +1595,7 @@ end
 
 -- Strip the extension from a file name (if present)
 function jobname(file)
-  local name = match(select(2, splitpath(file)), "^(.*)%.")
+  local name = match(basename(file), "^(.*)%.")
   return name or file
 end
 
@@ -1510,11 +1614,12 @@ function runtool(subdir, dir, envvar, command)
   return(
     run(
       typesetdir .. "/" .. subdir,
+      (forcedocepoch and setepoch() or "") ..
       os_setenv .. " " .. envvar .. "=." .. os_pathsep
         .. abspath(localdir) .. os_pathsep
         .. abspath(dir .. "/" .. subdir)
-        .. (typesetsearch and os_pathsep or "") ..
-      os_concat ..
+        .. (typesetsearch and os_pathsep or "")
+        .. os_concat ..
       command
     )
   )
@@ -1596,12 +1701,13 @@ function tex(file, dir)
 end
 
 function typesetpdf(file, dir)
-  local name = jobname(file)
+  local name = gsub(file, "%.[^.]+$", "")
   print("Typesetting " .. name)
   local errorlevel = typeset(file, dir)
   if errorlevel == 0 then
-    os_remove(name .. ".pdf")
-    cp(name .. ".pdf", typesetdir, ".")
+    name = name .. ".pdf"
+    os_remove(jobname(name))
+    cp(name, typesetdir, ".")
   else
     print(" ! Compilation failed")
   end
@@ -1658,15 +1764,27 @@ function help()
   print("   setversion Update version information in sources")
   print("")
   print("Valid options are:")
-  print("   --date|-d           Sets the date to insert into sources")
-  print("   --engine|-e         Sets the engine to use for running test")
-  print("   --force|-f          Force tests to run if engine is not set up")
-  print("   --halt-on-error|-H  Stops running tests after the first failure")
-  print("   --pdf|-p            Check/save PDF files")
-  print("   --quiet|-q          Suppresses TeX output when unpacking")
-  print("   --rerun|-r          Runs tests without any unpacking, etc.")
-  print("   --testfiledir|-t    Selects the specified testfile location")
-  print("   --version|-v        Sets the version to insert into sources")
+  local longest = 0
+  for k,v in pairs(option_list) do
+    if len(k) > longest then
+      longest = len(k)
+    end
+  end
+  -- Sort the options
+  local t = { }
+  for k,_ in pairs(option_list) do
+    insert(t, k)
+  end
+  sort(t)
+  for _,k in ipairs(t) do
+    local opt = option_list[k]
+    local filler = rep(" ", longest - len(k))
+    if opt["desc"] then -- Skip --help as it has no desc
+      print(
+        "   --" .. k .. "|-" .. opt["short"] .. filler .. opt["desc"]
+      )
+    end
+  end
   print("")
   print("See l3build.pdf for further details.")
 end
@@ -1674,7 +1792,7 @@ end
 function check(names)
   local errorlevel = 0
   if testfiledir ~= "" and direxists(testfiledir) then
-    if not optrerun then
+    if not options["rerun"] then
       checkinit()
     end
     local hide = true
@@ -1703,7 +1821,7 @@ function check(names)
       local errlevel = runcheck(name, hide)
       -- Return value must be 1 not errlevel
       if errlevel ~= 0 then
-        if opthalt then
+        if options["halt-on-error"] then
           return 1
         else
           errorlevel = 1
@@ -1817,7 +1935,7 @@ end
 
 function ctan(standalone)
   -- Always run tests for all engines
-  optengines = nil
+  options["engine"] = nil
   local function dirzip(dir, name)
     local zipname = name .. ".zip"
     local function tab_to_str(table)
@@ -1954,27 +2072,33 @@ function doc(files)
     cp(i, supportdir, typesetdir)
   end
   depinstall(typesetdeps)
-  unpack({sourcefiles, typesetsourcefiles})
+  unpack({sourcefiles, typesetsourcefiles}, {".", docfiledir})
   -- Main loop for doc creation
+  local done = {}
   for _, typesetfiles in ipairs({typesetdemofiles, typesetfiles}) do
     for _,i in ipairs(typesetfiles) do
       for _, dir in ipairs({unpackdir, typesetdir}) do
         for j,_ in pairs(tree(dir, i)) do
-          -- Allow for command line selection of files
-          local typeset = true
-          if files and next(files) then
-            typeset = false
-            for _,k in ipairs(files) do
-              if k == jobname(j) then
-                typeset = true
-                break
+          if not done[j] then
+            j = gsub(j, "^%./", "")
+            -- Allow for command line selection of files
+            local typeset = true
+            if files and next(files) then
+              typeset = false
+              for _,k in ipairs(files) do
+                if k == gsub(j, "%.[^.]+$", "") then
+                  typeset = true
+                  break
+                end
               end
             end
-          end
-          if typeset then
-            local errorlevel = typesetpdf(j, dir)
-            if errorlevel ~= 0 then
-              return errorlevel
+            if typeset then
+              local errorlevel = typesetpdf(j, dir)
+              if errorlevel ~= 0 then
+                return errorlevel
+              else
+                done[j] = true
+              end
             end
           end
         end
@@ -2008,7 +2132,7 @@ end
 
 function save(names)
   checkinit()
-  local engines = optengines or {stdengine}
+  local engines = options["engine"] or {stdengine}
   for _,name in pairs(names) do
     local engine
     for _,engine in pairs(engines) do
@@ -2017,11 +2141,11 @@ function save(names)
       local spdffile = name .. tlgengine .. pdfext
       local newfile  = name .. "." .. engine .. logext
       local pdffile  = name .. "." .. engine .. pdfext
-      local refext = ((optpdf and pdfext) or tlgext)
+      local refext = ((options["pdf"] and pdfext) or tlgext)
       if testexists(name) then
         print("Creating and copying " .. refext)
-        runtest(name, engine, false, lvtext, optpdf)
-        if optpdf then
+        runtest(name, engine, false, lvtext, options["pdf"])
+        if options["pdf"] then
           ren(testdir, pdffile, spdffile)
           cp(spdffile, testdir, testfiledir)
         else
@@ -2144,14 +2268,8 @@ function setversion(dir)
       rm(dir, file .. bakext)
     end
   end
-  local date = os_date("%Y-%m-%d")
-  if optdate then
-    date = optdate[1] or date
-  end
-  local version = -1
-  if optversion then
-    version = optversion[1] or version
-  end
+  local date = options["date"] or os_date("%Y-%m-%d")
+  local version = options["version"] or -1
   local dir = dir or "."
   for _,i in pairs(versionfiles) do
     for _,j in pairs(filelist(dir, i)) do
@@ -2163,12 +2281,12 @@ end
 
 -- Unpack the package files using an 'isolated' system: this requires
 -- a copy of the 'basic' DocStrip program, which is used then removed
-function unpack(sources)
+function unpack(sources, sourcedirs)
   local errorlevel = depinstall(unpackdeps)
   if errorlevel ~= 0 then
     return errorlevel
   end
-  errorlevel = bundleunpack({"."}, sources)
+  errorlevel = bundleunpack(sourcedirs, sources)
   if errorlevel ~= 0 then
     return errorlevel
   end
@@ -2183,7 +2301,7 @@ end
 
 -- Split off from the main unpack so it can be used on a bundle and not
 -- leave only one modules files
-bundleunpack = bundleunpack or function(sourcedir, sources)
+bundleunpack = bundleunpack or function(sourcedirs, sources)
   local errorlevel = mkdir(localdir)
   if errorlevel ~=0 then
     return errorlevel
@@ -2192,7 +2310,7 @@ bundleunpack = bundleunpack or function(sourcedir, sources)
   if errorlevel ~=0 then
     return errorlevel
   end
-  for _,i in ipairs(sourcedir or {"."}) do
+  for _,i in ipairs(sourcedirs or {"."}) do
     for _,j in ipairs(sources or {sourcefiles}) do
       for _,k in ipairs(j) do
         errorlevel = cp(k, i, unpackdir)
@@ -2225,7 +2343,7 @@ bundleunpack = bundleunpack or function(sourcedir, sources)
         os_concat ..
         unpackexe .. " " .. unpackopts .. " " .. name .. " < "
           .. localdir .. "/yes"
-          .. (optquiet and (" > " .. os_null) or "")
+          .. (options["quiet"] and (" > " .. os_null) or "")
       )
       if errorlevel ~=0 then
         return errorlevel