l3build (19aug15)

git-svn-id: svn://tug.org/texlive/trunk@38163 c570f23f-e606-0410-a88d-b1316a301751

1 files changed, 333 insertions, 138 deletions

diff --git a/Master/texmf-dist/tex/latex/l3build/l3build.lua b/Master/texmf-dist/tex/latex/l3build/l3build.lua
index f88ec640d16..c939faf11e6 100644
--- a/Master/texmf-dist/tex/latex/l3build/l3build.lua
+++ b/Master/texmf-dist/tex/latex/l3build/l3build.lua
@@ -17,8 +17,8 @@
 --]]
 
 -- Version information: should be identical to that in l3build.dtx
-release_date = "2015/07/28"
-release_ver  = "5704"
+release_date = "2015/08/18"
+release_ver  = "5866"
 
 -- "module" is a deprecated function in Lua 5.2: as we want the name
 -- for other purposes, and it should eventually be 'free', simply
@@ -146,16 +146,150 @@ makeindexexe  = makeindexexe  or "makeindex"
 makeindexopts = makeindexopts or ""
 
 -- Other required settings
-checkruns   = checkruns   or 1
-packtdszip  = packtdszip  or false -- Not actually needed but clearer
-scriptname  = scriptname  or "build.lua" -- Script used in each directory
-typesetcmds = typesetcmds or ""
+checkruns    = checkruns    or 1
+packtdszip   = packtdszip   or false -- Not actually needed but clearer
+scriptname   = scriptname   or "build.lua" -- Script used in each directory
+typesetcmds  = typesetcmds  or ""
 
 -- Extensions for various file types: used to abstract out stuff a bit
 logext = logext or ".log"
+lveext = lveext or ".lve"
 lvtext = lvtext or ".lvt"
 tlgext = tlgext or ".tlg"
 
+-- Run time options
+-- These are parsed into a global table, and all optional args
+-- are made available as a related global var
+
+function argparse()
+  local result = { }
+  local files  = { }
+  local long_options =
+    {
+      engine              = "engine",
+      ["halt-on-error"]   = "halt"  ,
+      ["halt-on-failure"] = "halt"  ,
+      help                = "help"
+    }
+  local short_options =
+    {
+      e = "engine",
+      h = "help"  ,
+      H = "halt"
+    }
+  local option_args =
+    {
+      engine = true ,
+      halt   = false,
+      help   = false
+    }
+  -- arg[1] is a special case: must be a command or "-h"/"--help"
+  -- Deal with this by assuming help and storing only apparently-valid
+  -- input
+  local a = arg[1]
+  result["target"] = "help"
+  if a then
+    -- No options are allowed in position 1, so filter those out
+    if not string.match(a, "^%-") then
+      result["target"] = a
+    end
+  end
+  -- Stop here if help is required
+  if result["target"] == "help" then
+    return result
+  end
+  -- An auxiliary to grab all file names into a table
+  local function remainder(num)
+    local i
+    local files = { }
+    for i = num, #arg do
+      table.insert(files, arg[i])
+    end
+    return files
+  end
+  -- Examine all other arguments
+  -- Use a while loop rather than for as this makes it easier
+  -- to grab arg for optionals where appropriate
+  local i = 2
+  while i <= #arg do
+    local a = arg[i]
+    -- Terminate search for options
+    if a == "--" then
+      files = remainder(i + 1)
+      break
+    end
+    -- Look for optionals
+    local opt, optarg
+    local opts
+    -- Look for and option and get it into a variable
+    if string.match(a, "^%-") then
+      if string.match(a, "^%-%-") then
+        opts = long_options
+        local pos = string.find(a, "=", 1, true)
+        if pos then
+          opt    = string.sub(a, 3, pos - 1)
+          optarg = string.sub(a, pos + 1)
+        else
+          opt = string.sub(a,3)
+        end
+      else
+        opts = short_options
+        opt  = string.sub(a, 2, 2)
+        -- Only set optarg if it is there
+        if #a > 2 then
+          optarg = string.sub(a, 3)
+        end
+      end
+      -- Now check that the option is valid and sort out the argument
+      -- if required
+      local optname = opts[opt]
+      if optname then
+        local reqarg = option_args[optname]
+        -- Tidy up arguments
+        if reqarg and not optarg then
+          optarg = arg[i + 1]
+          if not optarg then
+            io.stderr:write("Missing value for option " .. a .."\n")
+            return {"help"}
+          end
+          i = i + 1
+        end
+        if not reqarg and optarg then
+          io.stderr:write("Value not allowed for option " .. a .."\n")
+          return {"help"}
+        end
+      else
+        io.stderr:write("Unknown option " .. a .."\n")
+        return {"help"}
+      end
+      -- Store the result
+      if optarg then
+        local opts = result[optname] or { }
+        local match
+        for match in string.gmatch(optarg, "([^,%s]+)") do
+          table.insert(opts, match)
+        end
+        result[optname] = opts
+      else
+        result[optname] = true
+      end
+      i = i + 1
+    end
+    if not opt then
+      files = remainder(i)
+      break
+    end
+  end
+  result["files"] = files
+  return result
+end
+
+userargs = argparse()
+
+optengines = userargs["engine"]
+opthalt    = userargs["halt"]
+opthelp    = userargs["help"]
+
 -- Convert a file glob into a pattern for use by e.g. string.gub
 -- Based on https://github.com/davidm/lua-glob-pattern
 -- Simplified substantially: "[...]" syntax not supported as is not
@@ -243,6 +377,7 @@ if string.sub(package.config, 1, 1) == "\\" then
   os_diffext  = ".fc"
   os_diffexe  = "fc /n"
   os_grepexe  = "findstr /r"
+  os_newline  = "\r\n"
   os_null     = "nul"
   os_pathsep  = ";"
   os_setenv   = "set"
@@ -253,6 +388,7 @@ else
   os_diffext  = ".diff"
   os_diffexe  = "diff -c --strip-trailing-cr"
   os_grepexe  = "grep"
+  os_newline  = "\n"
   os_null     = "/dev/null"
   os_pathsep  = ":"
   os_setenv   = "export"
@@ -399,8 +535,7 @@ end
 
 -- Run a command in a given directory
 function run(dir, cmd)
-  local errorlevel = os.execute("cd " .. dir .. os_concat .. cmd)
-  return errorlevel
+  return os.execute("cd " .. dir .. os_concat .. cmd)
 end
 
 -- Deal with the fact that Windows and Unix use different path separators
@@ -415,18 +550,26 @@ end
 
 -- Do some subtarget for all modules in a bundle
 function allmodules(target)
-  local errorlevel = 0
+  local errorlevel
   for _,i in ipairs(modules) do
     print(
         "Running script " .. scriptname .. " with target \"" .. target .. "\" for module "
           .. i
       )
-    errorlevel = run(i, "texlua " .. scriptname .. " " .. target)
-    if errorlevel > 0 then
+    local engines = ""
+    if optengines then
+      engines = " --engine=" .. table.concat(optengines, ",")
+    end
+    errorlevel = run(
+      i, "texlua " .. scriptname .. " " .. target
+        .. (opthalt and " -H" or "")
+        .. engines
+    )
+    if errorlevel ~= 0 then
       return errorlevel
     end
   end
-  return errorlevel
+  return 0
 end
 
 -- Set up the check system files: needed for checking one or more tests and
@@ -533,7 +676,7 @@ end
 
 -- Convert the raw log file into one for comparison/storage: keeps only
 -- the 'business' part from the tests and removes system-dependent stuff
-function formatlog(logfile, newfile)
+function formatlog(logfile, newfile, engine)
   local function killcheck(line)
       -- Skip lines containing file dates
       if string.match(line, "[^<]%d%d%d%d/%d%d/%d%d") then
@@ -550,8 +693,14 @@ function formatlog(logfile, newfile)
     return false
   end
     -- Substitutions to remove some non-useful changes
-  local function normalize(line)
-    local line = line
+  local function normalize(line, maxprintline)
+    -- Allow for wrapped lines: preserve the content and wrap
+    if (string.len(line) == maxprintline) then
+      lastline = (lastline or "") .. line
+      return ""
+    end
+    local line = (lastline or "") .. line
+    lastline = ""
     -- Remove test file name from lines
     -- This needs to extract the base name from the log name,
     -- and one to allow for the case that there might be "-" chars
@@ -581,16 +730,18 @@ function formatlog(logfile, newfile)
     line = string.gsub(line, "\\csname\\endcsname ", "\\csname\\endcsname")
     -- Zap "on line <num>" and replace with "on line ..."
     line = string.gsub(line, "on line %d*", "on line ...")
-    -- Zap line numbers from \show, \showbox, \box_show and the like
-    line = string.gsub(line, "^l%.%d+ ", "l. ...")
     -- Tidy up to ^^ notation
     for i = 1, 31, 1 do
       line = string.gsub(line, string.char(i), "^^" .. string.char(64 + i))
     end
+    -- Zap line numbers from \show, \showbox, \box_show and the like
+    -- Two stages as line wrapping alters some of them and restore the break
+    line = string.gsub(line, "^l%.%d+ ", "l. ...")
+    line = string.gsub(line, "%.%.%.l%.%d+ ( *)%}$", "...\nl. ...%1}")
     -- Remove spaces at the start of lines: deals with the fact that LuaTeX
     -- uses a different number to the other engines
     line = string.gsub(line, "^%s+", "")
-    -- Remove 'normal' direction information on boxes with (u)p-TeX
+    -- Remove 'normal' direction information on boxes with (u)pTeX
     line = string.gsub(line, ", yoko direction", "")
     -- Unicode engines display chars in the upper half of the 8-bit range:
     -- tidy up to match pdfTeX
@@ -600,6 +751,13 @@ function formatlog(logfile, newfile)
     end
     return line
   end
+  local kpse = require("kpse")
+  kpse.set_program_name(engine)
+  local maxprintline = tonumber(kpse.expand_var("$max_print_line"))
+  if engine == "luatex" then
+    maxprintline = maxprintline + 1 -- Deal with an out-by-one error
+  end
+  local lastline = ""
   local newlog = ""
   local prestart = true
   local skipping = false
@@ -612,10 +770,10 @@ function formatlog(logfile, newfile)
       skipping = true
     elseif line == "TIMO" then
       skipping = false
-    elseif not prestart and not skipping and not killcheck(line) then
-      line = normalize(line)
-      if not string.match(line, "^ *$") then
-        newlog = newlog .. line .. "\n"
+    elseif not prestart and not skipping then
+      line = normalize(line, maxprintline)
+      if not string.match(line, "^ *$") and not killcheck(line) then
+        newlog = newlog .. line .. os_newline
       end
     end
   end
@@ -736,27 +894,37 @@ end
 
 -- Runs a single test: needs the name of the test rather than the .lvt file
 -- One 'test' here may apply to multiple engines
-function runcheck(name, engine, hide)
+function runcheck(name, hide)
   local checkengines = checkengines
-  if engine then
-    checkengines = {engine}
+  if optengines then
+    checkengines = optengines
   end
   local errorlevel = 0
   for _,i in ipairs(checkengines) do
-    cp(name .. tlgext, testfiledir, testdir)
-    runtest(name, i, hide)
     local testname = name .. "." .. i
     local difffile = testdir .. "/" .. testname .. os_diffext
     local newfile  = testdir .. "/" .. testname .. logext
     -- Use engine-specific file if available
     local tlgfile  = locate(
       {testfiledir, unpackdir},
-      {name ..  "." .. i .. tlgext, name .. tlgext}
+      {testname .. tlgext, name .. tlgext}
     )
-    if not tlgfile then
-      print("Error: failed to find " .. tlgext .. " file for " .. name .. "!")
-      os.exit(1)
+    if tlgfile then
+      cp(name .. tlgext, testfiledir, testdir)
+    else
+      -- Attempt to generate missing test goal from expectation
+      tlgfile = testdir .. "/" .. testname .. tlgext
+      if not locate({unpackdir, testfiledir}, {name .. lveext}) then
+        print(
+          "Error: failed to find " .. tlgext .. " or "
+          .. lveext .. " file for " .. name .. "!"
+        )
+        os.exit(1)
+      end
+      runtest(name, i, hide, lveext)
+      ren(testdir, testname .. logext, testname .. tlgext)
     end
+    runtest(name, i, hide, lvtext)
     if os_windows then
       tlgfile = unix_to_win(tlgfile)
     end
@@ -784,6 +952,10 @@ function runcheck(name, engine, hide)
     if errlevel == 0 then
       os.remove(difffile)
     else
+      if opthalt then
+        checkdiff()
+        return errlevel
+      end
       errorlevel = errlevel
     end
   end
@@ -792,8 +964,8 @@ end
 
 -- Run one of the test files: doesn't check the result so suitable for
 -- both creating and verifying .tlg files
-function runtest(name, engine, hide)
-  local lvtfile = name .. lvtext
+function runtest(name, engine, hide, ext)
+  local lvtfile = name .. (ext or lvtext)
   cp(lvtfile, fileexists(testfiledir .. "/" .. lvtfile)
     and testfiledir or unpackdir, testdir)
   local engine = engine or stdengine
@@ -807,7 +979,7 @@ function runtest(name, engine, hide)
   else
     format = ""
   end
-  -- Special casing for (u)p-TeX LaTeX formats
+  -- Special casing for (u)pTeX LaTeX formats
   if
     string.match(checkformat, "^latex$") and
     string.match(engine, "^u?ptex$") then
@@ -826,11 +998,11 @@ function runtest(name, engine, hide)
           .. (hide and (" > " .. os_null) or "")
       )
   end
-  formatlog(logfile, newfile)
+  formatlog(logfile, newfile, engine)
   -- Store secondary files for this engine
   for _,i in ipairs(filelist(testdir, name .. ".???")) do
     local ext = string.match(i, "%....")
-    if ext ~= lvtext and ext ~= tlgext and ext ~= logext then
+    if ext ~= lvtext and ext ~= tlgext and ext ~= lveext and ext ~= logext then
       if not fileexists(testsuppdir .. "/" .. i) then
         ren(
           testdir, i, string.gsub(
@@ -958,16 +1130,21 @@ typeset = typeset or function(file)
     return errorlevel
   else
     local name = stripext(file)
-    -- Return a non-zero errorlevel if something goes wrong
-    -- without having loads of nested tests
-    return(
-      biber(name)  +
-      bibtex(name) +
-      makeindex(name, ".glo", ".gls", ".glg", glossarystyle) +
-      makeindex(name, ".idx", ".ind", ".ilg", indexstyle)    +
-      tex(file) +
-      tex(file)
-    )
+    errorlevel = biber(name) + bibtex(name)
+    if errorlevel == 0 then
+      local function cycle(name)
+        return(
+          makeindex(name, ".glo", ".gls", ".glg", glossarystyle) +
+          makeindex(name, ".idx", ".ind", ".ilg", indexstyle)    +
+          tex(file)
+        )
+      end
+      errorlevel = cycle(name)
+      if errorlevel ~= 0 then
+        errorlevel = cycle(name)
+      end
+    end
+    return errorlevel
   end
 end
 
@@ -975,63 +1152,68 @@ end
 
 -- Simply print out how to use the build system
 help = help or function()
-  print ""
+  print("usage: " .. arg[0] .. " <command> [<options>] [<names>]")
+  print("")
+  print("The most commonly used l3build commands are:")
   if testfiledir ~= "" then
-    print " build check                 - run all automated tests for all engines"
+    print("   check      Run all automated tests")
   end
-  if module ~= "" and testfiledir ~= "" then
-    print " build check <name>          - check one test file <name> for all engines"
-    print " build check <name> <engine> - check one test file <name> for <engine>   "
-  end
-  print " build clean                 - clean out directory tree                  "
+  print("   clean      Clean out directory tree")
   if next(cmdchkfiles) ~= nil then
-    print " build cmdcheck              - check commands documented are defined     "
+    print("   cmdcheck   Check commands documented are defined")
   end
   if module == "" or bundle == "" then
-    print " build ctan                  - create CTAN-ready archive                 "
+    print("   ctan       Create CTAN-ready archive")
   end
-  print " build doc                   - runs all documentation files              "
-  print " build install               - install files in local texmf tree         "
+  print("   doc        Typesets all documentation files")
+  print("   install    Installs files into the local texmf tree")
   if module ~= "" and testfiledir ~= "" then
-    print " build save <name>           - save test log for <name> for all engines  "
-    print " build save <name> <engine>  - save test log for <name> for <engine>     "
+    print("   save       Saves test validation log")
   end
-  print ""
-end
-
-function check(name, engine)
-  local errorlevel = 0
-  if name then
-    errorlevel = checklvt(name, engine)
-  else
-    errorlevel = checkall()
-  end
-  return errorlevel
+  print("")
+  print("Valid options are:")
+  print("   --engine|-e         Sets the engine to use for running test")
+  print("   --halt-on-error|-H  Stops running tests after the first failure")
+  print("")
 end
 
-function checkall()
+function check(names)
   local errorlevel = 0
   if testfiledir ~= "" and direxists(testfiledir) then
-    local function execute(name)
-      local name = stripext(name)
-      print("  " .. name)
-      local errlevel = runcheck(name, nil, true)
-      if errlevel ~= 0 then
-        errorlevel = 1
+    checkinit()
+    local hide = true
+    if names and next(names) then
+      hide = false
+    end
+    local i
+    names = names or { }
+    -- No names passed: find all test files
+    if not next(names) then
+      for _,i in pairs(filelist(testfiledir, "*" .. lvtext)) do
+        table.insert(names, stripext(i))
+      end
+      for _,i in ipairs(filelist(unpackdir, "*" .. lvtext)) do
+        if fileexists(testfiledir .. "/" .. i) then
+          print("Duplicate test file: " .. i)
+          return 1
+        else
+          table.insert(stripext(i))
+        end
       end
     end
-    checkinit()
+    -- Actually run the tests
     print("Running checks on")
-    for _,i in ipairs(filelist(testfiledir, "*" .. lvtext)) do
-      execute(i)
-    end
-    for _,i in ipairs(filelist(unpackdir, "*" .. lvtext)) do
-      if fileexists(testfiledir .. "/" .. i) then
-        print("Duplicate test file: " .. i)
-        errorlevel = 1
-        break
-      else
-        execute(i)
+    local name
+    for _,name in ipairs(names) do
+      print("  " .. name)
+      local errlevel = runcheck(name, hide)
+      -- Return value must be 1 not errlevel
+      if errlevel ~= 0 then
+        if opthalt then 
+          return 1
+        else
+          errorlevel = 1
+        end
       end
     end
     if errorlevel ~= 0 then
@@ -1043,25 +1225,6 @@ function checkall()
   return errorlevel
 end
 
-function checklvt(name, engine)
-  checkinit()
-  if testexists(name) then
-    print("Running checks on " .. name)
-    local errorlevel = runcheck(name, engine)
-    if errorlevel ~= 0 then
-      checkdiff()
-    else
-      if engine then
-        print("  Check passes")
-      else
-        print("\n  All checks passed\n")
-      end
-    end
-  else
-    print("Test \"" .. name .. "\" not set up!")
-  end
-end
-
 -- A short auxiliary to print the list of differences for check
 function checkdiff()
   print("\n  Check failed with difference files")
@@ -1127,6 +1290,8 @@ function cmdcheck()
 end
 
 function ctan(standalone)
+  -- Always run tests for all engines
+  optengines = nil
   local function dirzip(dir, name)
     local zipname = name .. ".zip"
     local function tab_to_str(table)
@@ -1142,14 +1307,17 @@ function ctan(standalone)
     -- First, zip up all of the text files
     run(
         dir,
-        zipexe .. " " .. zipopts .. " -ll ".. zipname .. " " .. "." .. " -x"
-          .. binfiles .. " " .. exclude
+        zipexe .. " " .. zipopts .. " -ll ".. zipname .. " " .. "." 
+          .. ( 
+            (binfiles or  exclude) and (" -x" .. binfiles .. " " .. exclude)
+            or ""
+          )  
       )
     -- Then add the binary ones
     run(
         dir,
         zipexe .. " " .. zipopts .. " -g ".. zipname .. " " .. ". -i" ..
-          binfiles .. " -x" .. exclude
+          binfiles .. (exclude and (" -x" .. exclude) or "")
       )
   end
   local errorlevel
@@ -1236,7 +1404,7 @@ end
 
 -- Typeset all required documents
 -- Uses a set of dedicated auxiliaries that need to be available to others
-function doc()
+function doc(files)
   -- Set up
   cleandir(typesetdir)
   for _,i in ipairs({bibfiles, docfiles, sourcefiles, typesetfiles}) do
@@ -1252,9 +1420,23 @@ function doc()
   -- Main loop for doc creation
   for _,i in ipairs(typesetfiles) do
     for _,j in ipairs(filelist(".", i)) do
-      local errorlevel = typesetpdf(j)
-      if errorlevel ~= 0 then
-        return errorlevel
+      -- Allow for command line selection of files
+      local typeset = true
+      if files and next(files) then
+        local k
+        typeset = false
+        for _,k in ipairs(files) do
+          if k == stripext(j) then
+            typeset = true
+            break
+          end
+        end
+      end
+      if typeset then
+        local errorlevel = typesetpdf(j)
+        if errorlevel ~= 0 then
+          return errorlevel
+        end  
       end
     end
   end
@@ -1273,26 +1455,39 @@ function install()
   end
 end
 
-function save(name, engine)
-  local tlgfile = name .. (engine and ("." .. engine) or "") .. tlgext
-  local newfile = name .. "." .. (engine or stdengine) .. logext
+function save(names)
   checkinit()
-  if testexists(name) then
-    print("Creating and copying " .. tlgfile)
-    runtest(name, engine, false)
-    ren(testdir, newfile, tlgfile)
-    cp(tlgfile, testdir, testfiledir)
-    if fileexists(unpackdir .. "/" .. tlgfile) then
-      print(
-        "Saved " .. tlgext
-          .. " file overrides unpacked version of the same name"
-      )
+  local engines = optengines or {stdengine}
+  local name
+  for _,name in pairs(names) do
+    local engine
+    for _,engine in pairs(engines) do
+      local tlgengine = ((engine == stdengine and "") or "." .. engine)
+      local tlgfile = name .. tlgengine .. tlgext
+      local newfile = name .. "." .. engine .. logext
+      if testexists(name) then
+        print("Creating and copying " .. tlgfile)
+        runtest(name, engine, false, lvtext)
+        ren(testdir, newfile, tlgfile)
+        cp(tlgfile, testdir, testfiledir)
+        if fileexists(unpackdir .. "/" .. tlgfile) then
+          print(
+            "Saved " .. tlgext
+              .. " file overrides unpacked version of the same name"
+          )
+        end
+      elseif locate({unpackdir, testfiledir}, {name .. lveext}) then
+        print(
+          "Saved " .. tlgext .. " file overrides a "
+            .. lveext .. " file of the same name"
+        )
+      else
+        print(
+          "Test input \"" .. testfiledir .. "/" .. name .. lvtext
+            .. "\" not found"
+          )
+      end
     end
-  else
-    print(
-      "Test input \"" .. testfiledir .. "/" .. name .. lvtext
-        .. "\" not found"
-      )
   end
 end
 
@@ -1349,7 +1544,7 @@ end
 -- The overall main function
 --
 
-function stdmain(target, file, engine)
+function stdmain(target, files)
   local errorlevel
   -- If the module name is empty, the script is running in a bundle:
   -- apart from ctan all of the targets are then just mappings
@@ -1387,9 +1582,9 @@ function stdmain(target, file, engine)
     elseif target == "bundlectan" then
       errorlevel = bundlectan()
     elseif target == "doc" then
-      errorlevel = doc()
+      errorlevel = doc(files)
     elseif target == "check" and testfiledir ~= "" then
-      errorlevel = check(file, engine)
+      errorlevel = check(files)
     elseif target == "clean" then
       errorlevel = clean()
     elseif target == "cmdcheck" and next(cmdchkfiles) ~= nil then
@@ -1399,8 +1594,8 @@ function stdmain(target, file, engine)
     elseif target == "install" then
       errorlevel = install()
     elseif target == "save" and testfiledir ~= "" then
-      if file then
-        errorlevel = save(file, engine)
+      if next(files) then
+        errorlevel = save(files)
       else
         help()
       end
@@ -1419,4 +1614,4 @@ end
 main = main or stdmain
 
 -- Call the main function
-main(arg[1], arg[2], arg[3])
+main(userargs["target"], userargs["files"])