l3build (3aug18)

git-svn-id: svn://tug.org/texlive/trunk@48340 c570f23f-e606-0410-a88d-b1316a301751

9 files changed, 433 insertions, 249 deletions

diff --git a/Master/texmf-dist/scripts/l3build/l3build-arguments.lua b/Master/texmf-dist/scripts/l3build/l3build-arguments.lua
index 499e2c4031e..4416ab69cda 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-arguments.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-arguments.lua
@@ -47,6 +47,11 @@ option_list =
         desc  = "Sets the date to insert into sources",
         type  = "string"
       },
+    dirty =
+      {
+        desc = "Skip cleaning up the test area",
+        type = "boolean"
+      },
     ["dry-run"] =
       {
         desc = "Dry run for install",
@@ -85,12 +90,6 @@ option_list =
         short = "h",
         type  = "boolean"
       },
-    pdf =
-      {
-        desc  = "Check/save PDF files",
-        short = "p",
-        type  = "boolean"
-      },
     quiet =
       {
         desc  = "Suppresses TeX output when unpacking",
diff --git a/Master/texmf-dist/scripts/l3build/l3build-aux.lua b/Master/texmf-dist/scripts/l3build/l3build-aux.lua
index f081054d575..d93c00c6745 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-aux.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-aux.lua
@@ -100,10 +100,7 @@ function call(dirs, target, opts)
     if i == "." then
       text = " with configuration " .. opts["config"][1]
     end
-    print(
-      "Running script " .. scriptname .. " with target \"" .. target
-        .. "\"" .. text
-    )
+    print("Running l3build with target \"" .. target .. "\"" .. text )
     local errorlevel = run(
       i,
       "texlua " .. scriptname .. " " .. target .. s
diff --git a/Master/texmf-dist/scripts/l3build/l3build-check.lua b/Master/texmf-dist/scripts/l3build/l3build-check.lua
index 442bd916814..7cac25a7d93 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-check.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-check.lua
@@ -55,7 +55,9 @@ local remove           = os.remove
 -- Set up the check system files: needed for checking one or more tests and
 -- for saving the test files
 function checkinit()
-  cleandir(testdir)
+  if not options["dirty"] then
+    cleandir(testdir)
+  end
   depinstall(checkdeps)
   -- Copy dependencies to the test directory itself: this makes the paths
   -- a lot easier to manage, and is important for dealing with the log and
@@ -207,6 +209,9 @@ local function formatlog(logfile, newfile, engine, errlevels)
     if match(line, "^%.*\\special%{pdf: docinfo << /Creator") then
       return ""
     end
+    if match(line, "^%.*\\special%{dvipdfmx:config") then
+      return ""
+    end
     -- Remove the \special line possibly present in DVI mode for paper size
     if match(line, "^%.*\\special%{papersize") then
       return ""
@@ -482,6 +487,49 @@ local function formatlualog(logfile, newfile, luatex)
   close(newfile)
 end
 
+local function normalise_pdf(pdffile,npdffile)
+  local file = assert(open(pdffile, "rb"))
+  local contents = gsub(file:read("*all") .. "\n", "\r\n", "\n")
+  close(file)
+  local new_content = ""
+  local stream_content = ""
+  local binary = false
+  local stream = false
+  for line in gmatch(contents, "([^\n]*)\n") do
+    if stream then
+      if match(line,"endstream") then
+        stream = false
+        if binary then
+          new_content = new_content .. "[BINARY STREAM]" .. os_newline
+        else
+           new_content = new_content .. stream_content .. line .. os_newline
+        end
+        binary = false
+      else
+        for i = 0, 31 do
+          if match(line,char(i)) then
+            binary = true
+            break
+          end
+        end
+        if not binary and not match(line, "^ *$") then
+          stream_content = stream_content .. line .. os_newline
+        end
+      end
+    elseif match(line,"^stream$") then
+      binary = false
+      stream = true
+      stream_content = "stream" .. os_newline
+    elseif not match(line, "^ *$") then
+      new_content = new_content .. line .. os_newline
+    end
+  end
+  local newfile = open(npdffile, "w")
+  output(newfile)
+  write(new_content)
+  close(newfile)
+end
+
 -- Run one test which may have multiple engine-dependent comparisons
 -- Should create a difference file for each failed test
 function runcheck(name, hide)
@@ -489,64 +537,70 @@ function runcheck(name, hide)
   if options["engine"] then
     checkengines = options["engine"]
   end
+  -- Used for both .lvt and .pvt tests
+  local function check_and_diff(ext,engine,enginename,comp,pdftest)
+    runtest(name,engine,hide,ext,pdftest,true)
+    local errorlevel = comp(name,enginename)
+    if errorlevel ~= 0 and options["halt-on-error"] then
+      showfaileddiff()
+      return 1
+    end
+    return errorlevel
+  end
   local errorlevel = 0
-  for _,i in ipairs(checkengines) do
+  -- First check by log
+  if fileexists(testfiledir .. "/" .. name .. lvtext) then
+    for _,i in pairs(checkengines) do
     -- Allow for luatex == luajittex for .tlg purposes
-    local engine = i
-    if i == "luajittex" then
-      engine = "luatex"
-    end
-    checkpdf = setup_check(name, engine)
-    runtest(name, i, hide, lvtext, checkpdf, true)
-    -- Generation of results depends on test type
-    local errlevel
-    if checkpdf then
-      errlevel = compare_pdf(name, engine)
-    else
-      errlevel = compare_tlg(name, engine)
-    end
-    if errlevel ~= 0 and options["halt-on-error"] then
-      showfaileddiff()
-      if errlevel ~= 0 then
+      local engine = i
+      if i == "luajittex" then
+        engine = "luatex"
+      end
+      setup_check(name, engine)
+      local errlevel = 0
+      errlevel = check_and_diff(lvtext,i,engine,compare_tlg)
+      if errlevel ~= 0 and options["halt-on-error"] then
         return 1
       end
-    end
-    if errlevel > errorlevel then
-      errorlevel = errlevel
+      if errlevel > errorlevel then
+        errorlevel = errlevel
+      end
     end
   end
+  -- Then check by PDF
+  if fileexists(testfiledir .. "/" .. name .. pvtext) then
+    setup_check(name, stdengine)
+    errorlevel = check_and_diff(pvtext,stdengine,stdengine,compare_pdf,true)
+      + errorlevel
+  end
+  -- Return everything
   return errorlevel
 end
 
 function setup_check(name, engine)
   local testname = name .. "." .. engine
-  local pdffile = locate(
-    {testfiledir, unpackdir},
-    {testname .. pdfext, name .. pdfext}
-  )
   local tlgfile = locate(
     {testfiledir, unpackdir},
     {testname .. tlgext, name .. tlgext}
   )
+  local tpffile = locate(
+    {testfiledir, unpackdir},
+    {testname .. tpfext, name .. tpfext}
+  )
   -- Attempt to generate missing reference file from expectation
-  if not (pdffile or tlgfile) then
+  if not (tlgfile or tpffile) then
     if not locate({unpackdir, testfiledir}, {name .. lveext}) then
       print(
-        "Error: failed to find " .. pdfext .. ", " .. tlgext .. " or "
+        "Error: failed to find " .. tlgext .. ", " .. tlpext .. " or "
           .. lveext .. " file for " .. name .. "!"
       )
       exit(1)
     end
     runtest(name, engine, true, lveext, true, false)
-    pdffile = testdir .. "/" .. testname .. pdfext
-    -- If a PDF is generated use it for comparisons
-    if not fileexists(pdffile) then
-      pdffile = nil
-      ren(testdir, testname .. logext, testname .. tlgext)
-    end
+    ren(testdir, testname .. logext, testname .. tlgext)
   else
     -- Install comparison files found
-    for _,v in pairs({pdffile, tlgfile}) do
+    for _,v in pairs({tlgfile, tpffile}) do
       if v then
         cp(
           match(v, ".*/(.*)"),
@@ -556,36 +610,21 @@ function setup_check(name, engine)
       end
     end
   end
-  if pdffile then
-    local pdffile = match(pdffile, ".*/(.*)")
-    ren(
-      testdir,
-      pdffile,
-      gsub(pdffile, pdfext .. "$", ".ref" .. pdfext)
-    )
-    return true
-  else
-    return false
-  end
 end
 
-function compare_pdf(name, engine)
+function compare_pdf(name,engine,cleanup)
   local errorlevel
   local testname = name .. "." .. engine
-  local cmpfile    = testdir .. "/" .. testname .. os_cmpext
-  local pdffile    = testdir .. "/" .. testname .. pdfext
-  local refpdffile = locate(
-    {testdir}, {testname .. ".ref" .. pdfext, name .. ".ref" .. pdfext}
-  )
-  if not refpdffile then
-    return
+  local difffile = testdir .. "/" .. name .. pdfext .. os_diffext
+  local pdffile  = testdir .. "/" .. testname .. pdfext
+  local tpffile  = testdir .. "/" .. name .. tpfext
+  if not tpffile then
+    return 1
   end
-  errorlevel = execute(
-    os_cmpexe .. " " .. normalize_path(refpdffile)
-      .. " " .. pdffile .. " > " .. cmpfile
-  )
-  if errorlevel == 0 then
-    remove(cmpfile)
+  errorlevel = execute(os_diffexe .. " "
+    .. normalize_path(tpffile .. " " .. pdffile .. " > " .. difffile))
+  if errorlevel == 0 or cleanup then
+    remove(difffile)
   end
   return errorlevel
 end
@@ -631,7 +670,7 @@ end
 
 -- Run one of the test files: doesn't check the result so suitable for
 -- both creating and verifying .tlg files
-function runtest(name, engine, hide, ext, makepdf, breakout)
+function runtest(name, engine, hide, ext, pdfmode, breakout)
   local lvtfile = name .. (ext or lvtext)
   cp(lvtfile, fileexists(testfiledir .. "/" .. lvtfile)
     and testfiledir or unpackdir, testdir)
@@ -660,7 +699,7 @@ function runtest(name, engine, hide, ext, makepdf, breakout)
   end
   -- Special casing for XeTeX engine
   local checkopts = checkopts
-  if match(engine, "xetex") and not makepdf then
+  if match(engine, "xetex") and not pdfmode then
     checkopts = checkopts .. " -no-pdf"
   end
   -- Special casing for ConTeXt
@@ -683,6 +722,8 @@ function runtest(name, engine, hide, ext, makepdf, breakout)
   end
   local logfile = testdir .. "/" .. name .. logext
   local newfile = testdir .. "/" .. name .. "." .. engine .. logext
+  local pdffile = testdir .. "/" .. name .. pdfext
+  local npffile = testdir .. "/" .. name .. "." .. engine .. pdfext
   local asciiopt = ""
   for _,i in ipairs(asciiengines) do
     if realengine == i then
@@ -718,16 +759,27 @@ function runtest(name, engine, hide, ext, makepdf, breakout)
     )
     -- Break the loop if the result is stable
     if breakout and i < checkruns then
-      formatlog(logfile, newfile, engine, errlevels)
-      if compare_tlg(name,engine,true) == 0 then
-        break
+      if pdfmode then
+        normalise_pdf(pdffile,npffile)
+        if compare_pdf(name,engine,true) == 0 then
+          break
+        end
+      else
+        formatlog(logfile, newfile, engine, errlevels)
+        if compare_tlg(name,engine,true) == 0 then
+          break
+        end
       end
     end
   end
-  if makepdf and fileexists(testdir .. "/" .. name .. dviext) then
+  if pdfmode and fileexists(testdir .. "/" .. name .. dviext) then
     dvitopdf(name, testdir, engine, hide)
   end
-  formatlog(logfile, newfile, engine, errlevels)
+  if pdfmode then
+    normalise_pdf(pdffile,npffile)
+  else
+    formatlog(logfile, newfile, engine, errlevels)
+  end
   -- Store secondary files for this engine
   for _,filetype in pairs(auxfiles) do
     for _,file in pairs(filelist(testdir, filetype)) do
@@ -751,7 +803,8 @@ end
 
 -- Look for a test: could be in the testfiledir or the unpackdir
 function testexists(test)
-  return(locate({testfiledir, unpackdir}, {test .. lvtext}))
+  return(locate({testfiledir, unpackdir},
+    {test .. lvtext, test .. pvtext}))
 end
 
 -- Standard versions of the main targets for building modules
@@ -769,15 +822,36 @@ function check(names)
     names = names or { }
     -- No names passed: find all test files
     if not next(names) then
-      for _,i in pairs(filelist(testfiledir, "*" .. lvtext)) do
-        insert(names, jobname(i))
+      local excludenames = { }
+      for _,glob in pairs(excludetests) do
+        for _,name in pairs(filelist(testfiledir, glob .. lvtext)) do
+          excludenames[jobname(name)] = true
+        end
+        for _,name in pairs(filelist(unpackdir, glob .. lvtext)) do
+          excludenames[jobname(name)] = true
+        end
+        for _,name in pairs(filelist(testfiledir, glob .. pvtext)) do
+          excludenames[jobname(name)] = true
+        end
       end
-      for _,i in ipairs(filelist(unpackdir, "*" .. lvtext)) do
-        if fileexists(testfiledir .. "/" .. i) then
-          print("Duplicate test file: " .. i)
-          return 1
-        else
-          insert(names, jobname(i))
+      local function addname(name)
+        if not excludenames[jobname(name)] then
+          insert(names,jobname(name))
+        end
+      end
+      for _,glob in pairs(includetests) do
+        for _,name in pairs(filelist(testfiledir, glob .. lvtext)) do
+          addname(name)
+        end
+        for _,name in pairs(filelist(testfiledir, glob .. pvtext)) do
+          addname(name)
+        end
+        for _,name in pairs(filelist(unpackdir, glob .. lvtext)) do
+          if fileexists(testfiledir .. "/" .. name) then
+            print("Duplicate test file: " .. i)
+            return 1
+          end
+          addname(name)
         end
       end
       sort(names)
@@ -853,9 +927,6 @@ function checkdiff()
   for _,i in ipairs(filelist(testdir, "*" .. os_diffext)) do
     print("  - " .. testdir .. "/" .. i)
   end
-  for _,i in ipairs(filelist(testdir, "*" .. os_cmpext)) do
-    print("  - " .. testdir .. "/" .. i)
-  end
   print("")
 end
 
@@ -871,50 +942,44 @@ function showfaileddiff()
     print(content)
     print("-----------------------------------------------------------------------------------")
   end
-  for _,i in ipairs(filelist(testdir, "*" .. os_cmpext)) do
-    print("  - " .. testdir .. "/" .. i)
-  end
 end
 
 function save(names)
   checkinit()
   local engines = options["engine"] or {stdengine}
   for _,name in pairs(names) do
-    local engine
-    for _,engine in pairs(engines) do
-      local tlgengine = ((engine == stdengine and "") or "." .. engine)
-      local tlgfile  = name .. tlgengine .. tlgext
-      local spdffile = name .. tlgengine .. pdfext
-      local newfile  = name .. "." .. engine .. logext
-      local pdffile  = name .. "." .. engine .. pdfext
-      local refext = ((options["pdf"] and pdfext) or tlgext)
-      if testexists(name) then
-        print("Creating and copying " .. refext)
-        runtest(name, engine, false, lvtext, options["pdf"])
-        if options["pdf"] then
-          ren(testdir, pdffile, spdffile)
-          cp(spdffile, testdir, testfiledir)
+    if testexists(name) then
+      for _,engine in pairs(engines) do
+        local testengine = ((engine == stdengine and "") or "." .. engine)
+        if fileexists(testfiledir .. "/" .. name .. lvtext) then
+        -- Create one or more .tlg files        
+          print("Creating and copying " .. tlgext)
+          local tlgfile  = name .. testengine .. tlgext
+          local newfile  = name .. "." .. engine .. logext
+          runtest(name,engine,false,lvtext)
+          ren(testdir,newfile,tlgfile)
+          cp(tlgfile,testdir,testfiledir)
+          if fileexists(unpackdir .. "/" .. tlgfile) then
+            print("Saved " .. tlgext
+              .. " file overrides unpacked version of the same name")
+          end
         else
-          ren(testdir, newfile, tlgfile)
-          cp(tlgfile, testdir, testfiledir)
-        end
-        if fileexists(unpackdir .. "/" .. tlgfile) then
-          print(
-            "Saved " .. tlgext
-              .. " file overrides unpacked version of the same name"
-          )
+          -- Create one .tpf file
+          print("Creating and copying " .. tpfext)
+          local tpffile  = name .. tpfext
+          local newfile  = name .. "." .. engine .. pdfext
+          runtest(name,engine,false,pvtext,true)
+          ren(testdir,newfile,tpffile)
+          cp(tpffile,testdir,testfiledir)
         end
-      elseif locate({unpackdir, testfiledir}, {name .. lveext}) then
-        print(
-          "Saved " .. tlgext .. " file overrides a "
-            .. lveext .. " file of the same name"
-        )
-      else
-        print(
-          "Test input \"" .. testfiledir .. "/" .. name .. lvtext
-            .. "\" not found"
-        )
+        return 0
       end
+    elseif locate({unpackdir, testfiledir}, {name .. lveext}) then
+      print("Saved " .. tlgext .. " file overrides a "
+        .. lveext .. " file of the same name")
+    else
+      print('Test "'.. name .. '"not found')
+      return 1     
     end
   end
 end
diff --git a/Master/texmf-dist/scripts/l3build/l3build-file-functions.lua b/Master/texmf-dist/scripts/l3build/l3build-file-functions.lua
index 83b4bfb6539..510567e5342 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-file-functions.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-file-functions.lua
@@ -90,6 +90,67 @@ local function glob_to_pattern(glob)
     return match(char, "^%w$") and char or "%" .. char
   end
 
+  -- remove a leading "\" if one is present:
+  -- this can simple be discarded
+  local function unescape()
+    if char == "\\" then
+      i = i + 1
+      char = sub(glob,i,i)
+      if char == "" then
+        pattern = "[^]"
+        return false
+      end
+    end
+    return true
+  end
+
+  -- look for the body of a set:
+  -- char is already the first token
+  local function collect_set()
+    while true do
+      if char == "" then
+        pattern = "[^]"
+        return false
+      elseif char == "]" then
+        pattern = pattern .. "]"
+        break
+      else
+        if not unescape(char) then break end
+        local char_one = char
+        i = i + 1
+        char = sub(glob,i,i)
+        if char == "" then
+          pattern = "[^]"
+          return false
+        elseif char == "-" then
+          i = i + 1
+          char = sub(glob,i,i)
+          if char == "" then
+            pattern = "[^]"
+            return false
+          elseif char == "]" then
+            -- Not a range
+            pattern = pattern .. escape(char_one) .. "%-]"
+            return true
+          else
+            if not unescape(char) then break end
+            pattern = pattern .. escape(char_one) .. "-" .. escape(char)
+          end
+        elseif char == "]" then
+          -- Successfully done
+          pattern = pattern .. escape(char_one) .. "]"
+          return true
+        else
+          pattern = pattern .. escape(char_one)
+          i = i - 1 -- Back up one
+        end
+      end
+      i = i + 1
+      char = sub(glob,i,i)
+    end
+    return true
+  end
+
   -- Convert tokens.
   while true do
     i = i + 1
@@ -102,9 +163,24 @@ local function glob_to_pattern(glob)
     elseif char == "*" then
       pattern = pattern .. ".*"
     elseif char == "[" then
-      -- Ignored
-      print("[...] syntax not supported in globs!")
-    elseif char == "\\" then
+      -- Search inside a charset
+      i = i + 1
+      char = sub(glob,i,i)
+      if char == "" or char == "]" then -- Matches nothing
+        pattern = "[^]"
+        break
+      elseif char == "^" or char == "!" then
+        i = i + 1
+        char = sub(glob,i,i)
+        if char ~= "]" then -- Matches anything so ignore
+          pattern = pattern .. "[^"
+          if not collect_set() then break end
+        end
+      else
+        pattern = pattern .. "["
+        if not collect_set() then break end
+      end
+    elseif char == "\\" then -- Discarded
       i = i + 1
       char = sub(glob, i, i)
       if char == "" then
@@ -130,8 +206,6 @@ os_setenv  = "export"
 os_yes     = "printf 'y\\n%.0s' {1..200}"
 
 os_ascii   = "echo \"\""
-os_cmpexe  = getenv("cmpexe") or "cmp"
-os_cmpext  = getenv("cmpext") or ".cmp"
 os_diffext = getenv("diffext") or ".diff"
 os_diffexe = getenv("diffexe") or "diff -c --strip-trailing-cr"
 os_grepexe = "grep"
@@ -139,8 +213,6 @@ os_newline = "\n"
 
 if os_type == "windows" then
   os_ascii   = "@echo."
-  os_cmpexe  = getenv("cmpexe") or "fc /b"
-  os_cmpext  = getenv("cmpext") or ".cmp"
   os_concat  = "&"
   os_diffext = getenv("diffext") or ".fc"
   os_diffexe = getenv("diffexe") or "fc /n"
diff --git a/Master/texmf-dist/scripts/l3build/l3build-help.lua b/Master/texmf-dist/scripts/l3build/l3build-help.lua
index c7050c44210..18caf87daae 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-help.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-help.lua
@@ -22,6 +22,11 @@ for those people who are interested.
 
 --]]
 
+local insert = table.insert
+local match  = string.match
+local rep    = string.rep
+local sort   = table.sort
+
 function version()
   print(
     "\n" ..
@@ -31,46 +36,44 @@ function version()
 end
 
 function help()
+  local function setup_list(list)
+    local longest = 0
+    for k,v in pairs(list) do
+      if k:len() > longest then
+        longest = k:len()
+      end
+    end
+    -- Sort the options
+    local t = { }
+    for k,_ in pairs(list) do
+      insert(t, k)
+    end
+    sort(t)
+    return longest,t
+  end
+
   local scriptname = "l3build"
-  if not string.match(arg[0], "l3build(%.lua)$") then
+  if not match(arg[0], "l3build(%.lua)$") then
     scriptname = arg[0]
   end
   print("usage: " .. scriptname .. " <command> [<options>] [<names>]")
   print("")
   print("The most commonly used l3build commands are:")
-  if testfiledir ~= "" then
-    print("   check      Run all automated tests")
-  end
-  print("   clean      Clean out directory tree")
-  if module == "" or bundle == "" then
-    print("   ctan       Create CTAN-ready archive")
-  end
-  print("   doc        Typesets all documentation files")
-  print("   install    Installs files into the local texmf tree")
-  if module ~= "" and testfiledir ~= "" then
-    print("   save       Saves test validation log")
+  local longest,t = setup_list(target_list)
+  for _,k in ipairs(t) do
+    local target = target_list[k]
+    local filler = rep(" ", longest - k:len() + 1)
+    if target["desc"] then
+      print("   " .. k .. filler .. target["desc"])
+    end
   end
-  print("   tag        Update release tags in files")
-  print("   uninstall  Uninstalls files from the local texmf tree")
-  print("   unpack     Unpacks the source files into the build tree")
   print("")
   print("Valid options are:")
-  local longest = 0
-  for k,v in pairs(option_list) do
-    if k:len() > longest then
-      longest = k:len()
-    end
-  end
-  -- Sort the options
-  local t = { }
-  for k,_ in pairs(option_list) do
-    table.insert(t, k)
-  end
-  table.sort(t)
+  local longest,t = setup_list(option_list)
   for _,k in ipairs(t) do
     local opt = option_list[k]
-    local filler = string.rep(" ", longest - k:len() + 1)
-    if opt["desc"] then -- Skip --help as it has no desc
+    local filler = rep(" ", longest - k:len() + 1)
+    if opt["desc"] then
       if opt["short"] then
         print("   --" .. k .. "|-" .. opt["short"] .. filler .. opt["desc"])
       else
diff --git a/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua b/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua
index 7dac27aa983..8c39da91600 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua
@@ -22,7 +22,8 @@ for those people who are interested.
 
 --]]
 
-local exit             = os.exit
+local exit   = os.exit
+local insert = table.insert
 
 -- List all modules
 function listmodules()
@@ -34,7 +35,7 @@ function listmodules()
       assert(type(attr) == "table")
       if attr.mode == "directory" then
         if not exclmodules[entry] then
-          table.insert(modules, entry)
+          insert(modules, entry)
         end
       end
     end
@@ -42,91 +43,132 @@ function listmodules()
   return modules
 end
 
+target_list =
+  {
+    -- Some hidden targets
+    bundlecheck =
+      {
+        func = check,
+        pre  = function()
+            if names then
+              print("Bundle checks should not list test names")
+              help()
+              exit(1)
+            end
+            return 0
+          end
+      },
+    bundleunpack =
+      {
+        func = bundleunpack,
+        pre  = function() return(depinstall(unpackdeps)) end
+      },
+    -- Public targets
+    check =
+      {
+        bundle_target = true,
+        desc = "Run all automated tests",
+        func = check,
+      },
+    clean =
+      {
+        bundle_func = bundleclean,
+        desc = "Clean out directory tree",
+        func = clean
+      },
+    ctan =
+      {
+        bundle_func = ctan,
+        desc = "Create CTAN-ready archive",
+        func = ctan
+      },
+    doc =
+      {
+        desc = "Typesets all documentation files",
+        func = doc
+      },
+    install =
+      {
+        desc = "Installs files into the local textmf tree",
+        func = install
+      },
+    manifest =
+      {
+        desc = "Creates a manifest file",
+        func = manifest
+      },
+    save =
+      {
+        desc = "Saves test validation log",
+        func = save
+      },
+    tag =
+      {
+        bundle_func = function(names)
+            local modules = modules or listmodules()
+            local errorlevel = call(modules,"tag")
+            -- Deal with any files in the bundle dir itself
+            if errorlevel == 0 then
+              errorlevel = tag(names[1])
+            end
+            return errorlevel
+          end,
+        desc = "Updates release tags in files",
+        func = tag,
+        pre  = function(names)
+           if not names or #names ~=1 then
+             print("Tag name required")
+             help()
+             exit(1)
+           end
+           return 0
+         end
+      },
+    uninstall =
+      {
+        desc = "Uninstalls files from the local textmf tree",
+        func = uninstall
+      },
+    unpack=
+      {
+        bundle_target = true,
+        desc = "Unpacks the source files into the build tree",
+        func = unpack
+      }
+  }
+
 --
 -- The overall main function
 --
 
-function stdmain(target, names)
-  local errorlevel
-  -- If the module name is empty, the script is running in a bundle:
-  -- apart from ctan all of the targets are then just mappings
+function stdmain(target,names)
+  -- Deal with unknown targets up-front
+  if not target_list[target] then
+    help()
+    exit(1)
+  end
+  local errorlevel = 0
   if module == "" then
-    -- Detect all of the modules
     modules = modules or listmodules()
-    if target == "doc" then
-      errorlevel = call(modules, "doc")
-    elseif target == "check" then
-      errorlevel = call(modules, "bundlecheck")
-      if errorlevel ~=0 then
-        print("There were errors: checks halted!\n")
-      end
-    elseif target == "clean" then
-      errorlevel = bundleclean()
-    elseif target == "ctan" then
-      errorlevel = ctan()
-    elseif target == "install" then
-      errorlevel = call(modules, "install")
-    elseif target == "tag" then
-      if options["names"] and #options["names"] == 1 then
-        errorlevel = call(modules,"tag")
-        -- Deal with any files in the bundle dir itself
-        if errorlevel == 0 then
-          errorlevel = tag(options["names"][1])
-        end
-      else
-        print("Tag name required")
-        help()
-        exit(1)
-      end
-    elseif target == "uninstall" then
-      errorlevel = call(modules, "uninstall")
-    elseif target == "unpack" then
-      errorlevel = call(modules, "bundleunpack")
+    if target_list[target].bundle_func then
+      errorlevel = target_list[target].bundle_func(names)
     else
-      help()
+      -- Detect all of the modules
+      if target_list[target].bundle_target then
+        target = "bundle" .. target
+      end
+      errorlevel = call(modules,target)
     end
   else
-    if target == "bundleunpack" then -- 'Hidden' as only needed 'higher up'
-      depinstall(unpackdeps)
-      errorlevel = bundleunpack()
-    elseif target == "bundlecheck" then
-      errorlevel = check()
-    elseif target == "bundlectan" then
-      errorlevel = bundlectan()
-    elseif target == "doc" then
-      errorlevel = doc(names)
-    elseif target == "check" then
-      errorlevel = check(names)
-    elseif target == "clean" then
-      errorlevel = clean()
-    elseif target == "ctan" then
-      errorlevel = ctan()
-    elseif target == "install" then
-      errorlevel = install()
-    elseif target == "manifest" then
-      errorlevel = manifest()
-    elseif target == "save" then
-      if next(names) then
-        errorlevel = save(names)
-      else
-        help()
-      end
-    elseif target == "tag" then
-      if options["names"] and #options["names"] == 1 then
-        errorlevel = tag(options["names"][1])
-      else
-        print("Tag name required")
-        help()
-        exit(1)
-      end
-    elseif target == "uninstall" then
-      errorlevel = uninstall()
-    elseif target == "unpack" then
-      errorlevel = unpack()
-    else
-      help()
+    if target_list[target].pre then
+     errorlevel = target_list[target].pre(names)
+     if errorlevel ~= 0 then
+       exit(1)
+     end
     end
+    errorlevel = target_list[target].func(names)
   end
+  -- All done, finish up
   if errorlevel ~= 0 then
     exit(1)
   else
diff --git a/Master/texmf-dist/scripts/l3build/l3build-typesetting.lua b/Master/texmf-dist/scripts/l3build/l3build-typesetting.lua
index 229745dafc6..71fe0ea6c4b 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-typesetting.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-typesetting.lua
@@ -125,7 +125,7 @@ function makeindex(name, dir, inext, outext, logext, style)
         makeindexexe .. " " .. makeindexopts
           .. " -o " .. name .. outext
           .. (style and (" -s " .. style) or "")
-          .. " -t " .. name .. " "  .. name .. inext
+          .. " -t " .. name .. logext .. " "  .. name .. inext
       )
     )
   end
diff --git a/Master/texmf-dist/scripts/l3build/l3build-variables.lua b/Master/texmf-dist/scripts/l3build/l3build-variables.lua
index 38be1c7c5ea..98ce7fd6c0e 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-variables.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-variables.lua
@@ -117,6 +117,10 @@ checkengines = checkengines or {"pdftex", "xetex", "luatex"}
 checkformat  = checkformat  or "latex"
 stdengine    = stdengine    or "pdftex"
 
+-- The tests themselves
+includetests = includetests or {"*"}
+excludetests = excludetests or { }
+
 -- Configs for testing
 checkconfigs = checkconfigs or {"build"}
 
@@ -163,7 +167,6 @@ maxprintline = maxprintline or 79
 packtdszip   = packtdszip   or false
 typesetcmds  = typesetcmds  or ""
 typesetruns  = typesetruns  or 2
-versionform  = versionform  or ""
 recordstatus = recordstatus or false
 
 -- Extensions for various file types: used to abstract out stuff a bit
@@ -174,7 +177,9 @@ lveext = lveext or ".lve"
 lvtext = lvtext or ".lvt"
 pdfext = pdfext or ".pdf"
 psext  = psext  or ".ps"
+pvtext = pvtext or ".pvt"
 tlgext = tlgext or ".tlg"
+tpfext = tpfext or ".tpf"
 
 -- Manifest options
 manifestfile = manifestfile or "MANIFEST.md"
diff --git a/Master/texmf-dist/scripts/l3build/l3build.lua b/Master/texmf-dist/scripts/l3build/l3build.lua
index 1ded83cc5b6..5cefd6d5dc0 100755
--- a/Master/texmf-dist/scripts/l3build/l3build.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build.lua
@@ -2,7 +2,7 @@
 
 --[[
 
-File l3build.lua Copyright (C) 2014-2017 The LaTeX3 Project
+File l3build.lua Copyright (C) 2014-2018 The LaTeX3 Project
 
 It may be distributed and/or modified under the conditions of the
 LaTeX Project Public License (LPPL), either version 1.3c of this
@@ -25,7 +25,7 @@ for those people who are interested.
 --]]
 
 -- Version information
-release_date = "2018-05-10"
+release_date = "2018-08-02"
 
 -- File operations are aided by the LuaFileSystem module
 local lfs = require("lfs")
@@ -44,7 +44,7 @@ local exit             = os.exit
 
 -- l3build setup and functions
 kpse.set_program_name("kpsewhich")
-build_kpse_path = string.match(lookup("l3build.lua"),"(.*[/])")
+build_kpse_path = match(lookup("l3build.lua"),"(.*[/])")
 local function build_require(s)
   require(lookup("l3build-"..s..".lua", { path = build_kpse_path } ) )
 end
@@ -53,15 +53,6 @@ end
 build_require("arguments")
 build_require("help")
 
--- Filter out special cases early
-if options["target"] == "help" then
-  help()
-  exit(0)
-elseif options["target"] == "version" then
-  version()
-  exit(0)
-end
-
 build_require("file-functions")
 build_require("typesetting")
 build_require("aux")
@@ -75,6 +66,16 @@ build_require("manifest-setup")
 build_require("tagging")
 build_require("stdmain")
 
+-- This has to come after stdmain(),
+-- and that has to come after the functions are defined
+if options["target"] == "help" then
+  help()
+  exit(0)
+elseif options["target"] == "version" then
+  version()
+  exit(0)
+end
+
 -- Allow main function to be disabled 'higher up'
 main = main or stdmain