l3build (4aug18)

git-svn-id: svn://tug.org/texlive/trunk@48348 c570f23f-e606-0410-a88d-b1316a301751

15 files changed, 336 insertions, 277 deletions

diff --git a/Master/texmf-dist/doc/latex/l3build/CHANGELOG.md b/Master/texmf-dist/doc/latex/l3build/CHANGELOG.md
index b9a434787b0..2d59804c698 100644
--- a/Master/texmf-dist/doc/latex/l3build/CHANGELOG.md
+++ b/Master/texmf-dist/doc/latex/l3build/CHANGELOG.md
@@ -7,6 +7,21 @@ this project uses date-based 'snapshot' version identifiers.
 
 ## [Unreleased]
 
+## [2018-08-04]
+
+### Added
+- `ctanzip` variable
+- `--full` option
+
+### Changed
+- Run PDF-based tests for all engines
+- Tweaks to PDF-based normalisation: new `.tpf` files will be required
+- The `--halt-on-error|-H` setting now applies to multiple configs
+
+### Fixed
+- Testing using `.lve` files
+- Tagging with new approach to top-level targets
+
 ## [2018-08-02]
 
 ### Added
@@ -19,10 +34,10 @@ this project uses date-based 'snapshot' version identifiers.
 
 ### Changed
 - PDF-based testing now uses 'digested' PDF file for comparison,
-  working from dedicated .pvt input files
+  working from dedicated `.pvt` input files
 
 ### Removed
-- "--pdf" command line switch
+- `--pdf|-p` command line switch
 
 ## [2018-05-10]
 
@@ -127,8 +142,9 @@ this project uses date-based 'snapshot' version identifiers.
 - Rationalise short option names: removed `-d`, `-E`, `-r`
 - Target `cmdcheck`: specific to LaTeX3 kernel work
 
-[Unreleased]: https://github.com/latex3/l3build/compare/2018-08-02...HEAD
-[2018-08-02]: https://github.com/latex3/l3build/compare/2018-05-06...2018-08-02
+[Unreleased]: https://github.com/latex3/l3build/compare/2018-08-04...HEAD
+[2018-08-04]: https://github.com/latex3/l3build/compare/2018-08-02...2018-08-04
+[2018-08-02]: https://github.com/latex3/l3build/compare/2018-05-10...2018-08-02
 [2018-05-10]: https://github.com/latex3/l3build/compare/2018-05-06...2018-05-10
 [2018-05-06]: https://github.com/latex3/l3build/compare/2018-03-26...2018-05-06
 [2018-03-26]: https://github.com/latex3/l3build/compare/2018-03-24...2018-03-26
diff --git a/Master/texmf-dist/doc/latex/l3build/README.md b/Master/texmf-dist/doc/latex/l3build/README.md
index 43ac56bf0e9..1014ef3cd4a 100644
--- a/Master/texmf-dist/doc/latex/l3build/README.md
+++ b/Master/texmf-dist/doc/latex/l3build/README.md
@@ -1,7 +1,7 @@
 l3build: a testing and building system for LaTeX3
 =================================================
 
-Release 2018-08-02
+Release 2018-08-04
 
 Overview
 --------
diff --git a/Master/texmf-dist/doc/latex/l3build/l3build.pdf b/Master/texmf-dist/doc/latex/l3build/l3build.pdf
index c5675f9d9a9..cdbb633e42f 100644
--- a/Master/texmf-dist/doc/latex/l3build/l3build.pdf
+++ b/Master/texmf-dist/doc/latex/l3build/l3build.pdf
diff --git a/Master/texmf-dist/doc/man/man1/l3build.1 b/Master/texmf-dist/doc/man/man1/l3build.1
index fb85523ad0b..c3ef55a52c8 100644
--- a/Master/texmf-dist/doc/man/man1/l3build.1
+++ b/Master/texmf-dist/doc/man/man1/l3build.1
@@ -1,4 +1,4 @@
-.TH l3build 1 "2018-08-02"
+.TH l3build 1 "2018-08-04"
 .SH NAME
 l3build \- Checking and building packages
 .SH SYNOPSIS
diff --git a/Master/texmf-dist/doc/man/man1/l3build.man1.pdf b/Master/texmf-dist/doc/man/man1/l3build.man1.pdf
index 63568956eab..2c0b133a7be 100644
--- a/Master/texmf-dist/doc/man/man1/l3build.man1.pdf
+++ b/Master/texmf-dist/doc/man/man1/l3build.man1.pdf
diff --git a/Master/texmf-dist/scripts/l3build/l3build-arguments.lua b/Master/texmf-dist/scripts/l3build/l3build-arguments.lua
index 4416ab69cda..e693ab929df 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-arguments.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-arguments.lua
@@ -79,6 +79,11 @@ option_list =
         short = "f",
         type  = "boolean"
       },
+    full =
+      {
+        desc = "Install all files",
+        type = "boolean"
+      },
     ["halt-on-error"] =
       {
         desc  = "Stops running tests after the first failure",
diff --git a/Master/texmf-dist/scripts/l3build/l3build-check.lua b/Master/texmf-dist/scripts/l3build/l3build-check.lua
index 7cac25a7d93..f5d1716de9e 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-check.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-check.lua
@@ -83,9 +83,20 @@ function checkinit()
   execute(os_ascii .. ">" .. testdir .. "/ascii.tcx")
 end
 
+local function rewrite(source,result,processor,...)
+  local file = assert(open(source,"rb"))
+  local content = gsub(file:read("*all") .. "\n","\r\n","\n")
+  close(file)
+  local new_content = processor(content,...)
+  local newfile = open(result,"w")
+  output(newfile)
+  write(new_content)
+  close(newfile)
+end
+
 -- Convert the raw log file into one for comparison/storage: keeps only
 -- the 'business' part from the tests and removes system-dependent stuff
-local function formatlog(logfile, newfile, engine, errlevels)
+local function normalize_log(content,engine,errlevels)
   local maxprintline = maxprintline
   if engine == "luatex" or engine == "luajittex" then
     maxprintline = maxprintline + 1 -- Deal with an out-by-one error
@@ -239,14 +250,10 @@ local function formatlog(logfile, newfile, engine, errlevels)
     return line, lastline
   end
   local lastline = ""
-  local newlog = ""
+  local new_content = ""
   local prestart = true
   local skipping = false
-  -- Read the entire log file as a binary: deals with ^@/^[, etc.
-  local file = assert(open(logfile, "rb"))
-  local contents = gsub(file:read("*all") .. "\n", "\r\n", "\n")
-  close(file)
-  for line in gmatch(contents, "([^\n]*)\n") do
+  for line in gmatch(content, "([^\n]*)\n") do
     if line == "START-TEST-LOG" then
       prestart = false
     elseif line == "END-TEST-LOG" or
@@ -259,24 +266,23 @@ local function formatlog(logfile, newfile, engine, errlevels)
     elseif not prestart and not skipping then
       line, lastline = normalize(line, lastline)
       if not match(line, "^ *$") and not killcheck(line) then
-        newlog = newlog .. line .. os_newline
+        new_content = new_content .. line .. os_newline
       end
     end
   end
-  local newfile = open(newfile, "w")
-  output(newfile)
-  write(newlog)
   if recordstatus then
-    write('***************\n')
+    new_content = new_content .. '***************' .. os_newline
     for i = 1, checkruns do
-      write('Compilation ' .. i .. ' of test file completed with exit status ' .. errlevels[i] .. '\n')
+      new_content = new_content ..
+        'Compilation ' .. i .. ' of test file completed with exit status ' ..
+        errlevels[i] .. os_newline
     end
   end
-  close(newfile)
+  return new_content
 end
 
 -- Additional normalization for LuaTeX
-local function formatlualog(logfile, newfile, luatex)
+local function normalize_lua_log(content,luatex)
   local function normalize(line, lastline, dropping)
     -- Find \discretionary or \whatsit lines:
     -- These may come back later
@@ -468,34 +474,24 @@ local function formatlualog(logfile, newfile, luatex)
     -- uses a different number to the other engines
     return gsub(line, "^%s+", ""), ""
   end
-  local newlog = ""
+  local new_content = ""
   local lastline = ""
   local dropping = false
-  -- Read the entire log file as a binary: deals with ^@/^[, etc.
-  local file = assert(open(logfile, "rb"))
-  local contents = gsub(file:read("*all") .. "\n", "\r\n", "\n")
-  close(file)
-  for line in gmatch(contents, "([^\n]*)\n") do
+  for line in gmatch(content, "([^\n]*)\n") do
     line, lastline, dropping = normalize(line, lastline, dropping)
     if not match(line, "^ *$") then
-      newlog = newlog .. line .. os_newline
+      new_content = new_content .. line .. os_newline
     end
   end
-  local newfile = open(newfile, "w")
-  output(newfile)
-  write(newlog)
-  close(newfile)
+  return new_content
 end
 
-local function normalise_pdf(pdffile,npdffile)
-  local file = assert(open(pdffile, "rb"))
-  local contents = gsub(file:read("*all") .. "\n", "\r\n", "\n")
-  close(file)
+local function normalize_pdf(content)
   local new_content = ""
   local stream_content = ""
   local binary = false
   local stream = false
-  for line in gmatch(contents, "([^\n]*)\n") do
+  for line in gmatch(content,"([^\n]*)\n") do
     if stream then
       if match(line,"endstream") then
         stream = false
@@ -524,15 +520,13 @@ local function normalise_pdf(pdffile,npdffile)
       new_content = new_content .. line .. os_newline
     end
   end
-  local newfile = open(npdffile, "w")
-  output(newfile)
-  write(new_content)
-  close(newfile)
+  return new_content
 end
 
 -- Run one test which may have multiple engine-dependent comparisons
 -- Should create a difference file for each failed test
 function runcheck(name, hide)
+  if not testexists(name) then return 1 end
   local checkengines = checkengines
   if options["engine"] then
     checkengines = options["engine"]
@@ -548,30 +542,25 @@ function runcheck(name, hide)
     return errorlevel
   end
   local errorlevel = 0
-  -- First check by log
-  if fileexists(testfiledir .. "/" .. name .. lvtext) then
-    for _,i in pairs(checkengines) do
-    -- Allow for luatex == luajittex for .tlg purposes
-      local engine = i
-      if i == "luajittex" then
-        engine = "luatex"
-      end
-      setup_check(name, engine)
-      local errlevel = 0
-      errlevel = check_and_diff(lvtext,i,engine,compare_tlg)
-      if errlevel ~= 0 and options["halt-on-error"] then
-        return 1
-      end
-      if errlevel > errorlevel then
-        errorlevel = errlevel
-      end
+  for _,engine in pairs(checkengines) do
+    local enginename = engine
+    -- Allow for luatex == luajittex for .tlg/.tpf purposes
+    if engine == "luajittex" then
+      engine = "luatex"
+    end
+    setup_check(name,engine)
+    local errlevel = 0
+    if fileexists(testfiledir .. "/" .. name .. pvtext) then
+      errlevel = check_and_diff(pvtext,engine,enginename,compare_pdf,true)
+    else
+      errlevel = check_and_diff(lvtext,engine,enginename,compare_tlg)
+    end
+    if errlevel ~= 0 and options["halt-on-error"] then
+      return 1
+    end
+    if errlevel > errorlevel then
+      errorlevel = errlevel
     end
-  end
-  -- Then check by PDF
-  if fileexists(testfiledir .. "/" .. name .. pvtext) then
-    setup_check(name, stdengine)
-    errorlevel = check_and_diff(pvtext,stdengine,stdengine,compare_pdf,true)
-      + errorlevel
   end
   -- Return everything
   return errorlevel
@@ -596,7 +585,7 @@ function setup_check(name, engine)
       )
       exit(1)
     end
-    runtest(name, engine, true, lveext, true, false)
+    runtest(name, engine, true, lveext)
     ren(testdir, testname .. logext, testname .. tlgext)
   else
     -- Install comparison files found
@@ -613,15 +602,14 @@ function setup_check(name, engine)
 end
 
 function compare_pdf(name,engine,cleanup)
-  local errorlevel
   local testname = name .. "." .. engine
-  local difffile = testdir .. "/" .. name .. pdfext .. os_diffext
+  local difffile = testdir .. "/" .. testname .. pdfext .. os_diffext
   local pdffile  = testdir .. "/" .. testname .. pdfext
-  local tpffile  = testdir .. "/" .. name .. tpfext
+  local tpffile  = locate({testdir}, {testname .. tpfext, name .. tpfext})
   if not tpffile then
     return 1
   end
-  errorlevel = execute(os_diffexe .. " "
+  local errorlevel = execute(os_diffexe .. " "
     .. normalize_path(tpffile .. " " .. pdffile .. " > " .. difffile))
   if errorlevel == 0 or cleanup then
     remove(difffile)
@@ -650,8 +638,8 @@ function compare_tlg(name, engine,cleanup)
       lualogfile = testdir .. "/" .. testname .. ".tmp" .. logext
     end
     local luatlgfile = testdir .. "/" .. testname .. tlgext
-    formatlualog(tlgfile, luatlgfile, false)
-    formatlualog(logfile, lualogfile, true)
+    rewrite(tlgfile,luatlgfile,normalize_lua_log)
+    rewrite(logfile,lualogfile,normalize_lua_log,true)
     errorlevel = execute(os_diffexe .. " "
       .. normalize_path(luatlgfile .. " " .. lualogfile .. " > " .. difffile))
     if cleanup then
@@ -669,7 +657,7 @@ function compare_tlg(name, engine,cleanup)
 end
 
 -- Run one of the test files: doesn't check the result so suitable for
--- both creating and verifying .tlg files
+-- both creating and verifying
 function runtest(name, engine, hide, ext, pdfmode, breakout)
   local lvtfile = name .. (ext or lvtext)
   cp(lvtfile, fileexists(testfiledir .. "/" .. lvtfile)
@@ -677,13 +665,11 @@ function runtest(name, engine, hide, ext, pdfmode, breakout)
   local engine = engine or stdengine
   -- Set up the format file name if it's one ending "...tex"
   local realengine = engine
-  local format
+  local format = ""
   if
     match(checkformat, "tex$") and
     not match(engine, checkformat) then
     format = " -fmt=" .. gsub(engine, "(.*)tex$", "%1") .. checkformat
-  else
-    format = ""
   end
   -- Special casing for e-LaTeX format
   if
@@ -720,10 +706,11 @@ function runtest(name, engine, hide, ext, pdfmode, breakout)
       exit(1)
     end
   end
-  local logfile = testdir .. "/" .. name .. logext
-  local newfile = testdir .. "/" .. name .. "." .. engine .. logext
-  local pdffile = testdir .. "/" .. name .. pdfext
-  local npffile = testdir .. "/" .. name .. "." .. engine .. pdfext
+  local basename = testdir .. "/" .. name
+  local logfile = basename .. logext
+  local newfile = basename .. "." .. engine .. logext
+  local pdffile = basename .. pdfext
+  local npffile = basename .. "." .. engine .. pdfext
   local asciiopt = ""
   for _,i in ipairs(asciiengines) do
     if realengine == i then
@@ -760,12 +747,12 @@ function runtest(name, engine, hide, ext, pdfmode, breakout)
     -- Break the loop if the result is stable
     if breakout and i < checkruns then
       if pdfmode then
-        normalise_pdf(pdffile,npffile)
+        rewrite(pdffile,npffile,normalize_pdf)
         if compare_pdf(name,engine,true) == 0 then
           break
         end
       else
-        formatlog(logfile, newfile, engine, errlevels)
+        rewrite(logfile,newfile,normalize_log,engine,errlevels)
         if compare_tlg(name,engine,true) == 0 then
           break
         end
@@ -776,9 +763,9 @@ function runtest(name, engine, hide, ext, pdfmode, breakout)
     dvitopdf(name, testdir, engine, hide)
   end
   if pdfmode then
-    normalise_pdf(pdffile,npffile)
+    rewrite(pdffile,npffile,normalize_pdf)
   else
-    formatlog(logfile, newfile, engine, errlevels)
+    rewrite(logfile,newfile,normalize_log,engine,errlevels)
   end
   -- Store secondary files for this engine
   for _,filetype in pairs(auxfiles) do
@@ -794,6 +781,7 @@ function runtest(name, engine, hide, ext, pdfmode, breakout)
       end
     end
   end
+  return 0
 end
 
 -- A hook to allow additional tasks to run for the tests
@@ -807,8 +795,6 @@ function testexists(test)
     {test .. lvtext, test .. pvtext}))
 end
 
--- Standard versions of the main targets for building modules
-
 function check(names)
   local errorlevel = 0
   if testfiledir ~= "" and direxists(testfiledir) then
@@ -950,37 +936,37 @@ function save(names)
   for _,name in pairs(names) do
     if testexists(name) then
       for _,engine in pairs(engines) do
-        local testengine = ((engine == stdengine and "") or "." .. engine)
-        if fileexists(testfiledir .. "/" .. name .. lvtext) then
-        -- Create one or more .tlg files        
-          print("Creating and copying " .. tlgext)
-          local tlgfile  = name .. testengine .. tlgext
-          local newfile  = name .. "." .. engine .. logext
-          runtest(name,engine,false,lvtext)
-          ren(testdir,newfile,tlgfile)
-          cp(tlgfile,testdir,testfiledir)
-          if fileexists(unpackdir .. "/" .. tlgfile) then
-            print("Saved " .. tlgext
+        local testengine = ((engine == stdengine and "") or "." .. engine)        
+        local function save_test(test_ext,gen_ext,out_ext,pdfmode)
+          local out_file = name .. testengine .. out_ext
+          local gen_file = name .. "." .. engine .. gen_ext
+          print("Creating and copying " .. out_file)
+          runtest(name,engine,false,test_ext,pdfmode)
+          ren(testdir,gen_file,out_file)
+          cp(out_file,testdir,testfiledir)
+          if fileexists(unpackdir .. "/" .. out_file) then
+            print("Saved " .. out_ext
               .. " file overrides unpacked version of the same name")
+            return 1
           end
+          return 0
+        end
+        local errorlevel
+        if fileexists(testfiledir .. "/" .. name .. lvtext) then
+          errorlevel = save_test(lvtext,logext,tlgext)
         else
-          -- Create one .tpf file
-          print("Creating and copying " .. tpfext)
-          local tpffile  = name .. tpfext
-          local newfile  = name .. "." .. engine .. pdfext
-          runtest(name,engine,false,pvtext,true)
-          ren(testdir,newfile,tpffile)
-          cp(tpffile,testdir,testfiledir)
+          errorlevel = save_test(pvtext,pdfext,tpfext,true)
         end
-        return 0
+        if errorlevel ~=0 then return errorlevel end
       end
     elseif locate({unpackdir, testfiledir}, {name .. lveext}) then
       print("Saved " .. tlgext .. " file overrides a "
         .. lveext .. " file of the same name")
+      return 1
     else
       print('Test "'.. name .. '"not found')
       return 1     
     end
   end
+  return 0
 end
-
diff --git a/Master/texmf-dist/scripts/l3build/l3build-ctan.lua b/Master/texmf-dist/scripts/l3build/l3build-ctan.lua
index 875f50dc4fa..9d29f63f1c7 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-ctan.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-ctan.lua
@@ -22,9 +22,8 @@ for those people who are interested.
 
 --]]
 
-local gsub             = string.gsub
-local match            = string.match
-local insert           = table.insert
+local pairs = pairs
+local print = print
 
 -- Copy files to the main CTAN release directory
 function copyctan()
@@ -56,62 +55,6 @@ function copyctan()
   end
 end
 
--- Copy files to the correct places in the TDS tree
-function copytds()
-  local function install(source, dest, files, tool)
-    local moduledir = moduledir
-    -- For material associated with secondary tools (BibTeX, MakeIndex)
-    -- the structure needed is slightly different from those items going
-    -- into the tex/doc/source trees
-    if tool then
-      -- "base" is reserved for the tools themselves: make the assumption
-      -- in this case that the tdsroot name is the right place for stuff to
-      -- go (really just for the team)
-      if module == "base" then
-        moduledir = tdsroot
-      else
-        moduledir = module
-      end
-    end
-    -- Convert the file table(s) to a list of individual files
-    local filenames = { }
-    for _,i in ipairs(files) do
-      for _,j in ipairs(i) do
-        for file,_ in pairs(tree(source, j)) do
-          insert(filenames, file)
-        end
-      end
-    end
-    -- The target is only created if there are actual files to install
-    if next(filenames) ~= nil then
-      local installdir = tdsdir .. "/" .. dest .. "/" .. moduledir
-      mkdir(installdir)
-      for _,i in ipairs(filenames) do
-        cp(i, source, installdir)
-      end
-    end
-  end
-  install(
-    docfiledir,
-    "doc",
-    {bibfiles, demofiles, docfiles, pdffiles, textfiles, typesetlist}
-  )
-  install(unpackdir, "makeindex", {makeindexfiles}, true)
-  install(unpackdir, "bibtex/bst", {bstfiles}, true)
-  install(sourcefiledir, "source", {sourcelist})
-  install(unpackdir, "tex", {installfiles})
-  install(unpackdir, "scripts", {scriptfiles}, true)
-  -- Any script man files need special handling
-  for _,glob in pairs(scriptmanfiles) do
-    for file,_ in pairs(tree(docfiledir,glob)) do
-      -- Man files should have a single-digit extension: the type
-      local installdir = tdsdir .. "/doc/man/man"  .. match(file,".$")
-      mkdir(installdir)
-      cp(file,docfiledir,installdir)
-    end
-  end
-end
-
 -- Standard versions of the main targets for building modules
 
 function ctan()
@@ -162,7 +105,9 @@ function ctan()
     rmdir(tdsdir)
     mkdir(tdsdir)
     if standalone then
-      errorlevel = bundlectan()
+      errorlevel = install_files(tdsdir,true)
+      if errorlevel ~=0 then return errorlevel end
+      copyctan()
     else
       errorlevel = call(modules, "bundlectan")
     end
@@ -183,8 +128,8 @@ function ctan()
     if packtdszip then
       cp(ctanpkg .. ".tds.zip", tdsdir, ctandir)
     end
-    dirzip(ctandir, ctanpkg)
-    cp(ctanpkg .. ".zip", ctandir, currentdir)
+    dirzip(ctandir, ctanzip)
+    cp(ctanzip .. ".zip", ctandir, currentdir)
   else
     print("\n====================")
     print("Typesetting failed, zip stage skipped!")
@@ -193,53 +138,3 @@ function ctan()
   return errorlevel
 end
 
-function bundlectan()
-  -- Generate a list of individual file names excluding those in the second
-  -- argument: the latter is a table
-  local function excludelist(include, exclude, dir)
-    local include = include or { }
-    local exclude = exclude or { }
-    local dir = dir or currentdir
-    local includelist = { }
-    local excludelist = { }
-    for _,i in ipairs(exclude) do
-      for _,j in ipairs(i) do
-        for file,_ in pairs(tree(dir, j)) do
-          excludelist[file] = true
-        end
-      end
-    end
-    for _,i in ipairs(include) do
-      for file,_ in pairs(tree(dir, i)) do
-        if not excludelist[file] then
-          insert(includelist, file)
-        end
-      end
-    end
-    return includelist
-  end
-  unpack()
-  local errorlevel = doc()
-  if errorlevel == 0 then
-    -- Work out what PDF files are available
-    pdffiles = { }
-    for _,i in ipairs(typesetfiles) do
-      insert(pdffiles, (gsub(i, "%.%w+$", ".pdf")))
-    end
-    -- For the purposes here, any typesetting demo files need to be
-    -- part of the main typesetting list
-    local typesetfiles = typesetfiles
-    for _,v in pairs(typesetdemofiles) do
-      insert(typesetfiles, v)
-    end
-    typesetlist = excludelist(typesetfiles, {sourcefiles}, docfiledir)
-    sourcelist = excludelist(
-      sourcefiles, {bstfiles, installfiles, makeindexfiles, scriptfiles},
-      sourcefiledir
-    )
-    copyctan()
-    copytds()
-  end
-  return errorlevel
-end
-
diff --git a/Master/texmf-dist/scripts/l3build/l3build-file-functions.lua b/Master/texmf-dist/scripts/l3build/l3build-file-functions.lua
index 510567e5342..bc462d3ff33 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-file-functions.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-file-functions.lua
@@ -217,7 +217,6 @@ if os_type == "windows" then
   os_diffext = getenv("diffext") or ".fc"
   os_diffexe = getenv("diffexe") or "fc /n"
   os_grepexe = "findstr /r"
-  os_newline = "\n"
   if tonumber(luatex_version) < 100 or
      (tonumber(luatex_version) == 100
        and tonumber(luatex_revision) < 4) then
diff --git a/Master/texmf-dist/scripts/l3build/l3build-install.lua b/Master/texmf-dist/scripts/l3build/l3build-install.lua
index c18677e4271..2c4274454db 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-install.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-install.lua
@@ -28,71 +28,191 @@ local print = print
 local set_program = kpse.set_program_name
 local var_value   = kpse.var_value
 
+local gsub  = string.gsub
+local match = string.match
+
+local insert = table.insert
+
 local function gethome()
   set_program("latex")
   return options["texmfhome"] or var_value("TEXMFHOME")
 end
 
 function uninstall()
-  local function uninstall_files(target)
-    local installdir = gethome() .. target
+  local function uninstall_files(dir,subdir)
+    subdir = subdir or moduledir
+    dir = dir .. "/" .. subdir
+    local installdir = gethome() .. "/" .. dir
     if options["dry-run"] then
-      print("\n" .. "Installation root: " .. installdir)
       local files = filelist(installdir)
-      -- Deal with an empty directory
       if next(files) then
-        print("\n" .. "Files for removal:")
+        print("\n" .. "For removal from " .. installdir .. ":")
         for _,file in pairs(filelist(installdir)) do
           print("- " .. file)
         end
-      else
-        print("No files present")
       end
       return 0
     else
-      return rmdir(installdir)
+      if direxists(installdir) then
+        return rmdir(installdir)
+      end
+    end
+    return 0
+  end
+  local errorlevel = 0
+  -- Any script man files need special handling
+  local manfiles = { }
+  for _,glob in pairs(scriptmanfiles) do
+    for file,_ in pairs(tree(docfiledir,glob)) do
+      -- Man files should have a single-digit extension: the type
+      local installdir = gethome() .. "/doc/man/man"  .. match(file,".$")
+      if fileexists(installdir .. "/" .. file) then
+        if options["dry-run"] then
+          insert(manfiles,"man" .. match(file,".$") .. "/" ..
+           select(2,splitpath(file)))
+        else
+          errorlevel = errorlevel + rm(installdir,file)
+        end
+      end
+    end
+  end
+  if next(manfiles) then
+    print("\n" .. "For removal from " .. gethome() .. "/doc/man:")
+    for _,v in ipairs(manfiles) do
+      print("- " .. v)
     end
   end
-  return   uninstall_files("/tex/" .. moduledir)
-         + uninstall_files("/scripts/" .. module)
+  return   uninstall_files("doc")
+         + uninstall_files("source")
+         + uninstall_files("tex")
+         + uninstall_files("bibtex/bst",module)
+         + uninstall_files("makeindex",module)
+         + uninstall_files("scripts",module)
+         + errorlevel
 end
 
-
--- Locally install files: only deals with those extracted, not docs etc.
-function install()
-  local function install_files(files,target)
-    if not next(files) then
-      return 0
+function install_files(target,full,dry_run)
+  local function install_files(source,dir,files,subdir,tool)
+    subdir = subdir or moduledir
+    -- For material associated with secondary tools (BibTeX, MakeIndex)
+    -- the structure needed is slightly different from those items going
+    -- into the tex/doc/source trees
+    if tool and module == "base" then
+      subdir = nil
     end
-    local installdir = gethome() .. target
-    if options["dry-run"] then
-      print("\n" .. "Installation root: " .. installdir
-        .. "\n" .. "Installation files:"
-      )
-      for _,filetype in pairs(files) do
-        for _,file in pairs(filelist(unpackdir,filetype)) do
-          print("- " .. file)
+    dir = dir .. (subdir and "/" or "") .. subdir
+    local filenames = { }
+    for _,glob_table in pairs(files) do
+      for _,glob in pairs(glob_table) do
+        for file,_ in pairs(tree(source,glob)) do
+          insert(filenames,file)
         end
       end
-      return 0
-    else
-      errorlevel = cleandir(installdir)
-      if errorlevel ~= 0 then
-        return errorlevel
+    end
+    local errorlevel = 0
+    -- The target is only created if there are actual files to install
+    if next(filenames) then
+      local installdir = target .. "/" .. dir
+      if dry_run then
+        print("\n" .. "For installation in " .. installdir .. ":")
+      else
+        errorlevel = cleandir(installdir)
+        if errorlevel ~= 0 then return errorlevel end
       end
-      for _,filetype in pairs(files) do
-        errorlevel = cp(filetype, unpackdir, installdir)
-        if errorlevel ~= 0 then
-          return errorlevel
+      for _,file in ipairs(filenames) do
+        if dry_run then
+          print("- " .. select(2,splitpath(file)))
+        else
+          errorlevel = cp(file,source,installdir)
+          if errorlevel ~= 0 then return errorlevel end
         end
       end
     end
     return 0
   end
   local errorlevel = unpack()
-  if errorlevel ~= 0 then
-    return errorlevel
+  if errorlevel ~= 0 then return errorlevel end
+  errorlevel = install_files(unpackdir,"tex",{installfiles})
+    + install_files(unpackdir,"bibtex/bst",{bstfiles},module,true)
+    + install_files(unpackdir,"makeindex",{makeindexfiles},module,true)
+    + install_files(unpackdir,"scripts",{scriptfiles},module)
+  if errorlevel ~= 0 then return errorlevel end
+  if full then
+    errorlevel = doc()
+    if errorlevel ~= 0 then return errorlevel end
+
+    -- Creates a 'controlled' list of files
+    local function excludelist(dir,include,exclude)
+      include = include or { }
+      exclude = exclude or { }
+      dir = dir or currentdir
+      local includelist = { }
+      local excludelist = { }
+      for _,glob_table in pairs(exclude) do
+        for _,glob in pairs(glob_table) do
+          for file,_ in pairs(tree(dir,glob)) do
+            excludelist[file] = true
+          end
+        end
+      end
+      for _,glob in pairs(include) do
+        for file,_ in pairs(tree(dir,glob)) do
+          if not excludelist[file] then
+            insert(includelist, file)
+          end
+        end
+      end
+      return includelist
+    end
+
+    -- For the purposes here, any typesetting demo files need to be
+    -- part of the main typesetting list
+    local typesetfiles = typesetfiles
+    for _,glob in pairs(typesetdemofiles) do
+      insert(typesetfiles,glob)
+    end
+
+    -- Find PDF files
+    pdffiles = { }
+    for _,glob in pairs(typesetfiles) do
+      insert(pdffiles,(gsub(glob,"%.%w+$",".pdf")))
+    end
+
+    -- Set up lists: global as they are also needed to do CTAN releases
+    typesetlist = excludelist(docfiledir,typesetfiles,{sourcefiles})
+    sourcelist = excludelist(sourcefiledir,sourcefiles,
+      {bstfiles,installfiles,makeindexfiles,scriptfiles})
+    
+    errorlevel = install_files(sourcefiledir,"source",{sourcelist})
+      + install_files(docfiledir,"doc",
+          {bibfiles,demofiles,docfiles,pdffiles,textfiles,typesetlist})
+    if errorlevel ~= 0 then return errorlevel end
+
+    -- Any script man files need special handling
+    local manfiles = { }
+    for _,glob in pairs(scriptmanfiles) do
+      for file,_ in pairs(tree(docfiledir,glob)) do
+        if dry_run then
+          insert(manfiles,"man" .. match(file,".$") .. "/" ..
+            select(2,splitpath(file)))
+        else
+          -- Man files should have a single-digit extension: the type
+          local installdir = target .. "/doc/man/man"  .. match(file,".$")
+          errorlevel = errorlevel + mkdir(installdir)
+          errorlevel = errorlevel + cp(file,docfiledir,installdir)
+        end
+      end
+    end
+    if next(manfiles) then
+      print("\n" .. "For installation in " .. target .. "/doc/man:")
+      for _,v in ipairs(manfiles) do
+        print("- " .. v)
+      end
+    end
   end
-  return   install_files(installfiles, "/tex/" .. moduledir)
-         + install_files(scriptfiles, "/scripts/" .. module)
+  return errorlevel
+end
+
+function install()
+  return install_files(gethome(),options["full"],options["dry-run"])
 end
diff --git a/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua b/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua
index 8c39da91600..6398288a0a2 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua
@@ -109,7 +109,7 @@ target_list =
             local errorlevel = call(modules,"tag")
             -- Deal with any files in the bundle dir itself
             if errorlevel == 0 then
-              errorlevel = tag(names[1])
+              errorlevel = tag(names)
             end
             return errorlevel
           end,
diff --git a/Master/texmf-dist/scripts/l3build/l3build-tagging.lua b/Master/texmf-dist/scripts/l3build/l3build-tagging.lua
index 9a6f8cf5014..6ad11e68943 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-tagging.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-tagging.lua
@@ -28,7 +28,7 @@ local os_date = os.date
 local match   = string.match
 local gsub    = string.gsub
 
-function update_tag(filename,content,tagname,tagdate)
+update_tag = update_tag or function(filename,content,tagname,tagdate)
   return content
 end
 
@@ -61,8 +61,9 @@ local function update_file_tag(file,tagname,tagdate)
   return 0
 end
 
-function tag(tagname)
+function tag(tagnames)
   local tagdate = options["date"] or os_date("%Y-%m-%d")
+  local tagname = tagnames[1] or tagdate
   local dirs = remove_duplicates({currentdir, sourcefiledir, docfiledir})
   local errorlevel = 0
   for _,dir in pairs(dirs) do
diff --git a/Master/texmf-dist/scripts/l3build/l3build-variables.lua b/Master/texmf-dist/scripts/l3build/l3build-variables.lua
index 98ce7fd6c0e..f24b11acb3a 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-variables.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-variables.lua
@@ -159,6 +159,7 @@ end
 -- Other required settings
 asciiengines = asciiengines or {"pdftex"}
 checkruns    = checkruns    or 1
+ctanzip      = ctanzip      or ctanpkg .. "-ctan"
 epoch        = epoch        or 1463734800
 if flatten == nil then
   flatten = true
diff --git a/Master/texmf-dist/scripts/l3build/l3build.lua b/Master/texmf-dist/scripts/l3build/l3build.lua
index 5cefd6d5dc0..2107f024d3a 100755
--- a/Master/texmf-dist/scripts/l3build/l3build.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build.lua
@@ -25,7 +25,7 @@ for those people who are interested.
 --]]
 
 -- Version information
-release_date = "2018-08-02"
+release_date = "2018-08-04"
 
 -- File operations are aided by the LuaFileSystem module
 local lfs = require("lfs")
@@ -34,6 +34,7 @@ local lfs = require("lfs")
 
 local assert           = assert
 local ipairs           = ipairs
+local insert           = table.insert
 local lookup           = kpse.lookup
 local match            = string.match
 local next             = next
@@ -123,13 +124,28 @@ if options["target"] == "check" then
   if #checkconfigs > 1 then
     local errorlevel = 0
     local opts = options
+    local failed = { }
     for i = 1, #checkconfigs do
       opts["config"] = {checkconfigs[i]}
       errorlevel = call({"."}, "check", opts)
-      if errorlevel ~= 0 then exit(1) end
+      if errorlevel ~= 0 then
+        if options["halt-on-error"] then
+          exit(1)
+        else
+          insert(failed,checkconfigs[i])
+        end
+      end
+    end
+    if next(failed) then
+      print("  Failed tests for configs:")
+      for _,config in ipairs(failed) do
+        print("  - " .. config)
+      end
+      exit(1)
+    else
+      -- Avoid running the 'main' set of tests twice
+      exit(0)
     end
-    -- Avoid running the 'main' set of tests twice
-    exit(0)
   end
 end
 if #checkconfigs == 1 and
diff --git a/Master/texmf-dist/source/latex/l3build/l3build.dtx b/Master/texmf-dist/source/latex/l3build/l3build.dtx
index f15e1b970a9..5032e78ce5c 100644
--- a/Master/texmf-dist/source/latex/l3build/l3build.dtx
+++ b/Master/texmf-dist/source/latex/l3build/l3build.dtx
@@ -152,6 +152,7 @@
 \luavarseparator
 \luavarset{asciiengines}{\{"pdftex"\}}{Engines which should log as sure ASCII}
 \luavarset{checkruns}   {1}           {Number of runs to complete for a test before comparing the log}
+\luavarset{ctanzip}{ctanpkg ... "-ctan"}{Name of the zip file created for upload to CTAN}
 \luavarset{epoch}       {1463734800}  {Epoch (Unix date) to set for test runs}
 \luavarset{flatten}     {true}        {Switch to flatten any source structure when sending to CTAN}
 \luavarset{maxprintline}{79}          {Length of line to use in log files}
@@ -221,7 +222,7 @@
 %    }^^A
 % }
 %
-% \date{Released 2018-08-02}
+% \date{Released 2018-08-04}
 %
 % \maketitle
 % \tableofcontents
@@ -329,6 +330,8 @@
 % \item |--force| (|-f|) Force checks to run even if sanity
 %   checks fail, \emph{e.g.}~when |--engine| is not given in
 %   \luavar{checkengines}
+% \item |--full| Instructs the \texttt{install} target to include the 
+%   \texttt{doc} and \texttt{source} trees
 % \item |--halt-on-error| (|-H|) Specifies that checks
 %   should stop as soon as possible, rather than running all requested
 %   tests; the difference file is printed in the terminal directly in the case of failure
@@ -1122,10 +1125,10 @@
 %   \item Removal of blank lines
 % \end{itemize}
 %
-% Testing on the result of typesetting \texttt{.pvt} files is carried out using
-% only the standard engine, as aspects of PDF files are not identical between
-% engines. To allow platform-independence, PDF-based tests must use only
-% Type1 or OpenType fonts: Type3 fonts are system-dependent.
+% To allow platform-independence, PDF-based tests must use only Type~1 or
+% OpenType fonts: Type3 fonts are system-dependent. PDF files are
+% engine-specific, thus one |.tpf| file should be stored per engine to be
+% tested.
 %
 % \section{Release-focussed features}
 %
@@ -1251,16 +1254,22 @@
 %     \toprule
 %     Entry & Type \\
 %     \midrule
-%       \var{date}        & String  \\
-%       \var{engine}      & Table   \\
-%       \var{force}       & Boolean \\
-%       \var{halt}        & Boolean \\
-%       \var{help}        & Boolean \\
-%       \var{names}       & Table   \\
-%       \var{quiet}       & Boolean \\
-%       \var{rerun}       & Boolean \\
-%       \var{testfiledir} & Table   \\
-%       \var{version}     & String  \\
+%       \var{config}        & Table   \\
+%       \var{date}          & String  \\
+%       \var{dirty}         & Boolean \\
+%       \var{dry-run}       & Boolean \\
+%       \var{engine}        & Table   \\
+%       \var{epoch}         & String  \\
+%       \var{first}         & Boolean \\
+%       \var{force}         & Boolean \\
+%       \var{full}          & Boolean \\
+%       \var{halt-on-error} & Boolean \\
+%       \var{help}          & Boolean \\
+%       \var{names}         & Table   \\
+%       \var{quiet}         & Boolean \\
+%       \var{rerun}         & Boolean \\
+%       \var{shuffle}       & Boolean \\
+%       \var{texmfhome}     & String  \\
 %     \bottomrule
 %     \end{tabular}
 %   \end{center}
@@ -1452,6 +1461,17 @@
 %   |target| in this table is ignored.
 % \end{function}
 %
+% \begin{function}{install_files()}
+%   \begin{syntax}
+%     |install_files(|\meta{target},\meta{full},\meta{dry-run}|)|
+%   \end{syntax}
+%   Installs the files from the module into the TDS root \meta{target}.
+%   If \meta{full} is \texttt{true}, all files are copied: if it is
+%   \texttt{false}, the \texttt{doc} and \texttt{source} trees are skipped.
+%   If \meta{dry-run} is \texttt{true}, no files are copied, but instead the
+%   files which would be copied are reported.
+% \end{function}
+%
 % \subsection{Customising the target list}
 %
 % The targets known to \pkg{l3build} are stored in the global table