l3build (2apr15)

git-svn-id: svn://tug.org/texlive/trunk@36706 c570f23f-e606-0410-a88d-b1316a301751

3 files changed, 123 insertions, 47 deletions

diff --git a/Master/texmf-dist/doc/latex/l3build/l3build.pdf b/Master/texmf-dist/doc/latex/l3build/l3build.pdf
index 480919d6adf..3bd005b79d2 100644
--- a/Master/texmf-dist/doc/latex/l3build/l3build.pdf
+++ b/Master/texmf-dist/doc/latex/l3build/l3build.pdf
diff --git a/Master/texmf-dist/source/latex/l3build/l3build.dtx b/Master/texmf-dist/source/latex/l3build/l3build.dtx
index b75bd060034..9ac009cddf5 100644
--- a/Master/texmf-dist/source/latex/l3build/l3build.dtx
+++ b/Master/texmf-dist/source/latex/l3build/l3build.dtx
@@ -39,8 +39,8 @@
 \RequirePackage{expl3}
 \def\ExplFileName{l3build}
 \def\ExplFileDescription{L3 Regression test suite}
-\def\ExplFileDate{2015/02/01}
-\def\ExplFileVersion{5504}
+\def\ExplFileDate{2015/04/02}
+\def\ExplFileVersion{5564}
 \documentclass[full]{l3doc}
 \renewcommand\partname{Part}
 \usepackage{multicol,needspace}
@@ -73,6 +73,7 @@
 \def\allluavars{
 \luavarset{module}{""}{The name of the module.}
 \luavarset{bundle}{""}{The name of the bundle in which the module belongs.}
+\luavarset{ctanpkg}{bundle}{Name of the bundle on CTAN}
 \luavarseparator
 \luavarset{modules}{\{~\}}{The list of all modules in a bundle (when not auto-detecting)}
 \luavarset{exclmodules}{\{~\}}{Directories to be excluded from automatic module detection}
@@ -91,13 +92,15 @@
 \luavarset{tdsdir}     {distribdir .. "/tds"}       {Generated folder where files are organised for a TDS.}
 \luavarset{tdsroot}    {"latex"}{Root directory of the TDS structure for the bundle/module to be installed into.}
 \luavarseparator
+\luavarset{auxfiles}        {\{"*.aux", "*.toc"\}}
+  {Auxiliary typesetting files that should be auto-removed.}
 \luavarset{binaryfiles}     {\{"*.pdf", "*.zip"\}}
                             {Files to be added in binary mode to zip files.}
 \luavarset{checkfiles}      {\{~\}}{Extra files unpacked purely for tests}
 \luavarset{checksuppfiles}  { }{Files needed for performing regression tests.}
 \luavarset{cmdchkfiles}     {\{~\}}{Files need to perform command checking (\cls{l3doc}-based documentation only).}
 \luavarset{demofiles}       {\{~\}}{Files which show how to use a module.}
-\luavarset{cleanfiles}      {\{"*.pdf", "*.zip"\}}{Files to delete when cleaning.}
+\luavarset{cleanfiles}      {\{"*.log", "*.pdf", "*.zip"\}}{Files to delete when cleaning.}
 \luavarset{excludefiles}    {\{"*\string~"\}}            {Files to ignore entirely (default for Emacs backup files).}
 \luavarset{installfiles}    {\{"*.sty"\}}         {Files to install to the \TeX{} tree and similar tasks.}
 \luavarset{makeindexfiles}  {\{"*.ist"\}}{MakeIndex files to be included in a TDS-style zip}
@@ -259,6 +262,13 @@
 %              yyy.ins                 zoo.ins
 % \end{Verbatim}
 % All modules within a bundle must use the same build script name.
+% 
+% In a small number of cases, the name used by CTAN for a module or bundle is
+% different from that used in the installation tree. For example, the \LaTeXe{}
+% kernel is called \pkg{latex-base} by CTAN but is located inside
+% \texttt{\meta{texmf}/tex/latex/base}. This can be handled by using
+% \var{ctanpkg} for the name required by CTAN to override the standard
+% value.
 %
 % The |testfiles/| folder is local to each module, and its layout consists of a series of regression tests with their outputs.
 % \begin{Verbatim}
@@ -362,7 +372,7 @@
 % The `root' of the TDS structure is defined by \var{tdsroot}, which is \luavar{tdsroot} by default. Plain users would redefine this to |"plain"| (or perhaps |"generic"|), for example.
 % The build process for a |.tds.zip| file currently assumes a `standard'
 % structure in which all extracted files should be places inside the |tex|
-% tree in a single directory, as shown above.
+% tree in a single directory, as shown above. 
 %
 % \pagebreak[2]
 % The |doc| tree is constructed from:
@@ -378,6 +388,8 @@
 % Files that should always be excluded from the archive are matched against the \var{excludefiles} variable; by default this is \luavar{excludefiles}, which match Emacs' autosave files.
 %
 % Binary files should be specified with the \var{binaryfiles} variable (default \luavar{binaryfiles}); these are added to the zip archive without normalising line endings (text files are automatically converted to Unix-style line endings).
+% 
+% Any auxiliary files in the working directory may interfere with the automatic typesetting process. Those listed in \var{auxfiles} are automatically removed to prevent this, and are also deleted by the |clean| operation. (This seetting is most useful when typesetting manually and automatically may use different settings, for example enabling/disabling \pkg{hyperref}.)
 %
 % To create the archive, by default the binary \var{zipexe} is used (\luavar{zipexe}) with options \var{zipopts} (|-v -r -X|). The intermediate build directories \var{ctandir} and \var{tdsdir} are used to construct the archive.
 % \end{buildcmd}
@@ -604,6 +616,14 @@
 %   \item Correction of a \LuaTeX{} error message typo (|I''m going to assume|).
 % \end{itemize}
 %
+% \subsection{Generating test files with \pkg{DocStrip}}
+%
+% It is  possible to pack tests inside source files. Tests generated during the
+% unpacking process will be available to the \texttt{check} and \texttt{save}
+% commands as if they were stored in the \texttt{testfiledir}. Any explicit
+% test files inside \texttt{testfiledir} take priority over generated ones
+% with the same names.
+%
 % \section{Writing test files}
 % \label{sec:writing-tests}
 %
diff --git a/Master/texmf-dist/tex/latex/l3build/l3build.lua b/Master/texmf-dist/tex/latex/l3build/l3build.lua
index 5250161fb2c..1edf777cfa1 100644
--- a/Master/texmf-dist/tex/latex/l3build/l3build.lua
+++ b/Master/texmf-dist/tex/latex/l3build/l3build.lua
@@ -17,8 +17,8 @@
 --]]
 
 -- Version information: should be identical to that in l3build.dtx
-release_date = "2015/02/01"
-release_ver  = "5504"
+release_date = "2015/04/02"
+release_ver  = "5564"
 
 -- "module" is a deprecated function in Lua 5.2: as we want the name
 -- for other purposes, and it should eventually be 'free', simply
@@ -71,19 +71,22 @@ tdsroot     = tdsroot or "latex"
 -- Location for installation on CTAN or in TEXMFHOME
 if bundle == "" then
   moduledir = tdsroot .. "/" .. module
+  ctanpkg   = ctanpkg or module
 else
   moduledir = tdsroot .. "/" .. bundle .. "/" .. module
+  ctanpkg   = ctanpkg or bundle
 end
 
 -- File types for various operations
 -- Use Unix-style globs
 -- All of these may be set earlier, so a initialised conditionally
+auxfiles         = auxfiles         or {"*.aux", "*.toc"}
 binaryfiles      = binaryfiles      or {"*.pdf", "*.zip"}
 checkfiles       = checkfiles       or { } -- Extra files unpacked purely for tests
 checksuppfiles   = checksuppfiles   or { }
 cmdchkfiles      = cmdchkfiles      or { }
 demofiles        = demofiles        or { }
-cleanfiles       = cleanfiles       or {"*.pdf", "*.zip"}
+cleanfiles       = cleanfiles       or {"*.log", "*.pdf", "*.zip"}
 excludefiles     = excludefiles     or {"*~"}             -- Any Emacs stuff
 installfiles     = installfiles     or {"*.sty"}
 makeindexfiles   = makeindexfiles   or {"*.ist"}
@@ -425,7 +428,7 @@ function copyctan ()
   -- Do all of the copying in one go
   for _,i in ipairs ({demofiles, pdffiles, sourcefiles, textfiles, typesetlist}) do
     for _,j in ipairs (i) do
-      cp (j, ".", ctandir .. "/" .. bundle)
+      cp (j, ".", ctandir .. "/" .. ctanpkg)
     end
   end
 end
@@ -463,7 +466,7 @@ function copytds ()
       for _,i in ipairs (filenames) do
         cp (i, source, installdir)
       end
-    end  
+    end
   end
   install (".", "doc", {demofiles, pdffiles, textfiles, typesetlist})
   install (unpackdir, "makeindex", {makeindexfiles}, true)
@@ -546,13 +549,15 @@ function formatlog (logfile, newfile)
         line, "cm, mm, dd, cc, bp, or sp", "cm, mm, dd, cc, nd, nc, bp, or sp"
       )
     -- Normalise a case where a LuaTeX bug misprints \csname\endcsname
-    -- This comes before the normalisation of the appearance of 
+    -- This comes before the normalisation of the appearance of
     -- \csname\endcsname in case we alter that!
     line = string.gsub (line, "\\ycsnam\\rendcsnam", "\\csname\\endcsname")
     -- Normalise a case where fixing a TeX bug changes the message text
     line = string.gsub (line, "\\csname\\endcsname ", "\\csname\\endcsname")
     -- Zap "on line <num>" and replace with "on line ..."
     line = string.gsub (line, "on line %d*", "on line ...")
+    -- Zap line numbers from \show, \showbox, \box_show and the like
+    line = string.gsub (line, "^l.%d* (\\[^ ]*show)", "l. ... %1")
     -- Remove spaces at the start of lines: deals with the fact that LuaTeX
     -- uses a different number to the other engines
     line = string.gsub (line, "^%s+", "")
@@ -617,6 +622,18 @@ function formatlog (logfile, newfile)
   io.close (newfile)
 end
 
+-- Look for files, directory by directory, and return the first existing
+function locate (dirs, names)
+  for _,i in ipairs (dirs) do
+    for _,j in ipairs (names) do
+      local path = i .. "/" .. j
+      if fileexists (path) then
+        return (path)
+      end
+    end
+  end
+end
+
 -- List all modules
 function listmodules ()
   local modules = { }
@@ -650,9 +667,13 @@ function runcheck (name, engine, hide)
     local difffile = testdir .. "/" .. testname .. os_diffext
     local newfile  = testdir .. "/" .. testname .. logext
     -- Use engine-specific file if available
-    local tlgfile  = testfiledir .. "/" .. name ..  "." .. i .. tlgext
-    if not fileexists (tlgfile) then
-      tlgfile  = testfiledir .. "/" .. name .. tlgext
+    local tlgfile  = locate (
+      {testfiledir, unpackdir},
+      {name ..  "." .. i .. tlgext, name .. tlgext}
+    )
+    if not tlgfile then
+      print ("Error: failed to find " .. tlgext .. " file for " .. name .. "!")
+      os.exit (1)
     end
     if os_windows then
       tlgfile = unix_to_win (tlgfile)
@@ -672,7 +693,9 @@ end
 -- Run one of the test files: doesn't check the result so suitable for
 -- both creating and verifying .tlg files
 function runtest (name, engine, hide)
-  cp (name .. lvtext, testfiledir, testdir)
+  local lvtfile = name .. lvtext
+  cp (lvtfile, fileexists (testfiledir .. "/" .. lvtfile)
+    and testfiledir or unpackdir, testdir)
   local engine = engine or stdengine
   -- Set up the format file name if it's one ending "...tex"
   local format
@@ -682,7 +705,6 @@ function runtest (name, engine, hide)
     format = ""
   end
   local logfile = testdir .. "/" .. name .. logext
-  local lvtfile = name .. lvtext
   local newfile = testdir .. "/" .. name .. "." .. engine .. logext
   for i = 1, checkruns, 1 do
     run (
@@ -699,7 +721,11 @@ function runtest (name, engine, hide)
     local ext = string.match (i, "%....")
     if ext ~= lvtext and ext ~= tlgext and ext ~= logext then
       if not fileexists (testsuppdir .. "/" .. i) then
-        ren (testdir, i, string.gsub (i, name, name .. "." .. engine ))
+        ren (
+          testdir, i, string.gsub (
+            i, string.gsub (name, "%-", "%%-"), name .. "." .. engine
+          )
+        )
       end
     end
   end
@@ -711,12 +737,9 @@ function stripext (file)
   return name or file
 end
 
+-- Look for a test: could be in the testfiledir or the unpackdir
 function testexists (test)
-  if fileexists (testfiledir .. "/" .. test .. lvtext) then
-    return true
-  else
-    return false
-  end
+  return (locate ({testfiledir, unpackdir}, {test .. lvtext}))
 end
 
 -- Standard versions of the main targets for building modules
@@ -760,16 +783,28 @@ end
 function checkall ()
   local errorlevel = 0
   if testfiledir ~= "" and direxists (testfiledir) then
-    checkinit ()
-    print ("Running checks on")
-    for _,i in ipairs (filelist (testfiledir, "*" .. lvtext)) do
-      local name = stripext (i)
+    local function execute (name)
+      local name = stripext (name)
       print ("  " .. name)
       local errlevel = runcheck (name, nil, true)
       if errlevel ~= 0 then
         errorlevel = 1
       end
     end
+    checkinit ()
+    print ("Running checks on")
+    for _,i in ipairs (filelist (testfiledir, "*" .. lvtext)) do
+      execute (i)
+    end
+    for _,i in ipairs (filelist (unpackdir, "*" .. lvtext)) do
+      if fileexists (testfiledir .. "/" .. i) then
+        print ("Duplicate test file: " .. i)
+        errorlevel = 1
+        break
+      else
+        execute (i)
+      end
+    end
     if errorlevel ~= 0 then
       checkdiff ()
     else
@@ -780,8 +815,8 @@ function checkall ()
 end
 
 function checklvt (name, engine)
+  checkinit ()
   if testexists (name) then
-    checkinit ()
     print ("Running checks on " .. name)
     local errorlevel = runcheck (name, engine)
     if errorlevel ~= 0 then
@@ -823,6 +858,12 @@ function clean ()
   end
 end
 
+function auxclean ()
+  for _,i in ipairs (auxfiles) do
+    rm (".", i)
+  end
+end
+
 function bundleclean ()
   allmodules ("clean")
   for _,i in ipairs (cleanfiles) do
@@ -893,7 +934,7 @@ function ctan (standalone)
   end
   if errorlevel == 0 then
     rmdir (ctandir)
-    mkdir (ctandir .. "/" .. bundle)
+    mkdir (ctandir .. "/" .. ctanpkg)
     rmdir (tdsdir)
     mkdir (tdsdir)
     if standalone then
@@ -914,16 +955,16 @@ function ctan (standalone)
           cp (name, ".", dir)
           ren (dir, name, stripext (name))
         end
-        installtxt (j, ctandir .. "/" .. bundle)
+        installtxt (j, ctandir .. "/" .. ctanpkg)
         installtxt (j, tdsdir .. "/doc/" .. tdsroot .. "/" .. bundle)
       end
     end
-    dirzip (tdsdir, bundle .. ".tds")
+    dirzip (tdsdir, ctanpkg .. ".tds")
     if packtdszip then
-      cp (bundle .. ".tds.zip", tdsdir, ctandir)
+      cp (ctanpkg .. ".tds.zip", tdsdir, ctandir)
     end
-    dirzip (ctandir, bundle)
-    cp (bundle .. ".zip", ctandir, ".")
+    dirzip (ctandir, ctanpkg)
+    cp (ctanpkg .. ".zip", ctandir, ".")
   else
     print ("\n====================")
     print ("Typesetting failed, zip stage skipped!")
@@ -979,28 +1020,36 @@ function doc ()
     local name = stripext (file)
     -- A couple of short functions to deal with the repeated steps in a
     -- clear way
+    local function biber (name)
+      if fileexists (typesetdir .. "/" .. name .. ".bcf") then
+        return (run (typesetdir, "biber " .. name))
+      end
+    end
     local function makeindex (name, inext, outext, logext, style)
       if fileexists (typesetdir .. "/" .. name .. inext) then
-        run (
-          typesetdir ,
-          "makeindex -s " .. style .. " -o " .. name .. outext
-            .. " -t " .. name .. logext .. " "  .. name .. inext
-          )        
+        return (
+          run (
+            typesetdir ,
+            "makeindex -s " .. style .. " -o " .. name .. outext
+              .. " -t " .. name .. logext .. " "  .. name .. inext
+            )
+          )
       end
     end
     local function typeset (file)
-      local errorlevel =
+      return (
         os.execute (
-            os_setenv .. " TEXINPUTS=" .. typesetdir .. 
+            os_setenv .. " TEXINPUTS=" .. typesetdir ..
               os_pathsep .. localdir .. (typesetsearch and os_pathsep or "") ..
               os_concat ..
-            typesetexe .. " " .. typesetopts .. 
+            typesetexe .. " " .. typesetopts ..
               " -output-directory=" .. typesetdir ..
-              " \"" .. typesetcmds .. 
+              " \"" .. typesetcmds ..
               "\\input " .. typesetdir .. "/" .. file .. "\""
           )
-      return errorlevel
+      )
     end
+    auxclean ()
     os.remove (name .. ".pdf")
     print ("Typesetting " .. name)
     local errorlevel = typeset (file)
@@ -1008,6 +1057,7 @@ function doc ()
       print (" ! Compilation failed")
       return (errorlevel)
     else
+      biber (name)
       makeindex (name, ".glo", ".gls", ".glg", glossarystyle)
       makeindex (name, ".idx", ".ind", ".ilg", indexstyle)
       typeset (file)
@@ -1056,16 +1106,22 @@ end
 function save (name, engine)
   local tlgfile = name .. (engine and ("." .. engine) or "") .. tlgext
   local newfile = name .. "." .. (engine or stdengine) .. logext
-  if fileexists (testfiledir .. "/" .. name .. lvtext) then
-    checkinit ()
+  checkinit ()
+  if testexists (name) then
     print ("Creating and copying " .. tlgfile)
     runtest (name, engine, false)
     ren (testdir, newfile, tlgfile)
     cp (tlgfile, testdir, testfiledir)
+    if fileexists (unpackdir .. "/" .. tlgfile) then
+      print (
+        "Saved " .. tlgext
+          .. " file overrides unpacked version of the same name"
+      )
+    end
   else
     print (
-      "Test input \"" .. testfiledir .. "/" .. name .. lvtext ..
-        "\" not found"
+      "Test input \"" .. testfiledir .. "/" .. name .. lvtext
+        .. "\" not found"
       )
   end
 end
@@ -1082,7 +1138,7 @@ end
 
 -- Split off from the main unpack so it can be used on a bundle and not
 -- leave only one modules files
-function bundleunpack ()
+bundleunpack = bundleunpack or function ()
   mkdir (localdir)
   cleandir (unpackdir)
   for _,i in ipairs (sourcefiles) do