l3build (7feb19)

git-svn-id: svn://tug.org/texlive/trunk@49981 c570f23f-e606-0410-a88d-b1316a301751

8 files changed, 204 insertions, 102 deletions

diff --git a/Master/texmf-dist/scripts/l3build/l3build-arguments.lua b/Master/texmf-dist/scripts/l3build/l3build-arguments.lua
index e693ab929df..a6a13a43fee 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-arguments.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-arguments.lua
@@ -47,6 +47,11 @@ option_list =
         desc  = "Sets the date to insert into sources",
         type  = "string"
       },
+    debug =
+      {
+        desc = "Runs target in debug mode (not supported by all targets)",
+        type = "boolean"
+      },
     dirty =
       {
         desc = "Skip cleaning up the test area",
@@ -57,6 +62,11 @@ option_list =
         desc = "Dry run for install",
         type = "boolean"
       },
+    email =
+      {
+        desc = "Email address of CTAN uploader",
+        type = "string"
+      },
     engine =
       {
         desc  = "Sets the engine(s) to use for running test",
@@ -68,6 +78,12 @@ option_list =
         desc  = "Sets the epoch for tests and typesetting",
         type  = "string"
       },
+    file =
+      {
+        desc  = "Take the upload announcement from the given file",
+        short = "F",
+        type  = "string"
+      },
     first =
       {
         desc  = "Name of first test to run",
@@ -106,6 +122,12 @@ option_list =
         desc  = "Name of last test to run",
         type  = "string"
       },
+    message =
+      {
+        desc  = "Text for upload announcement message",
+        short = "m",
+        type  = "string"
+      },
     rerun =
       {
         desc  = "Skip setup: simply rerun tests",
diff --git a/Master/texmf-dist/scripts/l3build/l3build-check.lua b/Master/texmf-dist/scripts/l3build/l3build-check.lua
index 837b6356cbf..711d367331c 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-check.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-check.lua
@@ -541,9 +541,9 @@ function runcheck(name, hide)
     checkengines = options["engine"]
   end
   -- Used for both .lvt and .pvt tests
-  local function check_and_diff(ext,engine,enginename,comp,pdftest)
+  local function check_and_diff(ext,engine,comp,pdftest)
     runtest(name,engine,hide,ext,pdftest,true)
-    local errorlevel = comp(name,enginename)
+    local errorlevel = comp(name,engine)
     if errorlevel ~= 0 and options["halt-on-error"] then
       showfaileddiff()
       return 1
@@ -552,17 +552,12 @@ function runcheck(name, hide)
   end
   local errorlevel = 0
   for _,engine in pairs(checkengines) do
-    local enginename = engine
-    -- Allow for luatex == luajittex for .tlg/.tpf purposes
-    if match(engine,"^lua") then
-      engine = "luatex"
-    end
     setup_check(name,engine)
     local errlevel = 0
     if fileexists(testfiledir .. "/" .. name .. pvtext) then
-      errlevel = check_and_diff(pvtext,engine,enginename,compare_pdf,true)
+      errlevel = check_and_diff(pvtext,engine,compare_pdf,true)
     else
-      errlevel = check_and_diff(lvtext,engine,enginename,compare_tlg)
+      errlevel = check_and_diff(lvtext,engine,compare_tlg)
     end
     if errlevel ~= 0 and options["halt-on-error"] then
       return 1
@@ -626,7 +621,7 @@ function compare_pdf(name,engine,cleanup)
   return errorlevel
 end
 
-function compare_tlg(name, engine,cleanup)
+function compare_tlg(name,engine,cleanup)
   local errorlevel
   local testname = name .. "." .. engine
   local difffile = testdir .. "/" .. testname .. os_diffext
@@ -637,7 +632,7 @@ function compare_tlg(name, engine,cleanup)
   end
   -- Do additional log formatting if the engine is LuaTeX, there is no
   -- LuaTeX-specific .tlg file and the default engine is not LuaTeX
-  if engine == "luatex"
+  if match(engine,"^lua")
     and not match(tlgfile, "%.luatex" .. "%" .. tlgext)
     and not match(stdengine,"^lua")
     then
@@ -670,24 +665,25 @@ function runtest(name, engine, hide, ext, pdfmode, breakout)
   local lvtfile = name .. (ext or lvtext)
   cp(lvtfile, fileexists(testfiledir .. "/" .. lvtfile)
     and testfiledir or unpackdir, testdir)
+  local checkopts = checkopts
   local engine = engine or stdengine
-  local realengine = engine
-  local format = ""
-  if checkformat == "latex" then
-    -- Special case for e-LaTeX format
-    if engine == "etex" then
-      format = " -fmt=latex"
-    -- Use "...latex" formats for other engines
-    elseif not match(engine,"latex") then
-      format = " -fmt=" .. gsub(engine,"tex","latex")
-    end
-    -- Special case for (u)pTeX LaTeX
-    if match(engine,"^u?ptex$") then
-      realengine = "e" .. engine
-    end
+  local binary = engine
+  local format = gsub(engine,"tex$",checkformat)
+  -- Special binary/format combos
+  if specialformats[checkformat] and next(specialformats[checkformat]) then
+    local t = specialformats[checkformat]
+    if t[engine] and next(t[engine]) then
+      local t = t[engine]
+      binary    = t.binary  or binary
+      checkopts = t.options or checkopts
+      format    = t.format  or format
+    end
+  end
+  -- Finalise format string
+  if format ~= "" then
+    format = " --fmt=" .. format
   end
   -- Special casing for XeTeX engine
-  local checkopts = checkopts
   if match(engine, "xetex") and not pdfmode then
     checkopts = checkopts .. " -no-pdf"
   end
@@ -695,19 +691,8 @@ function runtest(name, engine, hide, ext, pdfmode, breakout)
   local function setup(file)
     return " -jobname=" .. name .. " " .. ' "\\input ' .. file .. '" '
   end
-  if match(checkformat, "^context$") then
-    format = ""
+  if match(checkformat,"^context$") then
     function setup(file) return ' "' .. file .. '" '  end
-    if match(engine,"^lua") then
-      realengine = "context"
-    elseif engine == "pdftex" then
-      realengine = "texexec"
-    elseif engine == "xetex" then
-      realengine = "texexec --xetex"
-    else
-      print("Engine incompatible with format")
-      exit(1)
-    end
   end
   local basename = testdir .. "/" .. name
   local logfile = basename .. logext
@@ -743,7 +728,7 @@ function runtest(name, engine, hide, ext, pdfmode, breakout)
       -- Ensure lines are of a known length
       os_setenv .. " max_print_line=" .. maxprintline
         .. os_concat ..
-      realengine .. format 
+      binary .. format 
         .. " " .. asciiopt .. " " .. checkopts
         .. setup(lvtfile)
         .. (hide and (" > " .. os_null) or "")
diff --git a/Master/texmf-dist/scripts/l3build/l3build-install.lua b/Master/texmf-dist/scripts/l3build/l3build-install.lua
index f32dcd003fd..a97a24e29ec 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-install.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-install.lua
@@ -93,13 +93,13 @@ function uninstall()
 end
 
 function install_files(target,full,dry_run)
-  local function install_files(source,dir,files,subdir,tool)
+  local function install_files(source,dir,files,subdir)
     subdir = subdir or moduledir
     -- For material associated with secondary tools (BibTeX, MakeIndex)
     -- the structure needed is slightly different from those items going
     -- into the tex/doc/source trees
-    if tool and module == "base" then
-      subdir = nil
+    if (dir == "makeindex" or match(dir,"$bibtex")) and module == "base" then
+      subdir = "latex"
     end
     dir = dir .. (subdir and ("/" .. subdir) or "")
     local filenames = { }
@@ -222,8 +222,8 @@ function install_files(target,full,dry_run)
   if errorlevel ~= 0 then return errorlevel end
 
   errorlevel = install_files(unpackdir,"tex",{installlist})
-    + install_files(unpackdir,"bibtex/bst",{bstfiles},module,true)
-    + install_files(unpackdir,"makeindex",{makeindexfiles},module,true)
+    + install_files(unpackdir,"bibtex/bst",{bstfiles},module)
+    + install_files(unpackdir,"makeindex",{makeindexfiles},module)
     + install_files(unpackdir,"scripts",{scriptfiles},module)
   
   return errorlevel
diff --git a/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua b/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua
index f2d8797c830..668b2d02d0a 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua
@@ -120,9 +120,8 @@ target_list =
         desc = "Updates release tags in files",
         func = tag,
         pre  = function(names)
-           if not names or #names ~=1 then
-             print("Tag name required")
-             help()
+           if names and #names > 1 then
+             print("Too many tags specified; exactly one required")
              exit(1)
            end
            return 0
diff --git a/Master/texmf-dist/scripts/l3build/l3build-tagging.lua b/Master/texmf-dist/scripts/l3build/l3build-tagging.lua
index 6ad11e68943..2c0c1641a40 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-tagging.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-tagging.lua
@@ -63,7 +63,10 @@ end
 
 function tag(tagnames)
   local tagdate = options["date"] or os_date("%Y-%m-%d")
-  local tagname = tagnames[1] or tagdate
+  local tagname = nil
+  if tagnames then
+    tagname = tagnames[1]
+  end
   local dirs = remove_duplicates({currentdir, sourcefiledir, docfiledir})
   local errorlevel = 0
   for _,dir in pairs(dirs) do
diff --git a/Master/texmf-dist/scripts/l3build/l3build-upload.lua b/Master/texmf-dist/scripts/l3build/l3build-upload.lua
index 68692b0af4c..4afee06b4ed 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-upload.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-upload.lua
@@ -22,6 +22,22 @@ for those people who are interested.
 
 --]]
 
+local pairs    = pairs
+local print    = print
+local tostring = tostring
+
+local close = io.close
+local flush = io.flush
+local open  = io.open
+local popen = io.popen
+local read  = io.read
+local write = io.write
+
+local os_type = os.type
+
+local len   = string.len
+local lower = string.lower
+local match = string.match
 
 -- UPLOAD()
 --
@@ -53,70 +69,83 @@ local curl_debug = curl_debug or false -- to disable posting
 -- For now, this is undocumented.
 
 local ctanupload = ctanupload or "ask"
+if options["dry-run"] then
+  ctanupload = false
+end
 -- if ctanupload is nil or false, only validation is attempted
 -- if ctanupload is true the ctan upload URL will be used after validation
 -- if upload is anything else, the user will be prompted whether to upload.
 -- For now, this is undocumented. I think I would prefer to keep it always set to ask for the time being.
 
-function upload()
+function upload(tagnames)
 
   local uploadfile = ctanzip..".zip"
 
+  -- Keep data local
+  local uploadconfig = uploadconfig
+
   -- try a sensible default for the package name:
   uploadconfig.pkg = uploadconfig.pkg or ctanpkg or nil
 
-  -- start building the curl command:
-  ctan_post = curlexe .. " "
+  -- Get data from command line if appropriate
+  if options["file"] then
+    local f = open(options["file"],"r")
+    uploadconfig.announcement = assert(f:read('*a'))
+    close(f)
+  end
+  uploadconfig.announcement = options["message"] or uploadconfig.announcement
+  uploadconfig.email = options["email"] or uploadconfig.email
 
-  -- build up the curl command field-by-field:
+  local tagnames = tagnames or { }
+  uploadconfig.version = tagnames[1] or uploadconfig.version
 
-  --         field                                   max  desc                                 mandatory  multi
-  --         ----------------------------------------------------------------------------------------------------
-  ctan_field("announcement", uploadconfig.announcement, 8192, "Announcement",                        false, false )
-  ctan_field("author",       uploadconfig.author,        128, "Author name",                         true,  false )
-  ctan_field("bugtracker",   uploadconfig.bugtracker,    255, "URL(s) of bug tracker",               false, true  )
-  ctan_field("ctanPath",     uploadconfig.ctanPath,      255, "CTAN path",                           true,  false )
-  ctan_field("description",  uploadconfig.description,  4096, "Short description of package",        false, false )
-  ctan_field("development",  uploadconfig.development,   255, "URL(s) of development channels",      false, true  )
-  ctan_field("email",        uploadconfig.email,         255, "Email of uploader",                   true,  false )
-  ctan_field("home",         uploadconfig.home,          255, "URL(s) of home page",                 false, true  )
-  ctan_field("license",      uploadconfig.license,      2048, "Package license(s)",                  true,  true  )
-  ctan_field("note",         uploadconfig.note,         4096, "Internal note to ctan",               false, false )
-  ctan_field("pkg",          uploadconfig.pkg,            32, "Package name",                        true,  false )
-  ctan_field("repository",   uploadconfig.repository,    255, "URL(s) of source repositories",       false, true  )
-  ctan_field("summary",      uploadconfig.summary,       128, "One-line summary of package",         true,  false )
-  ctan_field("support",      uploadconfig.support,       255, "URL(s) of support channels",          false, true  )
-  ctan_field("topic",        uploadconfig.topic,        1024, "Topic(s)",                            false, true  )
-  ctan_field("update",       uploadconfig.update,          8, "Boolean: true=update, false=new pkg", false, false )
-  ctan_field("uploader",     uploadconfig.uploader,      255, "Name of uploader",                    true,  false )
-  ctan_field("version",      uploadconfig.version,        32, "Package version",                     true,  false )
-
-  -- finish constructing the curl command:
-  ctan_post = ctan_post .. " --form 'file=@" .. tostring(uploadfile) .. ";filename=" .. tostring(uploadfile) .. "'"
-  ctan_post = ctan_post ..  " https://ctan.org/submit/"
+  local override_update_check = false
+  if uploadconfig.update == nil then
+    uploadconfig.update = true
+    override_update_check = true
+  end
 
   -- avoid lower level error from post command if zip file missing
-  local zip=io.open(trim_space(tostring(uploadfile)),"r")
+  local zip=open(trim_space(tostring(uploadfile)),"r")
   if zip~=nil then
-    io.close(zip)
+    close(zip)
   else
     error("Missing zip file '" .. tostring(uploadfile) .. "'")
   end
 
+  ctan_post = construct_ctan_post(uploadfile,options["debug"])
+
+  if options["debug"] then
+    fp_return = shell(ctan_post)
+    print('\n\nCURL COMMAND:')
+    print(ctan_post)
+    print("\n\nHTTP RESPONSE:")
+    print(fp_return)
+    return 1
+  end
+
   -- call post command to validate the upload at CTAN's validate URL
   local exit_status=0
   local fp_return=""
 
   -- use popen not execute so get the return body local exit_status=os.execute(ctan_post .. "validate")
   if (curl_debug==false) then
-    local fp = assert(io.popen(ctan_post .. "validate", 'r'))
-    fp_return = assert(fp:read('*a'))
-    fp:close()
+    print("Contacting CTAN for validation:")
+    fp_return = shell(ctan_post .. "validate")
   else
     fp_return="WARNING: curl_debug==true: posting disabled"
     print(ctan_post)
+    return 1
+  end
+  if override_update_check then
+    if match(fp_return,"non%-existent%spackage") then
+      print("Package not found on CTAN; re-validating as new package:")
+      uploadconfig.update = false
+      ctan_post = construct_ctan_post(uploadfile)
+      fp_return = shell(ctan_post .. "validate")
+    end
   end
-  if string.match(fp_return,"WARNING") or string.match(fp_return,"ERROR") then
+  if match(fp_return,"WARNING") or match(fp_return,"ERROR") then
     exit_status=1
   end
 
@@ -125,22 +154,20 @@ function upload()
     if (ctanupload ~=nil and ctanupload ~=false and ctanupload ~= true) then
       print("Validation successful, do you want to upload to CTAN? [y/n]" )
       local answer=""
-      io.write("> ")
-      io.flush()
-      answer=io.read()
-      if(string.lower(answer,1,1)=="y") then
+      write("> ")
+      flush()
+      answer=read()
+      if(lower(answer,1,1)=="y") then
         ctanupload=true
       end
     end
     if (ctanupload==true) then
-      local fp = assert(io.popen(ctan_post .. "upload", 'r'))
-      fp_return = assert(fp:read('*a'))
-      fp:close()
+      fp_return = shell(ctan_post .. "upload")
 --     this is just html, could save to a file
 --     or echo a cleaned up version
       print('Response from CTAN:')
       print(fp_return)
-      if string.match(fp_return,"WARNING") or string.match(fp_return,"ERROR") then
+      if match(fp_return,"WARNING") or match(fp_return,"ERROR") then
         exit_status=1
       end
     else
@@ -158,6 +185,57 @@ function trim_space(s)
 end
 
 
+function shell(s)
+  local h = assert(popen(s, 'r'))
+  t = assert(h:read('*a'))
+  h:close()
+  return t
+end
+
+function construct_ctan_post(uploadfile,debug)
+
+  -- start building the curl command:
+  ctan_post = curlexe .. " "
+
+  -- build up the curl command field-by-field:
+
+  --         field                                   max  desc                                 mandatory  multi
+  --         ----------------------------------------------------------------------------------------------------
+  ctan_field("announcement", uploadconfig.announcement, 8192, "Announcement",                        true,  false )
+  ctan_field("author",       uploadconfig.author,        128, "Author name",                         true,  false )
+  ctan_field("bugtracker",   uploadconfig.bugtracker,    255, "URL(s) of bug tracker",               false, true  )
+  ctan_field("ctanPath",     uploadconfig.ctanPath,      255, "CTAN path",                           true,  false )
+  ctan_field("description",  uploadconfig.description,  4096, "Short description of package",        false, false )
+  ctan_field("development",  uploadconfig.development,   255, "URL(s) of development channels",      false, true  )
+  ctan_field("email",        uploadconfig.email,         255, "Email of uploader",                   true,  false )
+  ctan_field("home",         uploadconfig.home,          255, "URL(s) of home page",                 false, true  )
+  ctan_field("license",      uploadconfig.license,      2048, "Package license(s)",                  true,  true  )
+  ctan_field("note",         uploadconfig.note,         4096, "Internal note to ctan",               false, false )
+  ctan_field("pkg",          uploadconfig.pkg,            32, "Package name",                        true,  false )
+  ctan_field("repository",   uploadconfig.repository,    255, "URL(s) of source repositories",       false, true  )
+  ctan_field("summary",      uploadconfig.summary,       128, "One-line summary of package",         true,  false )
+  ctan_field("support",      uploadconfig.support,       255, "URL(s) of support channels",          false, true  )
+  ctan_field("topic",        uploadconfig.topic,        1024, "Topic(s)",                            false, true  )
+  ctan_field("update",       uploadconfig.update,          8, "Boolean: true=update, false=new pkg", false, false )
+  ctan_field("uploader",     uploadconfig.uploader,      255, "Name of uploader",                    true,  false )
+  ctan_field("version",      uploadconfig.version,        32, "Package version",                     true,  false )
+
+  -- finish constructing the curl command:
+  local qq = '"'
+  if os_type == "windows" then
+    qq = '\"'
+  end
+  ctan_post = ctan_post .. ' --form ' .. qq .. 'file=@' .. tostring(uploadfile) .. ';filename=' .. tostring(uploadfile) .. qq
+  if debug then
+    ctan_post = ctan_post ..  ' https://httpbin.org/post'
+  else
+    ctan_post = ctan_post ..  ' https://ctan.org/submit/'
+  end
+
+  return ctan_post
+
+end
+
 function ctan_field(fname,fvalue,max,desc,mandatory,multi)
   if (type(fvalue)=="table" and multi==true) then
     for i, v in pairs(fvalue) do
@@ -170,7 +248,9 @@ end
 
 
 function ctan_single_field(fname,fvalue,max,desc,mandatory)
-  print('ctan-post: ' .. fname .. ' ' ..tostring(fvalue or '??'))
+  local fvalueprint = fvalue
+  if fvalue == nil then fvalueprint = '??' end
+  print('ctan-upload | ' .. fname .. ': ' ..tostring(fvalueprint))
   if ((fvalue==nil and mandatory) or (fvalue == 'ask')) then
     if (max < 256) then
       fvalue=input_single_line_field(fname)
@@ -183,13 +263,14 @@ function ctan_single_field(fname,fvalue,max,desc,mandatory)
     if (mandatory==true and (fvalue == nil or vs=="")) then
       error("The field " .. fname .. " must contain " .. desc)
     end
-    if (fvalue ~=nil and string.len(vs) > 0) then
-      if (max > 0 and string.len(vs) > max) then
+    if (fvalue ~=nil and len(vs) > 0) then
+      if (max > 0 and len(vs) > max) then
         error("The field " .. fname .. " is longer than " .. max)
       end
       vs = vs:gsub('"','\\"')
       vs = vs:gsub('`','\\`')
-      ctan_post=ctan_post .." --form " .. fname .. '="' .. vs .. '"'
+      vs = vs:gsub('\n','\\n')
+      ctan_post=ctan_post .. ' --form "' .. fname .. "=" .. vs .. '"'
     end
   else
     error("The value of the field '" .. fname .."' must be a scalar not a table")
@@ -206,9 +287,9 @@ function input_multi_line_field (name)
   local answer_line
   local return_count=0
   repeat
-    io.write("> ")
-    io.flush()
-    answer_line=io.read()
+    write("> ")
+    flush()
+    answer_line=read()
     if answer_line=="" then
       return_count=return_count+1
     else
@@ -229,9 +310,9 @@ function input_single_line_field(name)
 
   local field=""
 
-  io.write("> ")
-  io.flush()
-  field=io.read()
+  write("> ")
+  flush()
+  field=read()
   return field
 end
 
diff --git a/Master/texmf-dist/scripts/l3build/l3build-variables.lua b/Master/texmf-dist/scripts/l3build/l3build-variables.lua
index ad88aed00ce..1288a6b7ca7 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-variables.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-variables.lua
@@ -115,6 +115,17 @@ zipopts     = zipopts     or "-v -r -X"
 -- Engines for testing
 checkengines = checkengines or {"pdftex", "xetex", "luatex"}
 checkformat  = checkformat  or "latex"
+specialformats = specialformats or { }
+specialformats.context = specialformats.context or {
+    luatex = {binary = "context", format = ""},
+    pdftex = {binary = "texexec", format = ""},
+    xetex  = {binary = "texexec", format = "", options = "--xetex"}
+  }
+specialformats.latex = specialformats.latex or {
+    etex  = {format = "latex"},
+    ptex  = {binary = "eptex"},
+    uptex = {binary = "euptex"}
+  }
 stdengine    = stdengine    or "pdftex"
 
 -- The tests themselves
diff --git a/Master/texmf-dist/scripts/l3build/l3build.lua b/Master/texmf-dist/scripts/l3build/l3build.lua
index 3422f4d81a9..78db32d4130 100755
--- a/Master/texmf-dist/scripts/l3build/l3build.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build.lua
@@ -25,7 +25,7 @@ for those people who are interested.
 --]]
 
 -- Version information
-release_date = "2018-11-08"
+release_date = "2019-02-06"
 
 -- File operations are aided by the LuaFileSystem module
 local lfs = require("lfs")
@@ -37,6 +37,7 @@ local ipairs           = ipairs
 local insert           = table.insert
 local lookup           = kpse.lookup
 local match            = string.match
+local gsub             = string.gsub
 local next             = next
 local print            = print
 local select           = select
@@ -174,7 +175,7 @@ end
 if #checkconfigs == 1 and
    checkconfigs[1] ~= "build" and
    (options["target"] == "check" or options["target"] == "save") then
-   local config = "./" .. checkconfigs[1] .. ".lua"
+   local config = "./" .. gsub(checkconfigs[1],".lua$","") .. ".lua"
    if fileexists(config) then
      dofile(config)
      testdir = testdir .. "-" .. checkconfigs[1]