l3build (7feb19)

git-svn-id: svn://tug.org/texlive/trunk@49981 c570f23f-e606-0410-a88d-b1316a301751

14 files changed, 329 insertions, 125 deletions

diff --git a/Master/texmf-dist/doc/latex/l3build/CHANGELOG.md b/Master/texmf-dist/doc/latex/l3build/CHANGELOG.md
index a2c27f00465..d5e7a0e2411 100644
--- a/Master/texmf-dist/doc/latex/l3build/CHANGELOG.md
+++ b/Master/texmf-dist/doc/latex/l3build/CHANGELOG.md
@@ -7,6 +7,44 @@ this project uses date-based 'snapshot' version identifiers.
 
 ## [Unreleased]
 
+## [2019-02-06]
+
+### Added
+
+- Table-based control of binary/format combinations: `specialformats`
+  (see #84)
+- Switch `--debug` for chasing problems with the `upload` target
+
+### Changed
+
+- The `uploadconfig.update` field by default now automatically detects whether it
+  needs to be `true` or `false`
+- The `tag` target now allows no tag name to allow for setting this programmatically
+  within a `build.lua` script
+- Better support for multiple LuaTeX-like engines
+
+### Fixed
+
+- Uploading via Windows should now work
+
+## [2018-12-23]
+
+### Added
+
+- Switch `--email` for providing upload email address
+- Switch `-F|--file` for providing upload announcement from file
+- Switch `-m|--message` for providing upload announcement from command line
+
+### Changed
+
+- Enable `--dry-run` option for `upload` target
+- Enable tag/version to be passed as optional argument to `upload` target
+
+### Fixed
+
+- Packaging of some team-specific files
+- Handling of upload data containing newlines (Windows only)
+
 ## [2018-12-18]
 
 ### Changed
@@ -206,7 +244,9 @@ this project uses date-based 'snapshot' version identifiers.
 - Rationalise short option names: removed `-d`, `-E`, `-r`
 - Target `cmdcheck`: specific to LaTeX3 kernel work
 
-[Unreleased]: https://github.com/latex3/l3build/compare/2018-12-18...HEAD
+[Unreleased]: https://github.com/latex3/l3build/compare/2019-02-06...HEAD
+[2019-02-06]: https://github.com/latex3/l3build/compare/2018-12-23...2019-02-06
+[2018-12-23]: https://github.com/latex3/l3build/compare/2018-12-18...2018-12-23
 [2018-12-18]: https://github.com/latex3/l3build/compare/2018-11-08...2018-12-18
 [2018-11-08]: https://github.com/latex3/l3build/compare/2018-10-30...2018-11-08
 [2018-10-30]: https://github.com/latex3/l3build/compare/2018-10-25...2018-10-30
diff --git a/Master/texmf-dist/doc/latex/l3build/README.md b/Master/texmf-dist/doc/latex/l3build/README.md
index 2d88eb45457..8b4e4efb807 100644
--- a/Master/texmf-dist/doc/latex/l3build/README.md
+++ b/Master/texmf-dist/doc/latex/l3build/README.md
@@ -1,7 +1,7 @@
 l3build: a testing and building system for LaTeX3
 =================================================
 
-Release 2018-12-18
+Release 2019-02-06
 
 Overview
 --------
diff --git a/Master/texmf-dist/doc/latex/l3build/l3build.pdf b/Master/texmf-dist/doc/latex/l3build/l3build.pdf
index b26e510dd0e..2bc65c7ae7e 100644
--- a/Master/texmf-dist/doc/latex/l3build/l3build.pdf
+++ b/Master/texmf-dist/doc/latex/l3build/l3build.pdf
diff --git a/Master/texmf-dist/doc/man/man1/l3build.1 b/Master/texmf-dist/doc/man/man1/l3build.1
index 70c51b20544..5da14d7726a 100644
--- a/Master/texmf-dist/doc/man/man1/l3build.1
+++ b/Master/texmf-dist/doc/man/man1/l3build.1
@@ -1,4 +1,4 @@
-.TH l3build 1 "2018-12-18"
+.TH l3build 1 "2019-02-06"
 .SH NAME
 l3build \- Checking and building packages
 .SH SYNOPSIS
diff --git a/Master/texmf-dist/doc/man/man1/l3build.man1.pdf b/Master/texmf-dist/doc/man/man1/l3build.man1.pdf
index 4fbe5f737a4..8dc01a66373 100644
--- a/Master/texmf-dist/doc/man/man1/l3build.man1.pdf
+++ b/Master/texmf-dist/doc/man/man1/l3build.man1.pdf
diff --git a/Master/texmf-dist/scripts/l3build/l3build-arguments.lua b/Master/texmf-dist/scripts/l3build/l3build-arguments.lua
index e693ab929df..a6a13a43fee 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-arguments.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-arguments.lua
@@ -47,6 +47,11 @@ option_list =
         desc  = "Sets the date to insert into sources",
         type  = "string"
       },
+    debug =
+      {
+        desc = "Runs target in debug mode (not supported by all targets)",
+        type = "boolean"
+      },
     dirty =
       {
         desc = "Skip cleaning up the test area",
@@ -57,6 +62,11 @@ option_list =
         desc = "Dry run for install",
         type = "boolean"
       },
+    email =
+      {
+        desc = "Email address of CTAN uploader",
+        type = "string"
+      },
     engine =
       {
         desc  = "Sets the engine(s) to use for running test",
@@ -68,6 +78,12 @@ option_list =
         desc  = "Sets the epoch for tests and typesetting",
         type  = "string"
       },
+    file =
+      {
+        desc  = "Take the upload announcement from the given file",
+        short = "F",
+        type  = "string"
+      },
     first =
       {
         desc  = "Name of first test to run",
@@ -106,6 +122,12 @@ option_list =
         desc  = "Name of last test to run",
         type  = "string"
       },
+    message =
+      {
+        desc  = "Text for upload announcement message",
+        short = "m",
+        type  = "string"
+      },
     rerun =
       {
         desc  = "Skip setup: simply rerun tests",
diff --git a/Master/texmf-dist/scripts/l3build/l3build-check.lua b/Master/texmf-dist/scripts/l3build/l3build-check.lua
index 837b6356cbf..711d367331c 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-check.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-check.lua
@@ -541,9 +541,9 @@ function runcheck(name, hide)
     checkengines = options["engine"]
   end
   -- Used for both .lvt and .pvt tests
-  local function check_and_diff(ext,engine,enginename,comp,pdftest)
+  local function check_and_diff(ext,engine,comp,pdftest)
     runtest(name,engine,hide,ext,pdftest,true)
-    local errorlevel = comp(name,enginename)
+    local errorlevel = comp(name,engine)
     if errorlevel ~= 0 and options["halt-on-error"] then
       showfaileddiff()
       return 1
@@ -552,17 +552,12 @@ function runcheck(name, hide)
   end
   local errorlevel = 0
   for _,engine in pairs(checkengines) do
-    local enginename = engine
-    -- Allow for luatex == luajittex for .tlg/.tpf purposes
-    if match(engine,"^lua") then
-      engine = "luatex"
-    end
     setup_check(name,engine)
     local errlevel = 0
     if fileexists(testfiledir .. "/" .. name .. pvtext) then
-      errlevel = check_and_diff(pvtext,engine,enginename,compare_pdf,true)
+      errlevel = check_and_diff(pvtext,engine,compare_pdf,true)
     else
-      errlevel = check_and_diff(lvtext,engine,enginename,compare_tlg)
+      errlevel = check_and_diff(lvtext,engine,compare_tlg)
     end
     if errlevel ~= 0 and options["halt-on-error"] then
       return 1
@@ -626,7 +621,7 @@ function compare_pdf(name,engine,cleanup)
   return errorlevel
 end
 
-function compare_tlg(name, engine,cleanup)
+function compare_tlg(name,engine,cleanup)
   local errorlevel
   local testname = name .. "." .. engine
   local difffile = testdir .. "/" .. testname .. os_diffext
@@ -637,7 +632,7 @@ function compare_tlg(name, engine,cleanup)
   end
   -- Do additional log formatting if the engine is LuaTeX, there is no
   -- LuaTeX-specific .tlg file and the default engine is not LuaTeX
-  if engine == "luatex"
+  if match(engine,"^lua")
     and not match(tlgfile, "%.luatex" .. "%" .. tlgext)
     and not match(stdengine,"^lua")
     then
@@ -670,24 +665,25 @@ function runtest(name, engine, hide, ext, pdfmode, breakout)
   local lvtfile = name .. (ext or lvtext)
   cp(lvtfile, fileexists(testfiledir .. "/" .. lvtfile)
     and testfiledir or unpackdir, testdir)
+  local checkopts = checkopts
   local engine = engine or stdengine
-  local realengine = engine
-  local format = ""
-  if checkformat == "latex" then
-    -- Special case for e-LaTeX format
-    if engine == "etex" then
-      format = " -fmt=latex"
-    -- Use "...latex" formats for other engines
-    elseif not match(engine,"latex") then
-      format = " -fmt=" .. gsub(engine,"tex","latex")
-    end
-    -- Special case for (u)pTeX LaTeX
-    if match(engine,"^u?ptex$") then
-      realengine = "e" .. engine
-    end
+  local binary = engine
+  local format = gsub(engine,"tex$",checkformat)
+  -- Special binary/format combos
+  if specialformats[checkformat] and next(specialformats[checkformat]) then
+    local t = specialformats[checkformat]
+    if t[engine] and next(t[engine]) then
+      local t = t[engine]
+      binary    = t.binary  or binary
+      checkopts = t.options or checkopts
+      format    = t.format  or format
+    end
+  end
+  -- Finalise format string
+  if format ~= "" then
+    format = " --fmt=" .. format
   end
   -- Special casing for XeTeX engine
-  local checkopts = checkopts
   if match(engine, "xetex") and not pdfmode then
     checkopts = checkopts .. " -no-pdf"
   end
@@ -695,19 +691,8 @@ function runtest(name, engine, hide, ext, pdfmode, breakout)
   local function setup(file)
     return " -jobname=" .. name .. " " .. ' "\\input ' .. file .. '" '
   end
-  if match(checkformat, "^context$") then
-    format = ""
+  if match(checkformat,"^context$") then
     function setup(file) return ' "' .. file .. '" '  end
-    if match(engine,"^lua") then
-      realengine = "context"
-    elseif engine == "pdftex" then
-      realengine = "texexec"
-    elseif engine == "xetex" then
-      realengine = "texexec --xetex"
-    else
-      print("Engine incompatible with format")
-      exit(1)
-    end
   end
   local basename = testdir .. "/" .. name
   local logfile = basename .. logext
@@ -743,7 +728,7 @@ function runtest(name, engine, hide, ext, pdfmode, breakout)
       -- Ensure lines are of a known length
       os_setenv .. " max_print_line=" .. maxprintline
         .. os_concat ..
-      realengine .. format 
+      binary .. format 
         .. " " .. asciiopt .. " " .. checkopts
         .. setup(lvtfile)
         .. (hide and (" > " .. os_null) or "")
diff --git a/Master/texmf-dist/scripts/l3build/l3build-install.lua b/Master/texmf-dist/scripts/l3build/l3build-install.lua
index f32dcd003fd..a97a24e29ec 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-install.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-install.lua
@@ -93,13 +93,13 @@ function uninstall()
 end
 
 function install_files(target,full,dry_run)
-  local function install_files(source,dir,files,subdir,tool)
+  local function install_files(source,dir,files,subdir)
     subdir = subdir or moduledir
     -- For material associated with secondary tools (BibTeX, MakeIndex)
     -- the structure needed is slightly different from those items going
     -- into the tex/doc/source trees
-    if tool and module == "base" then
-      subdir = nil
+    if (dir == "makeindex" or match(dir,"$bibtex")) and module == "base" then
+      subdir = "latex"
     end
     dir = dir .. (subdir and ("/" .. subdir) or "")
     local filenames = { }
@@ -222,8 +222,8 @@ function install_files(target,full,dry_run)
   if errorlevel ~= 0 then return errorlevel end
 
   errorlevel = install_files(unpackdir,"tex",{installlist})
-    + install_files(unpackdir,"bibtex/bst",{bstfiles},module,true)
-    + install_files(unpackdir,"makeindex",{makeindexfiles},module,true)
+    + install_files(unpackdir,"bibtex/bst",{bstfiles},module)
+    + install_files(unpackdir,"makeindex",{makeindexfiles},module)
     + install_files(unpackdir,"scripts",{scriptfiles},module)
   
   return errorlevel
diff --git a/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua b/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua
index f2d8797c830..668b2d02d0a 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-stdmain.lua
@@ -120,9 +120,8 @@ target_list =
         desc = "Updates release tags in files",
         func = tag,
         pre  = function(names)
-           if not names or #names ~=1 then
-             print("Tag name required")
-             help()
+           if names and #names > 1 then
+             print("Too many tags specified; exactly one required")
              exit(1)
            end
            return 0
diff --git a/Master/texmf-dist/scripts/l3build/l3build-tagging.lua b/Master/texmf-dist/scripts/l3build/l3build-tagging.lua
index 6ad11e68943..2c0c1641a40 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-tagging.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-tagging.lua
@@ -63,7 +63,10 @@ end
 
 function tag(tagnames)
   local tagdate = options["date"] or os_date("%Y-%m-%d")
-  local tagname = tagnames[1] or tagdate
+  local tagname = nil
+  if tagnames then
+    tagname = tagnames[1]
+  end
   local dirs = remove_duplicates({currentdir, sourcefiledir, docfiledir})
   local errorlevel = 0
   for _,dir in pairs(dirs) do
diff --git a/Master/texmf-dist/scripts/l3build/l3build-upload.lua b/Master/texmf-dist/scripts/l3build/l3build-upload.lua
index 68692b0af4c..4afee06b4ed 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-upload.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-upload.lua
@@ -22,6 +22,22 @@ for those people who are interested.
 
 --]]
 
+local pairs    = pairs
+local print    = print
+local tostring = tostring
+
+local close = io.close
+local flush = io.flush
+local open  = io.open
+local popen = io.popen
+local read  = io.read
+local write = io.write
+
+local os_type = os.type
+
+local len   = string.len
+local lower = string.lower
+local match = string.match
 
 -- UPLOAD()
 --
@@ -53,70 +69,83 @@ local curl_debug = curl_debug or false -- to disable posting
 -- For now, this is undocumented.
 
 local ctanupload = ctanupload or "ask"
+if options["dry-run"] then
+  ctanupload = false
+end
 -- if ctanupload is nil or false, only validation is attempted
 -- if ctanupload is true the ctan upload URL will be used after validation
 -- if upload is anything else, the user will be prompted whether to upload.
 -- For now, this is undocumented. I think I would prefer to keep it always set to ask for the time being.
 
-function upload()
+function upload(tagnames)
 
   local uploadfile = ctanzip..".zip"
 
+  -- Keep data local
+  local uploadconfig = uploadconfig
+
   -- try a sensible default for the package name:
   uploadconfig.pkg = uploadconfig.pkg or ctanpkg or nil
 
-  -- start building the curl command:
-  ctan_post = curlexe .. " "
+  -- Get data from command line if appropriate
+  if options["file"] then
+    local f = open(options["file"],"r")
+    uploadconfig.announcement = assert(f:read('*a'))
+    close(f)
+  end
+  uploadconfig.announcement = options["message"] or uploadconfig.announcement
+  uploadconfig.email = options["email"] or uploadconfig.email
 
-  -- build up the curl command field-by-field:
+  local tagnames = tagnames or { }
+  uploadconfig.version = tagnames[1] or uploadconfig.version
 
-  --         field                                   max  desc                                 mandatory  multi
-  --         ----------------------------------------------------------------------------------------------------
-  ctan_field("announcement", uploadconfig.announcement, 8192, "Announcement",                        false, false )
-  ctan_field("author",       uploadconfig.author,        128, "Author name",                         true,  false )
-  ctan_field("bugtracker",   uploadconfig.bugtracker,    255, "URL(s) of bug tracker",               false, true  )
-  ctan_field("ctanPath",     uploadconfig.ctanPath,      255, "CTAN path",                           true,  false )
-  ctan_field("description",  uploadconfig.description,  4096, "Short description of package",        false, false )
-  ctan_field("development",  uploadconfig.development,   255, "URL(s) of development channels",      false, true  )
-  ctan_field("email",        uploadconfig.email,         255, "Email of uploader",                   true,  false )
-  ctan_field("home",         uploadconfig.home,          255, "URL(s) of home page",                 false, true  )
-  ctan_field("license",      uploadconfig.license,      2048, "Package license(s)",                  true,  true  )
-  ctan_field("note",         uploadconfig.note,         4096, "Internal note to ctan",               false, false )
-  ctan_field("pkg",          uploadconfig.pkg,            32, "Package name",                        true,  false )
-  ctan_field("repository",   uploadconfig.repository,    255, "URL(s) of source repositories",       false, true  )
-  ctan_field("summary",      uploadconfig.summary,       128, "One-line summary of package",         true,  false )
-  ctan_field("support",      uploadconfig.support,       255, "URL(s) of support channels",          false, true  )
-  ctan_field("topic",        uploadconfig.topic,        1024, "Topic(s)",                            false, true  )
-  ctan_field("update",       uploadconfig.update,          8, "Boolean: true=update, false=new pkg", false, false )
-  ctan_field("uploader",     uploadconfig.uploader,      255, "Name of uploader",                    true,  false )
-  ctan_field("version",      uploadconfig.version,        32, "Package version",                     true,  false )
-
-  -- finish constructing the curl command:
-  ctan_post = ctan_post .. " --form 'file=@" .. tostring(uploadfile) .. ";filename=" .. tostring(uploadfile) .. "'"
-  ctan_post = ctan_post ..  " https://ctan.org/submit/"
+  local override_update_check = false
+  if uploadconfig.update == nil then
+    uploadconfig.update = true
+    override_update_check = true
+  end
 
   -- avoid lower level error from post command if zip file missing
-  local zip=io.open(trim_space(tostring(uploadfile)),"r")
+  local zip=open(trim_space(tostring(uploadfile)),"r")
   if zip~=nil then
-    io.close(zip)
+    close(zip)
   else
     error("Missing zip file '" .. tostring(uploadfile) .. "'")
   end
 
+  ctan_post = construct_ctan_post(uploadfile,options["debug"])
+
+  if options["debug"] then
+    fp_return = shell(ctan_post)
+    print('\n\nCURL COMMAND:')
+    print(ctan_post)
+    print("\n\nHTTP RESPONSE:")
+    print(fp_return)
+    return 1
+  end
+
   -- call post command to validate the upload at CTAN's validate URL
   local exit_status=0
   local fp_return=""
 
   -- use popen not execute so get the return body local exit_status=os.execute(ctan_post .. "validate")
   if (curl_debug==false) then
-    local fp = assert(io.popen(ctan_post .. "validate", 'r'))
-    fp_return = assert(fp:read('*a'))
-    fp:close()
+    print("Contacting CTAN for validation:")
+    fp_return = shell(ctan_post .. "validate")
   else
     fp_return="WARNING: curl_debug==true: posting disabled"
     print(ctan_post)
+    return 1
+  end
+  if override_update_check then
+    if match(fp_return,"non%-existent%spackage") then
+      print("Package not found on CTAN; re-validating as new package:")
+      uploadconfig.update = false
+      ctan_post = construct_ctan_post(uploadfile)
+      fp_return = shell(ctan_post .. "validate")
+    end
   end
-  if string.match(fp_return,"WARNING") or string.match(fp_return,"ERROR") then
+  if match(fp_return,"WARNING") or match(fp_return,"ERROR") then
     exit_status=1
   end
 
@@ -125,22 +154,20 @@ function upload()
     if (ctanupload ~=nil and ctanupload ~=false and ctanupload ~= true) then
       print("Validation successful, do you want to upload to CTAN? [y/n]" )
       local answer=""
-      io.write("> ")
-      io.flush()
-      answer=io.read()
-      if(string.lower(answer,1,1)=="y") then
+      write("> ")
+      flush()
+      answer=read()
+      if(lower(answer,1,1)=="y") then
         ctanupload=true
       end
     end
     if (ctanupload==true) then
-      local fp = assert(io.popen(ctan_post .. "upload", 'r'))
-      fp_return = assert(fp:read('*a'))
-      fp:close()
+      fp_return = shell(ctan_post .. "upload")
 --     this is just html, could save to a file
 --     or echo a cleaned up version
       print('Response from CTAN:')
       print(fp_return)
-      if string.match(fp_return,"WARNING") or string.match(fp_return,"ERROR") then
+      if match(fp_return,"WARNING") or match(fp_return,"ERROR") then
         exit_status=1
       end
     else
@@ -158,6 +185,57 @@ function trim_space(s)
 end
 
 
+function shell(s)
+  local h = assert(popen(s, 'r'))
+  t = assert(h:read('*a'))
+  h:close()
+  return t
+end
+
+function construct_ctan_post(uploadfile,debug)
+
+  -- start building the curl command:
+  ctan_post = curlexe .. " "
+
+  -- build up the curl command field-by-field:
+
+  --         field                                   max  desc                                 mandatory  multi
+  --         ----------------------------------------------------------------------------------------------------
+  ctan_field("announcement", uploadconfig.announcement, 8192, "Announcement",                        true,  false )
+  ctan_field("author",       uploadconfig.author,        128, "Author name",                         true,  false )
+  ctan_field("bugtracker",   uploadconfig.bugtracker,    255, "URL(s) of bug tracker",               false, true  )
+  ctan_field("ctanPath",     uploadconfig.ctanPath,      255, "CTAN path",                           true,  false )
+  ctan_field("description",  uploadconfig.description,  4096, "Short description of package",        false, false )
+  ctan_field("development",  uploadconfig.development,   255, "URL(s) of development channels",      false, true  )
+  ctan_field("email",        uploadconfig.email,         255, "Email of uploader",                   true,  false )
+  ctan_field("home",         uploadconfig.home,          255, "URL(s) of home page",                 false, true  )
+  ctan_field("license",      uploadconfig.license,      2048, "Package license(s)",                  true,  true  )
+  ctan_field("note",         uploadconfig.note,         4096, "Internal note to ctan",               false, false )
+  ctan_field("pkg",          uploadconfig.pkg,            32, "Package name",                        true,  false )
+  ctan_field("repository",   uploadconfig.repository,    255, "URL(s) of source repositories",       false, true  )
+  ctan_field("summary",      uploadconfig.summary,       128, "One-line summary of package",         true,  false )
+  ctan_field("support",      uploadconfig.support,       255, "URL(s) of support channels",          false, true  )
+  ctan_field("topic",        uploadconfig.topic,        1024, "Topic(s)",                            false, true  )
+  ctan_field("update",       uploadconfig.update,          8, "Boolean: true=update, false=new pkg", false, false )
+  ctan_field("uploader",     uploadconfig.uploader,      255, "Name of uploader",                    true,  false )
+  ctan_field("version",      uploadconfig.version,        32, "Package version",                     true,  false )
+
+  -- finish constructing the curl command:
+  local qq = '"'
+  if os_type == "windows" then
+    qq = '\"'
+  end
+  ctan_post = ctan_post .. ' --form ' .. qq .. 'file=@' .. tostring(uploadfile) .. ';filename=' .. tostring(uploadfile) .. qq
+  if debug then
+    ctan_post = ctan_post ..  ' https://httpbin.org/post'
+  else
+    ctan_post = ctan_post ..  ' https://ctan.org/submit/'
+  end
+
+  return ctan_post
+
+end
+
 function ctan_field(fname,fvalue,max,desc,mandatory,multi)
   if (type(fvalue)=="table" and multi==true) then
     for i, v in pairs(fvalue) do
@@ -170,7 +248,9 @@ end
 
 
 function ctan_single_field(fname,fvalue,max,desc,mandatory)
-  print('ctan-post: ' .. fname .. ' ' ..tostring(fvalue or '??'))
+  local fvalueprint = fvalue
+  if fvalue == nil then fvalueprint = '??' end
+  print('ctan-upload | ' .. fname .. ': ' ..tostring(fvalueprint))
   if ((fvalue==nil and mandatory) or (fvalue == 'ask')) then
     if (max < 256) then
       fvalue=input_single_line_field(fname)
@@ -183,13 +263,14 @@ function ctan_single_field(fname,fvalue,max,desc,mandatory)
     if (mandatory==true and (fvalue == nil or vs=="")) then
       error("The field " .. fname .. " must contain " .. desc)
     end
-    if (fvalue ~=nil and string.len(vs) > 0) then
-      if (max > 0 and string.len(vs) > max) then
+    if (fvalue ~=nil and len(vs) > 0) then
+      if (max > 0 and len(vs) > max) then
         error("The field " .. fname .. " is longer than " .. max)
       end
       vs = vs:gsub('"','\\"')
       vs = vs:gsub('`','\\`')
-      ctan_post=ctan_post .." --form " .. fname .. '="' .. vs .. '"'
+      vs = vs:gsub('\n','\\n')
+      ctan_post=ctan_post .. ' --form "' .. fname .. "=" .. vs .. '"'
     end
   else
     error("The value of the field '" .. fname .."' must be a scalar not a table")
@@ -206,9 +287,9 @@ function input_multi_line_field (name)
   local answer_line
   local return_count=0
   repeat
-    io.write("> ")
-    io.flush()
-    answer_line=io.read()
+    write("> ")
+    flush()
+    answer_line=read()
     if answer_line=="" then
       return_count=return_count+1
     else
@@ -229,9 +310,9 @@ function input_single_line_field(name)
 
   local field=""
 
-  io.write("> ")
-  io.flush()
-  field=io.read()
+  write("> ")
+  flush()
+  field=read()
   return field
 end
 
diff --git a/Master/texmf-dist/scripts/l3build/l3build-variables.lua b/Master/texmf-dist/scripts/l3build/l3build-variables.lua
index ad88aed00ce..1288a6b7ca7 100644
--- a/Master/texmf-dist/scripts/l3build/l3build-variables.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build-variables.lua
@@ -115,6 +115,17 @@ zipopts     = zipopts     or "-v -r -X"
 -- Engines for testing
 checkengines = checkengines or {"pdftex", "xetex", "luatex"}
 checkformat  = checkformat  or "latex"
+specialformats = specialformats or { }
+specialformats.context = specialformats.context or {
+    luatex = {binary = "context", format = ""},
+    pdftex = {binary = "texexec", format = ""},
+    xetex  = {binary = "texexec", format = "", options = "--xetex"}
+  }
+specialformats.latex = specialformats.latex or {
+    etex  = {format = "latex"},
+    ptex  = {binary = "eptex"},
+    uptex = {binary = "euptex"}
+  }
 stdengine    = stdengine    or "pdftex"
 
 -- The tests themselves
diff --git a/Master/texmf-dist/scripts/l3build/l3build.lua b/Master/texmf-dist/scripts/l3build/l3build.lua
index 3422f4d81a9..78db32d4130 100755
--- a/Master/texmf-dist/scripts/l3build/l3build.lua
+++ b/Master/texmf-dist/scripts/l3build/l3build.lua
@@ -25,7 +25,7 @@ for those people who are interested.
 --]]
 
 -- Version information
-release_date = "2018-11-08"
+release_date = "2019-02-06"
 
 -- File operations are aided by the LuaFileSystem module
 local lfs = require("lfs")
@@ -37,6 +37,7 @@ local ipairs           = ipairs
 local insert           = table.insert
 local lookup           = kpse.lookup
 local match            = string.match
+local gsub             = string.gsub
 local next             = next
 local print            = print
 local select           = select
@@ -174,7 +175,7 @@ end
 if #checkconfigs == 1 and
    checkconfigs[1] ~= "build" and
    (options["target"] == "check" or options["target"] == "save") then
-   local config = "./" .. checkconfigs[1] .. ".lua"
+   local config = "./" .. gsub(checkconfigs[1],".lua$","") .. ".lua"
    if fileexists(config) then
      dofile(config)
      testdir = testdir .. "-" .. checkconfigs[1]
diff --git a/Master/texmf-dist/source/latex/l3build/l3build.dtx b/Master/texmf-dist/source/latex/l3build/l3build.dtx
index b62f2e42ece..d0bd63173a6 100644
--- a/Master/texmf-dist/source/latex/l3build/l3build.dtx
+++ b/Master/texmf-dist/source/latex/l3build/l3build.dtx
@@ -86,7 +86,7 @@
 \luavarset{docfiles}          {\{\}}{Files which are part of the documentation but should not be typeset}
 \luavarset{dynamicfiles}      {\{ \}}{Secondary files to cleared before each test is run}
 \luavarset{excludefiles}      {\{"*\string~"\}}{Files to ignore entirely (default for Emacs backup files)}
-\luavarset{installfiles}      {\{"*.sty","*.cls"\}}{Files to install to the \texttt{text} area of the \texttt{texmf} tree}
+\luavarset{installfiles}      {\{"*.sty","*.cls"\}}{Files to install to the \texttt{tex} area of the \texttt{texmf} tree}
 \luavarset{makeindexfiles}    {\{"*.ist"\}}{MakeIndex files to be included in a TDS-style zip}
 \luavarset{scriptfiles}       {\{ \}}{Files to install to the \texttt{scripts} area of the \texttt{texmf} tree}
 \luavarset{scriptmanfiles}    {\{ \}}{Files to install to the \texttt{doc/man} area of the \texttt{texmf} tree}
@@ -110,6 +110,7 @@
 \luavarset{checkengines}{\{"pdftex", "xetex", "luatex"\}}{Engines to check with \texttt{check} by default}
 \luavarset{stdengine}    {"pdftex"}{Engine to generate \texttt{.tlg} file from}
 \luavarset{checkformat}  {"latex"} {Format to use for tests}
+\luavarset{specialformats}{\meta{table}} {Non-standard engine/format combinations}
 \luavarseparator
 \luavarset{checkconfigs}{\{\}}{Configurations to use for tests}
 \luavarseparator
@@ -226,7 +227,7 @@
 %    }^^A
 % }
 %
-% \date{Released 2018-12-18}
+% \date{Released 2019-02-06}
 %
 % \maketitle
 % \tableofcontents
@@ -317,14 +318,16 @@
 % \item check \meta{name(s)}
 % \item clean
 % \item ctan
+% \item doc
 % \item doc \meta{name(s)}
 % \item install
 % \item manifest
 % \item save \meta{name(s)}
-% \item tag
+% \item tag \meta{tag}
 % \item uninstall
 % \item unpack
 % \item upload
+% \item upload \meta{tag}
 % \end{itemize}
 % These commands are described below.
 %
@@ -332,11 +335,14 @@
 % \begin{itemize}
 % \item |--config| (|-c|) Configuration(s) to use for testing
 % \item |--date| Date to use when tagging data
+% \item |--debug| Runs the target in debug mode (not supported by all targets)
 % \item |--dirty| Skip cleaning up of the test area
 % \item |--dry-run| Runs the \texttt{install} target but does not copy
 %   any files: simply lists those that would be installed
+% \item |--email| Sets the email address for CTAN upload
 % \item |--engine| (|-e|) Sets the engine to use for testing
 % \item |--epoch| Sets the epoch for typesetting and testing
+% \item |--file| (|-F|) Take the upload announcement from the given file
 % \item |--first| Name of the first test to run
 % \item |--force| (|-f|) Force checks to run even if sanity
 %   checks fail, \emph{e.g.}~when |--engine| is not given in
@@ -347,6 +353,7 @@
 %   should stop as soon as possible, rather than running all requested
 %   tests; the difference file is printed in the terminal directly in the case of failure
 %  \item |--last| Name of the last test to run
+% \item |--message| (|-m|) Text for upload announcement
 % \item |--quiet| (|-q|) Suppresses output from unpacking
 % \item |--rerun| Run tests without unpacking/set up
 % \item |--shuffle| Shuffle the order in which tests run
@@ -531,14 +538,15 @@
 %
 % \begin{buildcmd}{tag}
 % Modifies the contents of files specified by |tagfiles| using a search pattern
-% to automatically update the `release tag' (or package version) and date. The tag is given as a
-% command line option, whilst the optional date should be passed using
-% |--date| (|-d|); if not given, the date will default to the current date in
-% ISO format (YYYY-MM-DD). The standard set up has no search
-% pattern defined for this target and so no action will be taken \emph{unless}
-% tag substitution is set up.
-%
-% See Section~\ref{sec:tagging} for full details on this feature.
+% to automatically update the `release tag' (or package version) and date.
+% The tag is given as the command line option and the date using
+% |--date| (|-d|). If not given, the date will default to the current date in
+% ISO format (YYYY-MM-DD). If no option is given (i.e., no tag specified) the tag
+% is passed through as |nil| to allow setting the tag programmatically.
+%
+% The standard setup has no search pattern defined for this target and so no action
+% will be taken \emph{unless} tag substitution is set up by defining the Lua function
+% |update_tag()|. See Section~\ref{sec:tagging} for full details on this feature.
 % \end{buildcmd}
 %
 % \begin{buildcmd}{unpack}
@@ -758,6 +766,32 @@
 % A series of example layouts and matching |build.lua| files are available from
 % \url{https://github.com/latex3/l3build/tree/master/examples}.
 %
+% \subsection{Non-standard formats/binaries}
+%
+% The standard approach used by \pkg{l3build} is to use a combination
+% of \var{engine} and \var{checkformat} to generate the \emph{binary} and
+% \emph{format} combination used for tests. For example, when |pdftex| is
+% the \var{engine} and |latex| is the \var{checkformat}, the system call
+% used is
+% \begin{verbatim}
+%   pdftex --fmt=pdflatex
+% \end{verbatim}
+% \emph{i.e.}~the binary names is the same as the \var{engine}, and the format
+% is a simple substitution of the \var{checkformat} into \var{engine}, replacing
+% |tex|.
+%
+% For more complex set ups, \var{specialformats} should be used. This is a
+% table with one entry per \var{checkformat}. Each entry is itself a table,
+% and these contain a list of engines and settings for |binary|, |format|
+% and |options|. For example, for Con\TeX{}t and appropriate set up is
+% \begin{verbatim}
+% specialformats.context = {
+%   luatex = {binary = "context", format = ""},
+%   pdftex = {binary = "texexec", format = ""},
+%   xetex  = {binary = "texexec", format = "", options = "--xetex"}
+% }
+% \end{verbatim}
+%
 % \subsection{Output normalisation}
 % \label{sec:norm}
 %
@@ -1186,10 +1220,10 @@
 % replacement takes place, but setting up a |update_tag()| function
 % will allow this to happen. This function takes four input arguments:
 % \begin{enumerate}[nosep]
-%   \item The file name
-%   \item The full content of the file
-%   \item The tag name
-%   \item The tag date
+%   \item file name
+%   \item full content of the file
+%   \item tag name
+%   \item tag date
 % \end{enumerate}
 % The |update_tag()| function should return the (modified) contents
 % for writing to disk.
@@ -1222,8 +1256,9 @@
 %
 % To allow more complex tasks to take place, a hook |tag_hook()| is also
 % available. It will receive the tag name and date as arguments, and
-% may be used to carry out arbitrary tasks during tagging (for example,
-% setting a version control tag for an entire repository).
+% may be used to carry out arbitrary tasks after the main tagging process.
+% For example, this can be used to set a version control tag for an entire repository.
+%
 %
 % \subsection{Typesetting documentation}
 %
@@ -1295,8 +1330,34 @@
 % A description of this metadata is outlined in Table~\ref{tab:upload-setup},
 % and a simple example of an extract from a \texttt{build.lua} file using this is shown
 % in Figure~\ref{fig:uploadconfig}.
+%
 % Note that the \texttt{upload} target will \emph{not} execute the \texttt{ctan} target first.
 %
+% \paragraph{Announcement text}
+% It can be convenient not to include the announcement text within the |build.lua| file
+% directly. The command line argument |--message| (|-m|) allows the announcement to be
+% included as part of the |l3build| arguments, and |--file| (|-F|) reads the announcement
+% from a specified file. Note that if the announcement text is omitted a `silent update'
+% is performed; this should usually be performed for minor bug or documentation fixes only.
+%
+% \paragraph{Uploader details}
+% The CTAN team use the uploader email address as a form of low-security sanity
+% check that the upload is coming from a reputable source. Therefore, it is advisable not
+% to store this information within a public |build.lua| file. It can be set on the command
+% line with the |--email| option to \texttt{l3build}; alternatively, a private
+% configuration file could be used to add this information at upload time.
+%
+% \paragraph{The \texttt{update} field}
+% In most scenarios the |update| field does not need to be explicitly set. By default
+% \pkg{l3build} assumes that the package being uploaded already exists on CTAN
+% (|update=true|). If it does not, this is caught in the validation process before
+% uploading and automatically corrected. If you set |update| explicitly this will be passed
+% directly to CTAN in all circumstances, leading to errors if you attempt to update a
+% non-existing package or if you attempt to upload a new package with the same name as a
+% pre-existing one.
+%
+% If you have have difficulty with the upload process, add the option |--debug| to divert
+% the request from CTAN to a service that redirects the input back again so it can be examined.
 %
 % \begin{table}[p]
 %   \def\YES{\textbullet}
@@ -1307,6 +1368,7 @@
 %     \toprule
 %     Field & Req. & Multi & Description \\
 %     \midrule
+%  \texttt{announcement} & \YES &      & Announcement text                      \\
 %  \texttt{author      } & \YES &      & Author name (semicolon-separated for multiple) \\
 %  \texttt{ctanPath    } & \YES &      & CTAN path                              \\
 %  \texttt{email       } & \YES &      & Email address of uploader              \\
@@ -1316,7 +1378,6 @@
 %  \texttt{uploader    } & \YES &      & Name of uploader                       \\
 %  \texttt{version     } & \YES &      & Package version                        \\
 %  \midrule
-%  \texttt{announcement} &      &      & Announcement text                      \\
 %  \texttt{bugtracker  } &      & \YES & URL(s) of bug tracker                  \\
 %  \texttt{description } &      &      & Short description/abstract             \\
 %  \texttt{development } &      & \YES & URL(s) of development channels         \\
@@ -1342,8 +1403,6 @@
 %       summary     = "Mark vertical rules in margin of text",
 %       ctanPath    = "/macros/latex/contrib/vertbars",
 %       repository  = "https://github.com/wspr/herries-press/",
-%       uploader    = "Will Robertson",
-%       email       = [...],
 %       update      = true,
 %     }
 %   \end{lstlisting}
@@ -1373,13 +1432,16 @@
 %       \var{date}          & String  \\
 %       \var{dirty}         & Boolean \\
 %       \var{dry-run}       & Boolean \\
+%       \var{email}         & String  \\
 %       \var{engine}        & Table   \\
 %       \var{epoch}         & String  \\
+%       \var{file}          & string  \\
 %       \var{first}         & Boolean \\
 %       \var{force}         & Boolean \\
 %       \var{full}          & Boolean \\
 %       \var{halt-on-error} & Boolean \\
 %       \var{help}          & Boolean \\
+%       \var{message}       & string  \\
 %       \var{names}         & Table   \\
 %       \var{quiet}         & Boolean \\
 %       \var{rerun}         & Boolean \\