l3build now cmdline script (9mar18)

git-svn-id: svn://tug.org/texlive/trunk@46894 c570f23f-e606-0410-a88d-b1316a301751

3 files changed, 89 insertions, 161 deletions

diff --git a/Master/texmf-dist/source/latex/l3build/build.lua b/Master/texmf-dist/source/latex/l3build/build.lua
deleted file mode 100644
index 0a6bc7ceb55..00000000000
--- a/Master/texmf-dist/source/latex/l3build/build.lua
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env texlua
-
--- Build script for LaTeX3 "l3build" files
-
--- Identify the bundle and module
-module = "l3build"
-bundle = ""
-
--- Non-standard settings
-checkconfigs = {"build", "config-plain"}
-checkdeps    = { }
-checkengines = {"pdftex", "xetex", "luatex", "ptex", "uptex"}
-cleanfiles   = {"*.pdf", "*.tex", "*.zip"}
-installfiles = {"l3build*.lua", "regression-test.tex"}
-packtdszip   = true
-sourcefiles  = {"*.dtx", "*.lua", "*.ins"}
-typesetcmds  = "\\AtBeginDocument{\\DisableImplementation}"
-unpackdeps   = { }
-versionfiles = {"*.dtx", "*.md", "*.lua"}
-
--- Detail how to set the version automatically
-function setversion_update_line(line, date, version)
-  local date = string.gsub(date, "%-", "/")
-  -- .dtx file
-  if string.match(line, "^%% \\date{Released %d%d%d%d/%d%d/%d%d}$") then
-    line = string.gsub(line, "%d%d%d%d/%d%d/%d%d", date)
-  end
-  -- Markdown files
-  if string.match(
-    line, "^Release %d%d%d%d/%d%d/%d%d$"
-  ) then
-    line = "Release " .. date
-  end
-  -- l3build.lua
-  if string.match(line, "^release_date = \"%d%d%d%d/%d%d/%d%d\"$") then
-    line = "release_date = \"" .. date .. "\""
-  end
-  return line
-end
-
--- Load l3build itself: truly self-contained so no kpsewhich() here
-dofile("./l3build.lua")
diff --git a/Master/texmf-dist/source/latex/l3build/config-plain.lua b/Master/texmf-dist/source/latex/l3build/config-plain.lua
deleted file mode 100644
index b59b2c110ce..00000000000
--- a/Master/texmf-dist/source/latex/l3build/config-plain.lua
+++ /dev/null
@@ -1,2 +0,0 @@
-checkformat = "tex"
-testfiledir = "testfiles-plain"
\ No newline at end of file
diff --git a/Master/texmf-dist/source/latex/l3build/l3build.dtx b/Master/texmf-dist/source/latex/l3build/l3build.dtx
index 9b875e562a4..0dab5b0055e 100644
--- a/Master/texmf-dist/source/latex/l3build/l3build.dtx
+++ b/Master/texmf-dist/source/latex/l3build/l3build.dtx
@@ -1,6 +1,6 @@
 % \iffalse
 %
-% File l3build.dtx (C) Copyright 2014-2017 The LaTeX3 Project
+% File l3build.dtx (C) Copyright 2014-2018 The LaTeX3 Project
 %
 % It may be distributed and/or modified under the conditions of the
 % LaTeX Project Public License (LPPL), either version 1.3c of this
@@ -85,9 +85,12 @@
 \luavarset{demofiles}         {\{\}}{Files which show how to use a module}
 \luavarset{docfiles}          {\{\}}{Files which are part of the documentation but should not be typeset}
 \luavarset{excludefiles}      {\{"*\string~"\}}{Files to ignore entirely (default for Emacs backup files)}
-\luavarset{installfiles}      {\{"*.sty","*.cls"\}}{Files to install to the \TeX{} tree and similar task}
+\luavarset{installfiles}      {\{"*.sty","*.cls"\}}{Files to install to the \texttt{text} area of the \texttt{texmf} tree}
 \luavarset{makeindexfiles}    {\{"*.ist"\}}{MakeIndex files to be included in a TDS-style zip}
+\luavarset{scriptfiles}       {\{ \}}{Files to install to the \texttt{scripts} area of the \texttt{texmf} tree}
+\luavarset{scriptmanfiles}    {\{ \}}{Files to install to the \texttt{doc/man} area of the \texttt{texmf} tree}
 \luavarset{sourcefiles}       {\{"*.dtx", "*.ins"\}}{Files to copy for unpacking}
+\luavarset{tagfiles}          {\{"*.dtx"\}}{Files for automatic tagging}
 \luavarset{textfiles}         {\{"*.md", "*.txt"\}}{Plain text files to send to CTAN as-is}
 \luavarset{typesetdemofiles}  {\{\}}{Files to typeset before the documentation for inclusion in main documentation files}
 \luavarset{typesetfiles}      {\{"*.dtx"\}}{Files to typeset for documentation}
@@ -95,7 +98,6 @@
 \luavarset{typesetsourcefiles}{\{\}}{Files to copy to unpacking when typesetting.}
 \luavarset{unpackfiles}       {\{"*.ins"\}}{Files to run to perform unpacking.}
 \luavarset{unpacksuppfiles}   {\{\}}{Files needed to support unpacking when \enquote{sandboxed}}
-\luavarset{versionfiles}      {\{"*.dtx"\}}{Files for automatic version editing}
 \luavarseparator
 \luavarset{bakext}{".bak"}{Extension of backup files}
 \luavarset{dviext}{".dvi"}{Extension of DVI files}
@@ -115,7 +117,6 @@
 \luavarset{checkformat}  {"latex"} {Format to use for tests}
 \luavarseparator
 \luavarset{checkconfigs}{\{\}}{Configurations to use for tests}
-\luavarset{stdconfig}   {\meta{Main script}}{Standard test configuration}
 \luavarseparator
 \luavarset{typesetexe}{"pdflatex"}{Executable for compiling \texttt{doc(s)}}
 \luavarset{unpackexe} {"tex"}     {Executable for running \texttt{unpack}}
@@ -150,10 +151,8 @@
 \luavarset{flatten}     {true}        {Switch to flatten any source structure when sending to CTAN}
 \luavarset{maxprintline}{79}          {Length of line to use in log files}
 \luavarset{packtdszip}  {false}       {Switch to build a TDS-style zip file for CTAN}
-\luavarset{scriptname}  {"build.lua"} {Name of script used in dependencies}
 \luavarset{typesetcmds} {""}          {Instructions to be passed to \TeX{} when doing typesetting.}
 \luavarset{typsetcycles}{3}           {Number of cycles of typesetting to carry out.}
-\luavarset{versionform} {""}          {Nature of version strings for auto-replacement.}
 \luavarset{recordstatus}{false}       {Switch to include error level from test runs in \texttt{.tlg} files}
 \luavarset{manifestfile}        {"MANIFEST.md"} {Filename to use for the manifest file.}
 }
@@ -217,7 +216,7 @@
 %    }^^A
 % }
 %
-% \date{Released 2018/02/20}
+% \date{Released 2018-03-08}
 %
 % \maketitle
 % \tableofcontents
@@ -234,7 +233,7 @@
 %
 % Test files are written as standalone \TeX{} documents using the |regression-test.tex| setup file; documentation on writing these tests is discussed in Section~\ref{sec:writing-tests}.
 %
-% The |l3build.lua| script is not designed to be executed directly; each package will define its own |build.lua| script as a driver file which both sets variables (such as the name of the package) and then calls the main |l3build.lua| script internally.
+% Each package will define its own |build.lua| configuration file which both sets variables (such as the name of the package) and may also provide custom functions.
 %
 % \pagebreak[2]
 % A standard package layout might look something like the following:
@@ -305,37 +304,35 @@
 % \item doc \meta{name(s)}
 % \item install
 % \item save \meta{name(s)}
-% \item setversion
+% \item tag
 % \item unpack
 % \end{itemize}
 % These commands are described below.
 %
 % As well as these commands, the system recognises the options
 % \begin{itemize}
-%   \item \texttt{--config} (\texttt{-c}) Configuration(s) to use for testing
-% \item \texttt{--date} Date to use when setting version
+%   \item |--config| (|-c|) Configuration(s) to use for testing
+% \item |--date| Date to use when tagging
 %   data
-% \item \texttt{--dry-run} Runs the \texttt{install} target but does not copy
+% \item |--dry-run| Runs the \texttt{install} target but does not copy
 %   any files: simply lists those that would be installed
-% \item \texttt{--engine} (\texttt{-e}) Sets the engine to use for
+% \item |--engine| (|-e|) Sets the engine to use for
 %   testing
-% \item \texttt{--epoch} Sets the epoch for typesetting and testing
-% \item \texttt{--first} Name of the first test to run
-% \item \texttt{--force} (\texttt{-f}) Force checks to run even if sanity
-%   checks fail, \emph{e.g.}~when \texttt{--engine} is not given in
+% \item |--epoch| Sets the epoch for typesetting and testing
+% \item |--first| Name of the first test to run
+% \item |--force| (|-f|) Force checks to run even if sanity
+%   checks fail, \emph{e.g.}~when |--engine| is not given in
 %   \luavar{checkengines}
-% \item \texttt{--halt-on-error} (\texttt{-H}) Specifies that checks
+% \item |--halt-on-error| (|-H|) Specifies that checks
 %   should stop as soon as possible, rather than running all requested
 %   tests; the difference file is printed in the terminal directly in the case of failure
-%  \item \texttt{--last} Name of the last test to run
-% \item \texttt{--pdf} (\texttt{-p}) Test PDF file against a reference
+%  \item |--last| Name of the last test to run
+% \item |--pdf| (|-p|) Test PDF file against a reference
 %   version rather than using a log comparison
-% \item \texttt{--quiet} (\texttt{-q}) Suppresses output from unpacking
-% \item \texttt{--rerun} Run tests without unpacking/set up
-% \item \texttt{--shuffle} Shuffle the order in whichs tests run
-% \item \texttt{--texmfhome} Sets the location of the user tree for installing
-% \item \texttt{--version} (\texttt{-v}) Version string to use when setting
-%   version data
+% \item |--quiet| (|-q|) Suppresses output from unpacking
+% \item |--rerun| Run tests without unpacking/set up
+% \item |--shuffle| Shuffle the order in whichs tests run
+% \item |--texmfhome| Sets the location of the user tree for installing
 % \end{itemize}
 %
 % \begin{buildcmd}{check}
@@ -378,10 +375,10 @@
 % \begin{buildcmd}{check \meta{name(s)}}
 % Checks only the test \texttt{\meta{name(s)}.lvt}.
 % All engines specified by \var{checkengines} are tested unless the command
-% line option \texttt{--engine} (or \texttt{-e}) has been given to limit
+% line option |--engine| (or |-e|) has been given to limit
 % testing to a single engine. Normally testing is preceded by unpacking
 % source files and copying the result plus any additional support to the
-% test directory: this may be skipped using the \texttt{-s} option.
+% test directory: this may be skipped using the |-s| option.
 % \end{buildcmd}
 %
 % \begin{buildcmd}{check -p}
@@ -488,7 +485,7 @@
 % This command saves the output of the test to a |.tlg| file.
 % This file is then used in all subsequent checks against the \texttt{\meta{name}.lvt} test.
 %
-% If the \texttt{--engine} (or \texttt{-e}) is specified (one of |pdftex|, |xetex|, or |luatex|), the saved output is stored in \texttt{\meta{name}.\meta{engine}.tlg}. This is necessary if running the test through a different engine produces a different output.
+% If the |--engine| (or |-e|) is specified (one of |pdftex|, |xetex|, or |luatex|), the saved output is stored in \texttt{\meta{name}.\meta{engine}.tlg}. This is necessary if running the test through a different engine produces a different output.
 % A normalisation process is performed when checking to avoid common differences such as register allocation; full details are listed in section~\ref{sec:norm}.
 %
 % If the \var{recordstatus} variable is set \var{true}, additional information
@@ -528,15 +525,14 @@
 % Improvements to this process are planned for the future.
 % \end{buildcmd}
 %
-% \begin{buildcmd}{setversion}
-% Modifies the content of files specified by |versionfiles| to allow
-% automatic updating of the file date and version. The latter are
-% specified using the |-d| and |-v| command line options and if not
-% given will default to the current date in ISO format (YYYY-MM-DD) and |-1|,
-% respectively. As detailed below, the standard set up has no search pattern
-% defined for this target and so no action will be taken \emph{unless}
-% a version type for substitution is set up (using \var{versionform} or
-% by defining a custom function).
+% \begin{buildcmd}{tag}
+% Modifies the content of files specified by |tagfiles| to allow
+% automatic updating of the release tag and date. The tag is given as a
+% command line option, whilst the optional date should be passed using
+% |--date| (|-d|); if not given, the date will default to the current date in
+% ISO format (YYYY-MM-DD). As detailed below, the standard set up has no search
+% pattern defined for this target and so no action will be taken \emph{unless}
+% tag substitution is set up.
 % \end{buildcmd}
 %
 % \begin{buildcmd}{unpack}
@@ -560,9 +556,7 @@
 %
 % \begin{figure}[!b]
 %   \begin{lstlisting}[frame=single,language={[5.2]Lua},gobble = 6]
-%     #!/usr/bin/env texlua
-%
-%     -- Build script for breqn
+%     -- Build configuration for breqn
 %
 %     module = "breqn"
 %
@@ -570,11 +564,8 @@
 %     excludefiles = {"*/breqn-abbr-test.pdf",
 %                     "*/eqbreaks.pdf"}
 %     unpackopts  = "-interaction=batchmode"
-%
-%     kpse.set_program_name("kpsewhich")
-%     dofile(kpse.lookup("l3build.lua"))
 %   \end{lstlisting}
-% \caption{The build script for the \pkg{breqn} package.}
+% \caption{The build configuration for the \pkg{breqn} package.}
 % \label{fig:breqn}
 % \end{figure}
 %
@@ -584,8 +575,6 @@
 %
 % \begin{figure}[p]
 %   \begin{lstlisting}[frame=single,language={[5.2]Lua},gobble = 6]
-%     #!/usr/bin/env texlua
-%
 %     -- Build script for LaTeX3 "l3packages" files
 %
 %     -- Identify the bundle: there is no module as this is the "driver"
@@ -596,8 +585,8 @@
 %
 %     -- Load the common build code: this is the one place that a path
 %     -- needs to be hard-coded
-%     dofile (maindir .. "/l3build/l3build-config.lua")
-%     dofile (maindir .. "/l3build/l3build.lua")
+%     dofile(maindir .. "/l3build/l3build-config.lua")
+%     dofile(maindir .. "/l3build/l3build.lua")
 %   \end{lstlisting}
 % \caption{The build script for the \pkg{l3packages} bundle.}
 % \label{fig:bundle}
@@ -605,8 +594,6 @@
 %
 % \begin{figure}[p]
 %   \begin{lstlisting}[frame=single,language={[5.2]Lua},gobble = 6]
-%     #!/usr/bin/env texlua
-%
 %     -- Build script for LaTeX3 "xparse" files
 %
 %     -- Identify the bundle and module:
@@ -619,32 +606,26 @@
 %
 %     -- Load the common build code: this is the one place that a path
 %     -- needs to be hard-coded
-%     dofile (maindir .. "/l3build/l3build-config.lua")
-%     dofile (maindir .. "/l3build/l3build.lua")
+%     dofile(maindir .. "/l3build/l3build-config.lua")
+%     dofile(maindir .. "/l3build/l3build.lua")
 %   \end{lstlisting}
 % \caption{The build script for the \pkg{xparse} module.}
 % \label{fig:module}
 % \end{figure}
 %
-% \begin{figure}[p]
-%   \begin{lstlisting}[frame=single,gobble = 6]
-%     @echo off
-%     texlua build.lua %*
-%   \end{lstlisting}
-% \caption{Windows batch file wrapper for running the build process.}
-% \label{fig:batch}
-% \end{figure}
-%
-% Under a Unix-like platform, you may wish to run `|chmod +x build.lua|' on these files, which allows a simpler command line use. Instead of writing\\
-% \hspace*{2\parindent}|texlua build.lua check|\\ for example, you would simply write\\
-% \hspace*{2\parindent}|./build.lua check|\\
-% instead. (Or even omit the |./| depending on your path settings.)
-% Windows users can achieve a similar effect by creating a file |build.bat| as show in Figure~\ref{fig:batch}.
-%
 % \subsection{Variables}
 %
 % This section lists all variables defined in the |l3build.lua| script that are available for customisation.
 %
+% \subsection{Interaction between tests}
+%
+% Tests are run in a single directory, so whilst they are may be isolated from
+% the system \TeX{} tree they do share files. This may be significant if
+% installation-type files are generated during a test, for example by a 
+% |filecontents| environment in \LaTeX{}. Typically, you should set up your
+% tests such that they do not use the same names for such files: this may lead
+% to variable outcomes depending on the order in which tests are run.
+%
 % \luavartypeset
 %
 % \subsection{Multiple sets of tests}
@@ -683,10 +664,6 @@
 % be used. This works in the same way as |--engine|: it takes a comma list of
 % configurations to apply, overriding \var{checkconfigs}.
 %
-% Note that the setting \var{stdconfig} is used to determine the \emph{vanilla}
-% configuration: this will typically be the name of the main script (usually
-% |build| for a standard |build.lua| file).
-%
 % \subsection{Dependencies}
 %
 % If you have multiple packages that are developed separately but still interact in some way, it's often desirable to integrate them when performing regression tests.
@@ -760,7 +737,8 @@
 %   \item Removal of |./| at start of file names.
 %   \item Standardisation of the list of units known to \TeX{} (\pdfTeX{}
 %     and \LuaTeX{} add a small number of additional units which are not
-%     known to \TeX90 or \XeTeX{}).
+%     known to \TeX90 or \XeTeX{}, (u)p\TeX{} adds some additoonal non-standard
+%     ones)
 %   \item Standardisation of \verb*|\csname\endcsname | to |\csname\endcsname|
 %     (the former is formally correct, but the latter was produced for many
 %     years due to a \TeX{} bug).
@@ -771,6 +749,8 @@
 %     \texttt{\cs{\meta{type}}\meta{...}}
 %   \item Conversion of box numbers in |\show| lines
 %     \texttt{>~\cs{box}\meta{number}=} to \texttt{>~\cs{box}...=}
+%   \item Removal of some (u)p\TeX{} data where it is equivalent to
+%     \pdfTeX{} (|yoko direction|, |\displace 0.0|)
 % \end{itemize}
 %
 % \LuaTeX{} makes several additional changes to the log file. As normalising
@@ -1044,7 +1024,7 @@
 %
 % \subsection{Epoch setting}
 %
-% To produce predictable output when suing dates, the test system offers the
+% To produce predictable output when using dates, the test system offers the
 % ability to set the epoch to a known value. The \luavar{epoch} variable may
 % be given as a raw value (a simple integer) or as a date in ISO format.
 % The two flags \luavar{forcecheckepoch} and \luavar{forcedocepoch} then
@@ -1116,58 +1096,50 @@
 %
 % \section{Release-focussed features}
 %
-% \subsection{Automatic version modification}
+% \subsection{Automatic tagging}
 %
-% As detailed above, the |setversion| target will automatically edit
-% source files to modify date and version. This behaviour is governed by
-% variable \var{versionform}. As standard, no automatic replacement
-% takes place, but setting \var{versionform} will allow this to happen,
-% with options
+% As detailed above, the |tag| target will automatically edit
+% source files to modify date and release tag. As standard, no automatic
+% replacement takes place, but setting up a |updae_tag()| function
+% will allow this to happen. This takes four arguments:
 % \begin{itemize}
-%   \item |ProvidesPackage| --- Searches for lines using the \LaTeXe{}
-%     \cs{ProvidesPackage}, \cs{ProvidesClass} and \cs{ProvidesFile}
-%     identifiers (as a whole line).
-%   \item |ProvidesExplPackage| --- Searches for lines using the \pkg{expl3}
-%     \cs{ProvidesExplPackage}, \cs{ProvidesExplClass} and
-%     \cs{ProvidesExplFile} identifiers (at the start of a line).
-%   \item |filename| --- Searches for lines using |\def\filename|,
-%     |\def\filedate|, \dots, formulation.
-%   \item |ExplFileDate| --- Searches for lines using |\def\ExplFileDate|,
-%     \dots, formulation.
+%   \item The file name
+%   \item The full content of the file
+%   \item The tag name
+%   \item The tag date
 % \end{itemize}
-%
-% For more complex cases, the programmer may directly define the
-% Lua function |setversion_update_line()|, which takes as arguments the line of the
-% source, the supplied date and the supplied version. It should return a
-% (possibly unmodified) line and may use one, both or neither of the
-% date and version to update the line. Typically, |setversion_update_line| should
-% match to the exact pattern used by the programmer in the source files.
-% For example, for code using macros for the date and version
-% a suitable function might read as shown in Figure~\ref{fig:update-line}.
+% and should return the (modified) contents for writing to disk.
+% For example, the function used by |l3build| itself is
+% shown in Figure~\ref{fig:update-tag}.
 % \begin{figure}
 %   \begin{lstlisting}[frame=single,language={[5.2]Lua},gobble = 6]
-%     function setversion_update_line(line, date, version)
-%       -- No real regex so do it one type at a time
-%       for _,i in pairs({"Class", "File", "Package"}) do
-%         if string.match(
-%           line,
-%           "^\\Provides" .. i .. "{[a-zA-Z0-9%-]+}%[[^%]]*%]$"
-%         ) then
-%           line = string.gsub(line, "%[%d%d%d%d/%d%d/%d%d", "["
-%             .. string.gsub(date, "%-", "/")
-%           line = string.gsub(
-%             line, "(%[%d%d%d%d/%d%d/%d%d) [^ ]*", "%1 " .. version
-%           )
-%           break
-%         end
+%     -- Detail how to set the version automatically
+%     function update_tag(file,content,tagname,tagdate)
+%       if string.match(file, "%.dtx$") then
+%         return string.gsub(content,
+%           "\n%% \\date{Released %d%d%d%d/%d%d/%d%d}\n",
+%           "\n%% \\date{Released " .. tagname .. "}\n")
+%       elseif string.match(file, "%.md$") then
+%         return string.gsub(content,
+%           "\nRelease %d%d%d%d/%d%d/%d%d\n",
+%           "\nRelease " .. tagname .. "\n")
+%       elseif string.match(file, "%.lua$") then
+%         return string.gsub(content,
+%           '\nrelease_date = "%d%d%d%d/%d%d/%d%d"\n',
+%           '\nrelease_date = "' .. tagname .. '"\n')
 %       end
-%       return line
+%       return contents
 %     end
 %   \end{lstlisting}
-%   \caption{Example \texttt{setversion_update_line} function.}
-%   \label{fig:update-line}
+%   \caption{Example \texttt{update_tag} function.}
+%   \label{fig:update-tag}
 % \end{figure}
 %
+% To allow more complex tasks to take place, a hook |tag_hook()| is also
+% available. It will receive the tag name and date as arguments, and
+% may be used to carry out arbitrary tasks during tagging (for example,
+% setting a version control tag for an entire repository).
+%
 % \subsection{Typesetting documentation}
 %
 % As part of the overall build process, \pkg{l3build} will create PDF
@@ -1252,10 +1224,10 @@
 %     \midrule
 %       \var{date}        & String  \\
 %       \var{engine}      & Table   \\
-%       \var{files}       & Table   \\
 %       \var{force}       & Boolean \\
 %       \var{halt}        & Boolean \\
 %       \var{help}        & Boolean \\
+%       \var{names}       & Table   \\
 %       \var{pdf}         & Boolean \\
 %       \var{quiet}       & Boolean \\
 %       \var{rerun}       & Boolean \\