l3build (5may21)

git-svn-id: svn://tug.org/texlive/trunk@59091 c570f23f-e606-0410-a88d-b1316a301751

2 files changed, 74 insertions, 62 deletions

diff --git a/Master/texmf-dist/source/latex/l3build/l3build.dtx b/Master/texmf-dist/source/latex/l3build/l3build.dtx
index 198042469e6..9359b941b12 100644
--- a/Master/texmf-dist/source/latex/l3build/l3build.dtx
+++ b/Master/texmf-dist/source/latex/l3build/l3build.dtx
@@ -1,6 +1,6 @@
 % \iffalse
 %
-% File l3build.dtx (C) Copyright 2014-2020 The LaTeX3 Project
+% File l3build.dtx (C) Copyright 2014-2020 The LaTeX Project
 %
 % It may be distributed and/or modified under the conditions of the
 % LaTeX Project Public License (LPPL), either version 1.3c of this
@@ -114,6 +114,8 @@
 \luavarset{stdengine}    {"pdftex"}{Engine to generate \texttt{.tlg} file from}
 \luavarset{checkformat}  {"latex"} {Format to use for tests}
 \luavarset{specialformats}{\meta{table}} {Non-standard engine/format combinations}
+\luavarset{\detokenize{test_types}}        {\meta{table}} {Custom test variants}
+\luavarset{\detokenize{test_order}}        {\{"log", "pdf"\}} {Which kinds of tests to evaluate}
 \luavarseparator
 \luavarset{checkconfigs}{\{\}}{Configurations to use for tests}
 \luavarseparator
@@ -227,7 +229,7 @@
 % }
 %
 % \author{^^A
-%  The \LaTeX3 Project\thanks
+%  The \LaTeX{} Project\thanks
 %    {^^A
 %      E-mail:
 %        \href{mailto:latex-team@latex-project.org}
@@ -235,7 +237,7 @@
 %    }^^A
 % }
 %
-% \date{Released 2020-06-04}
+% \date{Released 2021-05-05}
 %
 % \maketitle
 % \tableofcontents
@@ -274,7 +276,7 @@
 % The \pkg{l3build} system is also capable of building and checking \emph{bundles} of packages.
 % To avoid confusion, we refer to either a standalone package or a package within a bundle as a \emph{module}.
 %
-% For example, within the \LaTeX3 project we have the \textsf{l3packages} bundle which contains the \textsf{xparse}, \textsf{xtemplate}, etc., modules.
+% For example, within the \LaTeX{} project we have the \textsf{l3packages} bundle which contains the \textsf{xparse}, \textsf{xtemplate}, etc., modules.
 % These are all built and distributed as one bundle for installation, distribution \emph{via} CTAN and so forth.
 %
 % Each module in a bundle will have its own build script, and a bundle build script brings them all together.
@@ -320,16 +322,19 @@
 % which is read during execution. In the current release of \pkg{l3build},
 % \texttt{build.lua} is read automatically and can access all of the global
 % functions provided by the script. Thus it may contain a simple list of
-% variable settings \emph{or} additionally custom code to change the build
-% process. A number of example scripts are given in Section~\ref{sec:examples}.
+% variable settings \emph{or} additional code to customize the build
+% process.
+%
+%The example scripts given in Section~\ref{sec:examples} largely cover the required knowledge in Lua programing.
+% For a more advanced usage, one may consult general Lua documentations including \url{http://www.lua.org/manual/5.3/manual.html} and for the few |texlua| specific additions see section 4.2 of the \LuaTeX{} manual available locally with |texdoc luatex| command line or at \url{https://www.pragma-ade.com/general/manuals/luatex.pdf}.
 %
 % \subsection{Main build targets}
 %
 % In the working directory of a bundle or module, \pkg{l3build} is run by executing
 % \begin{center}
-% \texttt{l3build \meta{targets} [\meta{option(s)}]}
+% \texttt{l3build \meta{target} [\meta{option(s)}]}
 % \end{center}
-% where \texttt{\meta{targets}} can be one of the following:
+% where \texttt{\meta{target}} can be one of the following:
 % \begin{itemize}[noitemsep]\ttfamily
 % \item check
 % \item check \meta{name(s)}
@@ -340,11 +345,10 @@
 % \item install
 % \item manifest
 % \item save \meta{name(s)}
-% \item tag \meta{tag}
+% \item tag [\meta{tag name}]
 % \item uninstall
 % \item unpack
-% \item upload
-% \item upload \meta{tag}
+% \item upload [\meta{version}]
 % \end{itemize}
 % These targets are described below.
 %
@@ -408,8 +412,8 @@
 %
 % Checking can be performed with any or all of the `engines' \texttt{pdftex}, \texttt{xetex}, and \texttt{luatex}.
 % By default, each test is executed with all three, being compared against the \texttt{.tlg} file produced from the \var{pdftex} engine (these defaults are controlled by the |checkengines| and |stdengine| variable respectively).
-% The format used for tests can be altered by setting \var{checkformat}: the default setting \texttt{latex} means that tests are run using \emph{e.g.}~\texttt{pdflatex}, whereas setting to \texttt{plain} will run tests using \emph{e.g.}~\texttt{pdftex}.
-% (Currently, this should be one of \texttt{latex} or \texttt{plain}.)
+% The format used for tests can be altered by setting \var{checkformat}: the default setting \texttt{latex} means that tests are run using \emph{e.g.}~\texttt{pdflatex}, whereas setting to \texttt{tex} will run tests using \emph{e.g.}~\texttt{pdftex}.
+% (Currently, this should be one of \texttt{latex} or \texttt{tex}.)
 % To perform the check, the engine typesets each test up to \var{checkruns} times.
 % More detail on this in the documentation on |save|.
 % Options passed to the binary are defined in the variable \var{checkopts}.
@@ -563,17 +567,14 @@
 % Improvements to this process are planned for the future.
 % \end{buildcmd}
 %
-% \begin{buildcmd}{tag}
-% Modifies the contents of files specified by |tagfiles| using a search pattern
-% to automatically update the `release tag' (or package version) and date.
-% The tag is given as the command line option and the date using
-% |--date| (|-d|). If not given, the date will default to the current date in
-% ISO format (YYYY-MM-DD). If no option is given (i.e., no tag specified) the tag
-% is passed through as |nil| to allow setting the tag programmatically.
-%
-% The standard setup has no search pattern defined for this target and so no action
-% will be taken \emph{unless} tag substitution is set up by defining the Lua function
-% |update_tag()|. See Section~\ref{sec:tagging} for full details on this feature.
+% \begin{buildcmd}{tag [\meta{tag name}]}
+% Apply the Lua |update_tag()| function to modify the contents of files specified by |tagfiles| to update the `release tag' (or package version) and date.
+% The tag is given as the optional command line argument \meta{tag name} and the date using
+% |--date| (or |-d|). If not given, the date will default to the current date in
+% ISO format (YYYY-MM-DD). If no \meta{tag name} is given, the tag will default to |nil|.
+% Both are passed as arguments to the |update_tag()| function.
+%
+% The standard setup does nothing unless tag update is set up by defining a custom |update_tag()| function. See Section~\ref{sec:tagging} for full details on this feature.
 % \end{buildcmd}
 %
 % \begin{buildcmd}{unpack}
@@ -588,11 +589,11 @@
 % By default this process allows files to be accessed in all standard |texmf| trees; this can be disabled by setting \var{unpacksearch} to |false|.
 % \end{buildcmd}
 %
-% \begin{buildcmd}{upload}
+% \begin{buildcmd}{upload [\meta{version}]}
 % This target uses \texttt{curl} to upload the package zip file (created using \texttt{ctan}) to CTAN.
 % To control the metadata used to upload the package, the \texttt{uploadconfig} table should be populated with a number of fields.
 % These are documented in Table~\ref{tab:upload-setup}.
-% Missing required fields will result in an interactive prompt for manual entry.
+% Missing required fields will result in an interactive prompt for manual entry. When given, \meta{version} overrides \texttt{uploadconfig.version}.
 %
 % See Section~\ref{sec:upload} for full details on this feature.
 % \end{buildcmd}
@@ -620,12 +621,12 @@
 % \end{figure}
 %
 % An example of a bundle build script for \pkg{l3packages} is shown in Figure~\ref{fig:bundle}.
-% Note for \LaTeX3 we use a common file to set all build variables in one place, and the path to the |l3build.lua| script is hard-coded so we always use our own most recent version of the script.
+% Note for \LaTeX{} we use a common file to set all build variables in one place, and the path to the |l3build.lua| script is hard-coded so we always use our own most recent version of the script.
 % An example of an accompanying module build script is shown in Figure~\ref{fig:module}.
 %
 % \begin{figure}[p]
 %   \begin{lstlisting}[frame=single,language={[5.2]Lua},gobble = 6]
-%     -- Build script for LaTeX3 "l3packages" files
+%     -- Build script for LaTeX "l3packages" files
 %
 %     -- Identify the bundle: there is no module as this is the "driver"
 %     bundle = "l3packages"
@@ -639,7 +640,7 @@
 %
 % \begin{figure}[p]
 %   \begin{lstlisting}[frame=single,language={[5.2]Lua},gobble = 6]
-%     -- Build script for LaTeX3 "xparse" files
+%     -- Build script for LaTeX "xparse" files
 %
 %     -- Identify the bundle and module:
 %     bundle = "l3packages"
@@ -656,28 +657,6 @@
 % A collection of full examples (source files in various layouts) are available
 % at \url{https://github.com/latex3/l3build/tree/master/examples}.
 %
-% \subsection{Backwards compatibility}
-%
-% Earlier releases of \pkg{l3build} required that the last line of
-% \texttt{build.lua} ran the main script, \emph{i.e.}~that \texttt{build.lua}
-% was what the user called rather than \texttt{l3build.lua}. To allow scripts
-% to support both forms \emph{for the transition}, a simple test may be
-% included as showing in Figure~\ref{fig:build-compat}.
-% \begin{figure}
-%   \begin{lstlisting}[frame=single,language={[5.2]Lua},gobble = 6]
-%     if not release_date then
-%       dofile(kpse.lookup("l3build.lua"))
-%     end
-%   \end{lstlisting}
-% \caption{Final lines for a \texttt{build.lua} script usable with both older
-%   and newer releases of \pkg{l3build}.}
-% \label{fig:build-compat}
-% \end{figure}
-%
-% Note that in time support for loading \pkg{l3build} by calling the
-% \texttt{build.lua} script \emph{may} be removed: the recommended approach for
-% new scripts is to run \texttt{l3build}.
-%
 % \subsection{Variables}
 %
 % This section lists all variables defined in the |l3build.lua| script that are available for customisation.
@@ -696,10 +675,10 @@
 % \subsection{Selective running of tests}
 %
 % The variables |includetests| and |excludetests| may be used to select which
-% tests are run: these variables take test \emph{names} not full file names.
+% tests are run: these variables take raw test \emph{names} not full file names.
 % The list of tests in |excludetests| overrides any matches in |includetests|,
 % meaning that tests can be disabled selectively. It also makes it possible
-% to disable test on for example a platform basis: the Lua core variable
+% to disable test on for example a platform basis: the |texlua| specific variable
 % |os.type| may be used to set |excludetests| only on some systems.
 %
 % \subsection{Multiple sets of tests}
@@ -741,10 +720,10 @@
 % \subsection{Dependencies}
 %
 % If you have multiple packages that are developed separately but still interact in some way, it's often desirable to integrate them when performing regression tests.
-% For \LaTeX3, for example, when we make changes to \pkg{l3kernel} it's important to check that the tests for \pkg{l3packages} still run correctly, so it's necessary to include the \pkg{l3kernel} files in the build process for \pkg{l3packages}.
+% For \LaTeX{}, for example, when we make changes to \pkg{l3kernel} it's important to check that the tests for \pkg{l3packages} still run correctly, so it's necessary to include the \pkg{l3kernel} files in the build process for \pkg{l3packages}.
 %
 % In other words, \pkg{l3packages} is \emph{dependent} on \pkg{l3kernel}, and this is specified in \pkg{l3build} by setting appropriately the variables \texttt{checkdeps}, \texttt{typesetdeps}, and \texttt{unpackdeps}.
-% The relevant parts of the \LaTeX3 repository is structured as the following.
+% The relevant parts of the \LaTeX{} repository is structured as the following.
 % \Needspace{3\baselineskip}
 % \begin{Verbatim}
 % l3/
@@ -763,7 +742,7 @@
 %                      xparse.ins
 %    support/
 % \end{Verbatim}
-% For \LaTeX3 build files, |maindir| is defined as top level folder |l3|, so all support files are located here, and the build directories will be created there.
+% For \LaTeX{} build files, |maindir| is defined as top level folder |l3|, so all support files are located here, and the build directories will be created there.
 % To set \pkg{l3kernel} as a dependency of \pkg{l3package}, within |l3packages/xparse/build.lua| the equivalent of the following is set:
 % \begin{Verbatim}
 %   maindir = "../.."
@@ -1246,6 +1225,38 @@
 % engine-specific, thus one |.tpf| file should be stored per engine to be
 % tested.
 %
+% \subsection{Custom tests}
+%
+% If neither the text-based methods nor PDF-based tests are sufficient,
+% there is the additional option of defining custom variants with individual
+% normalization rules.
+%
+% For this, the variant has to be registered in the \texttt{test_types} table
+% and then activated in \texttt{test_order}.
+%
+% Every element in \texttt{test_types} is a table with fields \texttt{test}
+% (the extension of the test file), \texttt{reference} (the extension of the
+% file the output is compared with), \texttt{generated} (extension of the
+% analyzed \LaTeX{} output file) and \texttt{rewrite} (A Lua function for
+% normalizing the output file, taking as parameters the name of the unnormalized
+% \LaTeX{} output file to be read, the name of the normalized file to be written,
+% the engine name and a potential errorcode).
+%
+% For example:
+% \begin{verbatim}
+% test_types = {
+%   mytest = {
+%     test = ".mylvt",
+%     reference = ".mytlg",
+%     generated = ".log",
+%     rewrite = function(source, normalized, engine, errorcode)
+%       -- In this example we just copy the logfile without any normalization
+%       os.execute(string.format("cp %s %s", source, normalized)
+%     end,
+%   },
+% }
+% test_order = {"mylvt", "log", "pdf"}
+% \end{verbatim}
 %
 % \section{Release-focussed features}
 %
@@ -1294,7 +1305,7 @@
 % \label{sec:tagging}
 %
 % The |tag| target can automatically edit
-% source files to modify date and release tag. As standard, no automatic
+% source files to modify date and release tag name. As standard, no automatic
 % replacement takes place, but setting up a |update_tag()| function
 % will allow this to happen. This function takes four input arguments:
 % \begin{enumerate}[nosep]
@@ -1334,7 +1345,7 @@
 %
 % To allow more complex tasks to take place, a hook |tag_hook()| is also
 % available. It will receive the tag name and date as arguments, and
-% may be used to carry out arbitrary tasks after the main tagging process.
+% may be used to carry out arbitrary tasks after all files have been updated.
 % For example, this can be used to set a version control tag for an entire repository.
 %
 %
@@ -1426,7 +1437,7 @@
 %
 % The CTAN upload process is backed by an API, which \pkg{l3build} can use
 % to send zip files for release. Along with the file, a variety of metadata
-% must be specified about the package, including the version, license, and so on.
+% must be specified about the package, including the version, license, and so on, explained at \url{https://www.ctan.org/upload}.
 % A description of this metadata is outlined in Table~\ref{tab:upload-setup},
 % and a simple example of an extract from a \texttt{build.lua} file using this is shown
 % in Figure~\ref{fig:uploadconfig}.
@@ -1666,7 +1677,7 @@
 %   \begin{syntax}
 %     |fileexists(|\meta{file}|)|
 %   \end{syntax}
-%   Tests if the \meta{file} exists; returns a boolean value.
+%   Tests if the \meta{file} exists and is readable; returns a boolean value.
 % \end{function}
 %
 % \begin{function}{filelist()}
@@ -1882,6 +1893,7 @@
 %     the |names| as an argument; this allows checking of the |name| data
 %     without impact on the main |func|.
 % \end{itemize}
+% The functions |func|, |bundle_func| and |pre| must return 0 on success.
 %
 % \subsection{Customising the manifest file}
 % \label{sec:manifest}
diff --git a/Master/texmf-dist/source/latex/l3build/l3build.ins b/Master/texmf-dist/source/latex/l3build/l3build.ins
index 483bab9d4a5..bdb0fb7a52f 100644
--- a/Master/texmf-dist/source/latex/l3build/l3build.ins
+++ b/Master/texmf-dist/source/latex/l3build/l3build.ins
@@ -1,6 +1,6 @@
 \iffalse meta-comment
 
-File l3build.ins Copyright (C) 2014-2018 The LaTeX3 Project
+File l3build.ins Copyright (C) 2014-2018,2021 The LaTeX Project
 
 It may be distributed and/or modified under the conditions of the
 LaTeX Project Public License (LPPL), either version 1.3c of this
@@ -32,7 +32,7 @@ license information is placed in the derived files.
 
 \preamble
 
-Copyright (C) 2014-2018 The LaTeX3 Project
+Copyright (C) 2014-2021 The LaTeX Project
 
 It may be distributed and/or modified under the conditions of
 the LaTeX Project Public License (LPPL), either version 1.3c of