summaryrefslogtreecommitdiff
path: root/macros/latex/required/tools/l3sys-query.dtx
diff options
context:
space:
mode:
Diffstat (limited to 'macros/latex/required/tools/l3sys-query.dtx')
-rw-r--r--macros/latex/required/tools/l3sys-query.dtx545
1 files changed, 545 insertions, 0 deletions
diff --git a/macros/latex/required/tools/l3sys-query.dtx b/macros/latex/required/tools/l3sys-query.dtx
new file mode 100644
index 0000000000..3cba5da4a6
--- /dev/null
+++ b/macros/latex/required/tools/l3sys-query.dtx
@@ -0,0 +1,545 @@
+% \iffalse
+%% Source File: l3sys-query.dtx
+%% Copyright (C) 2024
+%%
+%% The LaTeX Project and any individual authors listed elsewhere
+%% in this file.
+%%
+%% This file may be distributed under the terms of the LPPL.
+%% See README for details.
+%
+%<*dtx>
+ \ProvidesFile{l3sys-query.dtx}
+%</dtx>
+%<package>\ProvidesPackage{l3sys-query}
+%<driver> \ProvidesFile{l3sys-query.drv}
+% \fi
+% \ProvidesFile{l3sys-query.dtx}
+ [2024-03-28 v1.0a LaTeX2e interface for l3sys file queries]
+%
+% \iffalse
+%<*driver>
+\documentclass{l3doc}%{ltxdoc}
+\begin{document}
+\DocInput{l3sys-query.dtx}
+\end{document}
+%</driver>
+% \fi
+%
+% \GetFileInfo{l3sys-query.dtx}
+%
+% \title{\LaTeX\ document interface to the \pkg{l3sys-query} script:
+% System queries for LaTeX using Lua\thanks{This file
+% has version number \fileversion, last
+% revised \filedate.}}
+% \author{\LaTeX\ project}
+% \date{\filedate}
+%
+%
+% \maketitle
+%
+% \tableofcontents
+%
+% \section{Introduction}
+%
+% \TeX{} engines provide only very limited access to information about the system
+% they are used on: using primitives, one can for example get the size of a single
+% file, but not a list of files in a given location. For most documents, this is
+% not an issue as they are self-contained. However, for cases where
+% \enquote{dynamic} construction of parts of a document is needed based on file
+% lists or other system-dependent data, methods to obtain this from (restricted)
+% shell escape are desirable.
+%
+% Security considerations mean that directly querying the system shell is
+% problematic for general use. Instead, \emph{restricted} shell escape may be used
+% to get many details, provided a suitable tool is available to provide the
+% information in a platform-neutral and security-conscious way. The Java program
+% \texttt{texosquery}, written by Nicola Talbot, has been available for a number
+% of years to provide this facility. As well as file system insight,
+% \texttt{texosquery} also provides for example locale data and other system
+% information. However, the requirement for Java means that the script is not
+% automatically usable when a \TeX{} system is installed.
+%
+% The \LaTeX{} team have therefore provided a Lua-based script,
+% \texttt{l3sys-query}, which conforms to the security requirements of \TeX{} Live
+% using Lua to obtain the system information. This means that it can be used
+% \enquote{out of the box} across platforms. The facilities provided by
+% \texttt{l3sys-query} are more limited than \texttt{texosquery}, partly as some
+% information is available in modern \TeX{} systems using primitives, and partly
+% as the aim of \texttt{l3sys-query} is to provide information where there are
+% defined use cases. Requests for additional data interfaces are welcome.
+%
+% This package provides a document level \textit{Key-Value} interface
+% to The \texttt{l3sys-query} functionality, although as a convenience
+% it also summarizes the documentation for the L3 programming
+% interface layer (prvided by the \pkg{l3sys} module of
+% \pkg{l3kernel}, and of the command line Lua script
+% \texttt{l3sys-query} which is distributed separately and provides
+% the underlying functionality.
+%
+%
+%
+% \section{The document level key-value interface\label{sec:kv}}
+%
+% This provides an interface to \texttt{l3sys-query} based on
+% \pkg{l3keys} that allows the options to be specified and checked
+% individually. Internally the supplied key values are used to build
+% up the arguments to the commands described above in the L3
+% programming layer commands described in section~\ref{sec:expl3}
+%
+% Results containing path separators \emph{always}
+% use~|/|, irrespective of the platform in use.
+%
+%
+% \begin{function}
+% {\QueryWorkingDirectory}
+% \begin{syntax}
+% \cs{QueryWorkingDirectory} \Arg{result cmd}
+% \end{syntax}
+% Defines supplied command \meta{result cmd} to hold
+% the absolute path to the current working directory of the
+% \TeX\ system.
+% This is the directory (folder) from which \TeX\ was started, so usually
+% the location of the main document file.
+% \end{function}
+%
+% \begin{function}
+% {\QueryFiles, \QueryFilesTF}
+% \begin{syntax}
+% \cs{QueryFiles} \oarg{options} \Arg{spec} \Arg{function}
+% \cs{QueryFilesTF} \oarg{options} \Arg{spec} \Arg{function} \Arg{pre code} \Arg{empty list code}
+% \end{syntax}
+% This generates a file list based on the \meta{spec} and \meta{options}.
+% The command then applies \meta{function} to each item in the sequence of filenames.
+% The \meta{function} should be a macro body which will be passed the file path as |#1|.
+%
+% The TF version executes the T (\meta{pre code}) argument before iterating over the list
+% and the F (\meta{empty list code}) argument if the list is empty.
+% \end{function}
+%
+% Note that this interface in mapping directly over the sequence of
+% filenames does not allow some uses which are provided by the
+% programming interface described in the following section, which allows
+% the sequence to be manipulated before being used.
+%
+% The defined keys map very closely to the options of the
+% \texttt{l3sys-query} command which is described in
+% section~\ref{sec:lua}.
+%
+% \begin{itemize}
+% \item
+% The keys \texttt{recursive}, \texttt{ignore-case}, \texttt{reverse}, \texttt{pattern}
+% take no values and map directly to the command line options of the same name.
+%
+% \item
+% The key \texttt{sort} accepts the values \texttt{date} and \texttt{name}.
+%
+% \item
+% The key \texttt{type} accepts \texttt{d} or \texttt{f}.
+%
+% \item
+% The key \texttt{exclude} accepts a glob (or Lua pattern) matching
+% files to be excluded. The package arranges that the quoting of the
+% argument is automatically added if unrestricted shell escape is
+% enabled.
+%
+% \end{itemize}
+%
+% Within the main \meta{spec} argument, and the value of the \texttt{exclude} key,
+% the following characters may need special handling.
+% |~| may be used (which \texttt{kpathsea} uses to denote the users home directory).
+% |\%|, or at the top level |%|, may be used to produce a literal |%|
+% which may be especially useful if the \texttt{pattern} key is used
+% as \% is the escape character in Lua patterns.
+%
+% \subsection{Examples\label{sec:examples2e}}
+%
+% \begin{itemize}
+% \item
+% Include every |png| file in the current directory.
+% \begin{verbatim}
+% \QueryFiles{*.png}{\includegraphics{#1}\par}
+% \end{verbatim}
+%
+% \item
+% Input every \TeX\ file with a filename matching \texttt{chapter[0-9].tex}.
+% \begin{verbatim}
+% \QueryFiles[pattern]{chapter%d.*%.tex}{\input{#1}}
+% \end{verbatim}
+% \end{itemize}
+%
+% \section{The L3 programming layer interface\label{sec:expl3}}
+%
+% \begin{function}
+% {\sys_get_query:nN, \sys_get_query:nnN, \sys_get_query:nnnN}
+% \begin{syntax}
+% \cs{sys_get_query:nN} \Arg{cmd} \Arg{tl var}
+% \cs{sys_get_query:nnN} \Arg{cmd} \Arg{spec} \Arg{tl var}
+% \cs{sys_get_query:nnnN} \Arg{cmd} \Arg{options} \Arg{spec} \Arg{tl var}
+% \end{syntax}
+% Sets the \meta{tl var} to the information returned by the
+% \texttt{l3sys-query} \meta{cmd}, potentially supplying the \meta{options}
+% and \meta{spec} to the query call. The valid \meta{cmd} names are at present
+% \begin{itemize}
+% \item \texttt{pwd} Returns the absolute path to the current working directory
+% \item \texttt{ls} Returns a directory listing, using the \meta{spec} to
+% select files and applying the \meta{options} if given
+% \end{itemize}
+% The \meta{spec} should be a file glob and will automatically be passed to
+% the script without shell expansion.
+% \end{function}
+%
+% \begin{function}
+% {\sys_split_query:nN, \sys_split_query:nnN, \sys_split_query:nnnN}
+% \begin{syntax}
+% \cs{sys_split_query:nN} \Arg{cmd} \Arg{seq}
+% \cs{sys_split_query:nnN} \Arg{cmd} \Arg{spec} \Arg{seq}
+% \cs{sys_split_query:nnnN} \Arg{cmd} \Arg{options} \Arg{spec} \Arg{seq}
+% \end{syntax}
+% Works as described for \cs{sys_get_query:nnnN}, but sets the \meta{seq}
+% to contain one entry for each line returned by \texttt{l3sys-query}.
+% \end{function}
+%
+% \section{The \texttt{l3sys-query} Lua script\label{sec:lua}}
+% \subsection{The command line interface}
+%
+% The command line interface to
+% \begin{center}
+% \ttfamily
+% l3sys-query \meta{cmd} [\meta{option(s)}] [\meta{args}]
+% \end{center}
+% where \texttt{\meta{cmd}} can be one of the following:
+% \begin{itemize}[noitemsep]\ttfamily
+% \item ls
+% \item ls \meta{args}
+% \item pwd
+% \end{itemize}
+% The \meta{cmd} are described below. The result of the \meta{cmd} will be printed
+% to the terminal in an interactive run; in normal usage, this will be piped to
+% the calling \TeX{} process. Results containing path separators \emph{always}
+% use~|/|, irrespective of the platform in use.
+%
+% As well as these targets, the script recognizes the options
+% \begin{itemize}
+% \item |--exclude| Specification for directory entries to exclude
+% \item |--ignore-case| Ignores case when sorting directory listings
+% \item |--pattern| (|-p|) Treat the \meta{args} as Lua patterns rather
+% than converting from wildcards
+% \item |--recursive| (|-r|) Enables recursive searching during directory
+% listings
+% \item |--reverse| Causes sorting to go from highest to lowest rather
+% than lowest to highest
+% \item |--sort| Sets the method used to sort entries returned by |ls|
+% \item |--type| Selects the type of entry returned by |ls|
+% \end{itemize}
+% The action of these options on the appropriate \meta{cmd(s)} is detailed below.
+%
+% \subsubsection{\texttt{ls [\meta{args}]}}
+%
+% Lists the contents of one or more directories, in a manner somewhat reminiscent
+% of the Unix command |ls| or the Windows command |dir|. The exact nature of the
+% output will depend on the \meta{args}, if given, along with the prevailing
+% options. Note that the options names are inspired by ideas from the Unix
+% commands |ls| and |find| as well as the Windows command |dir|: they therefore do
+% not map directly to those of any one of the command line tools that they
+% somewhat mirror.
+%
+% When no \meta{args} are given, all entries in the current directory will be
+% listed, one per line in the output. This will include both files and
+% subdirectories. Each entry will include a path relative to the current
+% directory: for files \emph{in} the current directory, this will be |./|. The
+% order of results will be determined by the underlying operating system process:
+% unless requested \emph{via} an option, no sorting takes place.
+%
+% As standard, the \meta{args} are treated as a file/path name potentially
+% including |?| and |*| as wildcards, for example |*.png| or |file?.txt|.
+% \begin{verbatim}
+% l3sys-query ls '*.png'
+% \end{verbatim}
+% Some care is needed in preventing expansion of such wildcards by the shell or
+% \texttt{texlua} process: these are detailed in Section~\ref{sec:wildcard}. In
+% this section, |'| is used to indicate a character being used to suppress
+% expansion: this is for example normal on macOS and Linux.
+%
+% Removal of entries from the listing can be achieved using the |--exclude|
+% option, which should be given with a \meta{xarg}, for example
+% \begin{verbatim}
+% l3sys-query ls --exclude '*.bak' 'graphics/*'
+% \end{verbatim}
+% Directory entries starting |.| are traditionally hidden on Linux and macOS
+% systems: these \enquote{dot} entries are excluded from the output of
+% \texttt{l3sys-query}. The entries |.| and |..| for the current and parent
+% directory are also excluded from the results returned by \texttt{l3sys-query} as
+% they can always be assumed.
+%
+% For more complex matching, the \meta{args} can be treated as a Lua pattern using
+% the |--pattern| (|-p|) option; this also applies to the \meta{xarg} argument to
+% the |--exclude| option. For example, the equivalent to wildcard |*.png| could be
+% obtained using
+% \begin{verbatim}
+% l3sys-query ls --pattern '^.*%.png$'
+% \end{verbatim}
+%
+% The results returned by |ls| can be sorted using the |--sort| option. This can
+% be set to |none| (use the order from the file system: the default), |name| (sort
+% by file name) or |date| (sort by date last modified). The sorting order can be
+% reversed using |--reverse|. Sorting normally takes account of case: this can be
+% suppressed with the |--ignore-case| option.
+%
+% The listing can be filtered based on the type of entry using the |--type|
+% option. This takes a string argument, one of |d| (directory) or |f| (file).
+%
+% As standard, only the path specified as part of the \meta{args} is queried.
+% However, if the |--recursive| (|-r|) option is set, the query is applied within
+% all subdirectories. Subdirectories starting with~|.| (macOS and Linux hidden)
+% are excluded from recursion.
+%
+% For security reasons, only paths within the current working directory can be
+% queried, thus for example |graphics/*.png| will list all |png| files in the
+% |graphics| subdirectory, but |../graphics/*.png| will yield no output.
+%
+% \subsubsection{\texttt{pwd}}
+%
+% Returns the current working directory from which \texttt{l3sys-query} is run.
+% From within a \TeX{} run, this will (usually) be the directory containing the
+% main file, assuming a command such as
+% \begin{verbatim}
+% pdflatex main.tex
+% \end{verbatim}
+% The \texttt{pwd} command is unaffected by any options.
+%
+% \subsection{Spaces in arguments}
+%
+% Since \texttt{l3sys-query} is intended primarily for use with restricted shell
+% escape calls from \TeX{} processes, handling of spaces is unusual. It is not
+% possible to quote spaces in such a call, so for example whilst
+% \begin{verbatim}
+% l3sys-query ls "foo *"
+% \end{verbatim}
+% does work from the command prompt to find all files with names starting
+% \verb*|foo |, it would not work \emph{via} restricted shell escape. To
+% circumvent this, \texttt{l3sys-query} will collect all command line arguments
+% after any \meta{options}, and combine these as a space-separated \meta{args},
+% for example allowing
+% \begin{verbatim}
+% l3sys-query ls foo '*'
+% \end{verbatim}
+% to achieve the same result as the first example. The result is that the
+% \meta{args} will only every be interpreted by \texttt{l3sys-query} as a single
+% argument. It also means that spaces cannot be used at the start or end of the
+% argument, nor can multiple spaces appear between non-space arguments.
+%
+% \subsection{Wildcard expansion handling\label{sec:wildcard}}
+%
+% The handling of wildcards needs some further comment for those using
+% \texttt{l3sys-query} from the command line: the \pkg{expl3} interface described
+% in Section~\ref{sec:expl3} handles this aspect automatically for the user.
+%
+% On macOS and Linux, the shell normally expands globs, which include the
+% wildcards |*| and |?|, before passing arguments to the appropriate command.
+% This can be suppressed by surrounding the argument with |'| characters, hence
+% the formulation
+% \begin{verbatim}
+% l3sys-query ls '*.png'
+% \end{verbatim}
+% earlier.
+%
+% On Windows, the shell does no expansion, and thus arguments are passed as-is to
+% the relevant command. As such, |'| has no special meaning here. However, to
+% allow quoting of wildcards from the shell in a platform-neutral manner,
+% \texttt{l3sys-query} will strip exactly one set of |'| characters around each
+% argument before further processing.
+%
+% It is not possible to use |"| quotes at all in the argument passed to
+% \texttt{l3sys-query} from \TeX{}, as the \TeX{} system removes all |"| in
+% |\input| while handling space quoting.
+%
+% Restricted shell escape prevents shell expansion of wildcards entirely. On
+% non-Windows systems, it does this by ensuring that each argument is |'| quoted
+% to ensure further expansion. Thus a \TeX{} call such as
+% \begin{verbatim}
+% \input|"l3sys-query ls '*.png'"
+% \end{verbatim}
+% will work if |--shell-escape| is used as the argument is passed directly to the
+% shell, but in restricted shell escape will give an error such as:
+% \begin{verbatim}
+% ! I can't find file `"|l3sys-query ls '*.png'"'.
+% \end{verbatim}
+% The \LaTeX{} interfaces described above adjust the quoting used depending on the
+% |shell-escape| status.
+%
+%
+% \MaybeStop{}
+%
+% \section{The \LaTeXe\ package implementation}
+%
+% \begin{macrocode}
+%<*package>
+% \end{macrocode}
+%
+% \begin{macrocode}
+\ExplSyntaxOn
+%<@@=queryfiles>
+% \end{macrocode}
+%
+% The package should eventually work in restricted shell escape but will do
+% nothing useful if the process was started with |--no-shell-escape|.
+% \begin{macrocode}
+\sys_if_shell:F{
+ \PackageWarningNoLine{l3sys-query}
+ {Shell ~Escape ~is ~disabled.\MessageBreak All ~queries ~will ~return ~empty ~results}
+ }
+% \end{macrocode}
+%
+% \subsection{\cs{QueryWorkingDirectory}}
+% |\QueryWorkingDirectory| is a direct call to the |pwd| command provided by \texttt{l3sys-query}.
+% \begin{macrocode}
+\NewDocumentCommand\QueryWorkingDirectory {m} {
+ \sys_get_query:nN {pwd} #1
+}
+% \end{macrocode}
+%
+% \subsection{\cs{QueryFiles} and \cs{QueryFilesTF}}
+%
+% The declarations of these commands are done in two steps to allow catcode chanes
+% before the arguments are read. This allows the use of |%| and |^^| in patterns
+% at least if the command is not nested in another command argument. (|\%| may be used
+% to generate |%| in all cases).
+%
+% Variables for saving the current definition of |\%| and |~|.
+% \begin{macrocode}
+\tl_new:N\l_query_percent_tl
+\tl_new:N\l_query_tilde_tl
+% \end{macrocode}
+%
+% Allow |%| and |^^| at the top level.
+% \begin{macrocode}
+\NewDocumentCommand\QueryFiles {} {
+ \group_begin:
+ \char_set_catcode_other:N \%
+ \char_set_catcode_other:N \^
+ \QueryFiles_inner
+ }
+% \end{macrocode}
+%
+% \begin{macrocode}
+\char_set_catcode_active:N \~
+% \end{macrocode}
+%
+% \begin{macrocode}
+\NewDocumentCommand\QueryFiles_inner {O{}m}{
+ \group_end:
+ \tl_set:Nn\l_tmpa_tl{}
+ \cs_set_eq:NN \l_query_percent_tl \%
+ \cs_set_eq:NN \% \c_percent_str
+ \cs_set_eq:NN \l_query_tilde_tl ~
+ \cs_set_eq:NN ~ \c_tilde_str
+ \keys_set:nn{QueryFiles}{#1}
+ \exp_args:NnV\sys_split_query:nnnN {ls} \l_tmpa_tl {#2} \l_tmpa_seq
+ \cs_set_eq:NN \% \l_query_percent_tl
+ \cs_set_eq:NN ~ \l_query_tilde_tl
+ \seq_map_inline:Nn\l_tmpa_seq
+}
+% \end{macrocode}
+%
+% This duplicates rather than shares code so as to read the function
+% and TF arguments with normal catcode regime.
+% (This could probably be optimised.)
+% \begin{macrocode}
+\NewDocumentCommand\QueryFilesTF {} {
+ \group_begin:
+ \char_set_catcode_other:N \%
+ \QueryFilesTF_inner
+ }
+% \end{macrocode}
+%
+% \begin{macrocode}
+\NewDocumentCommand\QueryFilesTF_inner {O{}m}{
+ \group_end:
+ \tl_set:Nn\l_tmpa_tl{}
+ \cs_set_eq:NN \l_query_percent_tl \%
+ \cs_set_eq:NN \% \c_percent_str
+ \cs_set_eq:NN \l_query_tilde_tl ~
+ \cs_set_eq:NN ~ \c_tilde_str
+ \keys_set:nn{QueryFiles}{#1}
+ \exp_args:NnV\sys_split_query:nnnN {ls} \l_tmpa_tl {#2} \l_tmpa_seq
+ \cs_set_eq:NN \% \l_query_percent_tl
+ \cs_set_eq:NN ~ \l_query_tilde_tl
+ \seq_if_empty:NTF \l_tmpa_seq \use_iii:nnn \__queryfiles_aux:nnn
+}
+% \end{macrocode}
+%
+% \begin{macrocode}
+\char_set_catcode_space:N \~
+% \end{macrocode}
+%
+% \begin{macrocode}
+\cs_new:Npn \__queryfiles_aux:nnn #1#2#3 {
+ #2
+ \seq_map_inline:Nn\l_tmpa_seq {#1}
+ }
+% \end{macrocode}
+%
+% Defining the keys. Most take no value and simply add a \texttt{l3sys-query} \texttt{-{}-}
+% option to the command linebeing constructed.
+% \begin{macrocode}
+\keys_define:nn {QueryFiles} {
+% \end{macrocode}
+%
+% \begin{macrocode}
+recursive .code:n =\tl_put_right:Nn \l_tmpa_tl {--recursive ~ } ,
+recursive .value_forbidden:n = true ,
+% \end{macrocode}
+%
+% \begin{macrocode}
+ignore-case .code:n =\tl_put_right:Nn \l_tmpa_tl {--ignore-case ~ } ,
+ignore-case .value_forbidden:n = true ,
+% \end{macrocode}
+%
+% \begin{macrocode}
+reverse .code:n =\tl_put_right:Nn \l_tmpa_tl {--reverse ~ } ,
+reverse .value_forbidden:n = true ,
+% \end{macrocode}
+%
+% \begin{macrocode}
+exclude .code:n =\tl_put_right:Ne \l_tmpa_tl {
+ --exclude ~
+ \sys_if_shell_restricted:F'
+ \exp_not:n{#1}
+ \sys_if_shell_restricted:F'
+ ~ } ,
+exclude .value_required:n = true ,
+% \end{macrocode}
+%
+% The |type| key checks the supplied value is valid |d| for directories
+% or |f| for files The default behaviour lists both.
+% \begin{macrocode}
+type .choices:nn = {d,f}
+ {\tl_put_right:Nn \l_tmpa_tl {--type ~ #1 ~ }} ,
+% \end{macrocode}
+%
+% The |sort| key checks the supplied value is valid |date| or |name|.
+% \begin{macrocode}
+sort .choices:nn = {date,name}
+{\tl_put_right:Nn \l_tmpa_tl {--sort ~ #1 ~ }} ,
+% \end{macrocode}
+%
+% \begin{macrocode}
+pattern .code:n =\tl_put_right:Nn \l_tmpa_tl {--pattern ~ } ,
+pattern .value_forbidden:n = true ,
+% \end{macrocode}
+%
+% \begin{macrocode}
+}
+% \end{macrocode}
+%
+% \begin{macrocode}
+\ExplSyntaxOff
+% \end{macrocode}
+%
+% \begin{macrocode}
+%</package>
+% \end{macrocode}
+%
+% \Finale
+%