summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/doc/latex/datatool/datatool-user.tex
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/doc/latex/datatool/datatool-user.tex')
-rw-r--r--Master/texmf-dist/doc/latex/datatool/datatool-user.tex319
1 files changed, 286 insertions, 33 deletions
diff --git a/Master/texmf-dist/doc/latex/datatool/datatool-user.tex b/Master/texmf-dist/doc/latex/datatool/datatool-user.tex
index 885b832c301..a1bc741c631 100644
--- a/Master/texmf-dist/doc/latex/datatool/datatool-user.tex
+++ b/Master/texmf-dist/doc/latex/datatool/datatool-user.tex
@@ -99,11 +99,11 @@
\MakeShortVerb{"}
\DeleteShortVerb{\|}
- \title{User Manual for datatool bundle version~2.24}
+ \title{User Manual for datatool bundle version~2.25}
\author{Nicola L.C. Talbot\\
\url{http://www.dickimaw-books.com/}}
- \date{2016-01-12}
+ \date{2016-01-18}
\maketitle
\pagenumbering{roman}
@@ -118,7 +118,7 @@ The \styfmt{datatool} bundle comes with the following documentation:
\item[\url{datatool-code.pdf}]
Advanced users wishing to know more about the inner workings of
all the packages provided in the \styfmt{datatool} bundle should
- read \qt{Documented Code for datatool v2.24}
+ read \qt{Documented Code for datatool v2.25}
\item[INSTALL] Installation instructions.
@@ -2859,6 +2859,15 @@ external file, as described in \autoref{sec:loaddb}, or using the
commands described in \autoref{sec:newdb}, you can then iterate
through each row of the database and access elements in that row.
+\begin{important}
+The \cs{DTLforeach} command is provided as a convenient way of
+iterating through databases with the option to filter rows,
+cross-reference rows or have nested loops, but for large databases it can be
+\emph{extremely slow}, especially if the unstarred version is used.
+If you have a large database, consider pre-processing the data using
+an external script. See also \sectionref{sec:advancediter}.
+\end{important}
+
\begin{definition}[\DescribeMacro{\DTLforeach}]%
\cs{DTLforeach}\oarg{condition}\marg{db name}\marg{assign list}\marg{text}
\end{definition}
@@ -5224,36 +5233,6 @@ must be a command name. The type will be one of:
\item\DescribeMacro{\DTLcurrencytype}\cs{DTLcurrenttype} (currency).
\end{itemize}
-\begin{definition}[\DescribeMacro{\dtlforeachkey}]%
-\cs{dtlforeachkey}(\meta{key cs},\meta{col cs},\meta{type
-cs},\meta{header cs})\cs{in}\marg{db}\cs{do}\marg{body}
-\end{definition}
-This iterates through all the keys in the database given by
-\meta{db}. In each iteration, \meta{key cs} is set to the key,
-\meta{col cs} is set to the column index, \meta{type cs} is set to
-the data type (as for \cs{DTLgetdatatype}), \meta{header cs} is set
-to the header for that column, and then \meta{body} is done. Note
-that \meta{key cs}, \meta{col cs}, \meta{type cs} and \meta{header
-cs} must all be control sequences. No check is performed to
-determine if that control sequence already exists, and the control
-sequences are defined globally (since it's likely that
-\cs{dtlforeachkey} may be used within a \env{tabular} environment),
-so you need to make sure you don't override an existing command of
-the same name.
-
-\begin{definition}[\DescribeMacro{\dtlforcolumn}]%
-\cs{dtlforcolumn}\marg{cs}\marg{db}\marg{key}\marg{body}
-\end{definition}
-This iterates through the column given by \meta{key} in the
-database given by \meta{db} and applies \meta{body}. In each
-iteration, \meta{cs} (which must be a control sequence) is set to
-the current element in the column and may be used in \meta{body}.
-Alternatively, if you want to identify the column by its index
-rather than its key, use:
-\begin{definition}[\DescribeMacro{\dtlforcolumnidx}]%
-\cs{dtlforcolumnidx}\marg{cs}\marg{db}\marg{col index}\marg{body}
-\end{definition}
-
\begin{definition}[\DescribeMacro{\DTLifdbexists}]
\cs{DTLifdbexists}\marg{db name}\marg{true part}\marg{false part}
\end{definition}
@@ -5854,6 +5833,280 @@ The result is shown in \autoref{tab:join3}.
\end{table}
\end{example}
+\subsection{Advanced Iteration}
+\label{sec:advancediter}
+
+The \ics{DTLforeach} command described in \sectionref{sec:dbforeach}
+has some limitations, especially when trying to iterate through
+large databases. This section describes lower-level user commands
+that may be used for iteration instead of \cs{DTLforeach}.
+
+\begin{definition}[\DescribeMacro{\dtlforeachkey}]%
+\cs{dtlforeachkey}(\meta{key cs},\meta{col cs},\meta{type
+cs},\meta{header cs})\cs{in}\marg{db}\cs{do}\marg{body}
+\end{definition}
+This iterates through all the keys in the database given by
+\meta{db}. In each iteration, \meta{key cs} is set to the key,
+\meta{col cs} is set to the column index, \meta{type cs} is set to
+the data type (as for \cs{DTLgetdatatype}), \meta{header cs} is set
+to the header for that column, and then \meta{body} is done. Note
+that \meta{key cs}, \meta{col cs}, \meta{type cs} and \meta{header
+cs} must all be control sequences. No check is performed to
+determine if that control sequence already exists, and the control
+sequences are defined globally (since it's likely that
+\cs{dtlforeachkey} may be used within a \env{tabular} environment),
+so you need to make sure you don't override an existing command of
+the same name.
+
+\begin{definition}[\DescribeMacro{\dtlforcolumn}]%
+\cs{dtlforcolumn}\marg{cs}\marg{db}\marg{key}\marg{body}
+\end{definition}
+This iterates through the column given by \meta{key} in the
+database given by \meta{db} and applies \meta{body}. In each
+iteration, \meta{cs} (which must be a control sequence) is set to
+the current element in the column and may be used in \meta{body}.
+Alternatively, if you want to identify the column by its index
+rather than its key, use:
+\begin{definition}[\DescribeMacro{\dtlforcolumnidx}]%
+\cs{dtlforcolumnidx}\marg{cs}\marg{db}\marg{col index}\marg{body}
+\end{definition}
+Both \cs{dtlforcolumn} and \cs{dtlforcolumnidx} have a starred
+version that doesn't check for the existence of the given database.
+You may use \ics{dtlbreak} within \meta{body} to break out of the loop
+at the end of the current iteration.
+
+An alternative to \cs{DTLforeach} is to use \TeX's primitive
+\ics{loop} with \ics{DTLgetvalue}, but this may not be faster.
+For example, suppose I have a CSV
+file with a single column (labelled \qt{Word}) with 1000 rows of
+data (where the CSV file is called \texttt{test-data-1000.csv}).
+First, let's use the unstarred version of \cs{DTLforeach}:
+\begin{verbatim}
+\batchmode
+\documentclass{article}
+\usepackage{datatool}
+
+\DTLloaddb{data}{test-data-1000.csv}
+
+\begin{document}
+\DTLforeach{data}{\Word=Word}{\Word.\par}
+\end{document}
+\end{verbatim}
+On my 64bit Linux computer, this document took 4.942s to compile.
+Result from \texttt{time pdflatex test}:
+\begin{verbatim}
+real 0m4.942s
+user 0m4.934s
+sys 0m0.015s
+\end{verbatim}
+Now using the starred version of \cs{DTLforeach}:
+\begin{verbatim}
+\batchmode
+\documentclass{article}
+\usepackage{datatool}
+
+\DTLloaddb{data}{test-data-1000.csv}
+
+\begin{document}
+\DTLforeach*{data}{\Word=Word}{\Word.\par}
+\end{document}
+\end{verbatim}
+This took 2.138s to compile:
+\begin{verbatim}
+real 0m2.138s
+user 0m2.122s
+sys 0m0.020s
+\end{verbatim}
+Now using \TeX's \cs{loop}:
+\begin{verbatim}
+\batchmode
+\documentclass{article}
+\usepackage{datatool}
+
+\DTLloaddb{data}{test-data-1000.csv}
+
+\begin{document}
+
+\newcount\rowctr
+\loop
+ \advance\rowctr by 1\relax
+ \DTLgetvalue{\Word}{data}{\rowctr}{1}%
+ \Word.\par
+\ifnum\rowctr<\DTLrowcount{data}
+\repeat
+
+\end{document}
+\end{verbatim}
+(This takes advantage of the fact that I know I only have one column
+of data, so I only need to reference column~1 in \cs{DTLgetvalue}.)
+This takes 2.638s to compile:
+\begin{verbatim}
+real 0m2.638s
+user 0m2.622s
+sys 0m0.020s
+\end{verbatim}
+which is slightly longer than using \cs{DTLforeach*} (but not nearly
+as long as using the unstarred version). Another possible method is
+to use \cs{dtlgetrow} and \cs{dtlgetentryfromcurrentrow}:
+\begin{verbatim}
+\batchmode
+\documentclass{article}
+\usepackage{datatool}
+
+\DTLloaddb{data}{test-data-1000.csv}
+
+\begin{document}
+
+\newcount\rowctr
+\loop
+ \advance\rowctr by 1\relax
+ \dtlgetrow{data}{\rowctr}%
+ \dtlgetentryfromcurrentrow{\Word}{1}%
+ \Word.\par
+\ifnum\rowctr<\DTLrowcount{data}
+\repeat
+
+\end{document}
+\end{verbatim}
+This took 3.596s to compile:
+\begin{verbatim}
+real 0m3.596s
+user 0m3.582s
+sys 0m0.019s
+\end{verbatim}
+Another possibility is to use \cs{dtlforcolumnidx}, described above:
+\begin{verbatim}
+\batchmode
+\documentclass{article}
+\usepackage{datatool}
+
+\DTLloaddb{data}{test-data-1000.csv}
+
+\begin{document}
+
+\dtlforcolumnidx{\Word}{data}{1}{\Word.\par}
+
+\end{document}
+\end{verbatim}
+This took 2.093s to compile:
+\begin{verbatim}
+real 0m2.093s
+user 0m2.083s
+sys 0m0.013s
+\end{verbatim}
+So if you only want to iterate through one column, this is the
+fastest method, but it's still more efficient to pre-process the
+data using an external script that creates a \texttt{.tex} file that
+can be \cs{input} into the document.
+
+Note that the build time increases with extra columns, \emph{even if
+they're not required in the document}. For example, I created a new
+CSV file called \texttt{test-data-1000-5.csv} that had four extra
+columns (which were actually duplicates of the first column with
+different headers, for simplicity). Just iterating through the first
+column to obtain the same PDF as previously significantly increases
+the time taken. The only modification to the above examples was an
+edit to the \cs{DTLloaddb} line:
+\begin{verbatim}
+\DTLloaddb{data}{test-data-1000-5.csv}
+\end{verbatim}
+The fastest method using \cs{dtlforcolumnidx} took 25.688s:
+\begin{verbatim}
+real 0m25.688s
+user 0m25.712s
+sys 0m0.013s
+\end{verbatim}
+Using \cs{DTLforeach*} took 25.725s:
+\begin{verbatim}
+real 0m25.725s
+user 0m25.744s
+sys 0m0.018s
+\end{verbatim}
+The unstarred version took 35.665s:
+\begin{verbatim}
+real 0m35.665s
+user 0m35.692s
+sys 0m0.022s
+\end{verbatim}
+Using \cs{loop} and \cs{DTLgetvalue} took 27.844s:
+\begin{verbatim}
+real 0m27.844s
+user 0m27.866s
+sys 0m0.019s
+\end{verbatim}
+Using \cs{loop} and \cs{dtlgetrow} took 31.770s:
+\begin{verbatim}
+real 0m31.770s
+user 0m31.785s
+sys 0m0.028s
+\end{verbatim}
+
+On the other hand, if I want to only iterate through, say, the last
+100 rows of the data, it's simpler to use \cs{loop}. For example:
+\begin{verbatim}
+\batchmode
+\documentclass{article}
+
+\usepackage{datatool}
+
+\DTLloaddb{data}{test-data-1000-5.csv}
+
+\begin{document}
+\newcount\rowctr
+\rowctr=\numexpr\DTLrowcount{data}-100\relax
+\loop
+ \advance\rowctr by 1\relax
+ \DTLgetvalue{\Word}{data}{\rowctr}{1}%
+ \Word.\par
+\ifnum\rowctr<\DTLrowcount{data}
+\repeat
+\end{document}
+\end{verbatim}
+This took 25.124s:
+\begin{verbatim}
+real 0m25.124s
+user 0m25.138s
+sys 0m0.023s
+\end{verbatim}
+The equivalent using \cs{DTLforeach} is:
+\begin{verbatim}
+\batchmode
+\documentclass{article}
+
+\usepackage{datatool}
+
+\DTLloaddb{data}{test-data-1000-5.csv}
+
+\begin{document}
+\newcount\firstidx
+\firstidx=\numexpr\DTLrowcount{data}-100\relax
+\DTLforeach*{data}{\Word=Word}%
+ {\ifnum\DTLcurrentindex>\firstidx\relax\Word.\par\fi}
+\end{document}
+\end{verbatim}
+(Remember that the row count can't be used in the optional argument
+of \cs{DTLforeach} as it's only incremented when the condition is
+true.) This took 26.375s:
+\begin{verbatim}
+real 0m26.375s
+user 0m26.381s
+sys 0m0.029s
+\end{verbatim}
+This takes longer because it's still iterating over every row of the
+database and is applying the condition to each row.
+
+\minisec{Summary}
+\begin{itemize}
+\item If possible, use an external script to pre-process the data so
+that you can simply \cs{input} valid \LaTeX\ code into the document.
+\item If you only want to iterate through one column of the data,
+use \cs{dtlforcolumnidx}.
+\item Remove\footnote{Naturally, make a copy of the original data, if
+necessary.}\ unwanted columns and\slash or rows from the CSV file (and sort, if
+necessary) using the spreadsheet application (or whatever) that was
+used to generate the original CSV file.
+\end{itemize}
+
\chapter{Creating an index, glossary or list of acronyms
(\texorpdfstring{\sty{datagidx}}{datagidx} package)}
\label{sec:datagidx}