datatool (13jan16)

git-svn-id: svn://tug.org/texlive/trunk@39376 c570f23f-e606-0410-a88d-b1316a301751

19 files changed, 260 insertions, 22 deletions

diff --git a/Master/texmf-dist/doc/latex/datatool/CHANGES b/Master/texmf-dist/doc/latex/datatool/CHANGES
index 311f435b815..e29220008d8 100644
--- a/Master/texmf-dist/doc/latex/datatool/CHANGES
+++ b/Master/texmf-dist/doc/latex/datatool/CHANGES
@@ -1,3 +1,16 @@
+v2.24:
+
+  * datatool-base:
+
+    - added "utf8" package option and \dtlenableUTFviii, 
+      \dtldisableUTFviii
+
+    - added \dtlsetcharcode, \dtlsetlccharcode,
+      \dtlsetUTFviiicharcode, \dtlsetUTFviiilccharcode
+      \dtlsetdefaultUTFviiicharcode and
+      \dtlsetdefaultUTFviiilccharcode for use with the comparison
+      handlers.
+
 v2.23:
 
   * datatool:
diff --git a/Master/texmf-dist/doc/latex/datatool/README b/Master/texmf-dist/doc/latex/datatool/README
index 28ef7ab8756..38aad314e74 100644
--- a/Master/texmf-dist/doc/latex/datatool/README
+++ b/Master/texmf-dist/doc/latex/datatool/README
@@ -1,6 +1,6 @@
-LaTeX Bundle : datatool v2.23 
+LaTeX Bundle : datatool v2.24 
 
-Last Modified : 2015-07-11
+Last Modified : 2016-01-12
 
 Author        : Nicola Talbot
 
diff --git a/Master/texmf-dist/doc/latex/datatool/datatool-code.pdf b/Master/texmf-dist/doc/latex/datatool/datatool-code.pdf
index 5a1571a4282..e9a74d5c7a1 100644
--- a/Master/texmf-dist/doc/latex/datatool/datatool-code.pdf
+++ b/Master/texmf-dist/doc/latex/datatool/datatool-code.pdf
diff --git a/Master/texmf-dist/doc/latex/datatool/datatool-user.pdf b/Master/texmf-dist/doc/latex/datatool/datatool-user.pdf
index fac3cc3d181..5360d80b0c1 100644
--- a/Master/texmf-dist/doc/latex/datatool/datatool-user.pdf
+++ b/Master/texmf-dist/doc/latex/datatool/datatool-user.pdf
diff --git a/Master/texmf-dist/doc/latex/datatool/datatool-user.tex b/Master/texmf-dist/doc/latex/datatool/datatool-user.tex
index 91a26817787..885b832c301 100644
--- a/Master/texmf-dist/doc/latex/datatool/datatool-user.tex
+++ b/Master/texmf-dist/doc/latex/datatool/datatool-user.tex
@@ -1,5 +1,7 @@
 \documentclass[report,widecs]{nlctdoc}
 
+\usepackage[T1]{fontenc}
+\usepackage[utf8]{inputenc}
 \usepackage{ifthen}
 \usepackage[verbose=false]{datatool}
 \usepackage{datapie,databar,databib,person,datagidx}
@@ -7,10 +9,9 @@
 \usepackage{graphicx}
 \usepackage{colortbl}
 \usepackage{booktabs}
+\usepackage{metalogo}
 \usepackage{cmap}
 \usepackage{alltt}
-\usepackage[T1]{fontenc}
-\usepackage[utf8]{inputenc}
 \ifpdf
 \usepackage{mathpazo}
 \usepackage[scaled=.88]{helvet}
@@ -98,11 +99,11 @@
 \MakeShortVerb{"}
 \DeleteShortVerb{\|}
 
- \title{User Manual for datatool bundle version~2.23}
+ \title{User Manual for datatool bundle version~2.24}
  \author{Nicola L.C. Talbot\\
 \url{http://www.dickimaw-books.com/}}
 
- \date{2015-07-11}
+ \date{2016-01-12}
  \maketitle
 
 \pagenumbering{roman}
@@ -117,7 +118,7 @@ The \styfmt{datatool} bundle comes with the following documentation:
   \item[\url{datatool-code.pdf}]
   Advanced users wishing to know more about the inner workings of
   all the packages provided in the \styfmt{datatool} bundle should
-  read \qt{Documented Code for datatool v2.23}
+  read \qt{Documented Code for datatool v2.24}
 
   \item[INSTALL] Installation instructions.
 
@@ -3906,7 +3907,7 @@ environment:
 \bfseries Email 1 &
 \bfseries Email 2 &
 \bfseries Reg Num%
-\DTLforeach{emailDB}{\firstname=FirstName,\surname=Surname,%
+\DTLforeach*{emailDB}{\firstname=FirstName,\surname=Surname,%
 \emailI=Email1,\emailII=Email2,\regnum=RegNum}{%
 \\\firstname & \surname & \emailI & \emailII & \regnum}%
 \end{tabular}
@@ -3918,7 +3919,7 @@ This produces the following:\par\vskip\baselineskip\noindent
 \bfseries Email 1 &
 \bfseries Email 2 &
 \bfseries Reg Num\relax
-\DTLforeach{emailDB}{\firstname=FirstName,\surname
+\DTLforeach*{emailDB}{\firstname=FirstName,\surname
 =Surname,\emailI=Email1,\emailII=Email2,\regnum=RegNum}{\relax
 \\\firstname & \surname & \emailI & \emailII & \regnum}\relax
 \end{tabular}
@@ -4570,8 +4571,9 @@ First of all, display the unsorted data:
 \DTLdisplaydb{inversiondata}
 \end{verbatim}
 This produces:
-
+\begin{display}
 \DTLdisplaydb{inversiondata}
+\end{display}
 
 Now sort the data using the \ics{dtlwordindexcompare} handler:
 \begin{verbatim}
@@ -4583,10 +4585,10 @@ and display again:
 \DTLdisplaydb{inversiondata}
 \end{verbatim}
 which now produces:
-
+\begin{display}
 \DTLdisplaydb{inversiondata}
-
-~here are three entries with pre-inversion text as simply
+\end{display}
+There are three entries here with pre-inversion text that's simply
 \texttt{New York}. Since each of these three entries has the same
 pre-inversion text, they need to be sorted according to the type of
 inversion: person, place, subject. The fourth entry (New York, New
@@ -4601,7 +4603,221 @@ code for details on the syntax of the handler. (You may want to
 consider uploading your handler as a separate package to CTAN if you
 think it will be of general use.)
 
-There are two shortcut commands:
+As from version 2.24, the predefined handlers now have limited support for UTF-8
+characters. \emph{This is still experimental.} The support will
+automatically be switched on if the \sty{inputenc} package is loaded
+with the \pkgopt{utf8} option before loading \sty{datatool-base}
+(which is automatically loaded by \sty{datatool}). If \sty{inputenc}
+is loaded after \sty{datatool-base}, you can use the (boolean) \pkgopt{utf8}
+option when you load \sty{datatool-base} to enable it. For example,
+either:
+\begin{verbatim}
+\usepackage[utf8]{inputenc}
+\usepackage{datatool-base}
+\end{verbatim}
+or
+\begin{verbatim}
+\usepackage[utf8]{datatool-base}
+\usepackage[utf8]{inputenc}
+\end{verbatim}
+You can also enable this option after \sty{datatool-base} has been loaded
+using
+\begin{definition}[\DescribeMacro\dtlenableUTFviii]
+\cs{dtlenableUTFviii}
+\end{definition}
+or disable it with
+\begin{definition}[\DescribeMacro\dtldisableUTFviii]
+\cs{dtldisableUTFviii}
+\end{definition}
+For example:
+\begin{verbatim}
+\usepackage[utf8]{inputenc}
+\usepackage{datatool}
+\dtldisableUTFviii
+\end{verbatim}
+This option isn't required for \XeLaTeX, which treats each UTF-8
+character as an individual token.
+
+With regular (pdf)\LaTeX\ (as opposed to \XeLaTeX) each UTF-8 character is
+actually treated as two tokens that represent the first and second
+octet of the UTF-8 character. This means that it's not possible for
+\TeX\ to obtain a character code using the usual backtick method, so
+(with this option enabled) if a UTF-8 character is detected, the handlers 
+described above will pass both octet tokens to one of the following commands:
+\begin{definition}[\DescribeMacro\dtlsetUTFviiicharcode]
+\cs{dtlsetUTFviiicharcode}\marg{octet tokens}\marg{count}
+\end{definition}
+(for the case-sensitive handler \ics{dltcompare}) and
+\begin{definition}[\DescribeMacro\dtlsetUTFviiilccharcode]
+\cs{dtlsetUTFviiilccharcode}\marg{octet tokens}\marg{count}
+\end{definition}
+(for the case-insensitive handlers).
+
+By default the first simply does:
+\begin{definition}[\DescribeMacro\dtlsetdefaultUTFviiicharcode]
+\cs{dtlsetdefaultUTFviiicharcode}\marg{octet tokens}\marg{count}
+\end{definition}
+and the second does
+\begin{definition}[\DescribeMacro\dtlsetdefaultUTFviiilccharcode]
+\cs{dtlsetdefaultUTFviiilccharcode}\marg{octet tokens}\marg{count}
+\end{definition}
+
+In all the above \meta{octet tokens} are the two octets forming the
+UTF-8 character and \meta{count} is a count register in which to
+store the relevant character code.
+
+The default commands simply set the character code for certain
+common accented Latin characters (such as \'e) to the code for their 
+unaccented version (such as e). You can redefine
+\cs{dtlsetUTFviiicharcode} and \cs{dtlsetUTFviiilccharcode} to test
+for additional characters. For example, to add the Norwegian
+characters \ae, \o\ and \aa\ to the end of the Latin alphabet, you
+can do:
+\begin{verbatim}
+\renewcommand*{\dtlsetUTFviiicharcode}[2]{%
+  \ifstrequal{#1}{Æ}%
+  {%
+    #2=91\relax
+  }%
+  {%
+    \ifstrequal{#1}{Ø}%
+    {%
+      #2=92\relax
+    }%
+    {%
+      \ifstrequal{#1}{Å}%
+      {%
+        #2=93\relax
+      }%
+      {%
+        \ifstrequal{#1}{æ}%
+        {%
+          #2=123\relax
+        }%
+        {%
+          \ifstrequal{#1}{ø}%
+          {%
+            #2=124\relax
+          }%
+          {%
+            \ifstrequal{#1}{å}%
+            {%
+              #2=125\relax
+            }%
+            {%
+              \dtlsetdefaultUTFviiicharcode{#1}{#2}%
+            }%
+          }%
+        }%
+      }%
+    }%
+  }%
+}
+
+\renewcommand*{\dtlsetUTFviiilccharcode}[2]{%
+  \ifstrequal{#1}{Æ}%
+  {%
+    #2=123\relax
+  }%
+  {%
+    \ifstrequal{#1}{Ø}%
+    {%
+      #2=124\relax
+    }%
+    {%
+      \ifstrequal{#1}{Å}%
+      {%
+        #2=125\relax
+      }%
+      {%
+        \ifstrequal{#1}{æ}%
+        {%
+          #2=123\relax
+        }%
+        {%
+          \ifstrequal{#1}{ø}%
+          {%
+            #2=124\relax
+          }%
+          {%
+            \ifstrequal{#1}{å}%
+            {%
+              #2=125\relax
+            }%
+            {%
+              \dtlsetdefaultUTFviiilccharcode{#1}{#2}%
+            }%
+          }%
+        }%
+      }%
+    }%
+  }%
+}
+\end{verbatim}
+
+In the case where a~character is a single token (for example, with
+Latin-1 encoding), the code associated with that character is set
+using
+\begin{definition}[\DescribeMacro\dtlsetcharcode]
+\cs{dtlsetcharcode}\marg{c}\marg{count register}
+\end{definition}
+(for case-sensitive comparison) and
+\begin{definition}[\DescribeMacro\dtlsetlccharcode]
+\cs{dtlsetlccharcode}\marg{c}\marg{count register}
+\end{definition}
+(for case-insensitive comparison), where \meta{c} is the character
+and \meta{count register} is a count register. The default
+definitions are:
+\begin{verbatim}
+\newcommand*{\dtlsetcharcode}[2]{#2=`#1\relax}
+\newcommand*{\dtlsetlccharcode}[2]{#2=\lccode`#1\relax}
+\end{verbatim}
+So, for example, the letter \qt{e} has the code \number`e\ whereas
+(with the Latin-1 encoding) the letter \qt{\'e} has the code 233.
+This means that \qt{\'e} will be sorted after \qt{e}. It may be that
+you want \qt{\'e} to be treated the same as \qt{e} when making the
+comparison. In this case, you need to redefine \cs{dtlsetcharcode}
+and \cs{dtlsetlccharcode}. For example, for the case-sensitive
+comparisons:
+\begin{verbatim}
+\renewcommand*{\dtlsetcharcode}[2]{%
+  \ifstrequal{#1}{É}%
+  {%
+    #2=`E\relax
+  }%
+  {%
+    \ifstrequal{#1}{é}%
+    {%
+      #2=`e\relax
+    }%
+    {%
+      #2=`#1\relax
+    }%
+  }%
+}
+\end{verbatim}
+and for the case-insensitive comparisons:
+\begin{verbatim}
+\renewcommand*{\dtlsetlccharcode}[2]{%
+  \ifboolexpr
+  {%
+       test {\ifstrequal{#1}{é}}
+    or test {\ifstrequal{#1}{É}}
+  }%
+  {%
+    #2=`e\relax
+  }%
+  {%
+    #2=\lccode`#1\relax
+  }%
+}
+\end{verbatim}
+(Extra conditionals will need to be added for other diacritics.
+Alternative conditionals, such as \cs{ifx} or \cs{ifnum}, may be more 
+efficient. In the above I've used the \sty{etoolbox} commands for
+clarity.)
+
+There are two shortcut commands for \cs{dtlsort}:
 \begin{definition}[\DescribeMacro{\DTLsort}]%
 \cs{DTLsort}\oarg{replacement key list}\marg{sort criteria}\marg{db name}
 \end{definition}
@@ -4887,12 +5103,6 @@ This produces \autoref{tab:influencesort}.
 \section{Saving a Database to an External File}
 \label{sec:savedb}
 
-\begin{important}
-\TeX's write mechanism automatically inserts linebreaks every 80
-characters. This may cause problems if you have long entries in your
-database.
-\end{important}
-
 \begin{definition}[\DescribeMacro{\DTLsavedb}]%
 \cs{DTLsavedb}\marg{db name}\marg{filename}
 \end{definition}
diff --git a/Master/texmf-dist/doc/latex/datatool/samples/sample-autokeys.pdf b/Master/texmf-dist/doc/latex/datatool/samples/sample-autokeys.pdf
index ad495330892..dfba3d43cc7 100644
--- a/Master/texmf-dist/doc/latex/datatool/samples/sample-autokeys.pdf
+++ b/Master/texmf-dist/doc/latex/datatool/samples/sample-autokeys.pdf
diff --git a/Master/texmf-dist/doc/latex/datatool/samples/sample-barchart.pdf b/Master/texmf-dist/doc/latex/datatool/samples/sample-barchart.pdf
index aaf21393f64..c6aadfaa097 100644
--- a/Master/texmf-dist/doc/latex/datatool/samples/sample-barchart.pdf
+++ b/Master/texmf-dist/doc/latex/datatool/samples/sample-barchart.pdf
diff --git a/Master/texmf-dist/doc/latex/datatool/samples/sample-datatooltk.pdf b/Master/texmf-dist/doc/latex/datatool/samples/sample-datatooltk.pdf
index 3f82014ed9c..efc79437a63 100644
--- a/Master/texmf-dist/doc/latex/datatool/samples/sample-datatooltk.pdf
+++ b/Master/texmf-dist/doc/latex/datatool/samples/sample-datatooltk.pdf
diff --git a/Master/texmf-dist/doc/latex/datatool/samples/sample-dict.pdf b/Master/texmf-dist/doc/latex/datatool/samples/sample-dict.pdf
index 533efaba595..5339715bd16 100644
--- a/Master/texmf-dist/doc/latex/datatool/samples/sample-dict.pdf
+++ b/Master/texmf-dist/doc/latex/datatool/samples/sample-dict.pdf
diff --git a/Master/texmf-dist/doc/latex/datatool/samples/sample-europecv-bib.pdf b/Master/texmf-dist/doc/latex/datatool/samples/sample-europecv-bib.pdf
index 45461691083..25761b21490 100644
--- a/Master/texmf-dist/doc/latex/datatool/samples/sample-europecv-bib.pdf
+++ b/Master/texmf-dist/doc/latex/datatool/samples/sample-europecv-bib.pdf
diff --git a/Master/texmf-dist/doc/latex/datatool/samples/sample-gidx.pdf b/Master/texmf-dist/doc/latex/datatool/samples/sample-gidx.pdf
index a1d0cbdd44f..32fb87b3ab5 100644
--- a/Master/texmf-dist/doc/latex/datatool/samples/sample-gidx.pdf
+++ b/Master/texmf-dist/doc/latex/datatool/samples/sample-gidx.pdf
diff --git a/Master/texmf-dist/doc/latex/datatool/samples/sample-glossary.pdf b/Master/texmf-dist/doc/latex/datatool/samples/sample-glossary.pdf
index c2c7ed49280..7b88dbfa57a 100644
--- a/Master/texmf-dist/doc/latex/datatool/samples/sample-glossary.pdf
+++ b/Master/texmf-dist/doc/latex/datatool/samples/sample-glossary.pdf
diff --git a/Master/texmf-dist/doc/latex/datatool/samples/sample-index.pdf b/Master/texmf-dist/doc/latex/datatool/samples/sample-index.pdf
index 21697f30670..aaed390088c 100644
--- a/Master/texmf-dist/doc/latex/datatool/samples/sample-index.pdf
+++ b/Master/texmf-dist/doc/latex/datatool/samples/sample-index.pdf
diff --git a/Master/texmf-dist/doc/latex/datatool/samples/sample-mail-merge.pdf b/Master/texmf-dist/doc/latex/datatool/samples/sample-mail-merge.pdf
index 57406d47eab..b007bf31bbc 100644
--- a/Master/texmf-dist/doc/latex/datatool/samples/sample-mail-merge.pdf
+++ b/Master/texmf-dist/doc/latex/datatool/samples/sample-mail-merge.pdf
diff --git a/Master/texmf-dist/doc/latex/datatool/samples/sample-piechart.pdf b/Master/texmf-dist/doc/latex/datatool/samples/sample-piechart.pdf
index f431b620aaf..6ea6b930502 100644
--- a/Master/texmf-dist/doc/latex/datatool/samples/sample-piechart.pdf
+++ b/Master/texmf-dist/doc/latex/datatool/samples/sample-piechart.pdf
diff --git a/Master/texmf-dist/doc/latex/datatool/samples/sample-sort.pdf b/Master/texmf-dist/doc/latex/datatool/samples/sample-sort.pdf
index 1cf45c10e0e..9737968d009 100644
--- a/Master/texmf-dist/doc/latex/datatool/samples/sample-sort.pdf
+++ b/Master/texmf-dist/doc/latex/datatool/samples/sample-sort.pdf
diff --git a/Master/texmf-dist/doc/latex/datatool/samples/sample-sort.tex b/Master/texmf-dist/doc/latex/datatool/samples/sample-sort.tex
index bce750819dc..2c1f0b96f00 100644
--- a/Master/texmf-dist/doc/latex/datatool/samples/sample-sort.tex
+++ b/Master/texmf-dist/doc/latex/datatool/samples/sample-sort.tex
@@ -2,6 +2,10 @@
 
 \documentclass{article}
 
+\usepackage[margin=1in]{geometry}
+
+\usepackage[T1]{fontenc}
+\usepackage[utf8]{inputenc}
 \usepackage{datatool}
 
 % Sample database
@@ -20,8 +24,9 @@
   \DTLnewdbentry{data}{Word}{#1}%
 % Convert word to its definition and add
   {%
-    \def\thisword{#1}\@onelevel@sanitize\thisword
-    \dtlexpandnewvalue % ensure \theword gets expanded
+    \def\thisword{#1}%
+    \@onelevel@sanitize\thisword
+    \dtlexpandnewvalue % ensure \thisword gets expanded
     \DTLnewdbentry{data}{Definition}{\thisword}%
   }%
 }
@@ -69,8 +74,18 @@
 \newword{Alpha}
 \newword{seal}
 \newword{sea lion}
+\newword{elephant}
+\newword{Élite}
 
 \begin{document}
+This is a sample document testing datatool's comparison
+handlers. Note that UTF-8 characters are treated as two tokens
+(the first and second octets) with regular (pdf)\LaTeX, so the
+definition column on the following pages shows the two octets
+for the UTF-8 character É as ÃĽ, which is how \LaTeX\ sees them. XeLaTeX will treat it as a single
+token.
+
+\newpage
 
 \section{Unsorted data}
 
diff --git a/Master/texmf-dist/doc/latex/datatool/samples/sample-student-scores.pdf b/Master/texmf-dist/doc/latex/datatool/samples/sample-student-scores.pdf
index b73bfa7ef1f..ccff004636c 100644
--- a/Master/texmf-dist/doc/latex/datatool/samples/sample-student-scores.pdf
+++ b/Master/texmf-dist/doc/latex/datatool/samples/sample-student-scores.pdf
diff --git a/Master/texmf-dist/doc/latex/datatool/samples/sample-two-per-row.pdf b/Master/texmf-dist/doc/latex/datatool/samples/sample-two-per-row.pdf
index 5492735022c..7e6e918a0cb 100644
--- a/Master/texmf-dist/doc/latex/datatool/samples/sample-two-per-row.pdf
+++ b/Master/texmf-dist/doc/latex/datatool/samples/sample-two-per-row.pdf