unicode-data (7feb16)

git-svn-id: svn://tug.org/texlive/trunk@39607 c570f23f-e606-0410-a88d-b1316a301751

4 files changed, 154 insertions, 57 deletions

diff --git a/Master/texmf-dist/doc/generic/unicode-data/README.md b/Master/texmf-dist/doc/generic/unicode-data/README.md
index 4f284bcd62e..2bebefbb40e 100644
--- a/Master/texmf-dist/doc/generic/unicode-data/README.md
+++ b/Master/texmf-dist/doc/generic/unicode-data/README.md
@@ -32,8 +32,25 @@ From the file `UnicodeData.txt` the following properties are set:
   which an upper or lower case mapping is given
 - `\sfcode` 0 (ignored) for code points of Unicode classes "Pe"
   (closing punctuation marks) and "Pf" (final quotation marks)
-- `\Umathcode` for all letters in the base plane class 7 (var)
-  and for the supplementary plane class 0 (regular)
+- `\Umathcode` for all letters as TeX class 7 (var)
+
+File: `load-unicode-math-classes.tex`
+======================================
+
+From the file `MathClass.txt` the following mapping between
+Unicode math classes and TeX math types are set up
+- "L" (large)       `\mathop`
+- "B" (binary)      `\mathbin`
+- "V" (vary)        `\mathbin`
+- "R" (relation)    `\mathrel`
+- "O" (opening)     `\mathopen`
+- "C" (closing)     `\mathclose`
+- "P" (punctuation) `\mathpunct`
+- "A" (alphabetic)  `\mathalpha`
+
+For each code point processed, the result is of the form
+
+    \Umathcode <codepoint> = <type> 1 <codepoint>
 
 File: `load-unicode-xetex-classes.tex`
 ======================================
@@ -116,6 +133,15 @@ Changes
   - Drop use of Unicode math data at present: undesired effect on
     output if Unicode fonts are not loaded (classical OT1
     situation)
+- v1.2a (2016-02-05)
+  - Correct split between base and supplementary plane when
+    setting math codes
+- v1.3 (2016-02-06)
+  - New `load-unicode-math-classes.tex` loader
+  - Set all letters as math type 7 (var) in
+    `load-unicode-data.tex`
+- v1.3a (2016-02-06)
+  - Add defintion of `\loop` to `load-unicode-math-classes.tex`
 
 License and permission
 ======================
diff --git a/Master/texmf-dist/tex/generic/unicode-data/load-unicode-data.tex b/Master/texmf-dist/tex/generic/unicode-data/load-unicode-data.tex
index b9011421e23..e3629d0e181 100644
--- a/Master/texmf-dist/tex/generic/unicode-data/load-unicode-data.tex
+++ b/Master/texmf-dist/tex/generic/unicode-data/load-unicode-data.tex
@@ -33,8 +33,7 @@
 %   or lower case mapping is given
 % - \sfcode 0 (ignored) for code points of Unicode classes "Pe" (closing
 %   punctuation marks) and "Pf" (final quotation marks)
-% - \Umathcode for all letters in the base plane class 7 (var) and for
-%   the supplementary plane class 0 (regular)
+% - \Umathcode for all letters as math type 7 (var)
 %
 % =============================================================================
 %
@@ -58,7 +57,7 @@
   \catcode`\^=7 %
   \newlinechar=`\^^J %
   \message{^^J}%
-  \message{load-unicode-data.tex v1.2 (2016-02-02)^^J}%
+  \message{load-unicode-data.tex v1.3a (2016-02-06)^^J}%
   \message{Reading Unicode data^^J}%
 % The first stage of parsing is dealing with the fact that there are lots of
 % data items separated by |;|. Of those, only a few are needed so they are
@@ -112,13 +111,8 @@
         \parseunicodedataVI\lccode{#1}{#4}%
         \global\sfcode"#1=999 %
       \fi
-% Letters in base plane are class~$7$, those in the supplementary plane are
-% class~$0$.
-      \ifnum"#1<10000 %
-        \global\Umathcode"#1="7"01"#1 %
-      \else
-        \global\Umathcode"#1="0"01"#1 %
-      \fi
+% All letters in math mode should be variables.
+      \global\Umathcode"#1="7"01"#1 %
     \else
       \def\temp{#2}%
       \ifnum 0\ifx\temp\Pe 1\fi\ifx\temp\Pf 1\fi>0 %
@@ -197,48 +191,4 @@
     \fi
   \repeat
   \closein0 %
-% All of the other data files have some common aspects to their format. We
-% therefore begin with some shared code. First a check for a comment line:
-% these can be skipped. (Currently only |MathClass.txt| is used by this code
-% is also usable with the other Unicode data files.)
-  \edef\hash{\string#}%
-  \def\parseunicodedataI#1\relax{%
-    \unless\if\hash\firsttoken#1?\relax
-      \parseunicodedataII#1\relax
-    \fi
-  }%
-% The first entry in all of the files is a code point or range of code points:
-% set up to find a range. The definition of |\parseunicodedataIV| will depend on
-% the data being processed and may need to split the remainder of the line
-% further.
-  \def\parseunicodedataII#1;#2\relax{%
-    \parseunicodedataIII#1....\relax{#2}%
-  }%
-  \def\parseunicodedataIII#1..#2..#3\relax#4{%
-    \ifx\relax#2\relax
-      \parseunicodedataIV{#1}{#1}#4\relax
-    \else
-      \parseunicodedataIV{#1}{#2}#4\relax
-    \fi
-  }%
-% A shared routine for reading the data files: only one part of the parser
-% has to be altered.
-  \def\readandparse#1{%
-    \catcode`\#=12 %
-    \openin0=#1.txt %
-% Read two lines from the source file to extract the version information: it is
-% always the first two lines of the file.
-    \read0 to \unicodedataline
-    \message{\unicodedataline ^^J}%
-    \read0 to \unicodedataline
-    \message{\unicodedataline ^^J}%
-    \loop\unless\ifeof0 %
-      \read0 to \unicodedataline
-      \unless\ifx\unicodedataline\storedpar
-        \expandafter\parseunicodedataI\unicodedataline\relax
-      \fi
-    \repeat
-    \catcode`\#=6 %
-    \closein0 %
-  }%
 \endgroup
diff --git a/Master/texmf-dist/tex/generic/unicode-data/load-unicode-math-classes.tex b/Master/texmf-dist/tex/generic/unicode-data/load-unicode-math-classes.tex
new file mode 100644
index 00000000000..c9655082a3c
--- /dev/null
+++ b/Master/texmf-dist/tex/generic/unicode-data/load-unicode-math-classes.tex
@@ -0,0 +1,121 @@
+% File load-unicode-math-classes.tex
+%
+% Copyright 2015,2016 The LaTeX3 Project
+%
+% It may be distributed and/or modified under the conditions of
+% the LaTeX Project Public License (LPPL), either version 1.3c of
+% this license or (at your option) any later version. The latest
+% version of this license is in the file
+% http://www.latex-project.org/lppl.txt.
+%
+% Issues with this file should be reported at
+% https://github.com/latex3/unicode-data
+%
+% This file parses MathClass.txt, provided by the Unicode Consortium, and sets
+% up the following mapping between Unicode classes and TeX math types
+% - "L" (large)       \mathop
+% - "B" (binary)      \mathbin
+% - "V" (vary)        \mathbin
+% - "R" (relation)    \mathrel
+% - "O" (opening)     \mathopen
+% - "C" (closing)     \mathclose
+% - "P" (punctuation) \mathpunct
+% - "A" (alphabetic)  \mathalpha
+% 
+% For each code point processed, the result is of the form
+%
+%     \Umathcode <codepoint> = <type> 1 <codepoint>
+%
+% =============================================================================
+%
+% The data can only be loaded by Unicode engines. Currently this is limited to
+% XeTeX and LuaTeX, both of which define \Umathcode.
+\ifx\Umathcode\undefined
+  \expandafter\endinput
+\fi
+% Just in case, check for the e-TeX extensions.
+\ifx\eTeXversion\undefined
+  \expandafter\endinput
+\fi
+% This file can be loaded in IniTeX mode so the category codes of |{|, |}| and
+% |#| may not be correct. Everything is done in a group so that only the
+% settings we want to propagate are made available generally.
+\begingroup
+  \catcode`\{=1 %
+  \catcode`\}=2 %
+  \catcode`\#=6 %
+% Write some basic information to the log.
+  \catcode`\^=7 %
+  \newlinechar=`\^^J %
+  \message{^^J}%
+  \message{load-unicode-math-classes.tex v1.3a (2016-02-06)^^J}%
+  \message{Reading math class data^^J}%
+% The parser for data lines starts by skipping any comments (which start with
+% a |#| and which will be category code~$12$).
+  \edef\hash{\string#}%
+  \def\firsttoken#1#2\relax{#1}%
+  \def\parseunicodedataI#1\relax{%
+    \unless\if\hash\firsttoken#1?\relax
+      \parseunicodedataII#1\relax
+    \fi
+  }%
+% The first entry in each data line is a code point or range of code points:
+% set up to find a range if present.
+  \def\parseunicodedataII#1;#2\relax{%
+    \parseunicodedataIII#1....\relax{#2}%
+  }%
+  \def\parseunicodedataIII#1..#2..#3\relax#4{%
+    \ifx\relax#2\relax
+      \parseunicodedataIV{#1}{#1}#4\relax
+    \else
+      \parseunicodedataIV{#1}{#2}#4\relax
+    \fi
+  }%
+% Examine the Unicode class and if known set up the math code appropriately.
+  \chardef\mathclassL=1 %
+  \chardef\mathclassB=2 %
+  \chardef\mathclassV=2 %
+  \chardef\mathclassR=3 %
+  \chardef\mathclassO=4 %
+  \chardef\mathclassC=5 %
+  \chardef\mathclassP=6 %
+  \chardef\mathclassA=7 %
+  \def\parseunicodedataIV#1#2#3#4\relax{%
+    \begingroup
+      \count0="#1 %
+      \loop
+        \unless\ifnum\count0>"#2 %
+          \ifcsname mathclass#3\endcsname
+            \global\Umathcode\count0=\csname mathclass#3\endcsname 1 \count0 %
+          \fi
+        \advance\count0 by 1 %
+      \repeat
+    \endgroup
+  }%
+% From plain: may not be defined (yet).
+  \def\loop#1\repeat{\def\body{#1}\iterate}%
+  \def\iterate{%
+    \body
+      \let\next\iterate
+    \else
+      \let\next\relax
+    \fi
+    \next
+  }%
+  \let\repeat\fi
+% Now we can load the data.
+  \catcode`\#=12 %
+  \def\storedpar{\par}%
+  \openin0=MathClass.txt %
+  \read0 to \unicodedataline
+  \message{\unicodedataline ^^J}%
+  \read0 to \unicodedataline
+  \message{\unicodedataline ^^J}%
+  \loop\unless\ifeof0 %
+    \read0 to \unicodedataline
+    \unless\ifx\unicodedataline\storedpar
+      \expandafter\parseunicodedataI\unicodedataline\relax
+    \fi
+  \repeat
+  \closein0 %
+\endgroup
diff --git a/Master/texmf-dist/tex/generic/unicode-data/load-unicode-xetex-classes.tex b/Master/texmf-dist/tex/generic/unicode-data/load-unicode-xetex-classes.tex
index 3d8370fec02..c5b4d72f178 100644
--- a/Master/texmf-dist/tex/generic/unicode-data/load-unicode-xetex-classes.tex
+++ b/Master/texmf-dist/tex/generic/unicode-data/load-unicode-xetex-classes.tex
@@ -62,7 +62,7 @@
   \catcode`\^=7 %
   \newlinechar=`\^^J %
   \message{^^J}%
-  \message{load-unicode-xetex-classes.tex v1.2 (2016-02-02)^^J}%
+  \message{load-unicode-xetex-classes.tex v1.3a (2016-02-06)^^J}%
   \message{Reading Unicode east Asian character class data^^J}%
 % A string version of |#| will be needed to look for comment lines in the
 % source. Once that is done proper parsing can begin.