diff options
Diffstat (limited to 'Master/texmf-dist/source/latex/l3kernel/ltunicode.dtx')
-rw-r--r-- | Master/texmf-dist/source/latex/l3kernel/ltunicode.dtx | 669 |
1 files changed, 669 insertions, 0 deletions
diff --git a/Master/texmf-dist/source/latex/l3kernel/ltunicode.dtx b/Master/texmf-dist/source/latex/l3kernel/ltunicode.dtx new file mode 100644 index 00000000000..7dd0591d4b6 --- /dev/null +++ b/Master/texmf-dist/source/latex/l3kernel/ltunicode.dtx @@ -0,0 +1,669 @@ +% \iffalse meta-comment +% +% Copyright 2014-2015 +% The LaTeX3 Project and any individual authors listed elsewhere +% in this file. +% +% This file is part of the LaTeX base system. +% ------------------------------------------- +% +% It may be distributed and/or modified under the +% conditions of the LaTeX Project Public License, either version 1.3c +% of this license or (at your option) any later version. +% The latest version of this license is in +% http://www.latex-project.org/lppl.txt +% and version 1.3c or later is part of all distributions of LaTeX +% version 2005/12/01 or later. +% +% This file has the LPPL maintenance status "maintained". +% +% The list of all files belonging to the LaTeX base distribution is +% given in the file `manifest.txt'. See also `legal.txt' for additional +% information. +% +% The list of derived (unpacked) files belonging to the distribution +% and covered by LPPL is defined by the unpacking scripts (with +% extension .ins) which are part of the distribution. +% +% ----------------------------------------------------------------------------- +% +% The same approach as used in \pkg{DocStrip}: if \cs{documentclass} +% is undefined then skip the driver, allowing the file to be used directly. +% This works as the \cs{fi} is only seen if \LaTeX{} is not in use. The odd +% \cs{jobname} business allows the extraction to work with \LaTeX{} provided +% an appropriate \texttt{.ins} file is set up. +%<*gobble> +\ifx\jobname\relax + \let\documentclass\undefined +\fi +\begingroup\expandafter\expandafter\expandafter\endgroup +\expandafter\ifx\csname documentclass\endcsname\relax +\else + \csname fi\endcsname +%</gobble> +% +%<*driver> +\ProvidesFile{ltunicode.dtx} + [2015/01/01 v1.0 LaTeX Kernel (Unicode data)] +\documentclass{ltxdoc} +\begin{document} +\DocInput{\jobname.dtx} +\end{document} +%<*gobble> +\fi +%</gobble> +%</driver> +% \fi +% +% \GetFileInfo{ltunicode.dtx} +% \title{The \texttt{ltunicode.dtx} file\thanks +% {This file has version number \fileversion, dated \filedate.}\\ +% for use with \LaTeXe} +% \author{The \LaTeX3 Project} +% +% \maketitle +% +% This script extracts data from the Unicode Consortium files +% \texttt{UnicodeData.txt}, \texttt{EastAsianWidth.txt} and +% \texttt{LineBreak.txt} to be used for setting up \LaTeXe{} with sane +% default settings when using the Xe\TeX{} and Lua\TeX{} engines. Details +% of the process are included in the code comments. +% +% To create the extracted file, run this file in a location containing +% the three input data files using a plain \TeX{} system with the e-\TeX{} +% extensions enabled (\texttt{pdftex}, \texttt{xetex} or \texttt{luatex} +% in any modern \TeX{} distribution). +% +% \StopEventually{} +% +% \begin{macrocode} +%<*script> +% \end{macrocode} +% +% \section{General set up} +% +% The script is designed to work with plain \TeX{} and so |@| is made into +% a `letter' using the primitive approach. +% \begin{macrocode} +\catcode`\@=11 % +% \end{macrocode} +% +% \begin{macro}{\gobble} +% \begin{macro}{\firsttoken} +% Standard utilities. +% \begin{macrocode} +\long\def\gobble#1{} +\long\def\firsttoken#1#2\relax{#1} +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}{\storedpar} +% A simple piece of test set up: the final line of the read file will be +% tokenized by \TeX{} as \cs{par} which can be tested by \cs{ifx} provided +% we have an equivalent available. +% \begin{macrocode} +\def\storedpar{\par} +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\return} +% A stored |^^M| for string comparisons. +% \begin{macrocode} +\begingroup + \catcode`\^^M=12 % + \gdef\return{^^M}% +\endgroup% +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\sourceforhex} +% \begin{macro}{\sethex} +% \begin{macro}{\dohex} +% \begin{macro}{\hexdigit} +% Some parts of the code here will need to be able to convert integers +% to their hexadecimal equivalent. That is easiest to do for the requirements +% here using a modified version of some code from Appendix~D of \emph{The +% \TeX{}book}. +% \begin{macrocode} +\newcount\sourceforhex +\def\sethex#1#2{% + \def#1{}% + \sourceforhex=#2\relax + \ifnum\sourceforhex=0 % + \def#1{0}% + \else + \dohex#1% + \fi +} +\def\dohex#1{% + \begingroup + \count0=\sourceforhex + \divide\sourceforhex by 16 % + \ifnum\sourceforhex>0 % + \dohex#1% + \fi + \count2=\sourceforhex + \multiply\count2 by -16 % + \advance\count0 by\count2 + \hexdigit#1% + \expandafter\endgroup + \expandafter\def\expandafter#1\expandafter{#1}% +} +\def\hexdigit#1{% + \ifnum\count0<10 % + \edef#1{#1\number\count0}% + \else + \advance\count0 by -10 % + \edef#1{#1\ifcase\count0 A\or B\or C\or D\or E\or F\fi}% + \fi +} +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \begin{macro}{\unicoderead, \unicodewrite} +% Set up the streams for data. +% \begin{macrocode} +\newread\unicoderead +\newwrite\unicodewrite +% \end{macrocode} +% \end{macro} +% +% \section{Verbatim copying} +% +% \begin{macro}{\verbatimcopy} +% \begin{macro}{\endverbatimcopy} +% \begin{macro}{\verbatimcopy@auxii} +% \begin{macro}{\verbatimcopy@auxii} +% \begin{macro}{\verbatim@endmarker} +% Set up to read some material verbatim and write it to the output stream. +% There needs to be a dedicated `clean up first line' macro, but other than +% that life is simple enough. +% \begin{macrocode} +\begingroup + \catcode`\^^M=12 % + \gdef\verbatimcopy{% + \begingroup% + \catcode`\^^M=12 % + \catcode`\\=12 % + \catcode`\{=12 % + \catcode`\}=12 % + \catcode`\#=12 % + \catcode`\%=12 % + \catcode`\ =12 % + \endlinechar=`\^^M % + \verbatimcopy@auxi + }% + \gdef\verbatimcopy@auxi#1^^M{% + \expandafter\verbatimcopy@auxii\gobble#1^^M% + }% + \gdef\verbatimcopy@auxii#1^^M{% + \def\temp{#1}% + \ifx\temp\verbatim@endmarker% + \expandafter\endgroup% + \else% + \ifx\temp\empty\else% + \immediate\write\unicodewrite{#1}% + \fi% + \expandafter\verbatimcopy@auxii% + \fi% + }% +\endgroup% +\edef\verbatim@endmarker{\expandafter\gobble\string\\} +\edef\verbatim@endmarker{\verbatim@endmarker endverbatimcopy} +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \section{File header section} +% +% With the mechanisms set up, open the data file for writing. +% \begin{macrocode} +\immediate\openout\unicodewrite=ltunicode.ltx % +% \end{macrocode} +% There are various lines that now need to go at the start of the file. +% First, there is some header information. +% \begin{macrocode} +\verbatimcopy +%% This is the file `ltunicode.ltx', +%% generated using the script ltunicode.dtx. +%% +%% The data here are derived from the files +%% - UnicodeData.txt +%% which are maintained by the Unicode Consortium. +%% +%% Copyright 2014 +%% The LaTeX3 Project and any individual authors listed elsewhere +%% in this file. +%% +%% This file is part of the LaTeX base system. +%% ------------------------------------------- +%% +%% It may be distributed and/or modified under the +%% conditions of the LaTeX Project Public License, either version 1.3c +%% of this license or (at your option) any later version. +%% The latest version of this license is in +%% http://www.latex-project.org/lppl.txt +%% and version 1.3c or later is part of all distributions of LaTeX +%% version 2005/12/01 or later. +%% +%% This file has the LPPL maintenance status "maintained". +%% +%% The list of all files belonging to the LaTeX base distribution is +%% given in the file `manifest.txt'. See also `legal.txt' for additional +%% information. +\endverbatimcopy +% \end{macrocode} +% Automatically include the current date. +% \begin{macrocode} +\immediate\write\unicodewrite{% + \expandafter\gobble\string\%\expandafter\gobble\string\% + Generated on \the\year-\the\month-\the\day. +} +\immediate\write\unicodewrite{% + \expandafter\gobble\string\%\expandafter\gobble\string\% +} +% \end{macrocode} +% +% \section{Unicode character data} +% +% \begin{macro}{\parseunicodedata} +% \begin{macro}{\parseunicodedata@auxi} +% \begin{macro}{\parseunicodedata@auxii} +% \begin{macro}{\parseunicodedata@auxiii} +% The first step of parsing a line of data is to check that it's not come +% from a blank in the source, which will have been tokenized as \cs{par}. +% Assuming that is not the case, there are lots of data items separated by +% |;|. Of those, only a few are needed so they are picked out and everything +% else is dropped. +% \begin{macrocode} +\def\parseunicodedata#1{% + \ifx#1\storedpar + \else + \expandafter\parseunicodedata@auxi#1\relax + \fi +} +\def\parseunicodedata@auxi#1;#2;#3;#4;#5;#6;#7;#8;#9;{% + \parseunicodedata@auxii#1;#3; +} +\def\parseunicodedata@auxii#1;#2;#3;#4;#5;#6;#7;#8\relax{% + \parseunicodedata@auxiii{#1}{#2}{#6}{#7}% +} +% \end{macrocode} +% At this stage we have only four pieces of data +% \begin{enumerate} +% \item The code value +% \item The general class +% \item The uppercase mapping +% \item The lowercase mapping +% \end{enumerate} +% where both one or both of the last two may be empty. Everything here could +% be done in a single conditional within a \cs{write}, but that would be +% tricky to follow. Instead, a series of defined auxiliaries are used to +% show the flow. Notice that combining marks are treated as letters here +% (the second `letter' test). +% \begin{macrocode} +\def\parseunicodedata@auxiii#1#2#3#4{% + \if L\firsttoken#2?\relax + \expandafter\unicodeletter + \else + \if M\firsttoken#2?\relax + \expandafter\expandafter\expandafter\unicodeletter + \else + \expandafter\expandafter\expandafter\unicodenonletter + \fi + \fi + {#1}{#3}{#4}% +} +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \begin{macro}{\unicodeletter, \unicodenonletter} +% \begin{macro}{\writeunicodedata} +% For `letters', we always want to write the data to file, and the only +% question here is if the character has case mappings or these point back +% to the character itself. +% \begin{macrocode} +\def\unicodeletter#1#2#3{% + \writeunicodedata\L{#1}{#2}{#3}% +} +% \end{macrocode} +% Cased non-letters can also exist: they can be detected as they have at +% least one case mapping. Write these in much the same way as letters. +% \begin{macrocode} +\def\unicodenonletter#1#2#3{% + \ifx\relax#2#3\relax + \else + \writeunicodedata\C{#1}{#2}{#3}% + \fi +} +% \end{macrocode} +% Actually write the data. In all cases both upper- and lower-case mappings +% are given, so there is a need to test that both were actually available and +% if not set up to do nothing. +% \begin{macrocode} +\def\writeunicodedata#1#2#3#4{% + \immediate\write\unicodewrite{% + \space\space + \string#1\space + #2 % + \ifx\relax#3\relax + #2 % + \else + #3 % + \fi + \ifx\relax#4\relax + #2 % + \else + #4 % + \fi + \expandafter\gobble\string\% + }% +} +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% There is now a lead-in section which creates the macros which take the +% processed data and do the code assignments. Everything is done within a +% group so that there is no need to worry about names. +% \begin{macrocode} +\verbatimcopy +\begingroup +\endverbatimcopy +% \end{macrocode} +% Cased non-letters simply need to have the case mappings set. +% For letters, there are a few things to sort out. First, the case mappings are +% defined as for non-letters. Category code is then set to $11$ before a check +% to see if this is an upper case letter. If it is then the \cs{sfcode} is set +% to $999$. Finally there is a need to deal with Unicode math codes, where base +% plane letters are class $7$ but supplementary plane letters are class~$1$. +% Older versions of Xe\TeX{} used a different name here: easy to pick up as +% we know that this primitive must be defined in some way. There is also an issue +% with the supplementary plane and older Xe\TeX{} versions, which is dealt with +% using a check at run time. +% \begin{macrocode} +\verbatimcopy + \def\C#1 #2 #3 {% + \XeTeXcheck{#1}% + \global\uccode"#1="#2 % + \global\lccode"#1="#3 % + } + \def\L#1 #2 #3 {% + \C #1 #2 #3 % + \catcode"#1=11 % + \ifnum"#1="#3 % + \else + \global\sfcode"#1=999 % + \fi + \ifnum"#1<"10000 % + \global\Umathcode"#1="7"01"#1 % + \else + \global\Umathcode"#1="0"01"#1 % + \fi + } + \ifx\Umathcode\undefined + \let\Umathcode\XeTeXmathcode + \fi + \def\XeTeXcheck#1{} + \ifx\XeTeXversion\undefined + \else + \def\XeTeXcheck.#1.#2-#3\relax{#1} + \ifnum\expandafter\XeTeXcheck\XeTeXrevision.-\relax>996 % + \def\XeTeXcheck#1{} + \else + \def\XeTeXcheck#1{% + \ifnum"#1>"FFFF % + \long\def\XeTeXcheck##1\endgroup{\endgroup} + \expandafter\XeTeXcheck + \fi + } + \fi + \fi +\endverbatimcopy +% \end{macrocode} +% Read the data and write the resulting code assignments to the file. +% \begin{macrocode} +\openin\unicoderead=UnicodeData.txt % +\ifeof\unicoderead + \errmessage{Data file missing: UnicodeData.txt}% +\fi +\loop\unless\ifeof\unicoderead + \read\unicoderead to \unicodedataline + \parseunicodedata\unicodedataline +\repeat +% \end{macrocode} +% End the group for setting character codes and assign a couple of special +% cases. +% \begin{macrocode} +\verbatimcopy +\endgroup +\global\sfcode"2019=0 % +\global\sfcode"201D=0 % +\endverbatimcopy +% \end{macrocode} +% Lua\TeX{} and older versions of Xe\TeX{} stop here: character classes are a +% Xe\TeX{}-only concept. +% \begin{macrocode} +\verbatimcopy +\ifx\XeTeXcharclass\@undefined + \expandafter\endinput +\fi +\endverbatimcopy +% \end{macrocode} +% +% \section{Xe\TeX{} Character classes} +% +% The Xe\TeX{} engine includes the concept of character classes, which allow +% insertion of tokens into the input stream at defined boundaries. Setting +% up this data requires a two-part process as the information is split over +% two input files. +% +% \begin{macro}{\parseunicodedata} +% \begin{macro}{\parseunicodedata@auxi} +% \begin{macro}{\parseunicodedata@auxii} +% The parsing system is redefined to parse a detokenized input line which +% may be a comment starting with |#|. Assuming that is not the case, the +% data line with start with a code point potentially forming part of a range. +% The range is extracted and the width stored for each code point. +% \begin{macrocode} +\def\parseunicodedata#1{% + \ifx#1\return + \else + \wlog{#1}% + \if\expandafter\gobble\string\#\expandafter\firsttoken#1?\relax + \else + \expandafter\parseunicodedata@auxi#1\relax + \fi + \fi +} +\def\parseunicodedata@auxi#1;#2 #3\relax{% + \parseunicodedata@auxii#1....\relax{#2}% +} +\def\parseunicodedata@auxii#1..#2..#3\relax#4{% + \expandafter\gdef\csname EAW@#1\endcsname{#4}% + \ifx\relax#2\relax + \else + \count@="#1 % + \begingroup + \loop + \ifnum\count@<"#2 % + \advance\count@\@ne + \sethex\temp{\count@}% + \expandafter\gdef\csname EAW@\temp\endcsname{#4}% + \repeat + \endgroup + \fi +} +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% +% With the right parser in place, read the data file. +% \begin{macrocode} +\openin\unicoderead=EastAsianWidth.txt % +\ifeof\unicoderead + \errmessage{Data file missing: EastAsianWidth.txt}% +\fi +\loop\unless\ifeof\unicoderead + \readline\unicoderead to \unicodedataline + \parseunicodedata\unicodedataline +\repeat +% \end{macrocode} +% +% \begin{macro}{\parseunicodedata@auxii} +% \begin{macro}{\parseunicodedata@auxiii} +% \begin{macro}{\parseunicodedata@auxiv} +% \begin{macro}{\ID} +% \begin{macro}{\OP} +% \begin{macro}{\CL} +% \begin{macro}{\EX} +% \begin{macro}{\IS} +% \begin{macro}{\NS} +% \begin{macro}{\CM} +% The final file to read, |LineBreaking.txt|, uses the same format as +% |EastAsianWidth.txt|. As such, only the final parts of the parser have to be +% redefined. +% \begin{macrocode} +\def\parseunicodedata@auxii#1..#2..#3\relax#4{% + \parseunicodedata@auxiii{#1}{#4}% + \ifx\relax#2\relax + \else + \count@="#1 % + \begingroup + \loop + \ifnum\count@<"#2 % + \advance\count@\@ne + \sethex\temp{\count@}% + \expandafter\parseunicodedata@auxiii\expandafter{\temp}{#4}% + \repeat + \endgroup + \fi +} +% \end{macrocode} +% Adding data to the processed file depends on two factors: the +% classification in the line-breaking file and (possibly) the width data +% too. Any characters of class \texttt{ID} (ideograph) are stored: they +% always need special treatment. For characters of classes \texttt{OP} +% (opener), \texttt{CL} (closer), \texttt{EX} (exclamation), \texttt{IS} +% (infix sep) and \texttt{NS} (non-starter) the data is stored if the +% character is full, half or wide width. The same is true for +% \texttt{CM} (combining marks) characters, which need to be transparent +% to the mechanism. +% \begin{macrocode} +\def\parseunicodedata@auxiii#1#2{% + \ifcsname #2\endcsname + \ifnum\csname #2\endcsname=1 % + \parseunicodedata@auxiv{#1}{#2}% + \else + \ifnum 0% + \if F\csname EAW@#1\endcsname 1\fi + \if H\csname EAW@#1\endcsname 1\fi + \if W\csname EAW@#1\endcsname 1\fi + >0 % + \parseunicodedata@auxiv{#1}{#2}% + \fi + \fi + \fi +} +\def\parseunicodedata@auxiv#1#2{% + \immediate\write\unicodewrite{% + \space\space + \expandafter\string\csname #2\endcsname + \space + #1 % + \expandafter\gobble\string\% + }% +} +\def\ID{1} +\def\OP{2} +\def\CL{3} +\let\EX\CL +\let\IS\CL +\let\NS\CL +\def\CM{256} +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% Before actually reading the line breaking data file, the appropriate +% temporary code is added to the output. As described above, only a limited +% number of classes need to be covered: they are hard-coded as classes +% $1$, $2$ and $3$ following the convention adopted by plain Xe\TeX{}. +% \begin{macrocode} +\verbatimcopy +\begingroup + \def\ID#1 {\global\XeTeXcharclass"#1=1 \global\catcode"#1=11 } + \def\OP#1 {\global\XeTeXcharclass"#1=2 } + \def\CL#1 {\global\XeTeXcharclass"#1=3 } + \def\EX#1 {\global\XeTeXcharclass"#1=3 } + \def\IS#1 {\global\XeTeXcharclass"#1=3 } + \def\NS#1 {\global\XeTeXcharclass"#1=3 } + \def\CM#1 {\global\XeTeXcharclass"#1=256 } +\endverbatimcopy +% \end{macrocode} +% +% Read the line breaking data and save to the output. +% \begin{macrocode} +\openin\unicoderead=LineBreak.txt % +\ifeof\unicoderead + \errmessage{Data file missing: LineBreak.txt}% +\fi +\loop\unless\ifeof\unicoderead + \readline\unicoderead to \unicodedataline + \parseunicodedata\unicodedataline +\repeat +% \end{macrocode} +% +% Set up material to be inserted between character classes. Other than +% using \cs{hspace} here in place of \cs{hskip} this code is identical to +% that provided by plain Xe\TeX{}. +% \begin{macrocode} +\verbatimcopy +\endgroup +\gdef\xtxHanGlue{\hspace{0pt plus 0.1em}} +\gdef\xtxHanSpace{\hspace{0.2em plus 0.2em minus 0.1em}} +\global\XeTeXinterchartoks 0 1 = {\xtxHanSpace} +\global\XeTeXinterchartoks 0 2 = {\xtxHanSpace} +\global\XeTeXinterchartoks 0 3 = {\nobreak\xtxHanSpace} +\global\XeTeXinterchartoks 1 0 = {\xtxHanSpace} +\global\XeTeXinterchartoks 2 0 = {\nobreak\xtxHanSpace} +\global\XeTeXinterchartoks 3 0 = {\xtxHanSpace} +\global\XeTeXinterchartoks 1 1 = {\xtxHanGlue} +\global\XeTeXinterchartoks 1 2 = {\xtxHanGlue} +\global\XeTeXinterchartoks 1 3 = {\nobreak\xtxHanGlue} +\global\XeTeXinterchartoks 2 1 = {\nobreak\xtxHanGlue} +\global\XeTeXinterchartoks 2 2 = {\nobreak\xtxHanGlue} +\global\XeTeXinterchartoks 2 3 = {\xtxHanGlue} +\global\XeTeXinterchartoks 3 1 = {\xtxHanGlue} +\global\XeTeXinterchartoks 3 2 = {\xtxHanGlue} +\global\XeTeXinterchartoks 3 3 = {\nobreak\xtxHanGlue} +\endverbatimcopy +% \end{macrocode} +% +% Done: end the script. +% \begin{macrocode} +\bye +% \end{macrocode} +% +% \begin{macrocode} +%</script> +% \end{macrocode}
\ No newline at end of file |