diff options
Diffstat (limited to 'Master/texmf-dist/source/latex/base/utf8ienc.dtx')
-rw-r--r-- | Master/texmf-dist/source/latex/base/utf8ienc.dtx | 210 |
1 files changed, 174 insertions, 36 deletions
diff --git a/Master/texmf-dist/source/latex/base/utf8ienc.dtx b/Master/texmf-dist/source/latex/base/utf8ienc.dtx index 5478a029008..b67c9361eed 100644 --- a/Master/texmf-dist/source/latex/base/utf8ienc.dtx +++ b/Master/texmf-dist/source/latex/base/utf8ienc.dtx @@ -1,6 +1,6 @@ % \iffalse meta-comment % -% Copyright 1993-2017 +% Copyright 1993-2018 % The LaTeX3 Project and any individual authors listed elsewhere % in this file. % @@ -11,7 +11,7 @@ % conditions of the LaTeX Project Public License, either version 1.3c % of this license or (at your option) any later version. % The latest version of this license is in -% http://www.latex-project.org/lppl.txt +% https://www.latex-project.org/lppl.txt % and version 1.3c or later is part of all distributions of LaTeX % version 2005/12/01 or later. % @@ -30,14 +30,14 @@ % \iffalse %<*driver> \documentclass{ltxdoc} +\usepackage[latin1,utf8]{inputenc}% force utf8 to be re-loaded, to get version info \GetFileInfo{utf8.def} \title{Providing some UTF-8 support via \texttt{inputenc}} \date{\fileversion\space\filedate{} printed \today} \author{% - Frank Mittelbach \and Chris Rowley\thanks{Borrowing heavily from - code by David Carlisle and tables by Sebastian Rahtz; some table + David Carlisle \and Frank Mittelbach \and + Chris Rowley\thanks{Borrowing heavily from tables by Sebastian Rahtz; some table and code cleanup by Javier Bezos}} -\usepackage[utf8]{inputenc} \begin{document} \MaintainedByLaTeXTeam{latex} \maketitle @@ -52,7 +52,6 @@ % % \section{Introduction} % -% [The whole section is rather unfinished \ldots\ just like the code, sorry!] % % \subsection{Background and general stuff} % @@ -218,7 +217,7 @@ %<+ts1> \ProvidesFile{ts1enc.dfu} %<+x2> \ProvidesFile{x2enc.dfu} %<+all> \ProvidesFile{utf8enc.dfu} - [2017/01/28 v1.1t UTF-8 support for inputenc] + [2018/04/05 v1.2c UTF-8 support for inputenc] % \end{macrocode} % % \begin{macrocode} @@ -241,6 +240,7 @@ % \begin{macro}{\UTFviii@two@octets} % \begin{macro}{\UTFviii@three@octets} % \begin{macro}{\UTFviii@four@octets} +% \changes{v1.2a}{2018/03/24}{Macros made `\cs{long} for improved error messages}% % A UTF-8 char (that is not actually a 7-bit char, i.e.~a single % octet) is parsed as follows: each starting octet is an active % \TeX{} character token; each of these is defined below to be a @@ -263,19 +263,19 @@ % defined we simply execute the thing (which should then expand to % an encoding specific internal \LaTeX{} form). % \begin{macrocode} -\def\UTFviii@two@octets#1#2{\expandafter +\long\def\UTFviii@two@octets#1#2{\expandafter \UTFviii@defined\csname u8:#1\string#2\endcsname} % \end{macrocode} % \end{macro} % % \begin{macrocode} -\def\UTFviii@three@octets#1#2#3{\expandafter +\long\def\UTFviii@three@octets#1#2#3{\expandafter \UTFviii@defined\csname u8:#1\string#2\string#3\endcsname} % \end{macrocode} % \end{macro} % % \begin{macrocode} -\def\UTFviii@four@octets#1#2#3#4{\expandafter +\long\def\UTFviii@four@octets#1#2#3#4{\expandafter \UTFviii@defined\csname u8:#1\string#2\string#3\string#4\endcsname} % \end{macrocode} % \end{macro} @@ -289,18 +289,30 @@ \def\UTFviii@defined#1{% \ifx#1\relax % \end{macrocode} +% Test if the sequence is invalid UTF-8 or valid UTF-8 but without +% a \LaTeX\ definition. +% \begin{macrocode} + \if\relax\expandafter\UTFviii@checkseq\string#1\relax\relax +% \end{macrocode} % The endline character has a special definition within the % inputenc package (it is gobbling spaces). For this reason we % can't produce multiline strings without some precaution. % \changes{v1.1b}{2004/02/09}{No newlines allowed in error messages} % \changes{v1.1g}{2005/09/27}{Real spaces do not show up so use \cs{space}} % \changes{v1.1o}{2015/08/28}{Show Unicode number of character in hex} +% \changes{v1.2a}{2018/03/24}{Error message improved for non-UTF-8 sequences}% +% \begin{macrocode} + \UTFviii@undefined@err{#1}% +% \end{macrocode} +% +% \begin{macrocode} + \else + \PackageError{inputenc}{Invalid UTF-8 byte sequence}% + \UTFviii@invalid@help + \fi +% \end{macrocode} +% % \begin{macrocode} - \PackageError{inputenc}{Unicode\space char\space\expandafter - \UTFviii@splitcsname\string#1\relax - \MessageBreak - not\space set\space up\space - for\space use\space with\space LaTeX}\@eha \else\expandafter #1% \fi @@ -308,10 +320,76 @@ % \end{macrocode} % \end{macro} % +% \begin{macro}{\UTFviii@invalid@err} +% \begin{macro}{\UTFviii@invalid@help} +% \changes{v1.2a}{2018/03/24}{Macro added}% +% \begin{macrocode} +\def\UTFviii@invalid@err#1{% + \PackageError{inputenc}{Invalid UTF-8 byte \number`#1}% + \UTFviii@invalid@help} +% \end{macrocode} +% +% \begin{macrocode} +\def\UTFviii@invalid@help{% + The document does not appear to be in UTF-8 encoding.\MessageBreak + Try adding \noexpand\UseRawInputEncoding as the first line of the file\MessageBreak + or specify an encoding such as \noexpand\usepackage[latin1]{inputenc}\MessageBreak + in the document preamble.\MessageBreak + Alternatively, save the file in UTF-8 using your editor or another tool} +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}{\UTFviii@undefined@err} +% \changes{v1.2a}{2018/03/24}{Macro added}% +% \begin{macrocode} +\def\UTFviii@undefined@err#1{% + \PackageError{inputenc}{Unicode character \expandafter + \UTFviii@splitcsname\string#1\relax + \MessageBreak + not set up for use with LaTeX}% + {You may provide a definition with\MessageBreak + \noexpand\DeclareUnicodeCharacter}% + } +% \end{macrocode} +% \end{macro} +% +% +% \begin{macro}{\UTFviii@checkseq} +% \begin{macro}{\UTFviii@check@continue} +% \changes{v1.2a}{2018/03/24}{Macro added}% +% Check that the csname consists of a valid UTF-8 sequence. +% \begin{macrocode} +\def\UTFviii@checkseq#1:#2#3{% + \ifnum`#2<"80 % + \ifx\relax#3\else1\fi + \else + \ifnum`#2<"C0 % + 1 % + \else + \expandafter\expandafter\expandafter\UTFviii@check@continue + \expandafter\expandafter\expandafter#3% + \fi + \fi} +% \end{macrocode} +% +% \begin{macrocode} +\def\UTFviii@check@continue#1{% + \ifx\relax#1% + \else + \ifnum`#1<"80 1\else\ifnum`#1>"BF 1\fi\fi + \expandafter\UTFviii@check@continue + \fi +} +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% % \begin{macro}{\UTFviii@loop} -% This wonderful bit of code from Dr Carlisle defines the starting -% octets to call |\UTFviii@two@octets| etc as appropriate. The starting -% octet itself is passed directly as the first argument, the others +% This bit of code derived from \texttt{xmltex} defines the active character +% correspnding to starting octets to call |\UTFviii@two@octets| etc as appropriate. +% The starting octet itself is passed directly as the first argument, the others % are picked up later en route. % % The |\UTFviii@loop| loops through the numbers starting at @@ -321,10 +399,6 @@ % All this is done in a group so that temporary catcode changes % etc.~vanish after everything is set up. % -% It may be a good idea to add code to deal with `illegal utf8 octets': -% at present these will be handled by whatever code was in use for 8-bit -% input before this code is executed. -% % \begin{macrocode} \begingroup \catcode`\~13 @@ -341,6 +415,37 @@ \fi} % \end{macrocode} % +% Handle the single byte control characters. +% \changes{v1.2a}{2018/03/24}{Loop over C0 controls added}% +% C0 controls are valid UTF-8 but defined to give the ``Character not defined error'' +% They may be defined with |\DeclareUnicodeCharacter|. +% \begin{macrocode} + \def\UTFviii@tmp{\xdef~{\noexpand\UTFviii@undefined@err{:\string~}}} +% 0 ^^@ null + \count@"1 + \@tempcnta9 +% 9 ^^I tab +% 10 ^^J nl +\UTFviii@loop + \count@11 + \@tempcnta12 +\UTFviii@loop +% 12 ^^L +% 13 ^^M + \count@14 + \@tempcnta32 +\UTFviii@loop +% \end{macrocode} +% +% +% Bytes with leading bits |10| are not valid UTF-8 starting bytes +% \begin{macrocode} + \count@"80 + \@tempcnta"C2 + \def\UTFviii@tmp{\xdef~{\noexpand\UTFviii@invalid@err\string~}} +\UTFviii@loop +% \end{macrocode} +% % Setting up 2-byte UTF-8: % \begin{macrocode} \count@"C2 @@ -348,6 +453,7 @@ \def\UTFviii@tmp{\xdef~{\noexpand\UTFviii@two@octets\string~}} \UTFviii@loop % \end{macrocode} +% % Setting up 3-byte UTF-8: % \begin{macrocode} \count@"E0 @@ -362,8 +468,21 @@ \@tempcnta"F4 \def\UTFviii@tmp{\xdef~{\noexpand\UTFviii@four@octets\string~}} \UTFviii@loop +% \end{macrocode} +% +% Bytes above F4 are not valid UTF-8 starting bytes as they would encode numbers beyond +% the Unicode range +% \begin{macrocode} + \count@"F4 + \@tempcnta"100 + \def\UTFviii@tmp{\xdef~{\noexpand\UTFviii@invalid@err\string~}} +\UTFviii@loop +% \end{macrocode} +% +% \begin{macrocode} \endgroup % \end{macrocode} +% % \end{macro} % % For this case we must disable the warning generated by @@ -487,7 +606,7 @@ % \begin{macrocode} % \begingroup % \end{macrocode} -% The original code from David supported the convention that a +% The original code from \texttt{xmltex} supported the convention that a % Unicode slot number could be given either as a decimal or as a % hexadecimal (by starting with \texttt{x}). We do not do this so % this code is also removed. This could be reactivated if one @@ -500,18 +619,28 @@ % As |\count@| already contains the right value we make % |\parse@XML@charref| work without arguments. % \changes{v1.1g}{2005/09/27}{Real spaces do not show up so use \cs{space}} +% \changes{v1.2a}{2018/03/24}{Allow control characters if active} +% In the case single byte UTF-8 sequences, only allw definition if +% the character os already active. The definition of |\UTFviii@tmp| +% looks slightly strange but is designed for the sequence of |\expandafter| +% in |\DeclareUnicodeCharacter|. +% % \begin{macrocode} \ifnum\count@<"A0\relax - \PackageError{inputenc}{Cannot\space define\space Unicode\space - char\space value\space <\space 00A0}\@eha + \ifnum\catcode\count@=13 + \uccode`\~=\count@\uppercase{\def\UTFviii@tmp{\@empty\@empty~}}% + \else + \PackageError{inputenc}% + {Cannot define non-active Unicode char value < 00A0}% + \@eha + \def\UTFviii@tmp{\UTFviii@tmp}% + \fi % \end{macrocode} -% Do not ask us to provide an explanation for the code below, it is -% borrowed straight from \texttt{xmltex} by David and we trust him -% totally (and we are too lazy to reread the Unicode book to see if -% this is the correct algorithm).\footnote{We were hoping to also -% find in his work the \TeX{} code for going the other way: from -% UTF-8 octets to Unicode slot number, but no luck! -% This has now been added as \cs{decode@UTFviii}} +% The code below is derived from \texttt{xmltex} and generates the UTF-8 byte sequence +% for the number in |\count@|. +% +% The reverse operation (just used in error messages) +% has now been added as \cs{decode@UTFviii}. % \begin{macrocode} \else\ifnum\count@<"800\relax \parse@UTFviii@a,% @@ -534,7 +663,7 @@ % \end{macro} % % \begin{macro}{\parse@UTFviii@a} -% \ldots so somebody else can document this part :-) \ldots~David?:-))))! +% \ldots so somebody else can document this part :-) % \changes{v1.1b}{2004/02/09}{Space in the wrong place \cs{count @64}} % \begin{macrocode} \gdef\parse@UTFviii@a#1{% @@ -672,10 +801,11 @@ \else \gdef\UTFviii@splitcsname#1:#2\relax{% % \end{macrocode} +% \changes{v1.2b}{2018/03/26}{add percent as \cs{endlinechar} not -1 in the format}% % Need to pre-expand the argument to ensure cleanup in case of mal-formed UTF-8. % \begin{macrocode} #2 (U+\expandafter\UTFviii@hexnumber\expandafter{% - \the\numexpr\decode@UTFviii#2\relax}) + \the\numexpr\decode@UTFviii#2\relax})% } \fi % \end{macrocode} @@ -830,6 +960,9 @@ % % \subsection{The mapping table} % +% \begingroup +% \hfuzz=12pt ^^A just because +% % Note that the first argument must be a hex-digit number greater % than \texttt{00BF} and at most \texttt{10FFFF}. % @@ -1109,7 +1242,7 @@ % and corrected by Matthias Noe (\verb=a9931078@unet.univie.ac.at=) --- thanks. % \changes{v1.1j}{2007/11/09}{Added a few new unicode decls in cyrillic (pr/3988)} % \changes{v1.1k}{2007/11/11}{Added and further unicode decls in cyrillic} -% \changes{v1.1n}{2015/06/27}{correct accent http://tex.stackexchange.com/q/252521} +% \changes{v1.1n}{2015/06/27}{correct accent https://tex.stackexchange.com/q/252521} % \begin{macrocode} %<*all,x2,t2c,t2b,t2a,ot2,lcy> \DeclareUnicodeCharacter{0400}{\@tabacckludge`\CYRE} @@ -1473,7 +1606,12 @@ %<all,t1>\DeclareUnicodeCharacter{1E20}{\@tabacckludge=G} %<all,t1>\DeclareUnicodeCharacter{1E21}{\@tabacckludge=g} % \end{macrocode} -% +% \changes{v1.2c}{2018/04/05}{Add BOM} +% \begin{macrocode} +%<all,ts1,utf8>\DeclareUnicodeCharacter{FEFF}{\ifhmode\nobreak\fi} +% \end{macrocode} +% +% \endgroup % \subsection{Notes} % % \changes{v1.1e}{2004/05/22}{Added notes on inconsistency with `8-bit files'.} |