summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/source/latex/base/utf8ienc.dtx
diff options
context:
space:
mode:
authorKarl Berry <karl@freefriends.org>2018-04-06 21:33:45 +0000
committerKarl Berry <karl@freefriends.org>2018-04-06 21:33:45 +0000
commit588be4d1b7e254f46ac3c678ef1b3f38833f6491 (patch)
treec73b58d23db18c19737ff12c67e208fd7a89c4c6 /Master/texmf-dist/source/latex/base/utf8ienc.dtx
parentc436d9d03043bac0aac83254a8f1a2a45e4aaafd (diff)
latex2e (6apr18)
git-svn-id: svn://tug.org/texlive/trunk@47330 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/source/latex/base/utf8ienc.dtx')
-rw-r--r--Master/texmf-dist/source/latex/base/utf8ienc.dtx210
1 files changed, 174 insertions, 36 deletions
diff --git a/Master/texmf-dist/source/latex/base/utf8ienc.dtx b/Master/texmf-dist/source/latex/base/utf8ienc.dtx
index 5478a029008..b67c9361eed 100644
--- a/Master/texmf-dist/source/latex/base/utf8ienc.dtx
+++ b/Master/texmf-dist/source/latex/base/utf8ienc.dtx
@@ -1,6 +1,6 @@
% \iffalse meta-comment
%
-% Copyright 1993-2017
+% Copyright 1993-2018
% The LaTeX3 Project and any individual authors listed elsewhere
% in this file.
%
@@ -11,7 +11,7 @@
% conditions of the LaTeX Project Public License, either version 1.3c
% of this license or (at your option) any later version.
% The latest version of this license is in
-% http://www.latex-project.org/lppl.txt
+% https://www.latex-project.org/lppl.txt
% and version 1.3c or later is part of all distributions of LaTeX
% version 2005/12/01 or later.
%
@@ -30,14 +30,14 @@
% \iffalse
%<*driver>
\documentclass{ltxdoc}
+\usepackage[latin1,utf8]{inputenc}% force utf8 to be re-loaded, to get version info
\GetFileInfo{utf8.def}
\title{Providing some UTF-8 support via \texttt{inputenc}}
\date{\fileversion\space\filedate{} printed \today}
\author{%
- Frank Mittelbach \and Chris Rowley\thanks{Borrowing heavily from
- code by David Carlisle and tables by Sebastian Rahtz; some table
+ David Carlisle \and Frank Mittelbach \and
+ Chris Rowley\thanks{Borrowing heavily from tables by Sebastian Rahtz; some table
and code cleanup by Javier Bezos}}
-\usepackage[utf8]{inputenc}
\begin{document}
\MaintainedByLaTeXTeam{latex}
\maketitle
@@ -52,7 +52,6 @@
%
% \section{Introduction}
%
-% [The whole section is rather unfinished \ldots\ just like the code, sorry!]
%
% \subsection{Background and general stuff}
%
@@ -218,7 +217,7 @@
%<+ts1> \ProvidesFile{ts1enc.dfu}
%<+x2> \ProvidesFile{x2enc.dfu}
%<+all> \ProvidesFile{utf8enc.dfu}
- [2017/01/28 v1.1t UTF-8 support for inputenc]
+ [2018/04/05 v1.2c UTF-8 support for inputenc]
% \end{macrocode}
%
% \begin{macrocode}
@@ -241,6 +240,7 @@
% \begin{macro}{\UTFviii@two@octets}
% \begin{macro}{\UTFviii@three@octets}
% \begin{macro}{\UTFviii@four@octets}
+% \changes{v1.2a}{2018/03/24}{Macros made `\cs{long} for improved error messages}%
% A UTF-8 char (that is not actually a 7-bit char, i.e.~a single
% octet) is parsed as follows: each starting octet is an active
% \TeX{} character token; each of these is defined below to be a
@@ -263,19 +263,19 @@
% defined we simply execute the thing (which should then expand to
% an encoding specific internal \LaTeX{} form).
% \begin{macrocode}
-\def\UTFviii@two@octets#1#2{\expandafter
+\long\def\UTFviii@two@octets#1#2{\expandafter
\UTFviii@defined\csname u8:#1\string#2\endcsname}
% \end{macrocode}
% \end{macro}
%
% \begin{macrocode}
-\def\UTFviii@three@octets#1#2#3{\expandafter
+\long\def\UTFviii@three@octets#1#2#3{\expandafter
\UTFviii@defined\csname u8:#1\string#2\string#3\endcsname}
% \end{macrocode}
% \end{macro}
%
% \begin{macrocode}
-\def\UTFviii@four@octets#1#2#3#4{\expandafter
+\long\def\UTFviii@four@octets#1#2#3#4{\expandafter
\UTFviii@defined\csname u8:#1\string#2\string#3\string#4\endcsname}
% \end{macrocode}
% \end{macro}
@@ -289,18 +289,30 @@
\def\UTFviii@defined#1{%
\ifx#1\relax
% \end{macrocode}
+% Test if the sequence is invalid UTF-8 or valid UTF-8 but without
+% a \LaTeX\ definition.
+% \begin{macrocode}
+ \if\relax\expandafter\UTFviii@checkseq\string#1\relax\relax
+% \end{macrocode}
% The endline character has a special definition within the
% inputenc package (it is gobbling spaces). For this reason we
% can't produce multiline strings without some precaution.
% \changes{v1.1b}{2004/02/09}{No newlines allowed in error messages}
% \changes{v1.1g}{2005/09/27}{Real spaces do not show up so use \cs{space}}
% \changes{v1.1o}{2015/08/28}{Show Unicode number of character in hex}
+% \changes{v1.2a}{2018/03/24}{Error message improved for non-UTF-8 sequences}%
+% \begin{macrocode}
+ \UTFviii@undefined@err{#1}%
+% \end{macrocode}
+%
+% \begin{macrocode}
+ \else
+ \PackageError{inputenc}{Invalid UTF-8 byte sequence}%
+ \UTFviii@invalid@help
+ \fi
+% \end{macrocode}
+%
% \begin{macrocode}
- \PackageError{inputenc}{Unicode\space char\space\expandafter
- \UTFviii@splitcsname\string#1\relax
- \MessageBreak
- not\space set\space up\space
- for\space use\space with\space LaTeX}\@eha
\else\expandafter
#1%
\fi
@@ -308,10 +320,76 @@
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}{\UTFviii@invalid@err}
+% \begin{macro}{\UTFviii@invalid@help}
+% \changes{v1.2a}{2018/03/24}{Macro added}%
+% \begin{macrocode}
+\def\UTFviii@invalid@err#1{%
+ \PackageError{inputenc}{Invalid UTF-8 byte \number`#1}%
+ \UTFviii@invalid@help}
+% \end{macrocode}
+%
+% \begin{macrocode}
+\def\UTFviii@invalid@help{%
+ The document does not appear to be in UTF-8 encoding.\MessageBreak
+ Try adding \noexpand\UseRawInputEncoding as the first line of the file\MessageBreak
+ or specify an encoding such as \noexpand\usepackage[latin1]{inputenc}\MessageBreak
+ in the document preamble.\MessageBreak
+ Alternatively, save the file in UTF-8 using your editor or another tool}
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}{\UTFviii@undefined@err}
+% \changes{v1.2a}{2018/03/24}{Macro added}%
+% \begin{macrocode}
+\def\UTFviii@undefined@err#1{%
+ \PackageError{inputenc}{Unicode character \expandafter
+ \UTFviii@splitcsname\string#1\relax
+ \MessageBreak
+ not set up for use with LaTeX}%
+ {You may provide a definition with\MessageBreak
+ \noexpand\DeclareUnicodeCharacter}%
+ }
+% \end{macrocode}
+% \end{macro}
+%
+%
+% \begin{macro}{\UTFviii@checkseq}
+% \begin{macro}{\UTFviii@check@continue}
+% \changes{v1.2a}{2018/03/24}{Macro added}%
+% Check that the csname consists of a valid UTF-8 sequence.
+% \begin{macrocode}
+\def\UTFviii@checkseq#1:#2#3{%
+ \ifnum`#2<"80 %
+ \ifx\relax#3\else1\fi
+ \else
+ \ifnum`#2<"C0 %
+ 1 %
+ \else
+ \expandafter\expandafter\expandafter\UTFviii@check@continue
+ \expandafter\expandafter\expandafter#3%
+ \fi
+ \fi}
+% \end{macrocode}
+%
+% \begin{macrocode}
+\def\UTFviii@check@continue#1{%
+ \ifx\relax#1%
+ \else
+ \ifnum`#1<"80 1\else\ifnum`#1>"BF 1\fi\fi
+ \expandafter\UTFviii@check@continue
+ \fi
+}
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+%
% \begin{macro}{\UTFviii@loop}
-% This wonderful bit of code from Dr Carlisle defines the starting
-% octets to call |\UTFviii@two@octets| etc as appropriate. The starting
-% octet itself is passed directly as the first argument, the others
+% This bit of code derived from \texttt{xmltex} defines the active character
+% correspnding to starting octets to call |\UTFviii@two@octets| etc as appropriate.
+% The starting octet itself is passed directly as the first argument, the others
% are picked up later en route.
%
% The |\UTFviii@loop| loops through the numbers starting at
@@ -321,10 +399,6 @@
% All this is done in a group so that temporary catcode changes
% etc.~vanish after everything is set up.
%
-% It may be a good idea to add code to deal with `illegal utf8 octets':
-% at present these will be handled by whatever code was in use for 8-bit
-% input before this code is executed.
-%
% \begin{macrocode}
\begingroup
\catcode`\~13
@@ -341,6 +415,37 @@
\fi}
% \end{macrocode}
%
+% Handle the single byte control characters.
+% \changes{v1.2a}{2018/03/24}{Loop over C0 controls added}%
+% C0 controls are valid UTF-8 but defined to give the ``Character not defined error''
+% They may be defined with |\DeclareUnicodeCharacter|.
+% \begin{macrocode}
+ \def\UTFviii@tmp{\xdef~{\noexpand\UTFviii@undefined@err{:\string~}}}
+% 0 ^^@ null
+ \count@"1
+ \@tempcnta9
+% 9 ^^I tab
+% 10 ^^J nl
+\UTFviii@loop
+ \count@11
+ \@tempcnta12
+\UTFviii@loop
+% 12 ^^L
+% 13 ^^M
+ \count@14
+ \@tempcnta32
+\UTFviii@loop
+% \end{macrocode}
+%
+%
+% Bytes with leading bits |10| are not valid UTF-8 starting bytes
+% \begin{macrocode}
+ \count@"80
+ \@tempcnta"C2
+ \def\UTFviii@tmp{\xdef~{\noexpand\UTFviii@invalid@err\string~}}
+\UTFviii@loop
+% \end{macrocode}
+%
% Setting up 2-byte UTF-8:
% \begin{macrocode}
\count@"C2
@@ -348,6 +453,7 @@
\def\UTFviii@tmp{\xdef~{\noexpand\UTFviii@two@octets\string~}}
\UTFviii@loop
% \end{macrocode}
+%
% Setting up 3-byte UTF-8:
% \begin{macrocode}
\count@"E0
@@ -362,8 +468,21 @@
\@tempcnta"F4
\def\UTFviii@tmp{\xdef~{\noexpand\UTFviii@four@octets\string~}}
\UTFviii@loop
+% \end{macrocode}
+%
+% Bytes above F4 are not valid UTF-8 starting bytes as they would encode numbers beyond
+% the Unicode range
+% \begin{macrocode}
+ \count@"F4
+ \@tempcnta"100
+ \def\UTFviii@tmp{\xdef~{\noexpand\UTFviii@invalid@err\string~}}
+\UTFviii@loop
+% \end{macrocode}
+%
+% \begin{macrocode}
\endgroup
% \end{macrocode}
+%
% \end{macro}
%
% For this case we must disable the warning generated by
@@ -487,7 +606,7 @@
% \begin{macrocode}
% \begingroup
% \end{macrocode}
-% The original code from David supported the convention that a
+% The original code from \texttt{xmltex} supported the convention that a
% Unicode slot number could be given either as a decimal or as a
% hexadecimal (by starting with \texttt{x}). We do not do this so
% this code is also removed. This could be reactivated if one
@@ -500,18 +619,28 @@
% As |\count@| already contains the right value we make
% |\parse@XML@charref| work without arguments.
% \changes{v1.1g}{2005/09/27}{Real spaces do not show up so use \cs{space}}
+% \changes{v1.2a}{2018/03/24}{Allow control characters if active}
+% In the case single byte UTF-8 sequences, only allw definition if
+% the character os already active. The definition of |\UTFviii@tmp|
+% looks slightly strange but is designed for the sequence of |\expandafter|
+% in |\DeclareUnicodeCharacter|.
+%
% \begin{macrocode}
\ifnum\count@<"A0\relax
- \PackageError{inputenc}{Cannot\space define\space Unicode\space
- char\space value\space <\space 00A0}\@eha
+ \ifnum\catcode\count@=13
+ \uccode`\~=\count@\uppercase{\def\UTFviii@tmp{\@empty\@empty~}}%
+ \else
+ \PackageError{inputenc}%
+ {Cannot define non-active Unicode char value < 00A0}%
+ \@eha
+ \def\UTFviii@tmp{\UTFviii@tmp}%
+ \fi
% \end{macrocode}
-% Do not ask us to provide an explanation for the code below, it is
-% borrowed straight from \texttt{xmltex} by David and we trust him
-% totally (and we are too lazy to reread the Unicode book to see if
-% this is the correct algorithm).\footnote{We were hoping to also
-% find in his work the \TeX{} code for going the other way: from
-% UTF-8 octets to Unicode slot number, but no luck!
-% This has now been added as \cs{decode@UTFviii}}
+% The code below is derived from \texttt{xmltex} and generates the UTF-8 byte sequence
+% for the number in |\count@|.
+%
+% The reverse operation (just used in error messages)
+% has now been added as \cs{decode@UTFviii}.
% \begin{macrocode}
\else\ifnum\count@<"800\relax
\parse@UTFviii@a,%
@@ -534,7 +663,7 @@
% \end{macro}
%
% \begin{macro}{\parse@UTFviii@a}
-% \ldots so somebody else can document this part :-) \ldots~David?:-))))!
+% \ldots so somebody else can document this part :-)
% \changes{v1.1b}{2004/02/09}{Space in the wrong place \cs{count @64}}
% \begin{macrocode}
\gdef\parse@UTFviii@a#1{%
@@ -672,10 +801,11 @@
\else
\gdef\UTFviii@splitcsname#1:#2\relax{%
% \end{macrocode}
+% \changes{v1.2b}{2018/03/26}{add percent as \cs{endlinechar} not -1 in the format}%
% Need to pre-expand the argument to ensure cleanup in case of mal-formed UTF-8.
% \begin{macrocode}
#2 (U+\expandafter\UTFviii@hexnumber\expandafter{%
- \the\numexpr\decode@UTFviii#2\relax})
+ \the\numexpr\decode@UTFviii#2\relax})%
}
\fi
% \end{macrocode}
@@ -830,6 +960,9 @@
%
% \subsection{The mapping table}
%
+% \begingroup
+% \hfuzz=12pt ^^A just because
+%
% Note that the first argument must be a hex-digit number greater
% than \texttt{00BF} and at most \texttt{10FFFF}.
%
@@ -1109,7 +1242,7 @@
% and corrected by Matthias Noe (\verb=a9931078@unet.univie.ac.at=) --- thanks.
% \changes{v1.1j}{2007/11/09}{Added a few new unicode decls in cyrillic (pr/3988)}
% \changes{v1.1k}{2007/11/11}{Added and further unicode decls in cyrillic}
-% \changes{v1.1n}{2015/06/27}{correct accent http://tex.stackexchange.com/q/252521}
+% \changes{v1.1n}{2015/06/27}{correct accent https://tex.stackexchange.com/q/252521}
% \begin{macrocode}
%<*all,x2,t2c,t2b,t2a,ot2,lcy>
\DeclareUnicodeCharacter{0400}{\@tabacckludge`\CYRE}
@@ -1473,7 +1606,12 @@
%<all,t1>\DeclareUnicodeCharacter{1E20}{\@tabacckludge=G}
%<all,t1>\DeclareUnicodeCharacter{1E21}{\@tabacckludge=g}
% \end{macrocode}
-%
+% \changes{v1.2c}{2018/04/05}{Add BOM}
+% \begin{macrocode}
+%<all,ts1,utf8>\DeclareUnicodeCharacter{FEFF}{\ifhmode\nobreak\fi}
+% \end{macrocode}
+%
+% \endgroup
% \subsection{Notes}
%
% \changes{v1.1e}{2004/05/22}{Added notes on inconsistency with `8-bit files'.}