From 24e0da01ec79bfa74f109e457721fc6f51a04295 Mon Sep 17 00:00:00 2001
From: Karl Berry <karl@freefriends.org>
Date: Wed, 26 Sep 2018 20:48:48 +0000
Subject: babel (26sep18)

git-svn-id: svn://tug.org/texlive/trunk@48768 c570f23f-e606-0410-a88d-b1316a301751
---
 Master/texmf-dist/source/latex/babel/babel.dtx | 306 +++++++++++++++++--------
 1 file changed, 211 insertions(+), 95 deletions(-)

(limited to 'Master/texmf-dist/source/latex/babel/babel.dtx')

diff --git a/Master/texmf-dist/source/latex/babel/babel.dtx b/Master/texmf-dist/source/latex/babel/babel.dtx
index 24cfe5762b7..49c55ac35ea 100644
--- a/Master/texmf-dist/source/latex/babel/babel.dtx
+++ b/Master/texmf-dist/source/latex/babel/babel.dtx
@@ -31,7 +31,7 @@
 %
 % \iffalse
 %<*filedriver>
-\ProvidesFile{babel.dtx}[2018/09/02 v3.23 The Babel package]
+\ProvidesFile{babel.dtx}[2018/09/26 v3.24 The Babel package]
 \documentclass{ltxdoc}
 \GetFileInfo{babel.dtx}
 \usepackage{fontspec}
@@ -2071,6 +2071,35 @@ language name to be used by \textsf{fontspec} (eg, |Hindi|). Overrides
 the value in the |ini| file. Not so important, but sometimes still
 relevant.
 
+\bigskip
+
+A few options (only \luatex) set some properties of the writing system
+used by the language. These properties are \textit{always} applied to
+the script, no matter which language is active. Although somewhat
+inconsistent, this makes setting a language up easier in most typical
+cases.
+
+\Describe{mapfont=}{\texttt{direction}}
+Assigns the font for the writing direction of this language. More
+precisely, what |mapfont=direction| means is, ‘when a character has the
+same direction as the script for the “provided” language, then change
+its font to that set for this language’. There are 3 directions,
+following the bidi Unicode algorithm, namely, Arabic-like, Hebrew-like
+and left to right.\footnote{In future realeases an new value 
+(\texttt{script}) will be added.} So, there should be at most 3 
+directives of this kind.
+
+\Describe{intraspace=}{\meta{base} \meta{shrink} \meta{stretch}}
+Sets the interword space for the writing system of the language, in em
+units (so, |0 .1 0| is |0em plus .1em|). Like |\spaceskip|, the em unit
+applied is that of the current text (more precisely, the previous 
+glyph). Currently used only in Southeast Asian scrips, like Thai.
+
+\Describe{intrapenalty}{\meta{penalty}}
+Sets the interword penalty for the writing system of this language.
+Currently used only in Southeast Asian scrips, like Thai. Ignored if 0
+(which is the default value).
+
 \begin{note}
   (1) If you need shorthands, you can use |\useshorthands| and
   |\defineshorthand| as described above. (2) Captions and |\today| are
@@ -2216,7 +2245,14 @@ Listed words are saved expanded and therefore it relies on the
 LICR. Of course, it also works without the LICR if the input and the
 font encodings are the same, like in Unicode based engines.
 
-\Describe\babelpatterns{\texttt{[}\langlist\texttt{]}%
+\begin{note}
+  Using |\babelhyphenation| with Southeast Asian scripts is mostly
+  pointless. But with |\babelpatterns| (below) you may fine-tune line
+  breaking (only \luatex). Even if there are no pattern for the
+  language, you can add at least some typical cases.
+\end{note}
+
+\Describe{\babelpatterns}{\texttt{[}\langlist\texttt{]}%
     \marg{patterns}}
 
 \New{3.9m} \textit{In \textsf{luatex} only},\footnote{With
@@ -2400,12 +2436,10 @@ _\babelprovide[import, main]{arabic}_
   \end{document}
   \end{verbatim}
   \endgroup
-  What |mapfont=direction| means is, ‘when a character has the same
-  direction as the script for the “provided” language (|arabic| in
-  this case), then change its font to that set for this language’ (here
-  defined via |*arabic|, because Crimson does not provide Arabic
-  letters). Note Hebrew and Arabic, following the Unicode rules, have
-  different directions (internally ‘r’ and ‘al’, respectively).
+  In this example, and thanks to |mapfont=direction|, any Arabic letter
+  (because the language is |arabic|) changes its font to that set for
+  this language (here defined via |*arabic|, because Crimson does not
+  provide Arabic letters).
 \end{example}
 
 \begin{note}
@@ -2859,55 +2893,6 @@ as well as “non-European” digits. Also on the roadmap are R layouts
 (lists, footnotes, tables, column order), page and section numbering,
 and maybe kashida justification.
 
-As to Thai line breaking, here is the basic idea of what \luatex{} can
-do for us, with the Thai patterns and a little script (the final
-version will not be so little, of course). It replaces each
-discretionary by the equivalent to ZWJ.
-
-\begingroup
-\catcode`\_=13 \def_{\string_}
-\begin{verbatim}
-\documentclass{article}
-
-\usepackage{babel}
-
-\babelprovide[import, main]{thai}
-
-\babelfont{rm}{FreeSerif}
-
-\directlua{
-local GLYF = node.id'glyph'
-function insertsp (head)
-  local size = 0
-  for item in node.traverse(head) do
-    local i = item.id
-    if i == GLYF then
-      f = font.getfont(item.font)
-      size = f.size
-    elseif i == 7 then
-      local n = node.new(12, 0)
-      node.setglue(n, 0, size * 1) % 1 is a factor
-      node.insert_before(head, item, n)
-      node.remove(head, item)
-    end
-  end
-end
-
-luatexbase.add_to_callback('hyphenate',
-  function (head, tail)
-    lang.hyphenate(head)
-    insertsp(head)
-  end, 'insertsp')
-}
-
-\begin{document}
-
-(Thai text.)
-
-\end{document}
-\end{verbatim}
-\endgroup
-
 Useful additions would be, for example, time, currency, addresses and
 personal names.\footnote{See for example POSIX, ISO 14652 and the
 Unicode Common Locale Data Repository (CLDR). Those system, however,
@@ -2925,6 +2910,13 @@ chapters), in Hungarian ``from (1)'' is ``(1)-b\H{o}l'', but ``from
 
 \subsection{Tentative and experimental code}
 
+\textbf{Southeast Asian interword spacing}
+
+There is some preliminary interword spacing for Thai, Lao and Khemer
+in \luatex{} (provided there are hyphenation patters). It is activated
+automatically if a language with of one of these scripts are loaded
+with |\babelprovide|. See the sample on the \babel{} repository.
+
 \medskip
 \textbf{Old stuff}
 
@@ -3865,8 +3857,8 @@ help from Bernd Raichle, for which I am grateful.
 % \section{Tools}
 %
 %    \begin{macrocode}
-%<<version=3.23>>
-%<<date=2018/09/02>>
+%<<version=3.24>>
+%<<date=2018/09/26>>
 %    \end{macrocode}
 %
 % \textbf{Do not use the following macros in \texttt{ldf} files. They
@@ -4435,6 +4427,7 @@ help from Bernd Raichle, for which I am grateful.
 \DeclareOption{noconfigs}{}
 \DeclareOption{showlanguages}{}
 \DeclareOption{silent}{}
+\DeclareOption{mono}{}
 \DeclareOption{shorthands=off}{\bbl@tempa shorthands=\bbl@tempa}
 <@More package options@>
 %    \end{macrocode}
@@ -6744,6 +6737,7 @@ help from Bernd Raichle, for which I am grateful.
 %    \end{macrocode}
 %
 %  \end{macro}
+%
 % \subsection{Multiencoding strings}
 %
 % \changes{babel~3.9a}{2012/09/05}{Added tentative code for string
@@ -7714,12 +7708,10 @@ help from Bernd Raichle, for which I am grateful.
 %
 % \subsection{Creating languages}
 %
-% |\babelprovide| is a general purpose tool for creating
-% languages. Currently it just creates the language infrastructure,
-% but in the future it will be able to read data from |ini| files, as
-% well as to create variants. Unlike the \textsf{nil} pseudo-language,
-% captions are defined, but with a warning to invite the user to
-% provide the real string.
+% |\babelprovide| is a general purpose tool for creating and modifying
+% languages. It creates the language infrastructure, and
+% loads, if requested, an |ini| file. It may be used in conjunction to
+% previouly loaded |ldf| files.
 %
 % \changes{3.10}{2017/05/19}{Added \cs{babelprovide}}
 % \changes{3.13}{2017/08/30}{Added \cs{import}, which also reads
@@ -7759,6 +7751,7 @@ help from Bernd Raichle, for which I am grateful.
   \ifx\bbl@KVP@captions\@nil
     \let\bbl@KVP@captions\bbl@KVP@import
   \fi
+  % Load ini
   \bbl@ifunset{date#2}%
     {\bbl@provide@new{#2}}%
     {\bbl@ifblank{#1}%
@@ -7769,6 +7762,7 @@ help from Bernd Raichle, for which I am grateful.
          import=lang-tag}%
         {Use this macro as documented}}%
       {\bbl@provide@renew{#2}}}%
+  % Post tasks
   \bbl@exp{\\\babelensure[exclude=\\\today]{#2}}%
   \bbl@ifunset{bbl@ensure@\languagename}%
     {\bbl@exp{%
@@ -7804,6 +7798,33 @@ help from Bernd Raichle, for which I am grateful.
     \fi
     \bbl@exp{\\\bbl@add\\\bbl@mapselect{\\\bbl@mapdir{\languagename}}}%
   \fi
+  \ifcase\bbl@engine\or
+    \bbl@ifunset{bbl@intsp@\languagename}{}%
+      {\expandafter\ifx\csname bbl@intsp@\languagename\endcsname\@empty\else
+        \bbl@seaintraspace
+        \ifx\bbl@KVP@intraspace\@nil
+           \bbl@exp{%
+             \\\bbl@intraspace\bbl@cs{intsp@\languagename}\\\@@}%
+        \fi
+        \directlua{
+           Babel = Babel or {}
+           Babel.sea_ranges = Babel.sea_ranges or {}
+           Babel.set_chranges('\bbl@cs{sbcp@\languagename}', 
+                              '\bbl@cs{chrng@\languagename}')
+        }
+        \ifx\bbl@KVP@intrapenalty\@nil
+          \bbl@intrapenalty0\@@
+        \fi
+      \fi
+      \ifx\bbl@KVP@intraspace\@nil\else % We may override the ini
+        \expandafter\bbl@intraspace\bbl@KVP@intraspace\@@
+      \fi
+      \ifx\bbl@KVP@intrapenalty\@nil\else
+        \expandafter\bbl@intrapenalty\bbl@KVP@intrapenalty\@@
+      \fi}%
+  \or 
+    %% TODO - xetex
+  \fi
   \ifcase\bbl@engine\else
     \bbl@ifunset{bbl@dgnat@\languagename}{}%
       {\expandafter\ifx\csname bbl@dgnat@\languagename\endcsname\@empty\else
@@ -7820,6 +7841,15 @@ help from Bernd Raichle, for which I am grateful.
         \fi
       \fi}%
   \fi
+  \bbl@ifunset{bbl@rqtex@\languagename}{}%
+    {\expandafter\ifx\csname bbl@rqtex@\languagename\endcsname\@empty\else
+       \let\BabelBeforeIni\@gobbletwo
+       \chardef\atcatcode=\catcode`\@
+       \catcode`\@=11\relax
+       \InputIfFileExists{babel-\bbl@cs{rqtex@\languagename}.tex}{}{}%
+       \catcode`\@=\atcatcode
+       \let\atcatcode\relax
+     \fi}%
   \let\languagename\bbl@savelangname}
 \def\bbl@setdigits#1#2#3#4#5{%
   \bbl@exp{%
@@ -8034,12 +8064,17 @@ help from Bernd Raichle, for which I am grateful.
   \bbl@exportkey{sbcp}{identification.script.tag.bcp47}{}%
   \bbl@exportkey{sotf}{identification.script.tag.opentype}{DFLT}}
 \let\bbl@secline@typography\bbl@inikv
+\let\bbl@secline@characters\bbl@inikv
 \let\bbl@secline@numbers\bbl@inikv
 \def\bbl@after@ini{%
   \bbl@exportkey{lfthm}{typography.lefthyphenmin}{2}%
   \bbl@exportkey{rgthm}{typography.righthyphenmin}{3}%
   \bbl@exportkey{hyphr}{typography.hyphenrules}{}%
+  \bbl@exportkey{intsp}{typography.intraspace}{}%
+  \bbl@exportkey{jstfy}{typography.justify}{w}%
+  \bbl@exportkey{chrng}{characters.ranges}{}%
   \bbl@exportkey{dgnat}{numbers.digits.native}{}%
+  \bbl@exportkey{rqtex}{identification.require.babel}{}%
   \bbl@xin@{0.5}{\@nameuse{bbl@@kv@identification.version}}%
   \ifin@
     \bbl@warning{%
@@ -8149,7 +8184,7 @@ help from Bernd Raichle, for which I am grateful.
        range 0-9999.}%
       {There is little you can do. Sorry.}%
   \fi\fi\fi\fi}}
-\newcommand\BabelDateyyyy[1]{{\number#1}}
+\newcommand\BabelDateyyyy[1]{{\number#1}} % FIXME - add leading 0
 \def\bbl@replace@finish@iii#1{%
   \bbl@exp{\def\\#1####1####2####3{\the\toks@}}}
 \def\bbl@TG@@date{%
@@ -8182,7 +8217,7 @@ help from Bernd Raichle, for which I am grateful.
   \bbl@csarg\bbl@add@list{lsys@#1}{Script=\bbl@cs{sname@#1}}%
   \bbl@ifunset{bbl@lname@#1}{}%
     {\bbl@csarg\bbl@add@list{lsys@#1}{Language=\bbl@cs{lname@#1}}}%
-  \bbl@csarg\bbl@toglobal{lsys@#1}%}%
+  \bbl@csarg\bbl@toglobal{lsys@#1}%
   \bbl@exp{% TODO - should be global, but even local does its job
            % I'm still not sure -- must investigate 
     \<keys_if_exist:nnF>{fontspec-opentype}{Script/\bbl@cs{sname@#1}}%
@@ -8471,25 +8506,21 @@ help from Bernd Raichle, for which I am grateful.
 %  \end{macro}
 %
 %  \begin{macro}{\bibcite}
-%    The macro that is used in the |.aux| file to define citation
-%    labels. When packages such as \pkg{natbib} or \pkg{cite} are not
-%    loaded its second argument is used to typeset the citation
-%    label. In that case, this second argument can contain active
-%    characters but is used in an environment where
-%    |\@safe@activestrue| is in effect. This switch needs to be reset
-%    inside the |\hbox| which contains the citation label. In order to
-%    determine during \file{.aux} file processing which definition of
-%    |\bibcite| is needed we define |\bibcite| in such a way that it
-%    redefines itself with the proper definition.
+%  The macro that is used in the |.aux| file to
+%  define citation labels. When packages such as \pkg{natbib} or
+%  \pkg{cite} are not loaded its second argument is used to typeset the
+%  citation label. In that case, this second argument can contain
+%  active characters but is used in an environment where
+%  |\@safe@activestrue| is in effect. This switch needs to be reset
+%  inside the |\hbox| which contains the citation label. In order to
+%  determine during \file{.aux} file processing which definition of
+%  |\bibcite| is needed we define |\bibcite| in such a way that it
+%  redefines itself with the proper definition. We call
+%  |\bbl@cite@choice| to select the proper definition for |\bibcite|.
+%  This new definition is then activated.
 %
 %    \begin{macrocode}
   \bbl@redefine\bibcite{%
-%    \end{macrocode}
-%
-%    We call |\bbl@cite@choice| to select the proper definition for
-%    |\bibcite|. This new definition is then activated.
-%
-%    \begin{macrocode}
     \bbl@cite@choice
     \bibcite}
 %    \end{macrocode}
@@ -8509,28 +8540,19 @@ help from Bernd Raichle, for which I am grateful.
 %
 %  \begin{macro}{\bbl@cite@choice}
 %    The macro |\bbl@cite@choice| determines which definition of
-%    |\bibcite| is needed.
+%    |\bibcite| is needed. First we give |\bibcite| its default
+%    definition.
 %
 %    \begin{macrocode}
   \def\bbl@cite@choice{%
-%    \end{macrocode}
-%
-%    First we give |\bibcite| its default definition.
-%
-%    \begin{macrocode}
     \global\let\bibcite\bbl@bibcite
 %    \end{macrocode}
 %
 %    Then, when \pkg{natbib} is loaded we restore the original
-%    definition of |\bibcite|.
+%    definition of |\bibcite|. For \pkg{cite} we do the same.
 %
 %    \begin{macrocode}
     \@ifpackageloaded{natbib}{\global\let\bibcite\org@bibcite}{}%
-%    \end{macrocode}
-%
-%    For \pkg{cite} we do the same.
-%
-%    \begin{macrocode}
     \@ifpackageloaded{cite}{\global\let\bibcite\org@bibcite}{}%
 %    \end{macrocode}
 %
@@ -10153,6 +10175,13 @@ help from Bernd Raichle, for which I am grateful.
     it in the preamble with something like:\\%
     \string\renewcommand\@backslashchar#2{..}\\%
     Reported}}
+\def\bbl@tentative{\protect\bbl@tentative@i}
+\def\bbl@tentative@i#1{%
+  \bbl@warning{%
+    Some functions in '#1' are tentative.\\%
+    They might not work as expected and their behavior\\%
+    could change in the future.\\%
+    Reported}}
 \def\@nolanerr#1{%
   \bbl@error
     {You haven't defined the language #1\space yet}%
@@ -11498,6 +11527,93 @@ help from Bernd Raichle, for which I am grateful.
 %    \end{macrocode}
 %  \end{macro}
 %
+% \section{Southeast Asian scripts}
+%
+% \changes{babel~3.24}{2018/09/24}{Lua code for interword spacing
+%   in Southeast Asian scripts.}
+%
+% \textit{In progress.} Replace regular (ie, implicit) discretionaries
+% by spaceskips, based on the previous glyph (which I think makes
+% sense, because the hyphen and the previous char go always together).
+% Other discretionaries are not touched.
+%
+% For the moment, only 3 SA languages are 
+% activated by default (see Unicode UAX 14).
+%
+%    \begin{macrocode}
+\def\bbl@intraspace#1 #2 #3\@@{%
+  \directlua{
+    Babel = Babel or {}
+    Babel.intraspaces = Babel.intraspaces or {}
+    Babel.intraspaces['\csname bbl@sbcp@\languagename\endcsname'] = %
+       {b = #1, p = #2, m = #3}
+  }}
+\def\bbl@intrapenalty#1\@@{%
+  \directlua{
+    Babel = Babel or {}
+    Babel.intrapenalties = Babel.intrapenalties or {}
+    Babel.intrapenalties['\csname bbl@sbcp@\languagename\endcsname'] = #1
+  }}
+\begingroup
+\catcode`\%=12
+\catcode`\^=14
+\catcode`\'=12
+\catcode`\~=12
+\gdef\bbl@seaintraspace{^
+  \let\bbl@seaintraspace\relax
+  \directlua{
+    Babel = Babel or {}
+    Babel.sea_ranges = Babel.sea_ranges or {}
+    function Babel.set_chranges (script, chrng) 
+      local c = 0
+      for s, e in string.gmatch(chrng..' ', '(.-)%.%.(.-)%s') do
+        Babel.sea_ranges[script..c]={tonumber(s,16), tonumber(e,16)}
+        c = c + 1
+      end
+    end
+    function Babel.sea_disc_to_space (head)
+      local sea_ranges = Babel.sea_ranges
+      local last_char = nil
+      local quad = 655360      ^^ 10 pt = 655360 = 10 * 65536
+      for item in node.traverse(head) do
+        local i = item.id
+        if i == node.id'glyph' then
+          last_char = item
+        elseif i == 7 and item.subtype == 3 and last_char 
+            and last_char.char > 0x0C99 then
+          quad = font.getfont(last_char.font).size
+          for lg, rg in pairs(sea_ranges) do
+            if last_char.char > rg[1] and last_char.char < rg[2] then
+              lg = lg:sub(1, 4)
+              local intraspace = Babel.intraspaces[lg]
+              local intrapenalty = Babel.intrapenalties[lg]
+              local n
+              if intrapenalty ~= 0 then
+                n = node.new(14, 0)     ^^ penalty
+                n.penalty = intrapenalty
+                node.insert_before(head, item, n)
+              end    
+              n = node.new(12, 13)      ^^ (glue, spaceskip)
+              node.setglue(n, intraspace.b * quad,
+                              intraspace.p * quad,
+                              intraspace.m * quad)
+              node.insert_before(head, item, n)
+              node.remove(head, item)
+            end
+          end
+        end
+      end
+    end
+    luatexbase.add_to_callback('hyphenate',
+      function (head, tail)
+        lang.hyphenate(head)
+        Babel.sea_disc_to_space(head)
+      end,
+      'Babel.sea_disc_to_space')
+}}
+\endgroup
+%    \end{macrocode}
+%
 % Common stuff.
 %
 %    \begin{macrocode}
-- 
cgit v1.2.3

