babel (22apr19)

git-svn-id: svn://tug.org/texlive/trunk@50951 c570f23f-e606-0410-a88d-b1316a301751

8 files changed, 173 insertions, 57 deletions

diff --git a/Master/texmf-dist/tex/generic/babel/babel-bidi-basic-r.lua b/Master/texmf-dist/tex/generic/babel/babel-bidi-basic-r.lua
index c9c45087a99..0705daa99d6 100644
--- a/Master/texmf-dist/tex/generic/babel/babel-bidi-basic-r.lua
+++ b/Master/texmf-dist/tex/generic/babel/babel-bidi-basic-r.lua
@@ -34,6 +34,8 @@
 
 Babel = Babel or {}
 
+Babel.bidi_enabled = true
+
 require('babel-bidi.lua')
 
 local characters = Babel.characters
@@ -51,18 +53,6 @@ local function dir_mark(head, from, to, outer)
   node.insert_after(head, to, d)
 end
 
-function Babel.pre_otfload_v(head)
-  -- head = Babel.numbers(head)
-  head = Babel.bidi(head, true)
-  return head
-end
-
-function Babel.pre_otfload_h(head)
-  -- head = Babel.numbers(head)
-  head = Babel.bidi(head, false)
-  return head
-end
-
 function Babel.bidi(head, ispar)
   local first_n, last_n            -- first and last char with nums
   local last_es                    -- an auxiliary 'last' used with nums
diff --git a/Master/texmf-dist/tex/generic/babel/babel-bidi-basic.lua b/Master/texmf-dist/tex/generic/babel/babel-bidi-basic.lua
index 345c90a2fef..02e1406e4d1 100644
--- a/Master/texmf-dist/tex/generic/babel/babel-bidi-basic.lua
+++ b/Master/texmf-dist/tex/generic/babel/babel-bidi-basic.lua
@@ -34,22 +34,14 @@
 
 Babel = Babel or {}
 
+-- eg, Babel.fontmap[1][<prefontid>]=<dirfontid>
+
 Babel.fontmap = Babel.fontmap or {}
 Babel.fontmap[0] = {}      -- l
 Babel.fontmap[1] = {}      -- r
 Babel.fontmap[2] = {}      -- al/an
 
-function Babel.pre_otfload_v(head)
-  -- head = Babel.numbers(head)
-  head = Babel.bidi(head, true)
-  return head
-end
-
-function Babel.pre_otfload_h(head, gc, sz, pt, dir)
-  -- head = Babel.numbers(head)
-  head = Babel.bidi(head, false, dir)
-  return head
-end
+Babel.bidi_enabled = true
 
 require('babel-bidi.lua')
 
@@ -169,9 +161,9 @@ function Babel.bidi(head, ispar, hdir)
         end
       end
       d = d or 'l'
-      if inmath then d = ('TRT' == tex.mathdir) and 'r' or 'l' end
-      d_font = d_font or d
 
+      -- A short 'pause' in bidi for mapfont
+      d_font = d_font or d
       d_font = (d_font == 'l' and 0) or
                (d_font == 'nsm' and 0) or
                (d_font == 'r' and 1) or
@@ -183,8 +175,12 @@ function Babel.bidi(head, ispar, hdir)
 
       if new_d then
         table.insert(nodes, {nil, (outer == 'l') and 'l' or 'r', nil})
-        attr_d = node.get_attribute(item, ATDIR)
-        attr_d = attr_d % 3
+        if inmath then
+          attr_d = 0
+        else
+          attr_d = node.get_attribute(item, ATDIR)
+          attr_d = attr_d % 3
+        end
         if attr_d == 1 then
           outer_first = 'r'
           last = 'r'
diff --git a/Master/texmf-dist/tex/generic/babel/babel.def b/Master/texmf-dist/tex/generic/babel/babel.def
index 530548f8eec..1b65f99346f 100644
--- a/Master/texmf-dist/tex/generic/babel/babel.def
+++ b/Master/texmf-dist/tex/generic/babel/babel.def
@@ -41,7 +41,7 @@
     \wlog{File: #1 #4 #3 <#2>}%
     \let\ProvidesFile\@undefined}
 \fi
-\ProvidesFile{babel.def}[2019/04/03 3.29 Babel common definitions]
+\ProvidesFile{babel.def}[2019/04/22 3.30 Babel common definitions]
 \ifx\AtBeginDocument\@undefined
   \input plain.def\relax
 \fi
@@ -1304,7 +1304,11 @@
 \bbl@trace{Creating languages and reading ini files}
 \newcommand\babelprovide[2][]{%
   \let\bbl@savelangname\languagename
+  \edef\bbl@savelocaleid{\the\localeid}%
+  % Set name and locale id
   \def\languagename{#2}%
+  \bbl@id@assign
+  \chardef\localeid\@nameuse{bbl@id@@\languagename}%
   \let\bbl@KVP@captions\@nil
   \let\bbl@KVP@import\@nil
   \let\bbl@KVP@main\@nil
@@ -1314,6 +1318,7 @@
   \let\bbl@KVP@hyphenrules\@nil
   \let\bbl@KVP@mapfont\@nil
   \let\bbl@KVP@maparabic\@nil
+  \let\bbl@KVP@mapdigits\@nil
   \let\bbl@KVP@intraspace\@nil
   \let\bbl@KVP@intrapenalty\@nil
   \bbl@forkv{#1}{\bbl@csarg\def{KVP@##1}{##2}}%  TODO - error handling
@@ -1346,7 +1351,25 @@
         \\\foreignlanguage{\languagename}%
         {####1}}}}%
     {}%
-  % To override script and language names
+  % At this point all parameters are defined if 'import'. Now we
+  % execute some code depending on them. But what about if nothing was
+  % imported? We just load the very basic parameters: ids and a few
+  % more.
+  \bbl@ifunset{bbl@lname@#2}%
+    {\def\BabelBeforeIni##1##2{%
+       \begingroup
+         \catcode`\[=12 \catcode`\]=12 \catcode`\==12 %
+         \let\bbl@ini@captions@aux\@gobbletwo
+         \def\bbl@inidate ####1.####2.####3.####4\relax ####5####6{}%
+         \let\bbl@inikv@typography\bbl@iniskip
+         \bbl@read@ini{##1}%
+         \bbl@exportkey{chrng}{characters.ranges}{}%
+         \bbl@exportkey{dgnat}{numbers.digits.native}{}%
+       \endgroup}%           boxed, to avoid extra spaces:
+     {\setbox\z@\hbox{\InputIfFileExists{babel-#2.tex}{}{}}}}%
+    {}%
+  % -
+  % Override script and language names with script= and language=
   \ifx\bbl@KVP@script\@nil\else
     \bbl@csarg\edef{sname@#2}{\bbl@KVP@script}%
   \fi
@@ -1378,7 +1401,7 @@
     \fi
     \bbl@exp{\\\bbl@add\\\bbl@mapselect{\\\bbl@mapdir{\languagename}}}%
   \fi
-  % For Southeast Asian, if interspace in ini
+  % For Southeast Asian, if interspace in ini -- TODO: as hook
   \ifcase\bbl@engine\or
     \bbl@ifunset{bbl@intsp@\languagename}{}%
       {\expandafter\ifx\csname bbl@intsp@\languagename\endcsname\@empty\else
@@ -1430,7 +1453,7 @@
         \fi}%
     \fi
   \fi
-  % Native digits, if provided in ini
+  % Native digits, if provided in ini (TeX level, xe and lua)
   \ifcase\bbl@engine\else
     \bbl@ifunset{bbl@dgnat@\languagename}{}%
       {\expandafter\ifx\csname bbl@dgnat@\languagename\endcsname\@empty\else
@@ -1447,6 +1470,42 @@
         \fi
       \fi}%
   \fi
+  % Native digits (lua level).
+  \ifodd\bbl@engine
+    \ifx\bbl@KVP@mapdigits\@nil\else
+      \bbl@ifunset{bbl@dgnat@\languagename}{}%
+        {\RequirePackage{luatexbase}%
+         \bbl@activate@preotf
+         \directlua{
+           Babel = Babel or {}  %%% -> presets in luababel
+           Babel.digits_mapped = true
+           Babel.digits = Babel.digits or {}
+           Babel.digits[\the\localeid] =
+             table.pack(string.utfvalue('\bbl@cs{dgnat@\languagename}'))
+           if not Babel.numbers then
+             function Babel.numbers(head)
+               local LOCALE = luatexbase.registernumber'bbl@attr@locale'
+               local GLYPH = node.id'glyph'
+               local inmath = false
+               for item in node.traverse(head) do
+                 if not inmath and item.id == GLYPH then
+                   local temp = node.get_attribute(item, LOCALE)
+                   if Babel.digits[temp] then
+                     local chr = item.char
+                     if chr > 47 and chr < 58 then
+                       item.char = Babel.digits[temp][chr-47]
+                     end
+                   end
+                 elseif item.id == node.id'math' then
+                   inmath = (item.subtype == 0)
+                 end
+               end
+               return head
+             end
+           end
+        }}
+    \fi
+  \fi
   % To load or reaload the babel-*.tex, if require.babel in ini
   \bbl@ifunset{bbl@rqtex@\languagename}{}%
     {\expandafter\ifx\csname bbl@rqtex@\languagename\endcsname\@empty\else
@@ -1457,7 +1516,8 @@
        \catcode`\@=\atcatcode
        \let\atcatcode\relax
      \fi}%
-  \let\languagename\bbl@savelangname}
+  \let\languagename\bbl@savelangname
+  \chardef\localeid\bbl@savelocaleid\relax}
 \def\bbl@setdigits#1#2#3#4#5{%
   \bbl@exp{%
     \def\<\languagename digits>####1{%       ie, \langdigits
@@ -1567,7 +1627,7 @@
        {}}%                      so, l@<lang> is ok - nothing to do
     {\bbl@exp{\\\adddialect\<l@#1>\bbl@tempa}}}%  found in opt list or ini
 \def\bbl@read@ini#1{%
-  \openin1=babel-#1.ini
+  \openin1=babel-#1.ini        % FIXME - number must not be hardcoded
   \ifeof1
     \bbl@error
       {There is no ini file for the requested language\\%
@@ -1678,7 +1738,7 @@
 \ifcase\bbl@engine
   \bbl@csarg\def{inikv@date.gregorian.licr}#1=#2\@@{%  override
     \bbl@inidate#1...\relax{#2}{}}
-  \bbl@csarg\def{secpre@date.gregorian.licr}{%           discard uni
+  \bbl@csarg\def{secpre@date.gregorian.licr}{%         discard uni
     \ifcase\bbl@engine\let\bbl@savedate\@empty\fi}
 \fi
 \def\bbl@inidate#1.#2.#3.#4\relax#5#6{% TODO - ignore with 'captions'
@@ -2063,7 +2123,7 @@
       \fi
     \fi}
   \def\bbl@textdir#1{%
-    \bbl@setluadir{text}\textdir{#1}% TODO - ?\linedir
+    \bbl@setluadir{text}\textdir{#1}%
     \chardef\bbl@thetextdir#1\relax
     \setattribute\bbl@attr@dir{\numexpr\bbl@thepardir*3+#1}}
   \def\bbl@pardir#1{%
@@ -2076,9 +2136,9 @@
   % Activated every math with the package option bidi=:
   \def\bbl@mathboxdir{%
     \ifcase\bbl@thetextdir\relax
-      \everyhbox{\bgroup\aftergroup\egroup\textdir TLT\relax}%
+      \everyhbox{\textdir TLT\relax}%
     \else
-      \everyhbox{\bgroup\aftergroup\egroup\textdir TRT\relax}%
+      \everyhbox{\textdir TRT\relax}%
     \fi}
 \else % pdftex=0, xetex=2
   \AddBabelHook{babel-bidi}{afterextras}{\bbl@switchdir}
diff --git a/Master/texmf-dist/tex/generic/babel/babel.sty b/Master/texmf-dist/tex/generic/babel/babel.sty
index 62fd68065fc..7ee327b7c48 100644
--- a/Master/texmf-dist/tex/generic/babel/babel.sty
+++ b/Master/texmf-dist/tex/generic/babel/babel.sty
@@ -33,7 +33,7 @@
 %%
 
 \NeedsTeXFormat{LaTeX2e}[2005/12/01]
-\ProvidesPackage{babel}[2019/04/03 3.29 The Babel package]
+\ProvidesPackage{babel}[2019/04/22 3.30 The Babel package]
 \@ifpackagewith{babel}{debug}
   {\providecommand\bbl@trace[1]{\message{^^J[ #1 ]}}%
    \let\bbl@debug\@firstofone}
@@ -187,6 +187,46 @@
   \bbl@languages
 \fi
 \ifodd\bbl@engine
+  \def\bbl@activate@preotf{%
+    \let\bbl@activate@preotf\relax  % only once
+    \directlua{
+      Babel = Babel or {}
+      %
+      function Babel.pre_otfload_v(head)
+        if Babel.numbers and Babel.digits_mapped then
+          head = Babel.numbers(head)
+        end
+        if Babel.bidi_enabled then
+          head = Babel.bidi(head, false, dir)
+        end
+        return head
+      end
+      %
+      function Babel.pre_otfload_h(head, gc, sz, pt, dir)
+        if Babel.numbers and Babel.digits_mapped then
+          head = Babel.numbers(head)
+        end
+        if Babel.fixboxdirs then          % Temporary!
+          head = Babel.fixboxdirs(head)
+        end
+        if Babel.bidi_enabled then
+          head = Babel.bidi(head, false, dir)
+        end
+        return head
+      end
+      %
+      luatexbase.add_to_callback('pre_linebreak_filter',
+        Babel.pre_otfload_v,
+        'Babel.pre_otfload_v',
+        luatexbase.priority_in_callback('pre_linebreak_filter',
+          'luaotfload.node_processor') or nil)
+      %
+      luatexbase.add_to_callback('hpack_filter',
+        Babel.pre_otfload_h,
+        'Babel.pre_otfload_h',
+        luatexbase.priority_in_callback('hpack_filter',
+          'luaotfload.node_processor') or nil)
+    }}
   \let\bbl@tempa\relax
   \@ifpackagewith{babel}{bidi=basic}%
     {\def\bbl@tempa{basic}}%
@@ -200,17 +240,8 @@
     \directlua{
       require('babel-bidi.lua')
       require('babel-bidi-\bbl@tempa.lua')
-      luatexbase.add_to_callback('pre_linebreak_filter',
-        Babel.pre_otfload_v,
-        'Babel.pre_otfload_v',
-        luatexbase.priority_in_callback('pre_linebreak_filter',
-          'luaotfload.node_processor') or nil)
-      luatexbase.add_to_callback('hpack_filter',
-        Babel.pre_otfload_h,
-        'Babel.pre_otfload_h',
-        luatexbase.priority_in_callback('hpack_filter',
-          'luaotfload.node_processor') or nil)
-     }
+    }
+    \bbl@activate@preotf
   \fi
 \fi
 \bbl@trace{Defining option 'base'}
@@ -276,9 +307,11 @@
   \DeclareOption{bidi=basic}%
     {\let\bbl@beforeforeign\leavevmode
      \newattribute\bbl@attr@dir
-     % TODO - the same trick as bbl@severypar
-     % \everymath{\bbl@mathboxdir}%
-     % \everydisplay{\bbl@mathboxdir}%
+     % I don't like it, hackish:
+     \frozen@everymath\expandafter{%
+       \expandafter\bbl@mathboxdir\the\frozen@everymath}%
+     \frozen@everydisplay\expandafter{%
+       \expandafter\bbl@mathboxdir\the\frozen@everydisplay}%
      \bbl@exp{\output{\bodydir\pagedir\the\output}}%
      \AtEndOfPackage{\EnableBabelHook{babel-bidi}}}
 \else
diff --git a/Master/texmf-dist/tex/generic/babel/hyphen.cfg b/Master/texmf-dist/tex/generic/babel/hyphen.cfg
index cd1afb4d293..213ca7f27ce 100644
--- a/Master/texmf-dist/tex/generic/babel/hyphen.cfg
+++ b/Master/texmf-dist/tex/generic/babel/hyphen.cfg
@@ -37,7 +37,7 @@
     \wlog{File: #1 #4 #3 <#2>}%
     \let\ProvidesFile\@undefined}
 \fi
-\ProvidesFile{hyphen.cfg}[2019/04/03 3.29 Babel hyphens]
+\ProvidesFile{hyphen.cfg}[2019/04/22 3.30 Babel hyphens]
 \xdef\bbl@format{\jobname}
 \ifx\AtBeginDocument\@undefined
   \def\@empty{}
diff --git a/Master/texmf-dist/tex/generic/babel/luababel.def b/Master/texmf-dist/tex/generic/babel/luababel.def
index 85899a726ba..c3cd913bbf7 100644
--- a/Master/texmf-dist/tex/generic/babel/luababel.def
+++ b/Master/texmf-dist/tex/generic/babel/luababel.def
@@ -224,6 +224,11 @@
   end
 }
 \endgroup
+\ifx\newattribute\@undefined\else
+  \newattribute\bbl@attr@locale
+  \AddBabelHook{luatex}{beforeextras}{%
+    \setattribute\bbl@attr@locale\localeid}
+\fi
 \def\BabelStringsDefault{unicode}
 \let\luabbl@stop\relax
 \AddBabelHook{luatex}{encodedcommands}{%
@@ -546,6 +551,26 @@
   \expandafter\addto\csname extras#1\endcsname{%
     \babel@save\bbl@langfeatures
     \edef\bbl@langfeatures{#2,}}}
+\def\bbl@luafixboxdir{%
+  \setbox\z@\hbox{\textdir TLT}%
+  \directlua{
+    function Babel.first_dir(head)
+      for item in node.traverse_id(node.id'dir', head) do
+        return item
+      end
+      return nil
+    end
+    if Babel.first_dir(tex.box[0].head) then
+      function Babel.fixboxdirs(head)
+        local fd = Babel.first_dir(head)
+        if fd and fd.dir:sub(1,1) == '-' then
+          head = node.remove(head, fd)
+        end
+        return head
+      end
+    end
+  }}
+\AtBeginDocument{\bbl@luafixboxdir}
 \bbl@trace{Redefinitions for bidi layout}
 \ifx\@eqnnum\@undefined\else
   \ifx\bbl@attr@dir\@undefined\else
diff --git a/Master/texmf-dist/tex/generic/babel/nil.ldf b/Master/texmf-dist/tex/generic/babel/nil.ldf
index 4be4ec7477d..a50efbf34e7 100644
--- a/Master/texmf-dist/tex/generic/babel/nil.ldf
+++ b/Master/texmf-dist/tex/generic/babel/nil.ldf
@@ -32,7 +32,7 @@
 %% extension |.ins|) which are part of the distribution.
 %%
 
-\ProvidesLanguage{nil}[2019/04/03 3.29 Nil language]
+\ProvidesLanguage{nil}[2019/04/22 3.30 Nil language]
 \LdfInit{nil}{datenil}
 \ifx\l@nohyphenation\@undefined
    \@nopatterns{nil}
diff --git a/Master/texmf-dist/tex/generic/babel/switch.def b/Master/texmf-dist/tex/generic/babel/switch.def
index 1755128ad0a..f03f7486849 100644
--- a/Master/texmf-dist/tex/generic/babel/switch.def
+++ b/Master/texmf-dist/tex/generic/babel/switch.def
@@ -37,7 +37,7 @@
     \wlog{File: #1 #4 #3 <#2>}%
     \let\ProvidesFile\@undefined}
 \fi
-\ProvidesFile{switch.def}[2019/04/03 3.29 Babel switching mechanism]
+\ProvidesFile{switch.def}[2019/04/22 3.30 Babel switching mechanism]
 \ifx\AtBeginDocument\@undefined
   \input plain.def\relax
 \fi
@@ -58,8 +58,8 @@
   \countdef\last@language=19
   \def\addlanguage{\alloc@9\language\chardef\@cclvi}
 \fi
-\def\bbl@version{3.29}
-\def\bbl@date{2019/04/03}
+\def\bbl@version{3.30}
+\def\bbl@date{2019/04/22}
 \def\adddialect#1#2{%
   \global\chardef#1#2\relax
   \bbl@usehooks{adddialect}{{#1}{#2}}%
@@ -109,6 +109,15 @@
   \let\bbl@ifrestoring\@firstoftwo
   \expandafter\bbl@set@language\expandafter{\languagename}%
   \let\bbl@ifrestoring\@secondoftwo}
+\chardef\localeid\z@
+\def\bbl@id@last{0}    % No real need for a new counter
+\def\bbl@id@assign{%
+  \bbl@ifunset{bbl@id@@\languagename}%
+    {\count@\bbl@id@last\relax
+     \advance\count@\@ne
+     \bbl@csarg\chardef{id@@\languagename}\count@
+     \edef\bbl@id@last{\the\count@}}%
+    {}}
 \expandafter\def\csname selectlanguage \endcsname#1{%
   \ifnum\bbl@hymapsel=\@cclv\let\bbl@hymapsel\tw@\fi
   \bbl@push@language
@@ -175,6 +184,9 @@
     \babel@beginsave}%
   \bbl@usehooks{afterreset}{}%
   \languageshorthands{none}%
+  % set the locale id
+  \bbl@id@assign
+  \chardef\localeid\@nameuse{bbl@id@@\languagename}%
   % switch captions, date
   \ifcase\bbl@select@type
     \ifhmode