babel (9apr21)

git-svn-id: svn://tug.org/texlive/trunk@58797 c570f23f-e606-0410-a88d-b1316a301751

23 files changed, 390 insertions, 102 deletions

diff --git a/Master/texmf-dist/doc/latex/babel/README.md b/Master/texmf-dist/doc/latex/babel/README.md
index 723cf41d822..6d8f47c18c3 100644
--- a/Master/texmf-dist/doc/latex/babel/README.md
+++ b/Master/texmf-dist/doc/latex/babel/README.md
@@ -1,4 +1,4 @@
-## Babel 3.56
+## Babel 3.57
 
 This package manages culturally-determined typographical (and other)
 rules, and hyphenation patterns for a wide range of languages. Many
@@ -8,9 +8,9 @@ is a set of ini files for about 250 languages.
 
 The latest stable version is available on <https://ctan.org/pkg/babel>.
 
-Changes in version 3.56 are described in:
+Changes in version 3.57 are described in:
 
-https://github.com/latex3/babel/blob/master/news-guides/news/whats-new-in-babel-3.56.md
+https://github.com/latex3/babel/blob/master/news-guides/news/whats-new-in-babel-3.57.md
 
 Apart from the manual, you can find information on some aspects of babel at:
 
@@ -46,16 +46,16 @@ respective authors.
 
 ### Summary of Latest changes
 ```
-3.56   2021-03-24
-       * Transforms (\babelprehyphenation, \babelposthyphenation)
-         - Key 'space', in em units.
-         - Key 'spacefactor', based on the current word separation.
-         - Multiple 'insert's allowed.
-         - Definable in 'ini' files.
-         - Arbitrary characters in pattern.
-       * Fixes:
-         - 'insert' and 'data' didn't always set/get the expected node.
-         - Hyphen was not shown in Marathi (#123)
+3.57   2021-04-07
+       * Predefined transforms (lua):
+         - Arabic:     transliteration.dad
+         - Croatian:   digraphs.ligatures
+         - Greek:      diaeresis.hyphen
+         - Hindi:      transliteration.hk
+         - Hungarian:  digraphs.hyphen
+       * Transforms: {xxxx} syntax also in string=.
+       * Preliminary code for Uyghur hyphenation (lua).
+       * magyar as alternative to hungarian in \babelprovide.
 ```
 
 ### Previous changes
diff --git a/Master/texmf-dist/doc/latex/babel/babel.pdf b/Master/texmf-dist/doc/latex/babel/babel.pdf
index 588c656f93e..540babb7bb6 100644
--- a/Master/texmf-dist/doc/latex/babel/babel.pdf
+++ b/Master/texmf-dist/doc/latex/babel/babel.pdf
diff --git a/Master/texmf-dist/source/latex/babel/babel.dtx b/Master/texmf-dist/source/latex/babel/babel.dtx
index 9c8ff775c03..b21384f6a02 100644
--- a/Master/texmf-dist/source/latex/babel/babel.dtx
+++ b/Master/texmf-dist/source/latex/babel/babel.dtx
@@ -31,7 +31,7 @@
 %
 % \iffalse
 %<*filedriver>
-\ProvidesFile{babel.dtx}[2021/03/24 v3.56 The Babel package]
+\ProvidesFile{babel.dtx}[2021/04/07 v3.57 The Babel package]
 \documentclass{ltxdoc}
 \GetFileInfo{babel.dtx}
 \usepackage{fontspec}
@@ -3017,8 +3017,55 @@ spacing conventions and so on.\footnote{They are similar in concept,
 but not the same, as those in Unicode.}
 
 It currently embraces |\babelprehyphenation| and
-|\babelposthyphenation|, which have been available for several months.
-\New{3.56} In this version they can be defined in |ini| files, too.
+|\babelposthyphenation|.
+
+\New{3.57} Several \textsf{ini} files predefine some transforms. They
+are activated with the key |transforms| in |\babelprovide|, either if
+the locale is being defined with this macro or the languages has been
+previouly loaded as a class or package option, as the following example
+illustrates:
+\begin{verbatim}
+  \usepackage[magyar]{babel}
+  \babelprovide[_transforms = digraphs.hyphen_]{magyar}
+\end{verbatim}
+
+Here are the transforms currently predefined. (More to follow 
+in future releases.)
+
+\begingroup
+\def\trans#1#2#3{%
+  \vspace{1mm}%
+  \parbox[t]{2.4cm}{\strut#1}%
+  \parbox[t]{4.2cm}{\strut\ttfamily#2}%
+  \parbox[t]{6.6cm}{\strut#3}\par}
+\bigskip\hrule\nobreak\vspace{1mm}
+% \strut\hfil Transforms 
+% \medskip\hrule\nobreak
+
+\trans{Arabic}{transliteration.dad}{Applies the transliteration system
+devised by Yannis Haralambous for \textsf{dad} (simple and
+\TeX-friendly). Not yet complete, but sufficient for most texts.}
+
+\trans{Croatian}{digraphs.ligatures}{Ligatures \textit{DŽ}, \textit{Dž},
+\textit{dž}, \textit{LJ}, \textit{Lj}, \textit{lj}, \textit{NJ},
+\textit{Nj}, \textit{nj}. It assumes they exist. This is not the
+recommended way to make these transformations (the best way is with
+OTF features), but it can get you out of a hurry.}
+
+\trans{Greek}{diaeresis.hyphen}{Removes the diaeresis above iota and
+upsilon if hyphenated just before. It works with the
+three variants.}
+
+\trans{Hindi}{transliteration.hk}{The Harvard-Kyoto system to romanize
+Devanagari.}
+
+\trans{Hungarian}{digraphs.hyphen}{Hyphenates the long digraphs
+\textit{ccs}, \textit{ddz}, \textit{ggy}, \textit{lly}, \textit{nny},
+\textit{ssz}, \textit{tty} and \textit{zzs} as \textit{cs-cs},
+\textit{dz-dz}, etc.}
+
+\vspace{2mm}\hrule\nobreak
+\endgroup
 
 \Describe{\babelposthyphenation}{\marg{hyphenrules-name}%
           \marg{lua-pattern}\marg{replacement}}
@@ -3062,10 +3109,11 @@ future implementation may alternatively accept \textsf{lpeg}.
           \marg{lua-pattern}\marg{replacement}}
 
 \New{3.44-3-52} It is similar to the latter, but (as its name implies)
-applied before hyphenation. There are other differences: (1) the first
-argument is the locale instead the name of hyphenation patterns; (2) in
-the search patterns |=| has no special meaning, while \verb+|+ stands
-for an ordinary space; (3) in the replacement, discretionaries are not
+applied before hyphenation, which is particularly useful in
+transliterations. There are other differences: (1) the first argument
+is the locale instead of the name of the hyphenation patterns; (2) in the
+search patterns |=| has no special meaning, while \verb+|+ stands for
+an ordinary space; (3) in the replacement, discretionaries are not
 accepted.
 
 It handles glyphs and spaces.
@@ -3092,9 +3140,9 @@ This feature is activated with the first |\babelposthyphenation| or
   end of a line:
 \begin{verbatim}
 \babelprehyphenation{english}{|a|}
-  {}, {},                     % Keep first space and a
-  {insert, penalty = 10000},  % Insert penalty
-  {}                          % Keep last space
+  {}, {},                       % Keep first space and a
+  { insert, penalty = 10000 },  % Insert penalty
+  {}                            % Keep last space
 }
 \end{verbatim}
 \end{example}
@@ -4862,8 +4910,8 @@ help from Bernd Raichle, for which I am grateful.
 % \section{Tools}
 %
 %    \begin{macrocode}
-%<<version=3.56>>
-%<<date=2021/03/24>>
+%<<version=3.57>>
+%<<date=2021/04/07>>
 %    \end{macrocode}
 %
 % \textbf{Do not use the following macros in \texttt{ldf} files. They
@@ -10671,6 +10719,7 @@ help from Bernd Raichle, for which I am grateful.
   \let\bbl@KVP@intrapenalty\@nil
   \let\bbl@KVP@onchar\@nil
   \let\bbl@KVP@transforms\@nil
+  \global\let\bbl@release@transforms\@empty
   \let\bbl@KVP@alph\@nil
   \let\bbl@KVP@Alph\@nil
   \let\bbl@KVP@labels\@nil
@@ -10792,7 +10841,7 @@ help from Bernd Raichle, for which I am grateful.
           Babel.loc_to_scr[\the\localeid] =
             Babel.script_blocks['\bbl@cl{sbcp}']
         end}%
-      \ifx\bbl@mapselect\@undefined
+      \ifx\bbl@mapselect\@undefined  % TODO. almost the same as mapfont
         \AtBeginDocument{%
           \expandafter\bbl@add\csname selectfont \endcsname{{\bbl@mapselect}}%
           {\selectfont}}%
@@ -10820,7 +10869,7 @@ help from Bernd Raichle, for which I am grateful.
                  {See the manual for details.}}}%
     \bbl@ifunset{bbl@lsys@\languagename}{\bbl@provide@lsys{\languagename}}{}%
     \bbl@ifunset{bbl@wdir@\languagename}{\bbl@provide@dirs{\languagename}}{}%
-    \ifx\bbl@mapselect\@undefined
+    \ifx\bbl@mapselect\@undefined % TODO. See onchar
       \AtBeginDocument{%
         \expandafter\bbl@add\csname selectfont \endcsname{{\bbl@mapselect}}%
         {\selectfont}}%
@@ -10962,6 +11011,8 @@ help from Bernd Raichle, for which I am grateful.
          \let\atcatcode\relax
        \fi}%
   \fi
+  % == Release saved transforms ==
+  \bbl@release@transforms\relax % \relax closes the last item.
   % == main ==
   \ifx\bbl@KVP@main\@nil  % Restore only if not 'main'
     \let\languagename\bbl@savelangname
@@ -11339,8 +11390,6 @@ help from Bernd Raichle, for which I am grateful.
   \ifbbl@bcptoname
     \bbl@csarg\xdef{bcp@map@\bbl@cl{tbcp}}{\languagename}%
   \fi
-  % Finish here transforms, too
-  \bbl@release@transforms\relax  % \relax closes the last item.
   % Conditional
   \ifnum#1>\z@         % 0 = only info, 1, 2 = basic, (re)new
     \bbl@exportkey{lnbrk}{typography.linebreaking}{h}%
@@ -11721,10 +11770,10 @@ help from Bernd Raichle, for which I am grateful.
       \ifin@
         \in@{.0$}{#2$}&%
         \ifin@
-            \bbl@add\bbl@release@transforms{&%
+           \g@addto@macro\bbl@release@transforms{&%
               \relax\bbl@transforms@aux#1{\languagename}{#3}}&%
-          \else
-            \bbl@add\bbl@release@transforms{, {#3}}&%
+        \else
+           \g@addto@macro\bbl@release@transforms{, {#3}}&%
         \fi
       \fi
     \fi}
@@ -13360,10 +13409,10 @@ help from Bernd Raichle, for which I am grateful.
   Babel.linebreaking = Babel.linebreaking or {}
   Babel.linebreaking.before = {}
   Babel.linebreaking.after = {}
-  Babel.locale = {} % Free to use, indexed with \localeid
+  Babel.locale = {} % Free to use, indexed by \localeid
   function Babel.linebreaking.add_before(func)
     tex.print([[\noexpand\csname bbl@luahyphenate\endcsname]])
-    table.insert(Babel.linebreaking.before , func)
+    table.insert(Babel.linebreaking.before, func)
   end
   function Babel.linebreaking.add_after(func)
     tex.print([[\noexpand\csname bbl@luahyphenate\endcsname]])
@@ -13841,17 +13890,15 @@ end
         local locale = node.get_attribute(item, Babel.attr_locale)
 
         if lang == locale or lang == nil then
-          if (item.char ~= 124) then &% ie, not | = space
-            lang = lang or locale
-            word_string = word_string .. unicode.utf8.char(item.char)
-            word_nodes[#word_nodes+1] = item
-          end
+          lang = lang or locale
+          word_string = word_string .. unicode.utf8.char(item.char)
+          word_nodes[#word_nodes+1] = item
         else
           break
         end
 
       elseif item.id == 12 and item.subtype == 13 then
-        word_string = word_string .. '|'
+        word_string = word_string .. ' '
         word_nodes[#word_nodes+1] = item
 
       &% Ignore leading unrecognized nodes, too.
@@ -13865,7 +13912,7 @@ end
 
     &% Here and above we remove some trailing chars but not the
     &% corresponding nodes. But they aren't accessed. 
-    if word_string:sub(-1) == '|' then
+    if word_string:sub(-1) == ' ' then
       word_string = word_string:sub(1,-2)
     end
     word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '')
@@ -14050,6 +14097,16 @@ end
 
             if Babel.debug then
               Babel.debug_hyph(w, w_nodes, sc, first, last, last_match)
+              local ss = ''
+              for itt in node.traverse(head) do
+               if itt.id == 29 then
+                 ss = ss .. unicode.utf8.char(itt.char)
+               else
+                 ss = ss .. '{' .. itt.id .. '}'
+               end
+              end
+              print('*****************', ss)
+
             end
 
             local crep = r[rc]
@@ -14141,7 +14198,7 @@ end
                               crep.space[2] * quad,
                               crep.space[3] * quad)
               if mode == 0 then
-                placeholder = '|'
+                placeholder = ' '
               end
               head, new = node.insert_before(head, item, d)
 
@@ -14153,7 +14210,7 @@ end
                 crep.spacefactor[2] * base_font.parameters['space_stretch'],
                 crep.spacefactor[3] * base_font.parameters['space_shrink'])
               if mode == 0 then
-                placeholder = '|'
+                placeholder = ' '
               end
               head, new = node.insert_before(head, item, d)
 
@@ -14203,7 +14260,15 @@ end
   &% The following functions belong to the next macro
   function Babel.capture_func(key, cap)
     local ret = "[[" .. cap:gsub('{([0-9])}', "]]..m[%1]..[[") .. "]]"
-    ret = ret:gsub('{([0-9])|([^|]+)|(.-)}', Babel.capture_func_map)
+    local cnt
+    local u = unicode.utf8
+    ret, cnt = ret:gsub('{([0-9])|([^|]+)|(.-)}', Babel.capture_func_map)
+    if cnt == 0 then
+      ret = u.gsub(ret, '{(%x%x%x%x+)}', 
+            function (n)
+              return u.char(tonumber(n, 16))
+            end)
+    end
     ret = ret:gsub("%[%[%]%]%.%.", '')
     ret = ret:gsub("%.%.%[%[%]%]", '')
     return key .. [[=function(m) return ]] .. ret .. [[ end]]
@@ -14215,6 +14280,15 @@ end
 
   &% Handle the {n|abc|ABC} syntax in captures
   function Babel.capture_func_map(capno, from, to)
+    local u = unicode.utf8
+    from = u.gsub(from, '{(%x%x%x%x+)}', 
+         function (n)
+           return u.char(tonumber(n, 16))
+         end)
+    to = u.gsub(to, '{(%x%x%x%x+)}', 
+         function (n)
+           return u.char(tonumber(n, 16))
+         end)
     local froms = {}
     for s in string.utfcharacters(from) do
       table.insert(froms, s)
@@ -14319,6 +14393,7 @@ end
       local id = \the\csname bbl@id@@#1\endcsname
       &% Convert pattern:
       local patt = string.gsub([==[#2]==], '%s', '')
+      local patt = string.gsub(patt, '|', ' ')
       if not u.find(patt, '()', nil, true) then
         patt = '()' .. patt .. '()'
       end     
@@ -14334,7 +14409,7 @@ end
              end)
       lbkr[id] = lbkr[id] or {}
       table.insert(lbkr[id], { pattern = patt, replace = { \babeltempb } })
-    }&%
+    }&%    
   \endgroup}
 \endgroup
 \def\bbl@activateposthyphen{%
diff --git a/Master/texmf-dist/source/latex/babel/babel.ins b/Master/texmf-dist/source/latex/babel/babel.ins
index bb675249c6f..ddedf2c6678 100644
--- a/Master/texmf-dist/source/latex/babel/babel.ins
+++ b/Master/texmf-dist/source/latex/babel/babel.ins
@@ -26,7 +26,7 @@
 %% and covered by LPPL is defined by the unpacking scripts (with
 %% extension .ins) which are part of the distribution.
 %%
-\def\filedate{2021/03/24}
+\def\filedate{2021/04/07}
 \def\batchfile{babel.ins}
 \input docstrip.tex
 
diff --git a/Master/texmf-dist/source/latex/babel/bbcompat.dtx b/Master/texmf-dist/source/latex/babel/bbcompat.dtx
index efd3eb77dbf..ea7e8ecc5ab 100644
--- a/Master/texmf-dist/source/latex/babel/bbcompat.dtx
+++ b/Master/texmf-dist/source/latex/babel/bbcompat.dtx
@@ -30,7 +30,7 @@
 %
 % \iffalse
 %<*dtx>
-\ProvidesFile{bbcompat.dtx}[2021/03/24 v3.56]
+\ProvidesFile{bbcompat.dtx}[2021/04/07 v3.57]
 %</dtx>
 %
 %% File 'bbcompat.dtx'
diff --git a/Master/texmf-dist/source/latex/babel/locale.zip b/Master/texmf-dist/source/latex/babel/locale.zip
index 0f9331de0cd..2845c0f244f 100644
--- a/Master/texmf-dist/source/latex/babel/locale.zip
+++ b/Master/texmf-dist/source/latex/babel/locale.zip
diff --git a/Master/texmf-dist/tex/generic/babel/babel.def b/Master/texmf-dist/tex/generic/babel/babel.def
index 98495230347..7c5ec15a9bc 100644
--- a/Master/texmf-dist/tex/generic/babel/babel.def
+++ b/Master/texmf-dist/tex/generic/babel/babel.def
@@ -39,7 +39,7 @@
     \wlog{File: #1 #4 #3 <#2>}%
     \let\ProvidesFile\@undefined}
 \fi
-\ProvidesFile{babel.def}[2021/03/24 3.56 Babel common definitions]
+\ProvidesFile{babel.def}[2021/04/07 3.57 Babel common definitions]
 \ifx\AtBeginDocument\@undefined  % TODO. change test.
     % == Code for plain ==
 \def\@empty{}
@@ -397,8 +397,8 @@
 \fi
 \countdef\last@language=19  % TODO. why? remove?
 \def\addlanguage{\csname newlanguage\endcsname}
-\def\bbl@version{3.56}
-\def\bbl@date{2021/03/24}
+\def\bbl@version{3.57}
+\def\bbl@date{2021/04/07}
 \def\adddialect#1#2{%
   \global\chardef#1#2\relax
   \bbl@usehooks{adddialect}{{#1}{#2}}%
@@ -2336,6 +2336,7 @@
   \let\bbl@KVP@intrapenalty\@nil
   \let\bbl@KVP@onchar\@nil
   \let\bbl@KVP@transforms\@nil
+  \global\let\bbl@release@transforms\@empty
   \let\bbl@KVP@alph\@nil
   \let\bbl@KVP@Alph\@nil
   \let\bbl@KVP@labels\@nil
@@ -2457,7 +2458,7 @@
           Babel.loc_to_scr[\the\localeid] =
             Babel.script_blocks['\bbl@cl{sbcp}']
         end}%
-      \ifx\bbl@mapselect\@undefined
+      \ifx\bbl@mapselect\@undefined  % TODO. almost the same as mapfont
         \AtBeginDocument{%
           \expandafter\bbl@add\csname selectfont \endcsname{{\bbl@mapselect}}%
           {\selectfont}}%
@@ -2485,7 +2486,7 @@
                  {See the manual for details.}}}%
     \bbl@ifunset{bbl@lsys@\languagename}{\bbl@provide@lsys{\languagename}}{}%
     \bbl@ifunset{bbl@wdir@\languagename}{\bbl@provide@dirs{\languagename}}{}%
-    \ifx\bbl@mapselect\@undefined
+    \ifx\bbl@mapselect\@undefined % TODO. See onchar
       \AtBeginDocument{%
         \expandafter\bbl@add\csname selectfont \endcsname{{\bbl@mapselect}}%
         {\selectfont}}%
@@ -2627,6 +2628,8 @@
          \let\atcatcode\relax
        \fi}%
   \fi
+  % == Release saved transforms ==
+  \bbl@release@transforms\relax % \relax closes the last item.
   % == main ==
   \ifx\bbl@KVP@main\@nil  % Restore only if not 'main'
     \let\languagename\bbl@savelangname
@@ -2924,8 +2927,6 @@
   \ifbbl@bcptoname
     \bbl@csarg\xdef{bcp@map@\bbl@cl{tbcp}}{\languagename}%
   \fi
-  % Finish here transforms, too
-  \bbl@release@transforms\relax  % \relax closes the last item.
   % Conditional
   \ifnum#1>\z@         % 0 = only info, 1, 2 = basic, (re)new
     \bbl@exportkey{lnbrk}{typography.linebreaking}{h}%
@@ -3236,10 +3237,10 @@
       \ifin@
         \in@{.0$}{#2$}&%
         \ifin@
-            \bbl@add\bbl@release@transforms{&%
+           \g@addto@macro\bbl@release@transforms{&%
               \relax\bbl@transforms@aux#1{\languagename}{#3}}&%
-          \else
-            \bbl@add\bbl@release@transforms{, {#3}}&%
+        \else
+           \g@addto@macro\bbl@release@transforms{, {#3}}&%
         \fi
       \fi
     \fi}
diff --git a/Master/texmf-dist/tex/generic/babel/babel.sty b/Master/texmf-dist/tex/generic/babel/babel.sty
index ca2c203a858..8264f8ade70 100644
--- a/Master/texmf-dist/tex/generic/babel/babel.sty
+++ b/Master/texmf-dist/tex/generic/babel/babel.sty
@@ -33,7 +33,7 @@
 %%
 
 \NeedsTeXFormat{LaTeX2e}[2005/12/01]
-\ProvidesPackage{babel}[2021/03/24 3.56 The Babel package]
+\ProvidesPackage{babel}[2021/04/07 3.57 The Babel package]
 \@ifpackagewith{babel}{debug}
   {\providecommand\bbl@trace[1]{\message{^^J[ #1 ]}}%
    \let\bbl@debug\@firstofone
diff --git a/Master/texmf-dist/tex/generic/babel/hyphen.cfg b/Master/texmf-dist/tex/generic/babel/hyphen.cfg
index df8e10deb11..90f8b3d5d4f 100644
--- a/Master/texmf-dist/tex/generic/babel/hyphen.cfg
+++ b/Master/texmf-dist/tex/generic/babel/hyphen.cfg
@@ -37,10 +37,10 @@
     \wlog{File: #1 #4 #3 <#2>}%
     \let\ProvidesFile\@undefined}
 \fi
-\ProvidesFile{hyphen.cfg}[2021/03/24 3.56 Babel hyphens]
+\ProvidesFile{hyphen.cfg}[2021/04/07 3.57 Babel hyphens]
 \xdef\bbl@format{\jobname}
-\def\bbl@version{3.56}
-\def\bbl@date{2021/03/24}
+\def\bbl@version{3.57}
+\def\bbl@date{2021/04/07}
 \ifx\AtBeginDocument\@undefined
   \def\@empty{}
   \let\orig@dump\dump
diff --git a/Master/texmf-dist/tex/generic/babel/locale/ar/babel-ar.ini b/Master/texmf-dist/tex/generic/babel/locale/ar/babel-ar.ini
index 8ba86c631d0..de5c72b9d04 100644
--- a/Master/texmf-dist/tex/generic/babel/locale/ar/babel-ar.ini
+++ b/Master/texmf-dist/tex/generic/babel/locale/ar/babel-ar.ini
@@ -11,8 +11,8 @@
 
 [identification]
 charset = utf8
-version = 1.7
-date = 2020-10-11
+version = 1.8
+date = 2021-04-05
 name.local = العربية
 name.english = Arabic
 name.babel = arabic
@@ -176,4 +176,35 @@ superscriptingExponent = ×
 abjad = ا ب ج د ه‍ و ز ح ط ي ك ل م ن س ع ف ص ق ر ش ت ث خ ذ ض ظ غ
 maghrebi.abjad = ا ب ج د ه‍ و ز ح ط ي ك ل م ن ص ع ف ض ق ر س ت ث خ ذ ظ غ ش
 
+[transforms.prehyphenation]
+; Yannis Haralambous' system in his 'dad'. Only two rules are missing.
+; 3-letter
+transliteration.dad.1.0 = { {+}a{*} }
+transliteration.dad.1.2 =   { string = ٰ }
+transliteration.dad.1.2 =   { string = ّ }
+transliteration.dad.1.3 =   { remove }
+transliteration.dad.2.0 = { {+}([aui])N }
+transliteration.dad.2.2 =   { string = {1|aui|ًٌٍ} }
+transliteration.dad.2.2 =   { string = ّ }
+transliteration.dad.2.3 =   { remove }
+; 2-letter
+transliteration.dad.3.0 = { ([aui])N }
+transliteration.dad.3.1 =   { string = {1|aui|ًٌٍ} }
+transliteration.dad.3.2 =   { remove }
+transliteration.dad.4.0 = { ([tAa]){*} }
+transliteration.dad.4.1 =   { string = {1|tAa|ةٱٰ} }
+transliteration.dad.4.2 =   { remove } 
+transliteration.dad.5.0 = { ([tds])h }
+transliteration.dad.5.1 =   { string = {1|tds|ثذش} }
+transliteration.dad.5.2 =   { remove }
+transliteration.dad.6.0 = { {+}([aui]) }
+transliteration.dad.6.1 =   { string = {1|aui|َُِ} } 
+transliteration.dad.6.2 =   { string = ّ }
+transliteration.dad.7.0 = { '([AauiIbnf]) }
+transliteration.dad.7.1 =   { remove }
+transliteration.dad.7.2 =   { string = {1|AauiIbnf|آأؤإئٮںڡ} }
+; 1-letter
+transliteration.dad.8.0 = { ([{007C}AbtjHxdrzsSDTZ`RfqklmnhUIYaui+opCvgJe]) }
+transliteration.dad.8.1 =   { string = {1|{007C}AbtjHxdrzsSDTZ`RfqklmnhUIYaui+opCvgJe|ءابتجحخدرزسصضطظعغفقكلمنهوىيَُِّْپچڤگژۀ} }
+
 
diff --git a/Master/texmf-dist/tex/generic/babel/locale/el/babel-el-polyton.ini b/Master/texmf-dist/tex/generic/babel/locale/el/babel-el-polyton.ini
index 9d97ac573c1..dd1f9ad0069 100644
--- a/Master/texmf-dist/tex/generic/babel/locale/el/babel-el-polyton.ini
+++ b/Master/texmf-dist/tex/generic/babel/locale/el/babel-el-polyton.ini
@@ -4,8 +4,8 @@
 
 [identification]
 charset             = utf8
-version = 0.6
-date = 2020-06-30
+version = 0.7
+date = 2021-04-05
 name.local          = Ἐλληνικά
 ; Name from https://www.iana.org/assignments/language-subtag-registry/language-subtag-registry
 name.english        = Polytonic Greek
@@ -204,4 +204,7 @@ upper.ancient.3 = Ρ Σ Τ Υ Φ Χ Ψ Ω Ϡ
 upper.ancient.4 = ͵Α ͵Β ͵Γ ͵Δ ͵Ε ͵Ϛ ͵Ζ ͵Η ͵Θ
 upper.ancient.S.321 = ʹ
 
-
+[transforms.posthyphenation]
+diaeresis.hyphen.1.0 = { |([ϊΐῒῗϋΰῢῧ]) }
+diaeresis.hyphen.1.1 = { remove }
+diaeresis.hyphen.1.2 = { no = {1}, pre = -, post = {1|ϊΐῒῗϋΰῢῧ|ιίὶῖυύὺῦ} }
diff --git a/Master/texmf-dist/tex/generic/babel/locale/el/babel-el.ini b/Master/texmf-dist/tex/generic/babel/locale/el/babel-el.ini
index 30d1e4427cf..39b93e92ac3 100644
--- a/Master/texmf-dist/tex/generic/babel/locale/el/babel-el.ini
+++ b/Master/texmf-dist/tex/generic/babel/locale/el/babel-el.ini
@@ -11,8 +11,8 @@
 
 [identification]
 charset = utf8
-version = 1.5
-date = 2020-06-30
+version = 1.6
+date = 2021-04-05
 name.local = Ελληνικά
 name.english = Greek
 name.babel = greek
@@ -210,3 +210,8 @@ upper.ancient.3 = Ρ Σ Τ Υ Φ Χ Ψ Ω Ϡ
 upper.ancient.4 = ͵Α ͵Β ͵Γ ͵Δ ͵Ε ͵Ϛ ͵Ζ ͵Η ͵Θ
 upper.ancient.S.321 = ʹ
 
+[transforms.posthyphenation]
+diaeresis.hyphen.1.0 = { |([ϊΐϋΰ]) }
+diaeresis.hyphen.1.1 = { remove }
+diaeresis.hyphen.1.2 = { no = {1}, pre = -, post = {1|ϊΐϋΰ|ιίυύ} }
+
diff --git a/Master/texmf-dist/tex/generic/babel/locale/es/babel-es.ini b/Master/texmf-dist/tex/generic/babel/locale/es/babel-es.ini
index 733293f938f..5ed832dd28d 100644
--- a/Master/texmf-dist/tex/generic/babel/locale/es/babel-es.ini
+++ b/Master/texmf-dist/tex/generic/babel/locale/es/babel-es.ini
@@ -183,10 +183,3 @@ plusSign = +
 superscriptingExponent = ×
 
 [counters]
-
-[transforms.prehyphenation]
-; Experimental. Don't use in production
-percent.space.1.0 = { {d}{0025} }
-percent.space.1.1 = {}
-percent.space.1.2 = { insert, spacefactor = .5 .3 .5, data = 1 }
-percent.space.1.3 = {}
diff --git a/Master/texmf-dist/tex/generic/babel/locale/grc/babel-grc.ini b/Master/texmf-dist/tex/generic/babel/locale/grc/babel-grc.ini
index 03a525d346a..818b1ca7a0f 100644
--- a/Master/texmf-dist/tex/generic/babel/locale/grc/babel-grc.ini
+++ b/Master/texmf-dist/tex/generic/babel/locale/grc/babel-grc.ini
@@ -11,8 +11,8 @@
 
 [identification]
 charset = utf8
-version = 1.5
-date = 2020-06-30
+version = 1.6
+date = 2021-04-05
 name.local = Αρχαία ελληνικά
 name.english = Ancient Greek 
 name.babel = greek
@@ -185,4 +185,9 @@ upper.ancient.1 = Α Β Γ Δ Ε Ϛ Ζ Η Θ
 upper.ancient.2 = Ι Κ Λ Μ Ν Ξ Ο Π Ϟ
 upper.ancient.3 = Ρ Σ Τ Υ Φ Χ Ψ Ω Ϡ
 upper.ancient.4 = ͵Α ͵Β ͵Γ ͵Δ ͵Ε ͵Ϛ ͵Ζ ͵Η ͵Θ
-upper.ancient.S.321 = ʹ
\ No newline at end of file
+upper.ancient.S.321 = ʹ
+
+[transforms.posthyphenation]
+diaeresis.hyphen.1.0 = { |([ϊΐῒῗϋΰῢῧ]) }
+diaeresis.hyphen.1.1 = { remove }
+diaeresis.hyphen.1.2 = { no = {1}, pre = -, post = {1|ϊΐῒῗϋΰῢῧ|ιίὶῖυύὺῦ} }
\ No newline at end of file
diff --git a/Master/texmf-dist/tex/generic/babel/locale/hi/babel-hi.ini b/Master/texmf-dist/tex/generic/babel/locale/hi/babel-hi.ini
index 7419d2f4e8d..a230ec2a47f 100644
--- a/Master/texmf-dist/tex/generic/babel/locale/hi/babel-hi.ini
+++ b/Master/texmf-dist/tex/generic/babel/locale/hi/babel-hi.ini
@@ -12,8 +12,8 @@
 
 [identification]
 charset = utf8
-version = 1.8
-date = 2020-10-11
+version = 1.9
+date = 2021-04-05
 name.local = हिन्दी
 name.english = Hindi
 name.babel = hindi
@@ -201,3 +201,44 @@ superscriptingExponent = ×
 
 [counters]
 alphabetic = क ख ग घ ङ च छ ज झ ञ ट ठ ड ढ ण त थ द ध न प फ ब भ म य र ल व श ष स ह
+
+[transforms.prehyphenation]
+; Multiletter
+transliteration.hk.1.0  = { lRR } 
+transliteration.hk.1.1  =   { string = ॡ }
+transliteration.hk.1.2  =   { remove }
+transliteration.hk.1.3  =   { remove }
+transliteration.hk.2.0  = { OM } 
+transliteration.hk.2.1  =   { string = ॐ }
+transliteration.hk.2.2  =   { remove }
+transliteration.hk.3.0  = { {007C}{007C} } 
+transliteration.hk.3.1  =   { string = ॥ }
+transliteration.hk.3.2  =   { remove }
+transliteration.hk.4.0  = { ([lR])R }
+transliteration.hk.4.1  =   { string = {1|lR|ऌॠ} }
+transliteration.hk.4.2  =   { remove }
+transliteration.hk.5.0  = { a([iu]) }
+transliteration.hk.5.1  =   { remove }
+transliteration.hk.5.2  =   { string = {1|iu|ऐऔ} }
+; Xh
+transliteration.hk.6.0  = { ([kgcjTDtdpb])h }
+transliteration.hk.6.1  =   { string = {1|kgcjTDtdpb|खघछझठढथधफभ} }
+transliteration.hk.6.2  =   { remove }
+; Single letter
+transliteration.hk.7.0  = { ([MHaAiIuUReokgGcjJTDNtdnpbmyrlvzSshL{007C}']) }
+transliteration.hk.7.1  =   { string = {1|MHaAiIuUReokgGcjJTDNtdnpbmyrlvzSshL{007C}'|ंःअआइईउऊऋएओकगङचजञटडणतदनपबमयरलवशषसहळ।ऽ} }
+; Post-process vowels
+transliteration.hk.8.0  = { [{0915}-{0939}]([आइईउऊऋॠऌॡएऐओऔऎऒ]) }
+transliteration.hk.8.1  =   {}
+transliteration.hk.8.2  =   { string = {1|आइईउऊऋॠऌॡएऐओऔऎऒ|ािीुूृॄॢॣेैोौॆॊ} }
+transliteration.hk.9.0  = { [{0915}-{0939}]([{0915}-{0939}]) }
+transliteration.hk.9.1  =   {}
+transliteration.hk.9.2  =   { string = ^^^^094d{1} }
+; Duplicated (a trick)
+transliteration.hk.10.0 = { [{0915}-{0939}]([{0915}-{0939}]) }
+transliteration.hk.10.1 =   {}
+transliteration.hk.10.2 =   { string = ^^^^094d{1} }
+; Inherent a
+transliteration.hk.11.0 = { [{0915}-{0939}]{0905} }
+transliteration.hk.11.1 =   {}
+transliteration.hk.11.2 =   { remove }
diff --git a/Master/texmf-dist/tex/generic/babel/locale/hr/babel-hr.ini b/Master/texmf-dist/tex/generic/babel/locale/hr/babel-hr.ini
index 7692bbb3c33..dc9a604f1c9 100644
--- a/Master/texmf-dist/tex/generic/babel/locale/hr/babel-hr.ini
+++ b/Master/texmf-dist/tex/generic/babel/locale/hr/babel-hr.ini
@@ -11,8 +11,8 @@
 
 [identification]
 charset = utf8
-version = 1.2
-date = 2020-06-30
+version = 1.3
+date = 2021-04-05
 name.local = hrvatski
 name.english = Croatian
 name.babel = croatian
@@ -183,3 +183,17 @@ plusSign = +
 superscriptingExponent = ×
 
 [counters]
+
+[transforms.prehyphenation]
+digraphs.ligatures.1.0 = { D([Žž]) }
+digraphs.ligatures.1.1 = { remove }
+digraphs.ligatures.1.2 = { string = {1|Žž|DŽDž} }
+digraphs.ligatures.2.0 = { dž }
+digraphs.ligatures.2.1 = { string = dž }
+digraphs.ligatures.2.2 = { remove }
+digraphs.ligatures.3.0 = { ([LN])J }
+digraphs.ligatures.3.1 = { string = {1|LN|LJNJ} }
+digraphs.ligatures.3.2 = { remove }
+digraphs.ligatures.4.0 = { ([LlNn])j }
+digraphs.ligatures.4.1 = { string = {1|LlNn|LjljNjnj} }
+digraphs.ligatures.4.2 = { remove }
\ No newline at end of file
diff --git a/Master/texmf-dist/tex/generic/babel/locale/hu/babel-hu.ini b/Master/texmf-dist/tex/generic/babel/locale/hu/babel-hu.ini
index b681831d8d4..e6a69cd3415 100644
--- a/Master/texmf-dist/tex/generic/babel/locale/hu/babel-hu.ini
+++ b/Master/texmf-dist/tex/generic/babel/locale/hu/babel-hu.ini
@@ -11,8 +11,8 @@
 
 [identification]
 charset = utf8
-version = 1.3
-date = 2020-08-06
+version = 1.4
+date = 2021-04-05
 name.local = magyar
 name.english = Hungarian
 name.babel = hungarian
@@ -192,3 +192,21 @@ plusSign = +
 superscriptingExponent = ×
 
 [counters]
+
+[transforms.posthyphenation]
+digraphs.hyphen.1.0 = { ([czCZ])|{1}([sS]) }
+digraphs.hyphen.1.1 = {} 
+digraphs.hyphen.1.2 = { pre = {2}-, data = 1 }
+digraphs.hyphen.1.3 = {}
+digraphs.hyphen.1.4 = {}
+digraphs.hyphen.2.0 = { ([dsDS])|{1}([zZ]) }
+digraphs.hyphen.2.1 = {} 
+digraphs.hyphen.2.2 = { pre = {2}-, data = 1 }
+digraphs.hyphen.2.3 = {}
+digraphs.hyphen.2.4 = {}
+digraphs.hyphen.3.0 = { ([glntGLNT])|{1}([yY]) }
+digraphs.hyphen.3.1 = {} 
+digraphs.hyphen.3.2 = { pre = {2}-, data = 1 }
+digraphs.hyphen.3.3 = {}
+digraphs.hyphen.3.4 = {}
+
diff --git a/Master/texmf-dist/tex/generic/babel/locale/hu/babel-magyar.tex b/Master/texmf-dist/tex/generic/babel/locale/hu/babel-magyar.tex
new file mode 100644
index 00000000000..a774379bd90
--- /dev/null
+++ b/Master/texmf-dist/tex/generic/babel/locale/hu/babel-magyar.tex
@@ -0,0 +1,12 @@
+% This file is part of babel. For further details see:
+% https://www.ctan.org/pkg/babel
+\ifx\BabelBeforeIni\undefined
+  \PackageError{babel}%
+    {This file is a component of babel and cannot\MessageBreak
+     be loaded directly. I'll stop immediately}%
+    {Just use babel as documented.}%
+  \stop
+\fi
+\BabelBeforeIni{hu}{%
+}
+\endinput
\ No newline at end of file
diff --git a/Master/texmf-dist/tex/generic/babel/locale/mr/babel-mr.ini b/Master/texmf-dist/tex/generic/babel/locale/mr/babel-mr.ini
index 1178e561099..927dd8b4986 100644
--- a/Master/texmf-dist/tex/generic/babel/locale/mr/babel-mr.ini
+++ b/Master/texmf-dist/tex/generic/babel/locale/mr/babel-mr.ini
@@ -13,8 +13,8 @@
 
 [identification]
 charset = utf8
-version = 1.9
-date = 2021-03-16
+version = 1.10
+date = 2021-04-05
 name.local = मराठी
 name.english = Marathi
 name.babel = marathi
@@ -113,7 +113,7 @@ time.medium = [h]:[mm]:[ss] [a]
 time.short = [h]:[mm] [a]
 
 [typography]
-frenchspacing = yes
+frenchspacing = no
 hyphenrules = marathi
 lefthyphenmin = 2
 righthyphenmin = 2
diff --git a/Master/texmf-dist/tex/generic/babel/locale/ug/babel-ug.ini b/Master/texmf-dist/tex/generic/babel/locale/ug/babel-ug.ini
index 104cf00c468..395d9136094 100644
--- a/Master/texmf-dist/tex/generic/babel/locale/ug/babel-ug.ini
+++ b/Master/texmf-dist/tex/generic/babel/locale/ug/babel-ug.ini
@@ -12,8 +12,8 @@
 
 [identification]
 charset = utf8
-version = 1.5
-date = 2021-02-26
+version = 1.6
+date = 2021-04-05
 name.local = ئۇيغۇرچە
 name.english = Uyghur
 name.babel = uyghur
@@ -27,6 +27,7 @@ script.tag.opentype = arab
 level = 1
 encodings = 
 derivate = no
+require.babel = uyghur
 
 [captions]
 preface = كىرىش سۆز
diff --git a/Master/texmf-dist/tex/generic/babel/locale/ug/babel-uyghur.tex b/Master/texmf-dist/tex/generic/babel/locale/ug/babel-uyghur.tex
index 0963ec194e9..d3ee2eab655 100644
--- a/Master/texmf-dist/tex/generic/babel/locale/ug/babel-uyghur.tex
+++ b/Master/texmf-dist/tex/generic/babel/locale/ug/babel-uyghur.tex
@@ -9,4 +9,67 @@
 \fi
 \BabelBeforeIni{ug}{%
 }
+
+\newattribute\bblug@disc
+\bblug@disc=-1
+
+\bbl@luahyphenate
+
+% 1) Store discretionaries just after hyphenation as an attribute of the
+% next glyph, with the value of the disc penalty (assumed positive). Then
+% remove the discretionary. 
+% 2) After the shaping, restore the discretionaries.
+
+\directlua{
+Babel.locale[\the\localeid] = {}
+local ug = Babel.locale[\the\localeid]
+
+function ug.posthyphen(head)
+  local UGDISC = luatexbase.registernumber'bblug@disc'
+  for item in node.traverse(head) do
+    if item.id == 7 and item.subtype == 3 and
+        item.next and item.next.id == 29 and
+        item.next.lang == \the\l@uyghur\space then 
+      node.set_attribute(item.next, UGDISC, item.penalty)
+      node.remove(head, item)
+    end
+  end
+end
+
+ug.hyphen_sep = .09   % in em units
+% Note it can be a string, with several characters:
+ug.hyphen = unicode.utf8.char(0x0640)
+
+Babel.linebreaking.add_after(ug.posthyphen)
+
+function ug.hyphenate(head) 
+  local d, k
+  local quad = 655360
+  local UGDISC = luatexbase.registernumber'bblug@disc'
+  for item in node.traverse(head) do
+    if item.id == 29 and item.lang == \the\l@uyghur\space then
+      local ugdisc = node.get_attribute(item, UGDISC)
+      if ugdisc >= 0 then    
+        quad = font.getfont(item.font).size or quad
+        k = node.new(13, 1)  % (kern, userkern)
+        k.kern = ug.hyphen_sep * quad
+        d = node.new(7, 3)   % (disc, regular)
+        d.pre = Babel.str_to_nodes(
+                      function() return ug.hyphen end, 
+                      nil, item)
+        d.pre = node.insert_before(d.pre, d.pre, k)
+        d.penalty = ugdisc
+        head = node.insert_before(head, item, d)
+      end
+    end
+  end
+  return head
+end
+
+luatexbase.add_to_callback("pre_linebreak_filter",
+  ug.hyphenate, "Babel.locale.uyghur.hyphenate")
+luatexbase.add_to_callback("hpack_filter",
+  ug.hyphenate, "Babel.locale.uyghur.hyphenate")
+}
+
 \endinput
\ No newline at end of file
diff --git a/Master/texmf-dist/tex/generic/babel/luababel.def b/Master/texmf-dist/tex/generic/babel/luababel.def
index 0d00d5b238d..31f874fc4c5 100644
--- a/Master/texmf-dist/tex/generic/babel/luababel.def
+++ b/Master/texmf-dist/tex/generic/babel/luababel.def
@@ -333,10 +333,10 @@
   Babel.linebreaking = Babel.linebreaking or {}
   Babel.linebreaking.before = {}
   Babel.linebreaking.after = {}
-  Babel.locale = {} % Free to use, indexed with \localeid
+  Babel.locale = {} % Free to use, indexed by \localeid
   function Babel.linebreaking.add_before(func)
     tex.print([[\noexpand\csname bbl@luahyphenate\endcsname]])
-    table.insert(Babel.linebreaking.before , func)
+    table.insert(Babel.linebreaking.before, func)
   end
   function Babel.linebreaking.add_after(func)
     tex.print([[\noexpand\csname bbl@luahyphenate\endcsname]])
@@ -951,17 +951,15 @@ end
         local locale = node.get_attribute(item, Babel.attr_locale)
 
         if lang == locale or lang == nil then
-          if (item.char ~= 124) then &% ie, not | = space
-            lang = lang or locale
-            word_string = word_string .. unicode.utf8.char(item.char)
-            word_nodes[#word_nodes+1] = item
-          end
+          lang = lang or locale
+          word_string = word_string .. unicode.utf8.char(item.char)
+          word_nodes[#word_nodes+1] = item
         else
           break
         end
 
       elseif item.id == 12 and item.subtype == 13 then
-        word_string = word_string .. '|'
+        word_string = word_string .. ' '
         word_nodes[#word_nodes+1] = item
 
       &% Ignore leading unrecognized nodes, too.
@@ -975,7 +973,7 @@ end
 
     &% Here and above we remove some trailing chars but not the
     &% corresponding nodes. But they aren't accessed.
-    if word_string:sub(-1) == '|' then
+    if word_string:sub(-1) == ' ' then
       word_string = word_string:sub(1,-2)
     end
     word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '')
@@ -1160,6 +1158,16 @@ end
 
             if Babel.debug then
               Babel.debug_hyph(w, w_nodes, sc, first, last, last_match)
+              local ss = ''
+              for itt in node.traverse(head) do
+               if itt.id == 29 then
+                 ss = ss .. unicode.utf8.char(itt.char)
+               else
+                 ss = ss .. '{' .. itt.id .. '}'
+               end
+              end
+              print('*****************', ss)
+
             end
 
             local crep = r[rc]
@@ -1251,7 +1259,7 @@ end
                               crep.space[2] * quad,
                               crep.space[3] * quad)
               if mode == 0 then
-                placeholder = '|'
+                placeholder = ' '
               end
               head, new = node.insert_before(head, item, d)
 
@@ -1263,7 +1271,7 @@ end
                 crep.spacefactor[2] * base_font.parameters['space_stretch'],
                 crep.spacefactor[3] * base_font.parameters['space_shrink'])
               if mode == 0 then
-                placeholder = '|'
+                placeholder = ' '
               end
               head, new = node.insert_before(head, item, d)
 
@@ -1313,7 +1321,15 @@ end
   &% The following functions belong to the next macro
   function Babel.capture_func(key, cap)
     local ret = "[[" .. cap:gsub('{([0-9])}', "]]..m[%1]..[[") .. "]]"
-    ret = ret:gsub('{([0-9])|([^|]+)|(.-)}', Babel.capture_func_map)
+    local cnt
+    local u = unicode.utf8
+    ret, cnt = ret:gsub('{([0-9])|([^|]+)|(.-)}', Babel.capture_func_map)
+    if cnt == 0 then
+      ret = u.gsub(ret, '{(%x%x%x%x+)}',
+            function (n)
+              return u.char(tonumber(n, 16))
+            end)
+    end
     ret = ret:gsub("%[%[%]%]%.%.", '')
     ret = ret:gsub("%.%.%[%[%]%]", '')
     return key .. [[=function(m) return ]] .. ret .. [[ end]]
@@ -1325,6 +1341,15 @@ end
 
   &% Handle the {n|abc|ABC} syntax in captures
   function Babel.capture_func_map(capno, from, to)
+    local u = unicode.utf8
+    from = u.gsub(from, '{(%x%x%x%x+)}',
+         function (n)
+           return u.char(tonumber(n, 16))
+         end)
+    to = u.gsub(to, '{(%x%x%x%x+)}',
+         function (n)
+           return u.char(tonumber(n, 16))
+         end)
     local froms = {}
     for s in string.utfcharacters(from) do
       table.insert(froms, s)
@@ -1411,6 +1436,7 @@ end
       local id = \the\csname bbl@id@@#1\endcsname
       &% Convert pattern:
       local patt = string.gsub([==[#2]==], '%s', '')
+      local patt = string.gsub(patt, '|', ' ')
       if not u.find(patt, '()', nil, true) then
         patt = '()' .. patt .. '()'
       end
diff --git a/Master/texmf-dist/tex/generic/babel/nil.ldf b/Master/texmf-dist/tex/generic/babel/nil.ldf
index 4d4b1e326a5..6eb01e492c6 100644
--- a/Master/texmf-dist/tex/generic/babel/nil.ldf
+++ b/Master/texmf-dist/tex/generic/babel/nil.ldf
@@ -32,7 +32,7 @@
 %% extension |.ins|) which are part of the distribution.
 %%
 
-\ProvidesLanguage{nil}[2021/03/24 3.56 Nil language]
+\ProvidesLanguage{nil}[2021/04/07 3.57 Nil language]
 \LdfInit{nil}{datenil}
 \ifx\l@nil\@undefined
   \newlanguage\l@nil