babel (3feb20)

git-svn-id: svn://tug.org/texlive/trunk@53660 c570f23f-e606-0410-a88d-b1316a301751

1 files changed, 363 insertions, 215 deletions

diff --git a/Master/texmf-dist/source/latex/babel/babel.dtx b/Master/texmf-dist/source/latex/babel/babel.dtx
index 0d4360f493e..d0a6314f27d 100644
--- a/Master/texmf-dist/source/latex/babel/babel.dtx
+++ b/Master/texmf-dist/source/latex/babel/babel.dtx
@@ -1,7 +1,7 @@
 % \iffalse
 % This document requires lualatex
 %%
-%% Copyright (C) 2012-2019 Javier Bezos and Johannes L. Braams.
+%% Copyright (C) 2012-2020 Javier Bezos and Johannes L. Braams.
 %% Copyright (C) 1989-2012 Johannes L. Braams and
 %%           any individual authors listed elsewhere in this file.
 %% All rights reserved.
@@ -31,7 +31,7 @@
 %
 % \iffalse
 %<*filedriver>
-\ProvidesFile{babel.dtx}[2020/01/15 v3.38 The Babel package]
+\ProvidesFile{babel.dtx}[2020/02/03 v3.39 The Babel package]
 \documentclass{ltxdoc}
 \GetFileInfo{babel.dtx}
 \usepackage{fontspec}
@@ -235,21 +235,25 @@ Xe\largetex
 \part{User guide}
 
 \begin{itemize}
-\item This user guide focuses on \LaTeX. There are also some notes on
-  its use with Plain \TeX.
+\item This user guide focuses on internationalization and localization
+  with \LaTeX. There are also some notes on its use with Plain \TeX.
 
 \item  Changes and new features with relation to version 3.8 are
-  highlighted with \New{X.XX}\hspace{-.5em}. The most recent features
+  highlighted with \New{X.XX}\hspace{-.5em}, and there are some notes
+  for the latest versions in
+  \href{https://github.com/latex3/babel/wiki}{the \babel{} wiki}.
+  The most recent features
   could be still unstable. Please, report any issues you find in      
-  \texttt{https://github.com/latex3/babel/issues}, which is better 
+  \href{https://github.com/latex3/babel/issues}{GitHub}, which is better 
   than just complaining on an e-mail list or a web forum.
 
 \item If you are interested in the \TeX{} multilingual support, please
-  join the \textsf{kadingira} list on
-  \texttt{http://tug.org/mailman/listinfo/kadingira}. You can follow
+  join the  \href{http://tug.org/mailman/listinfo/kadingira}{kadingira
+  mail list}. You can follow
   the development of \babel{} in
-  \texttt{https://github.com/latex3/babel} (which provides some sample
-  files, too).
+  \href{https://github.com/latex3/babel}{GitHub} (which provides many
+  \href{https://github.com/latex3/babel/tree/master/samples}{sample
+  files}, too).
 
 \item  See section \ref{contribute} for contributing a language.
 
@@ -492,6 +496,40 @@ _\usepackage[vietnamese,danish]{babel}_
 \end{verbatim}
 \end{example}
 
+\subsection{Mostly monolingual documents}
+
+\New{3.39} Very often, multilingual documents consist of a main
+language with small pieces of text in another languages (words, idioms,
+short sentences). Typically, all you need is to set the line breaking
+rules and, perhaps, the font. In such a case, \babel{} now does not
+require declaring these secondary languages explicitly, because the
+basic settings are loaded on the fly when the language is selected (and
+also when provided in the optional argument of |\babelfont|, if used.)
+
+This is particularly useful, too, when there are short texts of this
+kind coming from an external source whose contents are not known on
+beforehand (for example, titles in a bibliography). At this regard, it
+is worth remembering that |\babelfont| does not load any font until
+required, so that it can be used just in case.
+
+\begin{example}
+  A trivial document is:
+\setengine{luatex/xetex}
+\begin{verbatim}
+\documentclass{article}
+\usepackage[english]{babel}
+
+\babelfont[russian]{rm}{FreeSerif}
+
+\begin{document}
+
+English. \foreignlanguage{russian}{Русский}.
+\foreignlanguage{spanish}{Español}
+
+\end{document}
+\end{verbatim}
+\end{example}
+
 \subsection{Modifiers}
 
 \New{3.9c} The basic behavior of some languages can be modified when
@@ -541,6 +579,7 @@ modifiers are a more general mechanism.
   cases, the error vanishes when the document is typeset again, but in
   more severe ones you will need to remove the |aux| file.
 \end{itemize}
+
 \subsection{Plain}
 
 In Plain, load languages styles with |\input| and then use
@@ -1188,9 +1227,17 @@ be the same as the option name as set in |\usepackage|!).
 
 \subsection{\texttt{ini} files}
 
-An alternative approach to define a language is by means of an
-\texttt{ini} file. Currently \babel{} provides about 200 of these
-files containing the basic data required for a language.
+An alternative approach to define a language (or, more precisely, a
+\textit{locale}) is by means of an \texttt{ini} file. Currently
+\babel{} provides about 200 of these files containing the basic data
+required for a locale. 
+
+|ini| files are not meant only for \babel, and they has been devised as
+a resource for other packages. To easy interoperability between \TeX{}
+and other systems, they are identified with the BCP 47 codes as
+preferred by the Unicode Common Language Data Repository, which was
+used as source for most of the data provided by these files, too (the
+main exception being the |\...name| strings).
 
 Most of them set the date, and many also the captions (Unicode and
 LICR). They will be evolving with the time to add more features
@@ -1280,6 +1327,19 @@ loads \textsf{luatexja}:
 \end{description}
 \end{note}
 
+\begin{note}
+  Wikipedia defines a \textit{locale} as follows: “In computing, a
+  locale is a set of parameters that defines the user’s language,
+  region and any special variant preferences that the user wants to see
+  in their user interface. Usually a locale identifier consists of at
+  least a language code and a country/region code.” \Babel{} is moving
+  gradually from the old and fuzzy concept of \textit{language} to the
+  more modern of \textit{locale}. Note each locale is by itself a
+  separate “language”, which explains why there are so many files. This
+  is on purpose, so that possible variants can be created and/or
+  redefined easily.
+\end{note}
+
 Here is the list (u means Unicode captions, and l means LICR
 captions):
 
@@ -1891,6 +1951,26 @@ zulu
 \endgroup
 \hrule
 
+\bigskip
+
+\textbf{Modifying and adding values to |ini| files}
+
+\New{3.39} There is a way to modify the values of |ini| files when they
+get loaded with |\babelprovide| and |import|. To set, say,
+|digits.native| in the |numbers| section, use something like
+|numbers/digits.native=abcdefghij|. Keys may be added, too. Without
+|import| you may modify the identification keys.
+
+This can be used to create private variants easily. All you need is to
+import the same |ini| file with a different locale name and different
+parameters. 
+
+% \begin{example}
+%   Let's assume you need only the basic features of `spanish`, so that
+%   the |ini| file is enough, and also a variant with your own
+%   hyphenation with  
+% \end{example}
+
 \subsection{Selecting fonts}
 
 \New{3.15} Babel provides a high level interface on top of |fontspec|
@@ -1988,22 +2068,18 @@ to select fonts in addition to the three basic families.
 \babelfont[hindi]{rm}{Shobhika}
 \end{verbatim}
   This makes sure the OpenType script for Devanagari is |deva| and not
-  |dev2| (\luatex{} does not detect automatically the correct 
-  script\footnote{And even with the correct code some fonts could be 
-  rendered incorrectly by \textsf{fontspec}, so double-check the 
-  results. \xetex{} fares better, but some fonts are still
-  problematic.}). You may also pass some options to \textsf{fontspec}:
-  with |silent|, the warnings about unavailable scripts or languages
-  are not shown (they are only really useful when the document format
-  is being set up).
+  |dev2|, in case it is not detected correctly. You may also pass some
+  options to \textsf{fontspec}: with |silent|, the warnings about
+  unavailable scripts or languages are not shown (they are only really
+  useful when the document format is being set up).
 \end{note}
 
 \begin{note}
   Directionality is a property affecting margins, indentation, column
-  order, etc., not just text. Therefore, it is under the direct
-  control of the language, which applies both the script and the
-  direction to the text. As a consequence, there is no need to set
-  \texttt{Script} when declaring a font (nor \texttt{Language}). In
+  order, etc., not just text. Therefore, it is under the direct control
+  of the language, which applies both the script and the direction to
+  the text. As a consequence, there is no need to set \texttt{Script}
+  when declaring a font with |\babelfont| (nor \texttt{Language}). In
   fact, it is even discouraged.
 \end{note}
 
@@ -2012,10 +2088,10 @@ to select fonts in addition to the three basic families.
   (|rm|, |sf|, |tt|, and the like). If a language is switched when an
   \textit{ad hoc} font is active, or you select the font with this
   command, neither the script nor the language is passed. You must
-  add them by hand. This is by design, for several reasons (for
+  add them by hand. This is by design, for several reasons —for
   example, each font has its own set of features and a generic setting
   for several of them could be problematic, and also a “lower-level”
-  font selection is useful).
+  font selection is useful.
 \end{note}
 
 \begin{note}
@@ -2028,53 +2104,45 @@ to select fonts in addition to the three basic families.
 \end{note}
 
 \begin{warning}
-  Do not use |\set|\textit{xxxx}|font| and |\babelfont| at the same
-  time.  |\babelfont| follows the standard \LaTeX{} conventions to set
-  the basic families -- define |\|\textit{xx}|default|, and activate
-  it with |\|\textit{xx}|family|. On the other hand,
-  |\set|\textit{xxxx}|font| in \textsf{fontspec} takes a different
-  approach, because |\|\textit{xx}|family| is redefined with the
-  family name hardcoded (so that |\|\textit{xx}|default| becomes
-  no-op). Of course, both methods are incompatible, and if you use
-  |\set|\textit{xxxx}|font|, font switching with |\babelfont| just
-  does \textit{not} work (nor the standard |\|\textit{xx}|default|,
-  for that matter). As of \New{3.34} there is an attempt to make them
-  compatible, but the language system will not be set by \babel and
-  should be set with |fontspec| if necessary.
+  Using |\set|\textit{xxxx}|font| and |\babelfont| at the same time is
+  discouraged, but very often works as expected. However, be aware with
+  |\set|\textit{xxxx}|font| the language system will not be set by
+  \babel{} and should be set with |fontspec| if necessary.
 \end{warning}
 
 \begin{troubleshooting}
-\trouble{Package fontspec Warning: 'Language 'LANG' not available for
-font 'FONT' with script 'SCRIPT' 'Default' language used instead'}
-\textit{Package fontspec Warning: 'Language 'LANG' not available for
-font 'FONT' with script 'SCRIPT' 'Default' language used instead'}.
-\textbf{This is \textit{not} and error.}
-This warning is shown by \textsf{fontspec}, not by \babel. It could be
-irrelevant for English, but not for many other languages, including
-Urdu and Turkish. This is a useful and harmless warning, and if
-everything is fine with your document the best thing you can do is just
-to ignore it altogether.
+  \trouble{Package fontspec Warning: 'Language 'LANG' not available for
+  font 'FONT' with script 'SCRIPT' 'Default' language used instead'}
+  \textit{Package fontspec Warning: 'Language 'LANG' not available for
+  font 'FONT' with script 'SCRIPT' 'Default' language used instead'}.
+
+  \textbf{This is \textit{not} and error.} This warning is shown by
+  \textsf{fontspec}, not by \babel. It could be irrelevant for English,
+  but not for many other languages, including Urdu and Turkish. This is
+  a useful and harmless warning, and if everything is fine with your
+  document the best thing you can do is just to ignore it altogether.
 \end{troubleshooting}
 
 \begin{troubleshooting}
-\trouble{Package babel Info: The following fonts are not babel standard families}
-\textit{Package babel Info: The following fonts are not babel
-standard families}.
-
-\textbf{This is \textit{not} and error.} \babel{} assumes that if you
-are using |\babelfont| for a family, very likely you want to define the
-rest of them. If you don't, you can find some inconsistencies between
-families. This checking is done at the beginning of the document, at a
-point where we cannot know which families will be used.
-
-Actually, there is no real need to use |\babelfont| in a monolingual
-document, if you set the language system in |\setmainfont| (or not,
-depending on what you want).
-
-As the message explains, \textit{there is nothing intrinsically wrong}
-with not defining all the families. In fact, there is nothing
-intrinsically wrong with not using |\babelfont| at all. But you must be
-aware that this may lead to some problems.
+  \trouble{Package babel Info: The following fonts are not babel
+  standard families} \textit{Package babel Info: The following fonts
+  are not babel standard families}.
+
+  \textbf{This is \textit{not} and error.} \babel{} assumes that if you
+  are using |\babelfont| for a family, very likely you want to define
+  the rest of them. If you don't, you can find some inconsistencies
+  between families. This checking is done at the beginning of the
+  document, at a point where we cannot know which families will be
+  used.
+
+  Actually, there is no real need to use |\babelfont| in a monolingual
+  document, if you set the language system in |\setmainfont| (or not,
+  depending on what you want).
+
+  As the message explains, \textit{there is nothing intrinsically
+  wrong} with not defining all the families. In fact, there is nothing
+  intrinsically wrong with not using |\babelfont| at all. But you must
+  be aware that this may lead to some problems.
 \end{troubleshooting}
 
 \subsection{Modifying a language}
@@ -2090,7 +2158,7 @@ sometimes necessary, but not always trivial.
 }
 \end{verbatim}
 As of 3.15, there is no need to hide spaces with \texttt{\%%
-} (\babel{} removes them), but it is advisable to do it.
+} (\babel{} removes them), but it is advisable to do so.
 \item The new way, which is found in |bulgarian|, |azerbaijani|,
   |spanish|, |french|, |turkish|, |icelandic|, |vietnamese| and a few
   more, as well as in languages created with |\babelprovide| and its
@@ -2177,6 +2245,17 @@ In most cases, you will only need to define a few macros.
 \end{verbatim}
 \end{example}
 
+\begin{example}
+  Locales with names based on BCP 47 codes can be created with
+  something like:
+\begin{verbatim}
+  \babelprovide[import=en-US]{enUS}
+\end{verbatim}
+  Note, however, mixing ways to identify locales can lead to problems.
+  For example, is \verb|yi| the name of the language spoken by the Yi
+  people or is it the code for Yiddish?
+\end{example}
+
 The main language is not changed (\texttt{danish} in this example).
 So, you must add |\selectlanguage{arhinish}| or other selectors where
 necessary.
@@ -2206,8 +2285,7 @@ There are about 200 |ini| files, with data taken from the |ldf| files
 and the CLDR provided by Unicode. Not all languages in the latter are
 complete, and therefore neither are the |ini| files. A few languages
 will show a warning about the current lack of suitability of the date
-format (\textsf{hindi}, \textsf{french}, \textsf{breton}, and
-\textsf{occitan}).
+format (\textsf{french}, \textsf{breton}, and \textsf{occitan}).
 
 Besides |\today|, this option defines an additional command for dates:
 |\<language>date|, which takes three arguments, namely, year, month and
@@ -2267,23 +2345,25 @@ the script, no matter which language is active. Although somewhat
 inconsistent, this makes setting a language up easier in most typical
 cases.
 
+\Describe{onchar=}{\texttt{ids} $\string|$ \texttt{fonts}}
+\New{3.38} This option is much like an ‘event’ called with a character
+belonging to the script of this locale is found. There are currently
+two ‘actions’, which can be used at the same time (separated by a
+space): with |ids| the |\language| and the |\localeid| are set to the
+values of this locale; with |fonts|, the fonts are changed to those of
+the this locale (as set with |\babelfont|). This option is not
+compatible with |mapfont|. Characters can be added with
+|\babelcharproperty|.
+
 \Describe{mapfont=}{\texttt{direction}}
 Assigns the font for the writing direction of this language (only with
-|bidi=basic|). More precisely, what |mapfont=direction| means is, ‘when
-a character has the same direction as the script for the “provided”
-language, then change its font to that set for this language’. There
-are 3 directions, following the bidi Unicode algorithm, namely,
-Arabic-like, Hebrew-like and left to right. So, there should be at most
-3 directives of this kind.
-
-\Describe{onchar=}{\texttt{ids} $\string|$ \texttt{fonts}}
-\New{3.38} This options is much like an ‘event’ called with a character
-belonging to the script of the current locale is found. There are two
-action, which can be used at the same time (separated by a space): with
-|ids| the |\language| and the |\localeid| are set to the values of this
-locale; with |fonts|, the fonts are changed to those of the current
-locale (as set with |\babelfont|). This option is not compatible with
-|mapfont|.
+|bidi=basic|). Whenever possible, instead of this option use |onchar|,
+based on the script, which usually makes more sense. More precisely,
+what |mapfont=direction| means is, ‘when a character has the same
+direction as the script for the “provided” language, then change its
+font to that set for this language’. There are 3 directions, following
+the bidi Unicode algorithm, namely, Arabic-like, Hebrew-like and left
+to right. So, there should be at most 3 directives of this kind.
 
 \Describe{intraspace=}{\meta{base} \meta{shrink} \meta{stretch}}
 Sets the interword space for the writing system of the language, in em
@@ -2393,6 +2473,12 @@ conditions are not met, write |\BabelEnsureInfo|.
 
 \subsection{Hyphenation and line breaking}
 
+\Babel{} deals with three kinds of line breaking rules: Western,
+typically the LGC group, South East Asian, like Thai, and CJK, but
+support depends on the engine: \textsf{pdftex} only deals with the
+former, \xetex{} also with the second one, while \luatex{} provides
+basic rules for the latter, too.
+
 \Describe{\babelhyphen}{%
   \colorbox{thegrey}{\ttfamily\hskip-.2em*\hskip-.2em}\marg{type}}
 \DescribeOther\babelhyphen{%
@@ -2515,6 +2601,55 @@ Unicode engines, spacing is based on the ``current'' em unit (the size
 of the previous char in \luatex, and the font size set by the last
 |\selectfont| in \xetex).
 
+\Describe{\babelposthyphenation}{\marg{hyphenrules-name}%
+          \marg{lua-pattern}\marg{replacement}}
+
+\New{3.37-3.39} With \luatex{} it is now possible to define
+non-standard hyphenation rules, like |f-f| $\to$ |ff-f|, repeated
+hyphens, ranked ruled (or more precisely, ‘penalized’ hyphenation
+points), and so on. No rules are currently provided by default, but
+they can be defined as shown in the following example, where |{1}| is
+the first captured char (between |()| in the pattern):
+\begin{verbatim}
+\babelposthyphenation{german}{([fmtrp]) | {1}}
+{
+  { no = {1}, pre = {1}{1}- }, % Replace first char with disc
+  remove,                      % Remove automatic disc (2nd node)
+  {}                           % Keep last char, untouched
+}
+\end{verbatim}
+
+In the replacements, a captured char may be mapped to another, too. For
+example, if the first capture reads |([ΐΰ])|, the replacement could be
+\verb+{1|ΐΰ|ίύ}+, which maps \textit{ΐ} to \textit{ί}, and \textit{ΰ}
+to \textit{ύ}, so that the diaeresis is removed.
+
+This feature is activated with the first |\babelposthyphenation|.
+
+See the \href{https://github.com/latex3/babel/wiki}{\babel\ wiki} for a
+more detailed description and some examples. It also describes an
+additional replacement type with the key |string|.
+
+\begin{example}
+  Although the main purpose of this command is non-standard
+  hyphenation, it may actually be used for other transformations (after
+  hyphenation is applied, so you must take discretionaries into
+  account). For example, you can use the |string| replacement to
+  replace a character (or series of them) by another character (or
+  series of them). Thus, to enter \textit{ž} as |zh| and \textit{š} as
+  |sh| in a newly created locale for transliterated Russian:
+\begin{verbatim}
+\babelprovide[hyphenrules=+]{russian-latin}   % Create locale
+\babelposthyphenation{russian-latin}{([sz])h} % Create rule
+{
+  { string = {1|sz|šž} },
+  remove
+}
+\end{verbatim}
+  In other words, it is a quite general tool. (A counterpart
+  |\babelprehyphenation| is on the way.)
+\end{example}
+
 \subsection{Selecting scripts}
 
 Currently \babel{} provides no standard interface to select
@@ -2589,6 +2724,14 @@ differ in the way `weak' numeric characters are ordered (eg, Arabic
   options described below).
 \end{warning}
 
+\begin{warning}
+  If characters to be mirrored are shown without changes with \luatex,
+  try with the following line:
+\begin{verbatim}
+\babeladjust{bidi.mirroring=off}
+\end{verbatim}
+\end{warning}
+
 There are some package options controlling bidi writing.
 
 \Describe{bidi=}{\texttt{default} $\string|$ \texttt{basic}
@@ -2612,20 +2755,13 @@ consider the intrinsic direction of scripts and weak directionality.)
 tentative, but it mostly works. For RL documents use the former, and
 for LR ones use the latter.
 
-\New{3.37} There is some experimental support for \textsf{luahbtex}
-(with |lualatex-dev|) and the latest releases of \textsf{luaotfload}
-(3.11), with |Renderer = Harfbuzz| in \textsf{fontspec}. Since it is
-based on \luatex, the option |basic| mostly works (You may need
-deactivate the |rtlm| or the |rtla| font features, or alternatively
-deactive mirroring in \babel{} with |\babeladjust|.)
-
 There are samples on GitHub, under \texttt{/required/babel/samples}.
 See particularly |lua-bidibasic.tex| and |lua-secenum.tex|.
 
 \begin{example}
   The following text comes from the Arabic Wikipedia (article about
   Arabia). Copy-pasting some text from the Wikipedia is a good way to
-  test this feature. Remember |basic-r| is available in \luatex{}
+  test this feature. Remember |basic| is available in \luatex{}
   only.
   \begingroup
 % If you are looking at the code to see how it has been written, you
@@ -2638,7 +2774,7 @@ See particularly |lua-bidibasic.tex| and |lua-secenum.tex|.
 \begin{verbatim}
 \documentclass{article}
 
-\usepackage[_bidi=basic-r_]{babel}
+\usepackage[_bidi=basic_]{babel}
 
 _\babelprovide[import, main]{arabic}_
 
@@ -2676,7 +2812,7 @@ _\babelprovide[import, main]{arabic}_
 
 \usepackage[english, _bidi=basic_]{babel}
 
-\babelprovide[_mapfont=direction_]{arabic}
+\babelprovide[_onchar=ids fonts_]{arabic}
 
 \babelfont{rm}{Crimson}
 \babelfont[*arabic]{rm}{FreeSerif}
@@ -2691,7 +2827,7 @@ Arabic as @1فصحى العصر@0 \textit{fuṣḥā l-ʻaṣr} (MSA) and
 \end{document}
 \end{verbatim}
   \endgroup
-  In this example, and thanks to |mapfont=direction|, any Arabic letter
+  In this example, and thanks to |onchar=ids fonts|, any Arabic letter
   (because the language is |arabic|) changes its font to that set for
   this language (here defined via |*arabic|, because Crimson does not
   provide Arabic letters).
@@ -2892,6 +3028,7 @@ command is available always in bidi documents, even without
 \subsection{Language attributes}
 
 \DescribeMacro{\languageattribute}
+
 This is a user-level command, to be used in the preamble of a
 document (after |\usepackage[...]{babel}|), that declares which
 attributes are to be used for a given language. It takes two
@@ -3088,12 +3225,14 @@ tables, which you can modify with the following macro (for example,
 to set them for glyphs in the PUA).
 
 \Describe{\babelcharproperty}{\marg{char-code}\oarg{to-char-code}%
-          \marg{propertry}\marg{value}}
+          \marg{property}\marg{value}}
 
 \New{3.32} Here, \marg{char-code} is a number (with \TeX{} syntax).
 With the optional argument, you can set a range of values. There are
 three properties (with a short name, taken from Unicode): |direction|
-(|bc|), |mirror| (|bmg|), |linebreak| (|lb|). The settings are global.
+(|bc|), |mirror| (|bmg|), |linebreak| (|lb|). The settings are global,
+and this command is allowed only in vertical mode (the preamble or
+between paragraphs).
 
 For example:
 \begin{verbatim}
@@ -3102,8 +3241,12 @@ For example:
 \babelcharproperty{`)}{linebreak}{cl} % or id, op, cl, ns, ex, in, hy
 \end{verbatim}
 
-This command is allowed only in vertical mode (the preamble or between
-paragraphs).
+\New{3.39} Another property is |locale|, which adds characters to the
+list used by |onchar| in |\babelprovide|, or, if the last argument is
+empty, removes them. The last argument is the locale name:
+\begin{verbatim}
+\babelcharproperty{`,}{locale}{english}
+\end{verbatim}
 
 \subsection{Tweaking some features}
 
@@ -3243,41 +3386,6 @@ See the code section for |\foreignlanguage*| (a new starred
 version of |\foreignlanguage|).
 
 \medskip
-\textbf{Modifying, and adding, values of |ini| files}
-
-\New{3.37} There is a way to modify the values of |ini| files when they
-get loaded with |\babelprovide|. To set, say, |digits.native| in the
-|numbers| section, use something like
-|numbers..digits.native=abcdefghij| (note the double dot between the
-section and the key name). New keys may be added, too.
-
-\medskip
-\textbf{Non-standard hyphenation}
-
-\New{3.37} With \luatex{} it is now possible to define non-standard
-hyohenation rules, like |f-f| $\to$ |ff-f|. No rules are currently
-provided by default, but they can be defined as shown in the following
-example:
-\begin{verbatim}
-\babelposthyphenation{ngerman}{([fmtrp]) | {1}}
-{
-  { no = {1}, pre = {1}{1}-}, % Replace first char with disc
-  remove,                     % Remove automatic disc
-  {}                          % Keep last char, untouched
-}
-\end{verbatim}
-
-This feature must be explicitly activated with:
-\begin{verbatim}
-\babeladjust{ hyphenation.extra = on }
-\end{verbatim}
-
-See the \babel{} wiki for a more detailed description and some examples:
-\begin{verbatim}
-https://github.com/latex3/babel/wiki
-\end{verbatim}
-
-\medskip
 \textbf{Old and deprecated stuff}
 
 A couple of tentative macros were provided by \babel{} ($\ge$3.9g) with
@@ -4222,8 +4330,8 @@ help from Bernd Raichle, for which I am grateful.
 % \section{Tools}
 %
 %    \begin{macrocode}
-%<<version=3.38>>
-%<<date=2020/01/15>>
+%<<version=3.39>>
+%<<date=2020/02/03>>
 %    \end{macrocode}
 %
 % \textbf{Do not use the following macros in \texttt{ldf} files. They
@@ -4715,7 +4823,6 @@ help from Bernd Raichle, for which I am grateful.
 %
 % \changes{babel~3.30}{2019/04/22}{Callbacks aren't specific to
 %   bidi any more, so they are moved.}
-% \changes{babel~3.32}{2019/05/30}{Consider Harf.}
 %
 %    \begin{macrocode}
 \ifx\bbl@languages\@undefined\else
@@ -4737,8 +4844,6 @@ help from Bernd Raichle, for which I am grateful.
   \bbl@languages
 \fi
 \ifodd\bbl@engine
-  % Harftex is evolving, so the callback is not harcoded, just in case
-  \def\bbl@harfpreline{Harf pre_linebreak_filter callback}%
   \def\bbl@activate@preotf{%
     \let\bbl@activate@preotf\relax  % only once
     \directlua{
@@ -4768,23 +4873,14 @@ help from Bernd Raichle, for which I am grateful.
         Babel.pre_otfload_v,
         'Babel.pre_otfload_v',
         luatexbase.priority_in_callback('pre_linebreak_filter',
-           '\bbl@harfpreline') 
-        or luatexbase.priority_in_callback('pre_linebreak_filter',
-           'luaotfload.node_processor')
-        or nil)
+          'luaotfload.node_processor') or nil)
       %
       luatexbase.add_to_callback('hpack_filter',
         Babel.pre_otfload_h,
         'Babel.pre_otfload_h',
         luatexbase.priority_in_callback('hpack_filter',
-           '\bbl@harfpreline') 
-        or luatexbase.priority_in_callback('hpack_filter',
-           'luaotfload.node_processor')
-        or nil)
-    }%
-    \@ifpackageloaded{harfload}%
-      {\directlua{ Babel.mirroring_enabled = false }}%
-      {}}
+          'luaotfload.node_processor') or nil)
+    }}
   \let\bbl@tempa\relax
   \@ifpackagewith{babel}{bidi=basic}%
     {\def\bbl@tempa{basic}}%
@@ -6771,20 +6867,8 @@ help from Bernd Raichle, for which I am grateful.
       \ifx\bbl@known@attribs\@undefined
         \in@false
       \else
-%    \end{macrocode}
-%
-%    Now we need to see if the attribute occurs in the list of
-%    already selected attributes.
-%
-%    \begin{macrocode}
         \bbl@xin@{,\bbl@tempc-##1,}{,\bbl@known@attribs,}%
       \fi
-%    \end{macrocode}
-%
-%    When the attribute was in the list we issue a warning; this might
-%    not be the users intention.
-%
-%    \begin{macrocode}
       \ifin@
         \bbl@warning{%
           You have more than once selected the attribute '##1'\\%
@@ -7349,6 +7433,7 @@ help from Bernd Raichle, for which I am grateful.
   \edef\bbl@L{\zap@space#1 \@empty}%
   \edef\bbl@G{\zap@space#2 \@empty}%
   \bbl@startcmds@ii}
+\let\bbl@startcommands\StartBabelCommands
 %    \end{macrocode}
 %
 %    Parse the encoding info to get the label, input, and font parts.
@@ -7474,6 +7559,7 @@ help from Bernd Raichle, for which I am grateful.
   \endgroup
   \endgroup
   \bbl@scafter}
+\let\bbl@endcommands\EndBabelCommands
 %    \end{macrocode}
 %
 %    Now we define commands to be used inside |\StartBabelCommands|.
@@ -7602,7 +7688,7 @@ help from Bernd Raichle, for which I am grateful.
   \newcommand\SetHyphenMap[1]{%
     \bbl@forlang\bbl@tempa{%
       \expandafter\bbl@stringdef
-        \csname\bbl@tempa @bbl@hyphenmap\endcsname{##1}}}
+        \csname\bbl@tempa @bbl@hyphenmap\endcsname{##1}}}%
 %<</Macros local to BabelCommands>>
 %    \end{macrocode}
 %
@@ -8213,7 +8299,7 @@ help from Bernd Raichle, for which I am grateful.
   \let\bbl@savelangname\languagename
   \edef\bbl@savelocaleid{\the\localeid}%
   % Set name and locale id
-  \def\languagename{#2}%
+  \edef\languagename{#2}%
   % \global\@namedef{bbl@lcname@#2}{#2}%
   \bbl@id@assign
   \let\bbl@KVP@captions\@nil
@@ -8230,7 +8316,7 @@ help from Bernd Raichle, for which I am grateful.
   \let\bbl@KVP@onchar\@nil
   \let\bbl@KVP@chargroups\@nil
   \bbl@forkv{#1}{%  TODO - error handling
-    \in@{..}{##1}%
+    \in@{/}{##1}%
     \ifin@
       \bbl@renewinikey##1\@@{##2}%
     \else
@@ -8279,8 +8365,10 @@ help from Bernd Raichle, for which I am grateful.
          \bbl@read@ini{##1}{basic data}%
          \bbl@exportkey{chrng}{characters.ranges}{}%
          \bbl@exportkey{dgnat}{numbers.digits.native}{}%
+         \bbl@exportkey{lnbrk}{typography.linebreaking}{h}%
          \bbl@exportkey{hyphr}{typography.hyphenrules}{}%
          \bbl@exportkey{intsp}{typography.intraspace}{}%
+         \endinput
        \endgroup}%           boxed, to avoid extra spaces:
      {\setbox\z@\hbox{\InputIfFileExists{babel-#2.tex}{}{}}}}%
     {}%
@@ -8301,10 +8389,15 @@ help from Bernd Raichle, for which I am grateful.
         Babel.locale_mapped = true
         Babel.linebreaking.add_before(Babel.locale_map)
         Babel.loc_to_scr = {}
-        Babel.chr_to_loc = {}
+        Babel.chr_to_loc = Babel.chr_to_loc or {}
       end}%
     \bbl@xin@{ ids }{ \bbl@KVP@onchar\space}%
     \ifin@
+      \ifx\bbl@starthyphens\@undefined % Needed if no explicit selection
+        \AddBabelHook{babel-onchar}{beforestart}{{\bbl@starthyphens}}%
+      \fi
+      \bbl@exp{\\\bbl@add\\\bbl@starthyphens
+        {\\\bbl@patterns@lua{\languagename}}}%
       % TODO - error/warning if no script
       \directlua{
         if Babel.script_blocks['\bbl@cs{sbcp@\languagename}'] then
@@ -8323,7 +8416,7 @@ help from Bernd Raichle, for which I am grateful.
         if Babel.script_blocks['\bbl@cs{sbcp@\languagename}'] then
           Babel.loc_to_scr[\the\localeid] =
             Babel.script_blocks['\bbl@cs{sbcp@\languagename}']
-        end}
+        end}%
       \ifx\bbl@mapselect\@undefined
         \AtBeginDocument{%
           \expandafter\bbl@add\csname selectfont \endcsname{{\bbl@mapselect}}%
@@ -8343,9 +8436,6 @@ help from Bernd Raichle, for which I am grateful.
     \fi
     % TODO - catch non-valid values
   \fi
-%   \ifx\bbl@KVP@chargroups\@nil\else
-%      \bbl@chargroups
-%   \fi
   % == mapfont ==
   % For bidi texts, to switch the font based on direction
   \ifx\bbl@KVP@mapfont\@nil\else
@@ -8430,7 +8520,7 @@ help from Bernd Raichle, for which I am grateful.
                return head
              end
            end
-        }}
+        }}%
     \fi
   \fi
   % == require.babel in ini ==
@@ -8497,7 +8587,7 @@ help from Bernd Raichle, for which I am grateful.
   \@namedef{date#1}{}% marks lang exists - required by \StartBabelCommands
   \@namedef{extras#1}{}%
   \@namedef{noextras#1}{}%
-  \StartBabelCommands*{#1}{captions}%
+  \bbl@startcommands*{#1}{captions}%
     \ifx\bbl@KVP@captions\@nil %      and also if import, implicit
       \def\bbl@tempb##1{%             elt for \bbl@captionslist
         \ifx##1\@empty\else
@@ -8520,7 +8610,7 @@ help from Bernd Raichle, for which I am grateful.
       \bbl@savetoday
       \bbl@savedate
     \fi
-  \EndBabelCommands
+  \bbl@endcommands
   \bbl@exp{%
     \def\<#1hyphenmins>{%
       {\bbl@ifunset{bbl@lfthm@#1}{2}{\@nameuse{bbl@lfthm@#1}}}%
@@ -8620,7 +8710,7 @@ help from Bernd Raichle, for which I am grateful.
     \def\bbl@inipreread##1=##2\@@{%
       \bbl@trim@def\bbl@tempa{##1}% Redundant below !!
       % Move trims here ??
-      \bbl@ifunset{bbl@KVP@\bbl@section..\bbl@tempa}%
+      \bbl@ifunset{bbl@KVP@\bbl@section/\bbl@tempa}%
         {\expandafter\bbl@inireader\bbl@tempa=##2\@@}%
         {}}%
     \let\bbl@inireader\bbl@iniskip
@@ -8643,6 +8733,7 @@ help from Bernd Raichle, for which I am grateful.
     \@nameuse{bbl@renew@\bbl@section}%
     \global\bbl@csarg\let{renew@\bbl@section}\relax
     \@nameuse{bbl@secpost@\bbl@section}%  
+%    \bbl@csarg\bbl@toglobal{inikeys@\languagename}%
   \fi}
 \def\bbl@iniline#1\bbl@iniline{%
   \@ifnextchar[\bbl@inisec{\@ifnextchar;\bbl@iniskip\bbl@inipreread}#1\@@}% ]
@@ -8665,14 +8756,14 @@ help from Bernd Raichle, for which I am grateful.
   \@nameuse{bbl@secpost@\bbl@section}%  ends previous section
   \def\bbl@section{#1}%                 starts current section
   \def\bbl@elt##1##2{%
-    \@namedef{bbl@KVP@#1..##1}{}}%
+    \@namedef{bbl@KVP@#1/##1}{}}%
   \@nameuse{bbl@renew@#1}%
   \@nameuse{bbl@secpre@#1}%  pre-section `hook'
   \bbl@ifunset{bbl@inikv@#1}%
     {\let\bbl@inireader\bbl@iniskip}%
     {\bbl@exp{\let\\\bbl@inireader\<bbl@inikv@#1>}}}
 \let\bbl@renewlist\@empty
-\def\bbl@renewinikey#1..#2\@@#3{%
+\def\bbl@renewinikey#1/#2\@@#3{%
   \bbl@ifunset{bbl@renew@#1}%
     {\bbl@add@list\bbl@renewlist{#1}}%
     {}%
@@ -8732,6 +8823,7 @@ help from Bernd Raichle, for which I am grateful.
   \bbl@exportkey{lfthm}{typography.lefthyphenmin}{2}%
   \bbl@exportkey{rgthm}{typography.righthyphenmin}{3}%
   \bbl@exportkey{prehc}{typography.prehyphenchar}{}%
+  \bbl@exportkey{lnbrk}{typography.linebreaking}{h}%
   \bbl@exportkey{hyphr}{typography.hyphenrules}{}%
   \bbl@exportkey{intsp}{typography.intraspace}{}%
   \bbl@exportkey{jstfy}{typography.justify}{w}%
@@ -9016,9 +9108,7 @@ help from Bernd Raichle, for which I am grateful.
 \@namedef{bbl@ADJ@layout.lists@on}{%
   \bbl@adjust@layout{\let\list\bbl@OL@list}}
 \@namedef{bbl@ADJ@hyphenation.extra@on}{%
-  \directlua{
-    Babel.linebreaking.add_after(Babel.post_hyphenate_replace)
-  }}
+  \bbl@activateposthyphen}
 %    \end{macrocode}
 %
 % \section{The kernel of Babel (\texttt{babel.def} for \LaTeX only)}
@@ -9851,11 +9941,7 @@ help from Bernd Raichle, for which I am grateful.
 %      and so on, but bidi text does not work out of the box and some
 %      development is necessary. It also provides tools to properly
 %      set left-to-right and right-to-left page layouts. As Lua\TeX-ja
-%      shows, vertical typesetting is possible, too. Its main drawback
-%      is font handling is often considered to be less mature than
-%      \xetex{}, mainly in Indic scripts (but there are steps to make
-%      HarfBuzz, the \xetex{} font engine, available in \luatex{}; see
-%      <https://github.com/tatzetwerk/luatex-harfbuzz>).
+%      shows, vertical typesetting is possible, too.
 %    \end{itemize}
 %
 % \changes{babel~3.15}{2017/10/30}{Use an attribute instead of tex language
@@ -10014,18 +10100,20 @@ help from Bernd Raichle, for which I am grateful.
     \newtoks\everypar
     \everypar=\bbl@severypar
     \bbl@severypar{\bbl@xeeverypar\the\everypar}}
-  \@ifpackagewith{babel}{bidi=bidi}%
-    {\let\bbl@textdir@i\@gobbletwo
-     \let\bbl@xebidipar\@empty
-     \AddBabelHook{bidi}{foreign}{%
-       \def\bbl@tempa{\def\BabelText####1}%
-       \ifcase\bbl@thetextdir
-         \expandafter\bbl@tempa\expandafter{\BabelText{\LR{##1}}}%
-       \else
-         \expandafter\bbl@tempa\expandafter{\BabelText{\RL{##1}}}%
-       \fi}
-     \def\bbl@pardir#1{\ifcase#1\relax\setLR\else\setRL\fi}}
-    {}%
+  \def\bbl@tempb{%
+    \let\bbl@textdir@i\@gobbletwo
+    \let\bbl@xebidipar\@empty
+    \AddBabelHook{bidi}{foreign}{%
+      \def\bbl@tempa{\def\BabelText########1}%
+      \ifcase\bbl@thetextdir
+        \expandafter\bbl@tempa\expandafter{\BabelText{\LR{####1}}}%
+      \else
+        \expandafter\bbl@tempa\expandafter{\BabelText{\RL{####1}}}%
+      \fi}
+    \def\bbl@pardir##1{\ifcase##1\relax\setLR\else\setRL\fi}}
+  \@ifpackagewith{babel}{bidi=bidi}{\bbl@tempb}{}%
+  \@ifpackagewith{babel}{bidi=bidi-l}{\bbl@tempb}{}%
+  \@ifpackagewith{babel}{bidi=bidi-r}{\bbl@tempb}{}%
 \fi
 %    \end{macrocode}
 %
@@ -10406,7 +10494,6 @@ help from Bernd Raichle, for which I am grateful.
   \edef\languagename{%
     \ifnum\escapechar=\expandafter`\string#1\@empty
     \else\string#1\@empty\fi}%
-  % \@namedef{bbl@lcname@#1}{#1}%
   \select@language{\languagename}%
   % write to auxs
   \expandafter\ifx\csname date\languagename\endcsname\relax\else
@@ -10423,8 +10510,13 @@ help from Bernd Raichle, for which I am grateful.
   % set name
   \edef\languagename{#1}%
   \bbl@fixname\languagename
+  \expandafter\ifx\csname date\languagename\endcsname\relax
+    \IfFileExists{babel-\languagename.tex}%
+      {\babelprovide{\languagename}}%
+      {}%
+  \fi
   \bbl@iflanguage\languagename{%
-    \expandafter\ifx\csname date\languagename\endcsname\relax
+     \expandafter\ifx\csname date\languagename\endcsname\relax
       \bbl@error
         {Unknown language `#1'. Either you have\\%
          misspelled its name, it has not been installed,\\%
@@ -10725,8 +10817,12 @@ help from Bernd Raichle, for which I am grateful.
 \def\foreign@language#1{%
   % set name
   \edef\languagename{#1}%
-  % \@namedef{bbl@lcname@#1}{#1}%
   \bbl@fixname\languagename
+  \expandafter\ifx\csname date\languagename\endcsname\relax
+    \IfFileExists{babel-\languagename.tex}%
+      {\babelprovide{\languagename}}%
+      {}%
+  \fi
   \bbl@iflanguage\languagename{%
     \expandafter\ifx\csname date\languagename\endcsname\relax
       \bbl@warning   % TODO - why a warning, not an error?
@@ -10945,6 +11041,7 @@ help from Bernd Raichle, for which I am grateful.
 \let\uselocale\setlocale
 \let\locale\setlocale
 \let\selectlocale\setlocale
+\let\localename\setlocale
 \let\textlocale\setlocale
 \let\textlanguage\setlocale
 \let\languagetext\setlocale
@@ -11570,6 +11667,12 @@ help from Bernd Raichle, for which I am grateful.
 \bbl@trace{Font handling with fontspec}
 \@onlypreamble\babelfont
 \newcommand\babelfont[2][]{%  1=langs/scripts 2=fam
+  \bbl@foreach{#1}{%
+    \expandafter\ifx\csname date##1\endcsname\relax
+    \IfFileExists{babel-##1.tex}%
+      {\babelprovide{##1}}%
+      {}%
+    \fi}%
   \edef\bbl@tempa{#1}%
   \def\bbl@tempb{#2}%  Used by \bbl@bblfont
   \ifx\fontspec\@undefined
@@ -12457,6 +12560,9 @@ help from Bernd Raichle, for which I am grateful.
 % \changes{babel~3.37}{2019/12/07}{Added code for non-standard
 %   hyphenation.}
 %
+% First, some general code for line breaking, used by
+% |\babelposthyphenation|.
+%
 % \textit{In progress.} Replace regular (ie, implicit) discretionaries
 % by spaceskips, based on the previous glyph (which I think makes
 % sense, because the hyphen and the previous char go always together).
@@ -12641,7 +12747,7 @@ help from Bernd Raichle, for which I am grateful.
 \def\bbl@provide@intraspace{%
   \bbl@ifunset{bbl@intsp@\languagename}{}%
     {\expandafter\ifx\csname bbl@intsp@\languagename\endcsname\@empty\else
-       \bbl@xin@{\bbl@cs{sbcp@\languagename}}{Hant,Hans,Jpan,Kore,Kana}%
+       \bbl@xin@{\bbl@cs{lnbrk@\languagename}}{c}%
        \ifin@           % cjk
          \bbl@cjkintraspace
          \directlua{
@@ -12738,7 +12844,7 @@ Babel.script_blocks = {
               {0x2B740, 0x2B81F}, {0x2B820, 0x2CEAF},
               {0x2CEB0, 0x2EBEF}, {0x2F800, 0x2FA1F}},
   ['Hebr'] = {{0x0590, 0x05FF}},
-  ['Japa'] = {{0x3000, 0x303F}, {0x3040, 0x309F}, {0x30A0, 0x30FF},
+  ['Jpan'] = {{0x3000, 0x303F}, {0x3040, 0x309F}, {0x30A0, 0x30FF},
               {0x4E00, 0x9FAF}, {0xFF00, 0xFFEF}},
   ['Khmr'] = {{0x1780, 0x17FF}, {0x19E0, 0x19FF}},
   ['Knda'] = {{0x0C80, 0x0CFF}},
@@ -12760,10 +12866,11 @@ Babel.script_blocks = {
   ['Thai'] = {{0x0E00, 0x0E7F}},
   ['Tibt'] = {{0x0F00, 0x0FFF}},
   ['Vaii'] = {{0xA500, 0xA63F}},
-  ['Yiii']= {{0xA490, 0xA4CF}, {0xA000, 0xA48F}}
+  ['Yiii'] = {{0xA000, 0xA48F}, {0xA490, 0xA4CF}}
 }
 
 Babel.script_blocks.Hant = Babel.script_blocks.Hans
+Babel.script_blocks.Kana = Babel.script_blocks.Jpan
 
 function Babel.locale_map(head)
   if not Babel.locale_mapped then return head end
@@ -12789,7 +12896,7 @@ function Babel.locale_map(head)
         end
       end
       % Now, take action
-      if toloc then
+      if toloc and toloc > -1 then
         if Babel.locale_props[toloc].lg then
           item.lang = Babel.locale_props[toloc].lg
           node.set_attribute(item, LOCALE, toloc)
@@ -12856,6 +12963,12 @@ end
     Babel.Babel.cjk_characters[\the\count@]['c'] = '#1'
   }}
 \let\bbl@chprop@lb\bbl@chprop@linebreak
+\def\bbl@chprop@locale#1{%
+  \directlua{
+    Babel.chr_to_loc = Babel.chr_to_loc or {}
+    Babel.chr_to_loc[\the\count@] =
+      \bbl@ifblank{#1}{-1000}{\the\@nameuse{bbl@id@@#1}}\space
+  }}
 %    \end{macrocode}
 %
 % Post-handling hyphenation patterns for non-standard rules, like |ff|
@@ -13050,13 +13163,40 @@ end
     return head
   end
 
-  &% Used below
+  &% The following functions belong to the next macro
+
+  &% This table stores capture maps, numbered consecutively
+  Babel.capture_maps = {}
+
   function Babel.capture_func(key, cap)
     local ret = "[[" .. cap:gsub('{([0-9])}', "]]..m[%1]..[[") .. "]]"
+    ret = ret:gsub('{([0-9])|([^|]+)|(.-)}', Babel.capture_func_map)
     ret = ret:gsub("%[%[%]%]%.%.", '')
     ret = ret:gsub("%.%.%[%[%]%]", '')
     return key .. [[=function(m) return ]] .. ret .. [[ end]]
   end
+
+  function Babel.capt_map(from, mapno)
+    return Babel.capture_maps[mapno][from] or from
+  end
+
+  &% Handle the {n|abc|ABC} syntax in captures
+  function Babel.capture_func_map(capno, from, to)
+    local froms = {}
+    for s in string.utfcharacters(from) do
+      table.insert(froms, s)
+    end
+    local cnt = 1
+    table.insert(Babel.capture_maps, {})
+    local mlen = table.getn(Babel.capture_maps)
+    for s in string.utfcharacters(to) do
+      Babel.capture_maps[mlen][froms[cnt]] = s
+      cnt = cnt + 1
+    end
+    return "]]..Babel.capt_map(m[" .. capno .. "]," ..
+           (mlen) .. ").." .. "[["
+  end
+
 }
 %    \end{macrocode}
 %
@@ -13064,7 +13204,10 @@ end
 % defined above for converting strings to functions returning a string.
 % These functions handle the |{|\textit{n}|}| syntax. For example,
 % |pre={1}{1}-| becomes |function(m) return m[1]..m[1]..'-' end|, where
-% |m| are the matches returned after applying the pattern. The way it
+% |m| are the matches returned after applying the pattern. With a
+% mapped capture the functions are similar to
+% |function(m) return Babel.capt_map(m[1],1) end|, where the last
+% argument identifies the mapping to be applied to |m[1]|. The way it
 % is carried out is somewhat tricky, but the effect in not dissimilar
 % to lua |load| – save the code as string in a TeX macro, and expand
 % this macro at the appropriate place. As |\directlua| does not take
@@ -13074,6 +13217,7 @@ end
 %    \begin{macrocode}
 \catcode`\#=6
 \gdef\babelposthyphenation#1#2#3{&%
+  \bbl@activateposthyphen
   \begingroup
     \def\babeltempa{\bbl@add@list\babeltempb}&%
     \let\babeltempb\@empty
@@ -13106,6 +13250,11 @@ end
     }&%
   \endgroup}
 \endgroup
+\def\bbl@activateposthyphen{%
+  \let\bbl@activateposthyphen\relax
+  \directlua{
+    Babel.linebreaking.add_after(Babel.post_hyphenate_replace)
+  }}
 %    \end{macrocode}
 %
 % \subsection{Layout}
@@ -19714,12 +19863,6 @@ Babel.fontmap[2] = {}      -- al/an
 Babel.bidi_enabled = true
 Babel.mirroring_enabled = true
 
--- Temporary:
-
-if harf then 
-  Babel.mirroring_enabled = false
-end
-
 require('babel-data-bidi.lua')
 
 local characters = Babel.characters
@@ -20010,7 +20153,10 @@ function Babel.bidi(head, ispar, hdir)
         item = nodes[r][1]    -- MIRRORING
         if Babel.mirroring_enabled and item.id == GLYPH
              and temp == 'r' and characters[item.char] then
-          item.char = characters[item.char].m or item.char
+          local font_mode = font.fonts[item.font].properties.mode
+          if font_mode ~= 'harf' and font_mode ~= 'plug' then
+            item.char = characters[item.char].m or item.char
+          end
         end
       end
       first_on = nil
@@ -20321,6 +20467,8 @@ Babel.cjk_class = setmetatable ( Babel.cjk_characters, {
   __index = function(_, k)
     if (k >= 0xAC00  and k <= 0xD7A3)      -- H2/H3
         or (k >= 0x2E80  and k <= 0x9FFF)
+        or (k >= 0xA000  and k <= 0xA48F)  -- Yi
+        or (k >= 0xA490  and k <= 0xA4CF)  -- Yi
         or (k >= 0xF900  and k <= 0xFAFF)
         or (k >= 0xFE10  and k <= 0xFE1F) 
         or (k >= 0xFE30  and k <= 0xFE6F)