babel (2sep18)

git-svn-id: svn://tug.org/texlive/trunk@48551 c570f23f-e606-0410-a88d-b1316a301751

1 files changed, 269 insertions, 223 deletions

diff --git a/Master/texmf-dist/source/latex/babel/babel.dtx b/Master/texmf-dist/source/latex/babel/babel.dtx
index 8706a4c546f..24cfe5762b7 100644
--- a/Master/texmf-dist/source/latex/babel/babel.dtx
+++ b/Master/texmf-dist/source/latex/babel/babel.dtx
@@ -31,7 +31,7 @@
 %
 % \iffalse
 %<*filedriver>
-\ProvidesFile{babel.dtx}[2018/06/05 v3.22 The Babel package]
+\ProvidesFile{babel.dtx}[2018/09/02 v3.23 The Babel package]
 \documentclass{ltxdoc}
 \GetFileInfo{babel.dtx}
 \usepackage{fontspec}
@@ -235,7 +235,9 @@ Javier Bezos
 
 \item  Changes and new features with relation to version 3.8 are
   highlighted with \New{X.XX}\hspace{-.5em}. The most recent features
-  could be still unstable. Please, report any issues you find.
+  could be still unstable. Please, report any issues you find on      
+  \texttt{https://github.com/latex3/latex2e/issues}, which is better 
+  than just complaining on an e-mail list or a web forum.
 
 \item If you are interested in the \TeX{} multilingual support, please
   join the \textsf{kadingira} list on
@@ -526,7 +528,7 @@ In Plain, load languages styles with |\input| and then use
   Not all languages provide a |sty| file and some of them are not
   compatible with Plain.\footnote{Even in the \babel{} kernel there
   were some macros not compatible with plain. Hopefully these issues
-  will be fixed soon.}
+  have been fixed.}
 \end{warning}
 
 \subsection{Basic language selectors}
@@ -579,7 +581,9 @@ extra definitions and the hyphenation rules for the language,
 the language to auxiliary files (i.e., the surrounding language is
 still in force), and (3) it works even if the language has not been
 set as package option (but in such a case it only sets the hyphenation
-patterns and a warning is shown).
+patterns and a warning is shown). With the |bidi| option, it also 
+enters in horizontal mode (this is not done always for backwards 
+compatibility).
 
 \subsection{Auxiliary language selectors}
 
@@ -613,7 +617,9 @@ This environment was originally intended for intermixing left-to-right
 typesetting with right-to-left typesetting in engines not supporting a
 change in the writing direction inside a line. However, by default it
 never complied with the documented behavior and it is just a version as
-environment of |\foreignlanguage|.
+environment of |\foreignlanguage|, except when the option |bidi| is 
+set -- in this case, |\foreignlanguage| emits a |\leavevmode|, while 
+|otherlanguage*| does not.
 
 \Describe{\begin\menv{hyphenrules}}{\marg{language}\Eenv{hyphenrules}}
 
@@ -774,8 +780,11 @@ list of characters as their arguments.
 The command |\shorthandoff| sets the |\catcode| for each of the
 characters in its argument to other (12); the command |\shorthandon|
 sets the |\catcode| to active (13). Both commands only work on `known'
-shorthand characters. If a character is not known to be a shorthand
-character its category code will be left unchanged.
+shorthand characters.
+
+% TODO - Is the following true ????
+% If a character is not known to be a shorthand character its category
+% code will be left unchanged.
 
 \New{3.9a} However, |\shorthandoff| does not behave as
 you would expect with characters like |~| or |^|, because they
@@ -806,7 +815,7 @@ are always activated.
 
 Currently, if the package option |shorthands| is used, you must include any
 character to be activated with |\useshorthands|. This restriction will
-be lifted in a future release.
+be lifted in a future release.  % <- TODO
 
 \Describe\defineshorthand{\texttt{[}\langlist\texttt{]}%
      \marg{shorthand}\marg{code}}
@@ -947,6 +956,10 @@ which is let, like the standard |~|, to a non breaking
 space.\footnote{This declaration serves to nothing, but it is
 preserved for backward compatibility.}
 
+\Describe\ifbabelshorthand{\marg{character}\marg{true}\marg{false}}
+
+\New{3.23} Tests if a character has been made a shorthand.
+
 \subsection{Package options}
 
 \New{3.9a}
@@ -1076,7 +1089,8 @@ can take the following values:
   really useful, but who knows.}
 \end{description}
 
-\Describe{bidi=}{}
+\Describe{bidi=}{\texttt{default} $\string|$ \texttt{basic}
+  $\string|$ \texttt{basic-r}}
 
 \New{3.14} Selects the bidi algorithm to be used in \luatex{} and
 \xetex{}. See sec.~\ref{bidi}.
@@ -1146,7 +1160,7 @@ for auxiliary tasks).
 \documentclass{book}
 
 \usepackage{babel}
-\babelprovide[import=ka, main]{georgian}
+\babelprovide[import, main]{georgian}
 
 \babelfont{rm}{DejaVu Sans}
 
@@ -1432,10 +1446,11 @@ captions):
 \bigskip
 
 In some contexts (currently |\babelfont|) an \texttt{ini} file may be
-loaded by its name. Here is the list of the names currently
-supported. With these languages, |\babelfont| loads (if not done
-before) the language and script names (even if the language is defined
-as a package option with an \textsf{ldf} file).
+loaded by its name. Here is the list of the names currently supported.
+With these languages, |\babelfont| loads (if not done before) the
+language and script names (even if the language is defined as a package
+option with an \textsf{ldf} file). These are also the names recognized 
+by |\babelprovide| with a valueless |import|.
 
 \begingroup
 \bigskip\hrule\nobreak
@@ -1818,7 +1833,7 @@ Mono} \fi}
 
 \usepackage[swedish, bidi=default]{babel}
 
-\babelprovide[import=he]{hebrew}
+\babelprovide[import]{hebrew}
 
 _\babelfont{rm}{FreeSerif}_
 
@@ -1993,6 +2008,15 @@ and hyphenmins. For example:
 Unicode engines load the UTF-8 variants, while 8-bit engines load the
 LICR (ie, with macros like |\'| or |\ss|) ones.
 
+\New{3.23} It may be used without a value. In such a case, the |ini|
+file set in the corresponding |babel-<language>.tex| (where
+|<language>| is the last argument in |\babelprovide|) is imported. See
+the list of recognized languages above. So, the previous example could 
+be written:
+\begin{verbatim}
+\babelprovide[import]{hungarian}
+\end{verbatim}
+
 There are about 200 |ini| files, with data taken from the |ldf| files
 and the CLDR provided by Unicode. Not all languages in the latter are
 complete, and therefore neither are the |ini| files. A few languages
@@ -2000,10 +2024,10 @@ will show a warning about the current lack of suitability of the date
 format (\textsf{hindi}, \textsf{french}, \textsf{breton}, and
 \textsf{occitan}).
 
-Besides |\today|, there is a |\<language>date| macro with three
-arguments: year, month and day numbers. In fact, |\today| calls
-|\<language>today|, which in turn calls
-|\<language>date{\the\year}{\the\month}{\the\day}|.
+Besides |\today|, this option defines an additional command for dates:
+|\<language>date|, which takes three arguments, namely, year, month and
+day numbers. In fact, |\today| calls |\<language>today|, which in turn
+calls |\<language>date{\the\year}{\the\month}{\the\day}|.
 
 \Describe{captions=}{\meta{language-tag}}
 Loads only the strings. For example:
@@ -2039,7 +2063,8 @@ one. Only in newly defined languages.
 \Describe{script=}{\meta{script-name}} \New{3.15} Sets the script name
 to be used by \textsf{fontspec} (eg, |Devanagari|). Overrides the
 value in the |ini| file. This value is particularly important because
-it sets the writing direction.
+it sets the writing direction, so you must use it if for some reason 
+the default value is wrong.
 
 \Describe{language=}{\meta{language-name}} \New{3.15} Sets the
 language name to be used by \textsf{fontspec} (eg, |Hindi|). Overrides
@@ -2049,26 +2074,26 @@ relevant.
 \begin{note}
   (1) If you need shorthands, you can use |\useshorthands| and
   |\defineshorthand| as described above. (2) Captions and |\today| are
-  ``ensured'' with |\babelensure| (this is be the default in
+  ``ensured'' with |\babelensure| (this is the default in
   |ini|-based languages).
 \end{note}
 
 \subsection{Digits}
 
-\New{3.20} A few |ini| files define a field named |digits.native|. When
-it is present, two macros are created: |\<language>digits| and
-|\<language>counter| (only \xetex{} and \luatex{}). With the first, a
-string of ‘Latin’ digits are converted to the native digits of that
-language; the second takes a counter name as argument. With option
-|maparabic| in |\babelprovide|, |\arabic| is redefined to produce the
-native digits (this is done \textit{globally}, to avoid inconsistencies
-in, for example, page numbering).
+\New{3.20} About thirty |ini| files define a field named
+|digits.native|. When it is present, two macros are created:
+|\<language>digits| and |\<language>counter| (only \xetex{} and
+\luatex{}). With the first, a string of ‘Latin’ digits are converted to
+the native digits of that language; the second takes a counter name as
+argument. With the option |maparabic| in |\babelprovide|, |\arabic| is
+redefined to produce the native digits (this is done \textit{globally},
+to avoid inconsistencies in, for example, page numbering).
 
 For example:
 \begin{verbatim}
-\babelprovide[import=te]{telugu}  % Telugu better with XeTeX
+\babelprovide[import]{telugu}  % Telugu better with XeTeX
   % Or also, if you want:
-  % \babelprovide[import=te, maparabic]{telugu}
+  % \babelprovide[import, maparabic]{telugu}
 \babelfont{rm}{Gautami}
 \begin{document}
 \telugudigits{1234}
@@ -2246,11 +2271,12 @@ bug described in the previous paragraph.
 If non-ASCII encodings are not loaded (or no encoding at all), it is
 no-op (also |\TeX| and |\LaTeX| are not redefined); otherwise,
 |\ensureascii| switches to the encoding at the beginning of the
-document if ASCII-savvy, or else the last ASCII-savvy encoding
-loaded. For example, if you load |LY1,LGR|, then it is set to |LY1|,
-but if you load |LY1,T2A| it is set to |T2A|. The symbol encodings
-|TS1|, |T3|, and |TS3| are not taken into account, since they are not
-used for ``ordinary'' text.
+document if ASCII-savvy, or else the last ASCII-savvy encoding loaded.
+For example, if you load |LY1,LGR|, then it is set to |LY1|, but if you
+load |LY1,T2A| it is set to |T2A|. The symbol encodings |TS1|, |T3|,
+and |TS3| are not used for ``ordinary'' text, so they should be
+selected with some other encoding, but they are taken into account,
+just in case.
 
 The foregoing rules (which are applied ``at begin document'') cover
 most of cases. No asumption is made on characters above
@@ -2268,14 +2294,16 @@ differ in the way `weak' numeric characters are ordered (eg, Arabic
 \%123 \textit{vs} Hebrew 123\%).
 
 \begin{warning}
-  Setting bidi text has many subtleties (see for example
-  <https://www.w3.org/TR/html-bidi/>). \textit{This means the \babel{}
-  bidi code may take some time before it is truly stable.}\footnote{A
-  basic stable version for \luatex{} is planned before (Northern)
-  Summer 2018. Other engines must wait very likely until (Northern)
-  Winter.} \textit{This is particularly true for graphical elements,
-  including the |picture| environment and PDF or PS based graphics.}
-
+  The current code for \textbf{text} in \luatex{} should be considered
+  essentially stable, but, of course, it is not bug free and there
+  could be improvements in the future, because setting bidi text has
+  many subtleties (see for example <https://www.w3.org/TR/html-bidi/>).
+  A basic stable version for other engines must wait very likely until
+  (Northern) Winter. This applies to text, but \textbf{graphical}
+  elements, including the |picture| environment and PDF or PS based
+  graphics, are not yet correctly handled. Also, indexes and the like 
+  are under study, as well as math.
+  
   An effort is being made to avoid incompatibilities in the future
   (this one of the reason currently bidi must be explicitly requested
   as a package option, with a certain bidi model, and also the |layout|
@@ -2284,29 +2312,30 @@ differ in the way `weak' numeric characters are ordered (eg, Arabic
 
 There are some package options controlling bidi writing.
 
-\Describe{bidi=}{\texttt{default} $\string|$ \texttt{basic-r}
-  $\string|$ \texttt{basic}}
+\Describe{bidi=}{\texttt{default} $\string|$ \texttt{basic}
+  $\string|$ \texttt{basic-r}}
 
 \New{3.14} Selects the bidi algorithm to be used. With |default| the
 bidi mechanism is just activated (by default it is not), but every
 change must by marked up. In \xetex{} and \pdftex{} this is the only
 option. In \luatex, |basic-r| provides a simple and fast method for R
-text, which handles numbers and unmarked L text within an R context.
-\New{3.19} Finally, |basic| suports both L and R text (see
-\ref{bidibasic}). (They are named \texttt{basic} mainly because they
-only consider the intrinsic direction of scripts and weak
-directionality.)
+text, which handles numbers and unmarked L text within an R context in
+typical cases. \New{3.19} Finally, |basic| supports both L and R text.
+(They are named \texttt{basic} mainly because they only consider the
+intrinsic direction of scripts and weak directionality.)
+
+There are samples on GitHub, under \texttt{/required/babel/samples}.
+See particularly |lua-bidibasic.tex| and |lua-secenum.tex|.
 
 \begin{example}
   The following text comes from the Arabic Wikipedia (article about
   Arabia). Copy-pasting some text from the Wikipedia is a good way to
-  test this feature, which will be improved in the future. Remember
-  |basic-r| is available in \luatex{} only.\footnote{At the time of
-  this writing some Arabic fonts are not rendered correctly by the
-  default \luatex{} font loader, with misplaced kerns inside some
-  words, so double check the resulting text. It seems a fix is on the
-  way, but in the meanwhile you could have a look at the workaround
-  available on GitHub, under \texttt{/required/babel/samples}}
+  test this feature. Remember |basic-r| is available in \luatex{}
+  only.\footnote{At the time of this writing some Arabic fonts are not
+  rendered correctly by the default \luatex{} font loader, with
+  misplaced kerns inside some words, so double check the resulting
+  text. Have a look at the workaround available on GitHub, under
+  \texttt{/required/babel/samples}}
   \begingroup
 % If you are looking at the code to see how it has been written, you
 % will be disappointed :-). The following example is built ad hoc to
@@ -2320,7 +2349,7 @@ directionality.)
 
 \usepackage[_bidi=basic-r_]{babel}
 
-_\babelprovide[import=ar, main]{arabic}_
+_\babelprovide[import, main]{arabic}_
 
 \babelfont{rm}{FreeSerif}
 
@@ -2336,6 +2365,65 @@ _\babelprovide[import=ar, main]{arabic}_
 \endgroup
 \end{example}
 
+\begin{example}
+  With |bidi=basic| \textit{both} L and R text can be
+  mixed without explicit markup (the latter will be only necessary in
+  some special cases where the Unicode algorithm fails). It is used much
+  like |bidi=basic-r|, but with R text inside L text you may want to map
+  the font so that the correct features are in force. This is accomplised
+  with an option in |\babelprovide|, as illustrated:
+    \begingroup
+  % If you are looking at the code to see how it has been written, you
+  % will be disappointed :-). The following example is built ad hoc to
+  % emulate the final result to avoid dependencies, and therefore it's
+  % not "real" code.
+  \setmonofont[Scale=.87,Script=Arabic]{DejaVu Sans Mono} \catcode`@=13
+  \def@#1{\ifcase#1\relax \egroup \or \bgroup\textdir TRT \else
+  \bgroup\textdir TLT \pardir TLT \fi}
+  \begin{verbatim}
+  \documentclass{book}
+
+  \usepackage[english, _bidi=basic_]{babel}
+
+  \babelprovide[_mapfont=direction_]{arabic}
+
+  \babelfont{rm}{Crimson}
+  \babelfont[*arabic]{rm}{FreeSerif}
+
+  \begin{document}
+
+  Most Arabic speakers consider the two varieties to be two registers
+  of one language, although the two registers can be referred to in
+  Arabic as @1فصحى العصر@0 \textit{fuṣḥā l-ʻaṣr} (MSA) and
+  @1فصحى التراث@0 \textit{fuṣḥā t-turāth} (CA).
+
+  \end{document}
+  \end{verbatim}
+  \endgroup
+  What |mapfont=direction| means is, ‘when a character has the same
+  direction as the script for the “provided” language (|arabic| in
+  this case), then change its font to that set for this language’ (here
+  defined via |*arabic|, because Crimson does not provide Arabic
+  letters). Note Hebrew and Arabic, following the Unicode rules, have
+  different directions (internally ‘r’ and ‘al’, respectively).
+\end{example}
+
+\begin{note}
+  Boxes are “black boxes”. Numbers inside an |\hbox| (as for example
+  in a |\ref|) do not know anything about the surrounding chars. So,
+  |\ref{A}-\ref{B}| are not rendered in the visual order A-B, but in the
+  wrong one B-A (because the hyphen does not “see” the digits inside the
+  |\hbox|’es). If you need |\ref| ranges, the best option is to define
+  a dedicated macro like this (to avoid explicit direction changes in the
+  body; here |\texthe| must be defined to select the main language):
+  \begin{verbatim}
+  \newcommand\refrange[2]{\babelsublr{\texthe{\ref{#1}}-\texthe{\ref{#2}}}}
+  \end{verbatim}
+
+  In a future a more complete method, reading recursively boxed text, may
+  be added.
+\end{note}
+
 \Describe{layout=}{\texttt{sectioning} $\string|$
                    \texttt{counters}  $\string|$
                    \texttt{lists}  $\string|$
@@ -2406,39 +2494,49 @@ required by all engines.
 
 \item[extras] is used for miscelaneous readjustments which do not fit into
   the previous groups. Currently redefines in \luatex{} |\underline| and
-  |LaTeX2e| \New{3.19}.
+  |\LaTeX2e| \New{3.19}.
 \end{description}
 
 \Describe{\babelsublr}{\marg{lr-text}}
 
 Digits in \pdftex{} must be marked up explicitly (unlike \luatex{} with
-|bidi=basic-r| and, usually, \xetex{}). This command is provided to set
-\marg{lr-text} in L mode if necessary. It's intended for what Unicode
-calls weak characters, because words are best set with the
-corresponding language. For this reason, there is no |rl| counterpart.
+|bidi=basic| or |bidi=basic-r| and, usually, \xetex{}). This command is
+provided to set \marg{lr-text} in L mode if necessary. It's intended
+for what Unicode calls weak characters, because words are best set with
+the corresponding language. For this reason, there is no |rl|
+counterpart.
 
 Any |\babelsublr| in \textit{explicit} L mode is ignored. However, with
 |bidi=basic| and \textit{implicit} L, it first returns to R and then
-switches to explicit L. This is by design to provide the proper behaviour
-in the most usual cases — but if you need to use |\ref| in an L text
-inside R, it must be marked up explictly.
+switches to explicit L. To clarify this point, consider, in an R 
+context:
+\begin{verbatim}
+RTL A ltr text \thechapter{} and still ltr RTL B
+\end{verbatim}
+There are \textit{three} R blocks and \textit{two} L blocks, and the
+order is \textit{RTL B and still ltr 1 ltr text RTL A}. This is by
+design to provide the proper behaviour in the most usual cases — but if
+you need to use |\ref| in an L text inside R, the L text must be marked
+up explictly; for example:
+\begin{verbatim}
+RTL A \foreignlanguage{english}{ltr text \thechapter{} and still ltr} RTL B
+\end{verbatim}
 
 \Describe{\BabelPatchSection}{\marg{section-name}}
 
-Mainly for bidi text, but it could be useful in other
-cases. |\BabelPatchSection| and the corresponding option
-|layout=sectioning| takes a more logical approach (at least in many
-cases) because it applies the global language to the section format
-(including the |\chaptername| in |\chapter|), while the section text
-is still the current language. The latter is passed to tocs and marks,
-too, and with |sectioning| in |layout| they both reset the “global”
-language to the main one, while the text uses the “local”
-language
-
-With |layout=sectioning| all the standard sectioning
-commands are redefined, but with this command you can set them
-individually if necessary (but note then tocs and marks are not
-touched).
+Mainly for bidi text, but it could be useful in other cases.
+|\BabelPatchSection| and the corresponding option |layout=sectioning|
+takes a more logical approach (at least in many cases) because it
+applies the global language to the section format (including the
+|\chaptername| in |\chapter|), while the section text is still the
+current language. The latter is passed to tocs and marks, too,
+and with |sectioning| in |layout| they both reset the “global” language
+to the main one, while the text uses the “local” language.
+
+With |layout=sectioning| all the standard sectioning commands are
+redefined (it also “isolates” the page number in heads, for a proper
+bidi behavior), but with this command you can set them individually if
+necessary (but note then tocs and marks are not touched).
 
 \Describe{\BabelFootnote}{\marg{cmd}\marg{local-language}%
   \marg{before}\marg{after}}
@@ -2581,7 +2679,7 @@ requiring a command to switch the language. Its default value is
 |toc,lof,lot|, but you may redefine it with |\renewcommand| (it's up
 to you to make sure no toc type is duplicated).
 
-\subsection{Languages supported by \babel}
+\subsection{Languages supported by \babel{} with \textsf{ldf} files}
 
 In the following table most of the languages supported by \babel{} with
 and |.ldf| file are listed, together with the names of the option which
@@ -2656,6 +2754,12 @@ you can create a file with extension |.dn|:
 Then you preprocess it with |devnag| \m{file}, which creates
 \m{file}|.tex|; you can then typeset the latter with \LaTeX.
 
+\begin{note}
+  Please, for info about the support in luatex for some complex scripts,
+  see the wiki, on \texttt{https://github.com/latex3/latex2e/wiki/%
+  Babel:-Remarks-on-the-luatex-support-for-some-scripts}.
+\end{note}
+
 \subsection{Tips, workarounds, know issues and notes}
 
 \begin{itemize}
@@ -2767,7 +2871,7 @@ discretionary by the equivalent to ZWJ.
 
 \usepackage{babel}
 
-\babelprovide[import=th, main]{thai}
+\babelprovide[import, main]{thai}
 
 \babelfont{rm}{FreeSerif}
 
@@ -2820,66 +2924,6 @@ chapters), in Hungarian ``from (1)'' is ``(1)-b\H{o}l'', but ``from
 ``3.$^{\textrm{\scriptsize er}}$ \'{\i}tem'', and so on.
 
 \subsection{Tentative and experimental code}
-\label{bidibasic}
-
-\textbf{Option \texttt{bidi=basic}}
-
-\New{3.19} With this package option \textit{both} L and R text can be
-mixed without explicit markup (the latter will be only necessary in
-some special cases where the Unicode algorithm fails). It is used much
-like |bidi=basic-r|, but with R text inside L text you may want to map
-the font so that the correct features are in force. This is accomplised
-with an option in |\babelprovide|, as illustrated:
-  \begingroup
-% If you are looking at the code to see how it has been written, you
-% will be disappointed :-). The following example is built ad hoc to
-% emulate the final result to avoid dependencies, and therefore it's
-% not "real" code.
-\setmonofont[Scale=.87,Script=Arabic]{DejaVu Sans Mono} \catcode`@=13
-\def@#1{\ifcase#1\relax \egroup \or \bgroup\textdir TRT \else
-\bgroup\textdir TLT \pardir TLT \fi}
-\begin{verbatim}
-\documentclass{book}
-
-\usepackage[english, _bidi=basic_]{babel}
-
-\babelprovide[_mapfont=direction_]{arabic}
-
-\babelfont{rm}{Crimson}
-\babelfont[*arabic]{rm}{FreeSerif}
-
-\begin{document}
-
-Most Arabic speakers consider the two varieties to be two registers
-of one language, although the two registers can be referred to in
-Arabic as @1فصحى العصر@0 \textit{fuṣḥā l-ʻaṣr} (MSA) and
-@1فصحى التراث@0 \textit{fuṣḥā t-turāth} (CA).
-
-\end{document}
-\end{verbatim}
-\endgroup
-What |mapfont=direction| means is, ‘when a character has the same
-direction as the script for the “provided” language (|arabic| in
-this case), then change its font to that set for this language’ (here
-defined via |*arabic|, because Crimson does not provide Arabic
-letters).
-
-Boxes are “black boxes”. Numbers inside an |\hbox| (as for example
-in a |\ref|) do not know anything about the surrounding chars. So,
-|\ref{A}-\ref{B}| are not rendered in the visual order A-B, but in the
-wrong one B-A (because the hyphen does not “see” the digits inside the
-|\hbox|’es). If you need |\ref| ranges, the best option is to define
-a dedicated macro like this (to avoid explicit direction changes in the
-body; here |\texthe| must be defined to select the main language):
-\begin{verbatim}
-\newcommand\refrange[2]{\babelsublr{\texthe{\ref{#1}}-\texthe{\ref{#2}}}}
-\end{verbatim}
-
-In a future a more complete method, reading recursively boxed text, may
-be added.
-
-There are samples on GitHub, under \texttt{/required/babel/samples}:
-|lua-bidibasic.tex| and |lua-secenum.tex|.
 
 \medskip
 \textbf{Old stuff}
@@ -3634,7 +3678,7 @@ assignment is wrong, fix it directly.
 
 \subsection{Changes in \babel\ version 3.9}
 
-Most of changes in version 3.9 are related to bugs, either to fix them
+Most of changes in version 3.9 were related to bugs, either to fix them
 (there were lots), or to provide some alternatives. Even new features
 like |\babelhyphen| are intended to solve a certain problem (in this
 case, the lacking of a uniform syntax and behavior for shorthands
@@ -3673,45 +3717,6 @@ corresponding place. A selective list follows:
   called as global options.
 \end{itemize}
 
-\subsection{Changes in \babel\ version 3.7}
-
-In \babel\ version 3.7 a number of bugs that were found in
-version~3.6 are fixed. Also a number of changes and additions
-have occurred:
-\begin{itemize}
-\item Shorthands are expandable again. The disadvantage is that
-  one has to type |'{}a| when the acute accent is used as a
-  shorthand character. The advantage is that a number of other
-  problems (such as the breaking of ligatures, etc.) have
-  vanished.
-\item Two new commands, |\shorthandon| and |\shorthandoff| have
-  been introduced to enable to temporarily switch off one or more
-  shorthands.
-\item Support for typesetting Hebrew (and potential support for
-  typesetting other right-to-left written languages) is now
-  available thanks to Rama Porrat and Boris Lavva.
-\item A language attribute has been added to the |\mark...|
-  commands in order to make sure that a Greek header line comes
-  out right on the last page before a language switch.
-\item Hyphenation pattern files are now read \emph{inside a
-  group}; therefore any changes a pattern file needs to make to
-  lowercase codes, uppercase codes, and category codes are kept
-  local to that group. If they are needed for the language, these
-  changes will need to be repeated and stored in |\extras...|
-\item The concept of language attributes is introduced. It is intended
-  to give the user some control over the features a
-  language-definition file provides. Its first use is for the Greek
-  language, where the user can choose the πολυτονικό (``polytonikó'' or
-  multi-accented) Greek way of typesetting texts.
-\item The environment \Lenv{hyphenrules} is introduced.
-\item The syntax of the file \file{language.dat} has been
-  extended to allow (optionally) specifying the font
-  encoding to be used while processing the patterns file.
-\item The command |\providehyphenmins| should now be used in
-  language definition files in order to be able to keep any
-  settings provided by the pattern file.
-\end{itemize}
-
 \DocInput{babel.dtx}
 
 \section{Acknowledgements}
@@ -3835,6 +3840,7 @@ help from Bernd Raichle, for which I am grateful.
 %
 % Most keys are self-explanatory.
 % \begin{description}
+% \itemsep=-\parskip
 % \item[charset] the encoding used in the ini file.
 % \item[version] of the ini file
 % \item[level] ``version'' of the ini specification . which keys are
@@ -3859,8 +3865,8 @@ help from Bernd Raichle, for which I am grateful.
 % \section{Tools}
 %
 %    \begin{macrocode}
-%<<version=3.22>>
-%<<date=2018/06/05>>
+%<<version=3.23>>
+%<<date=2018/09/02>>
 %    \end{macrocode}
 %
 % \textbf{Do not use the following macros in \texttt{ldf} files. They
@@ -4497,7 +4503,7 @@ help from Bernd Raichle, for which I am grateful.
 %    wrapped (in |babel.def|) to define only those given.
 %
 %    A bit of optimization: if there is no |shorthands=|, then
-%    |\bbl@ifshorthands| is always true, and it is always false if
+%    |\bbl@ifshorthand| is always true, and it is always false if
 %    |shorthands| is empty. Also, some code makes sense only with
 %    |shorthands=...|.
 %
@@ -4520,7 +4526,7 @@ help from Bernd Raichle, for which I am grateful.
 \else
 %    \end{macrocode}
 %
-%    The following macro tests if a shortand is one of the allowed
+%    The following macro tests if a shorthand is one of the allowed
 %    ones.
 %
 %    \begin{macrocode}
@@ -5095,7 +5101,7 @@ help from Bernd Raichle, for which I am grateful.
 %    case you need them for some reason).
 %
 %    \begin{macrocode}
-\def\bbl@evargs{,% don't delete the comma
+\def\bbl@evargs{,% <- don't delete this comma
   everylanguage=1,loadkernel=1,loadpatterns=1,loadexceptions=1,%
   adddialect=2,patterns=2,defaultcommands=0,encodedcommands=2,write=0,%
   beforeextras=0,afterextras=0,stopcommands=0,stringprocess=0,%
@@ -6161,6 +6167,15 @@ help from Bernd Raichle, for which I am grateful.
 \fi
 %    \end{macrocode}
 %
+% \changes{babel~3.23}{2018/08/26}{Added \cs{ifbabelshorthand}}
+%
+% You may want to test if a character is a shorthand. Note it does not
+% test whether the shorthand is on or off.
+%
+%    \begin{macrocode}
+\newcommand\ifbabelshorthand[3]{\bbl@ifunset{bbl@active@\string#1}{#3}{#2}}
+%    \end{macrocode}
+%
 % \changes{babel~3.9a}{2012/12/27}{Removed redundant system declarations}
 %
 %  \begin{macro}{\bbl@prim@s}
@@ -6169,12 +6184,11 @@ help from Bernd Raichle, for which I am grateful.
 % \changes{babel~3.9a}{2012/07/29}{\cs{bbl@pr@m@s} rewritten to
 %    take into account catcodes for both the quote and the hat}
 %
-%    One of the internal macros that are involved in substituting
-%    |\prime| for each right quote in mathmode is |\prim@s|. This
-%    checks if the next character is a right quote. When the right
-%    quote is active, the definition of this macro needs to be adapted
-%    to look also for an active right quote; the hat could be active,
-%    too.
+% One of the internal macros that are involved in substituting |\prime|
+% for each right quote in mathmode is |\prim@s|. This checks if the
+% next character is a right quote. When the right quote is active, the
+% definition of this macro needs to be adapted to look also for an
+% active right quote; the hat could be active, too.
 %
 %    \begin{macrocode}
 \def\bbl@prim@s{%
@@ -7717,6 +7731,7 @@ help from Bernd Raichle, for which I am grateful.
 %   only with direction}
 % \changes{babel~3.20}{2018/05/01}{Handle native digits (TeX level).
 %   New option - maparabic.}
+% \changes{babel~3.23}{2018/09/01}{Valueless import}
 %
 %    \begin{macrocode}
 \bbl@trace{Creating languages and reading ini files}
@@ -7733,6 +7748,14 @@ help from Bernd Raichle, for which I am grateful.
   \let\bbl@KVP@mapfont\@nil
   \let\bbl@KVP@maparabic\@nil
   \bbl@forkv{#1}{\bbl@csarg\def{KVP@##1}{##2}}%  TODO - error handling
+  \ifx\bbl@KVP@import\@nil\else
+    \bbl@exp{\\\bbl@ifblank{\bbl@KVP@import}}%
+      {\begingroup
+         \def\BabelBeforeIni##1##2{\gdef\bbl@KVP@import{##1}\endinput}%
+         \InputIfFileExists{babel-#2.tex}{}{}%
+       \endgroup}%
+      {}%
+  \fi
   \ifx\bbl@KVP@captions\@nil
     \let\bbl@KVP@captions\bbl@KVP@import
   \fi
@@ -7787,8 +7810,13 @@ help from Bernd Raichle, for which I am grateful.
         \expandafter\expandafter\expandafter
         \bbl@setdigits\csname bbl@dgnat@\languagename\endcsname
         \ifx\bbl@KVP@maparabic\@nil\else
-          \expandafter\let\expandafter\@arabic
-            \csname bbl@counter@\languagename\endcsname
+          \ifx\bbl@latinarabic\@undefined
+            \expandafter\let\expandafter\@arabic
+              \csname bbl@counter@\languagename\endcsname
+          \else    % ie, if layout=counters, which redefines \@arabic
+            \expandafter\let\expandafter\bbl@latinarabic
+              \csname bbl@counter@\languagename\endcsname
+          \fi
         \fi
       \fi}%
   \fi
@@ -8150,16 +8178,17 @@ help from Bernd Raichle, for which I am grateful.
     {}%
   \bbl@csarg\let{lsys@#1}\@empty
   \bbl@ifunset{bbl@sname@#1}{\bbl@csarg\gdef{sname@#1}{Default}}{}%
-  \bbl@ifunset{bbl@sotf#1}{\bbl@csarg\gdef{sotf@#1}{DFLT}}{}%
+  \bbl@ifunset{bbl@sotf@#1}{\bbl@csarg\gdef{sotf@#1}{DFLT}}{}%
   \bbl@csarg\bbl@add@list{lsys@#1}{Script=\bbl@cs{sname@#1}}%
   \bbl@ifunset{bbl@lname@#1}{}%
     {\bbl@csarg\bbl@add@list{lsys@#1}{Language=\bbl@cs{lname@#1}}}%
-  \bbl@csarg\bbl@toglobal{lsys@#1}}%
- %  \bbl@exp{% TODO - should be global
- %    \<keys_if_exist:nnF>{fontspec-opentype/Script}{\bbl@cs{sname@#1}}%
- %      {\\\newfontscript{\bbl@cs{sname@#1}}{\bbl@cs{sotf@#1}}}%
- %    \<keys_if_exist:nnF>{fontspec-opentype/Language}{\bbl@cs{lname@#1}}%
- %      {\\\newfontlanguage{\bbl@cs{lname@#1}}{\bbl@cs{lotf@#1}}}}}
+  \bbl@csarg\bbl@toglobal{lsys@#1}%}%
+  \bbl@exp{% TODO - should be global, but even local does its job
+           % I'm still not sure -- must investigate 
+    \<keys_if_exist:nnF>{fontspec-opentype}{Script/\bbl@cs{sname@#1}}%
+      {\\\newfontscript{\bbl@cs{sname@#1}}{\bbl@cs{sotf@#1}}}%
+    \<keys_if_exist:nnF>{fontspec-opentype}{Language/\bbl@cs{lname@#1}}%
+      {\\\newfontlanguage{\bbl@cs{lname@#1}}{\bbl@cs{lotf@#1}}}}}
 %    \end{macrocode}
 %
 % The following |ini| reader ignores everything but the
@@ -8554,6 +8583,9 @@ help from Bernd Raichle, for which I am grateful.
 %    it into the token registers}
 % \changes{babel~3.9t}{2017/04/23}{Refactored \cs{markright} and
 %    \cs{markboth}}
+% \changes{babel~3.23}{2018/09/01}{Trick to isolate the bidi in page 
+%    numbers}
+%
 %
 %    We check whether the argument is empty; if it is, we just make
 %    sure the scratch token register is empty.  Next, we store the
@@ -8571,7 +8603,9 @@ help from Bernd Raichle, for which I am grateful.
      \g@addto@macro\@resetactivechars{%
        \set@typeset@protect
        \expandafter\select@language@x\expandafter{\bbl@main@language}%
-       \let\protect\noexpand}%
+       \let\protect\noexpand
+       \edef\thepage{%
+         \noexpand\babelsublr{\unexpanded\expandafter{\thepage}}}}%
    \fi}
   {\bbl@redefine\markright#1{%
      \bbl@ifblank{#1}%
@@ -8579,7 +8613,7 @@ help from Bernd Raichle, for which I am grateful.
        {\toks@{#1}%
         \bbl@exp{%
           \\\org@markright{\\\protect\\\foreignlanguage{\languagename}%
-            {\\\protect\\\bbl@restore@actives\the\toks@}}}}}
+            {\\\protect\\\bbl@restore@actives\the\toks@}}}}}%
 %    \end{macrocode}
 %
 %  \end{macro}
@@ -8888,13 +8922,12 @@ help from Bernd Raichle, for which I am grateful.
 %  Because documents may use non-ASCII font encodings, we make sure
 %  that the logos of \TeX\ and \LaTeX\ always come out in the right
 %  encoding. There is a list of non-ASCII encodings. Unfortunately,
-%  \textsf{fontenc} deletes its package options, so we must guess
-%  which encodings has been loaded by traversing |\@filelist| to
-%  search for \m{enc}|enc.def|. If a non-ASCII has been loaded, we
-%  define versions of |\TeX| and |\LaTeX| for them using
-%  |\ensureascii|. The default ASCII encoding is set, too (in reverse
-%  order): the ``main'' encoding (when the document begins), the last
-%  loaded, or |OT1|.
+%  \textsf{fontenc} deletes its package options, so we must guess which
+%  encodings has been loaded by traversing |\@filelist| to search for
+%  \m{enc}|enc.def|. If a non-ASCII has been loaded, we define versions
+%  of |\TeX| and |\LaTeX| for them using |\ensureascii|. The default
+%  ASCII encoding is set, too (in reverse order): the ``main'' encoding
+%  (when the document begins), the last loaded, or |OT1|.
 %
 %  \begin{macro}{\ensureascii}
 %
@@ -8902,10 +8935,12 @@ help from Bernd Raichle, for which I am grateful.
 %    \cs{textlatin} and friends}
 % \changes{babel~3.9j}{2014/03/17}{Moved misplaced code - it should be
 %    executed only with LaTeX}
+% \changes{babel~3.23}{2018/08/28}{Added TS1, T3, TS3}
 %
 %    \begin{macrocode}
 \bbl@trace{Encoding and fonts}
-\newcommand\BabelNonASCII{LGR,X2,OT2,OT3,OT6,LHE,LWN,LMA,LMC,LMS,LMU,}
+\newcommand\BabelNonASCII
+  {LGR,X2,OT2,OT3,OT6,LHE,LWN,LMA,LMC,LMS,LMU,TS1,T3,TS3}
 \let\org@TeX\TeX
 \let\org@LaTeX\LaTeX
 \let\ensureascii\@firstofone
@@ -18100,6 +18135,9 @@ function Babel.bidi(head, ispar, hdir)
   local nodes = {}
   local outer_first = nil
 
+  local glue_d = nil
+  local glue_i = nil
+
   local has_en = false
   local first_et = nil
 
@@ -18186,10 +18224,23 @@ function Babel.bidi(head, ispar, hdir)
         new_d = false
       end
 
+      if glue_d then
+        if (d == 'l' and 'l' or 'r') ~= glue_d then
+           table.insert(nodes, {glue_i, 'on', nil})
+        end
+        glue_d = nil
+        glue_i = nil
+      end
+
     elseif item.id == DIR then
       d = nil
       new_d = true
 
+    elseif item.id == node.id'glue' and item.subtype == 13 then
+      glue_d = d
+      glue_i = item
+      d = nil
+
     else
       d = nil
     end
@@ -18249,13 +18300,6 @@ function Babel.bidi(head, ispar, hdir)
       end
       prev_d = d
       table.insert(nodes, {item, d, outer_first})
-    else
-      -- Not sure about the following. Looks too 'ad hoc', but it's
-      -- required for numbers, so that 89 19 becomes 19 89. It also
-      -- affects n+cs/es+n.
-      if prev_d == 'an' or prev_d == 'en' then
-        table.insert(nodes, {item, 'on', nil})
-      end
     end
 
     outer_first = nil
@@ -18377,7 +18421,7 @@ function Babel.bidi(head, ispar, hdir)
     end
 
   end
-
+  
   return node.prev(head) or head
 end
 %</basic>
@@ -18736,9 +18780,11 @@ end
          \noexpand#1%
       \fi
       \noexpand\protect
-      \expandafter\noexpand\csname\bbl@stripslash#1 \endcsname
+      \expandafter\noexpand\csname
+         \expandafter\@gobble\string#1 \endcsname
    }%
-   \expandafter\new@command\csname\bbl@stripslash#1 \endcsname
+   \expandafter\new@command\csname
+      \expandafter\@gobble\string#1 \endcsname
 }
 \def\x@protect#1{%
    \ifx\protect\@typeset@protect\else