From 933416e3a978851a263168f5daad76eea18de483 Mon Sep 17 00:00:00 2001 From: Norbert Preining Date: Tue, 20 Jul 2021 03:00:52 +0000 Subject: CTAN sync 202107200300 --- macros/latex/required/babel/base/README.md | 21 +- macros/latex/required/babel/base/babel.dtx | 1667 +++++++++++++------------ macros/latex/required/babel/base/babel.ins | 3 +- macros/latex/required/babel/base/babel.pdf | Bin 846875 -> 852005 bytes macros/latex/required/babel/base/bbcompat.dtx | 2 +- 5 files changed, 879 insertions(+), 814 deletions(-) (limited to 'macros/latex/required') diff --git a/macros/latex/required/babel/base/README.md b/macros/latex/required/babel/base/README.md index 0d63b0df47..777d19bd2b 100644 --- a/macros/latex/required/babel/base/README.md +++ b/macros/latex/required/babel/base/README.md @@ -1,4 +1,4 @@ -## Babel 3.61 +## Babel 3.62 This package manages culturally-determined typographical (and other) rules, and hyphenation patterns for a wide range of languages. Many @@ -8,9 +8,9 @@ is a set of ini files for about 250 languages. The latest stable version is available on . -Changes in version 3.61 are described in: +Changes in version 3.62 are described in: -https://latex3.github.io/babel/news/whats-new-in-babel-3.61.html +https://latex3.github.io/babel/news/whats-new-in-babel-3.62.html Apart from the manual, you can find information on some aspects of babel at: @@ -46,16 +46,13 @@ respective authors. ### Summary of Latest changes ``` -3.61 2021-06-28 - * Improved justification=kashida/elongated: hboxes (lua). - * Transform danda.nobreak for several Indic scripts (lua): - Assamese, Bengali, Gujarati, Hindi, Kannada, Malayalam, - Marathi, Oriya, Tamil, Telugu. - * Improved \babelprovide when used to reconfigure a language. +3.62 2021-07-19 + * No more errors with unknown languages in aux file. + * Preliminary support for \AddToHook. + * Tentative extension for provide=. * Fixes: - - Partial fix for #114 (bad breaks and spacing with - \selectlanguage). - - \shorthandoff*{^} caused error for \section command (#129). + - Locale dependent quotes with CJK (#131). + - Babel and hyperref prevent changes to the \LaTeX logo (#138). ``` ### Previous changes diff --git a/macros/latex/required/babel/base/babel.dtx b/macros/latex/required/babel/base/babel.dtx index b6aea49d74..e88326ebe2 100644 --- a/macros/latex/required/babel/base/babel.dtx +++ b/macros/latex/required/babel/base/babel.dtx @@ -32,7 +32,7 @@ % % \iffalse %<*filedriver> -\ProvidesFile{babel.dtx}[2021/06/28 v3.61 The Babel package] +\ProvidesFile{babel.dtx}[2021/07/19 v3.62 The Babel package] \documentclass{ltxdoc} \GetFileInfo{babel.dtx} \usepackage{fontspec} @@ -697,6 +697,14 @@ This command can be used as environment, too. code with an additional grouping level. \end{warning} +\begin{warning} + |\selectlanguage| should not be used inside some boxed environments + (like floats or |minipage|) to switch the language if you need the + information written to the |aux| be correctly synchronized. This + rarely happens, but if it were the case, you must use |otherlanguage| + instead. +\end{warning} + \Describe{\foreignlanguage}{\oarg{option-list}\marg{language}\marg{text}} The command |\foreignlanguage| takes two arguments; the second argument @@ -936,6 +944,12 @@ If you do not need shorthands, or prefer an alternative approach of your own, you may want to switch them off with the package option |shorthands=off|, as described below. +\begin{warning} + It is worth emphasizing these macros are meant for temporary changes. + Whenever possible and if there are not conflicts with other packages, + shorthands must be always enabled (or disabled). +\end{warning} + \Describe{\useshorthands}{% \colorbox{thegrey}{\ttfamily\hskip-.2em*\hskip-.2em}% \marg{char}} @@ -1264,7 +1278,6 @@ option (by its name in |language.dat|). There are two main uses: classes and packages, and as a last resort in case there are, for some reason, incompatible languages. It can be used if you just want to select the hyphenation patterns of a single language, too. -% TODO: example \Describe\AfterBabelLanguage{\marg{option-name}\marg{code}} @@ -1292,6 +1305,13 @@ be the same as the option name as set in |\usepackage|!). \end{verbatim} \end{example} +\begin{note} + With a recent version of \LaTeX, an alternative method to execute + some code just after an |ldf| file is loaded is with |\AddToHook| and + the hook |file/after/.ldf|. \Babel{} does not predeclare + it, and you have to do it yourself with |\NewHook| or |\ProvideHook|. +\end{note} + \begin{warning} Currently this option is not compatible with languages loaded on the fly. @@ -3120,12 +3140,12 @@ standard system devised by Ljudevit Gaj.} \Describe{\babelposthyphenation}{\marg{hyphenrules-name}% \marg{lua-pattern}\marg{replacement}} -\New{3.37-3.39} \textit{With \luatex{}} it is now possible to define +\New{3.37-3.39} \textit{With \luatex{}} it is possible to define non-standard hyphenation rules, like |f-f| $\to$ |ff-f|, repeated hyphens, ranked ruled (or more precisely, ‘penalized’ hyphenation -points), and so on. Only a few rules are currently provided (see -below), but they can be defined as shown in the following example, -where |{1}| is the first captured char (between |()| in the pattern): +points), and so on. A few rules are currently provided (see above), but +they can be defined as shown in the following example, where |{1}| is +the first captured char (between |()| in the pattern): \begin{verbatim} \babelposthyphenation{german}{([fmtrp]) | {1}} { @@ -3690,15 +3710,24 @@ options are also used (eg, |\ProsodicMarksOn| in \textsf{latin}). events. Some hooks are predefined when \luatex{} and \xetex{} are used. +\New{3.62} This is not the only way to inject code at those points. The +events listed below can be used as a hook name in |\AddToHook| in the +form |babel/|\marg{name}, but there is a limitation, because the +parameters passed with the \babel{} mechanism are not allowed. The +|\AddToHook| mechanism does \textit{not} replace the current one in +`babel`. Its main advantage is you can reconfigure `babel` even before +loading it. See the example below. + \Describe{\AddBabelHook}{\oarg{lang}\marg{name}\marg{event}\marg{code}} -The same name can be applied to several events. Hooks may be enabled -and disabled for all defined events with |\EnableBabelHook|\marg{name}, -|\DisableBabelHook|\marg{name}. Names containing the string |babel| are -reserved (they are used, for example, by |\useshortands*| to add a hook -for the event |afterextras|). \New{3.33} They may be also applied to a -specific language with the optional argument; language-specific -settings are executed after global ones. +The same name can be applied to several events. Hooks with a certain +\marg{name} may be enabled and disabled for all defined events with +|\EnableBabelHook|\marg{name}, |\DisableBabelHook|\marg{name}. Names +containing the string |babel| are reserved (they are used, for example, +by |\useshortands*| to add a hook for the event |afterextras|). +\New{3.33} They may be also applied to a specific language with the +optional argument; language-specific settings are executed after global +ones. Current events are the following; in some of them you can use one to three \TeX{} parameters (|#1|, |#2|, |#3|), with the meaning given: @@ -3771,6 +3800,27 @@ ones, they only have a single hook and replace a default definition. file. Used by \file{luababel.def}. \end{description} +\begin{example} +The generic unlocalized \LaTeX{} hooks are predefined, so that you can +write: +\begin{verbatim} +\AddToHook{babel/afterextras}{\frenchspacing} +\end{verbatim} +which is executed always after the extras for the language being +selected (and just before the non-localized hooks defined with +|\AddBabelHook|). + +In addition, locale-specific hooks in the form +|babel/|\marg{hook-name}|/|\marg{language-name} are \textit{recognized} +(executed just before the localized \babel{} hooks), but they are \textit{not +predefined}. You have to do it yourself. For example, to set +|\frenchspacing| only in |bengali|: +\begin{verbatim} +\ProvideHook{babel/afterextras/bengali} +\AddToHook{babel/afterextras/bengali}{\frenchspacing} +\end{verbatim} +\end{example} + \Describe{\BabelContentsFiles}{} \New{3.9a} This macro contains a list of ``toc'' types requiring a command to switch the language. Its default value is @@ -3951,7 +4001,8 @@ direction with |bidi.text|). \item \verb|\bibitem| is out of sync with \verb|\selectlanguage| in the \file{.aux} file. The reason is \verb|\bibitem| uses \verb|\immediate| (and others, in fact), while - \verb|\selectlanguage| doesn't. There is no known workaround. + \verb|\selectlanguage| doesn't. There is a similar issue with floats, + too. There is no known workaround. \item Babel does not take into account |\normalsfcodes| and (non-)French spacing is not always properly (un)set by languages. However, problems are unlikely to happen and therefore @@ -4973,8 +5024,8 @@ help from Bernd Raichle, for which I am grateful. % \section{Tools} % % \begin{macrocode} -%<> -%<> +%<> +%<> % \end{macrocode} % % \textbf{Do not use the following macros in \texttt{ldf} files. They @@ -5174,6 +5225,8 @@ help from Bernd Raichle, for which I am grateful. % % \begin{macro}{\bbl@replace} % +% Returns implicitly |\toks@| with the modified string. +% % \begin{macrocode} \def\bbl@replace#1#2#3{% in #1 -> repl #2 by #3 \toks@{}% @@ -5293,6 +5346,16 @@ help from Bernd Raichle, for which I am grateful. \fi} % \end{macrocode} % +% An alternative to |\IfFormatAtLeastTF| for old versions. Temporary. +% +% \begin{macrocode} +\ifx\IfFormatAtLeastTF\@undefined + \def\bbl@ifformatlater{\@ifl@t@r\fmtversion} +\else + \let\bbl@ifformatlater\IfFormatAtLeastTF +\fi +% \end{macrocode} +% % The following adds some code to |\extras...| both before and after, % while avoiding doing it twice. It's somewhat convoluted, to deal with % |#|'s. @@ -5348,8 +5411,8 @@ help from Bernd Raichle, for which I am grateful. % % \begin{macro}{\last@language} % -% Another counter is used to store the last language defined. For -% pre-3.0 formats an extra counter has to be allocated. +% Another counter is used to keep track of the allocated languages. +% \TeX{} and \LaTeX{} reserves for this purpose the count 19. % % \begin{macro}{\addlanguage} % @@ -5358,8 +5421,7 @@ help from Bernd Raichle, for which I am grateful. % % \begin{macrocode} %<<*Define core switching macros>> -%<<*Define core switching macros>> -\countdef\last@language=19 % TODO. why? remove? +\countdef\last@language=19 \def\addlanguage{\csname newlanguage\endcsname} %<> % \end{macrocode} @@ -5369,7 +5431,7 @@ help from Bernd Raichle, for which I am grateful. % % Now we make sure all required files are loaded. When the command % |\AtBeginDocument| doesn't exist we assume that we are dealing -% with a plain-based format or \LaTeX2.09. In that case the file +% with a plain-based format. In that case the file % \file{plain.def} is needed (which also defines % |\AtBeginDocument|, and therefore it is not loaded twice). We % need the first part when the format is created, and |\orig@dump| @@ -5405,6 +5467,7 @@ help from Bernd Raichle, for which I am grateful. \ifx\directlua\@undefined\else \directlua{ Babel = Babel or {} Babel.debug = true }% + \input{babel-debug.tex}% \fi} {\providecommand\bbl@trace[1]{}% \let\bbl@debug\@gobble @@ -5665,9 +5728,6 @@ help from Bernd Raichle, for which I am grateful. % \begin{macrocode} \ifx\bbl@opt@provide\@nnil\else % Tests. Ignore. \chardef\bbl@iniflag\@ne - \bbl@replace\bbl@opt@provide{;}{,} - \bbl@add\bbl@opt@provide{,import} - \show\bbl@opt@provide \fi % % \end{macrocode} @@ -6295,10 +6355,10 @@ help from Bernd Raichle, for which I am grateful. % % \begin{macro}{\substitutefontfamily} % +% This command is deprecated. Use the tools provides by \LaTeX. % The command |\substitutefontfamily| creates an % \file{.fd} file on the fly. The first argument is an encoding % mnemonic, the second and third arguments are font family names. -% This command is deprecated. Use the tools provides by \LaTeX. % % \begin{macrocode} \def\substitutefontfamily#1#2#3{% @@ -6328,52 +6388,48 @@ help from Bernd Raichle, for which I am grateful. % % Because documents may use non-ASCII font encodings, we make sure % that the logos of \TeX\ and \LaTeX\ always come out in the right -% encoding. There is a list of non-ASCII encodings. Unfortunately, -% \textsf{fontenc} deletes its package options, so we must guess which -% encodings has been loaded by traversing |\@filelist| to search for -% \m{enc}|enc.def|. If a non-ASCII has been loaded, we define versions -% of |\TeX| and |\LaTeX| for them using |\ensureascii|. The default -% ASCII encoding is set, too (in reverse order): the ``main'' encoding -% (when the document begins), the last loaded, or |OT1|. +% encoding. There is a list of non-ASCII encodings. Requested +% encodings are currently stored in |\@fontenc@load@list|. If a +% non-ASCII has been loaded, we define versions of |\TeX| and |\LaTeX| +% for them using |\ensureascii|. The default ASCII encoding is set, +% too (in reverse order): the ``main'' encoding (when the document +% begins), the last loaded, or |OT1|. % % \begin{macro}{\ensureascii} % % \begin{macrocode} \bbl@trace{Encoding and fonts} -\newcommand\BabelNonASCII{LGR,X2,OT2,OT3,OT6,LHE,LWN,LMA,LMC,LMS,LMU,PU,PD1} +\newcommand\BabelNonASCII{LGR,X2,OT2,OT3,OT6,LHE,LWN,LMA,LMC,LMS,LMU} \newcommand\BabelNonText{TS1,T3,TS3} \let\org@TeX\TeX \let\org@LaTeX\LaTeX \let\ensureascii\@firstofone \AtBeginDocument{% - \in@false - \bbl@foreach\BabelNonASCII{% is there a text non-ascii enc? - \ifin@\else - \lowercase{\bbl@xin@{,#1enc.def,}{,\@filelist,}}% + \def\@elt#1{,#1,}% + \edef\bbl@tempa{\expandafter\@gobbletwo\@fontenc@load@list}% + \let\@elt\relax + \let\bbl@tempb\@empty + \def\bbl@tempc{OT1}% + \bbl@foreach\BabelNonASCII{% LGR loaded in a non-standard way + \bbl@ifunset{T@#1}{}{\def\bbl@tempb{#1}}}% + \bbl@foreach\bbl@tempa{% + \bbl@xin@{#1}{\BabelNonASCII}% + \ifin@ + \def\bbl@tempb{#1}% Store last non-ascii + \else\bbl@xin@{#1}{\BabelNonText}% Pass + \ifin@\else + \def\bbl@tempc{#1}% Store last ascii + \fi \fi}% - \ifin@ % if a text non-ascii has been loaded - \def\ensureascii#1{{\fontencoding{OT1}\selectfont#1}}% - \DeclareTextCommandDefault{\TeX}{\org@TeX}% - \DeclareTextCommandDefault{\LaTeX}{\org@LaTeX}% - \def\bbl@tempb#1\@@{\uppercase{\bbl@tempc#1}ENC.DEF\@empty\@@}% - \def\bbl@tempc#1ENC.DEF#2\@@{% - \ifx\@empty#2\else - \bbl@ifunset{T@#1}% - {}% - {\bbl@xin@{,#1,}{,\BabelNonASCII,\BabelNonText,}% - \ifin@ - \DeclareTextCommand{\TeX}{#1}{\ensureascii{\org@TeX}}% - \DeclareTextCommand{\LaTeX}{#1}{\ensureascii{\org@LaTeX}}% - \else - \def\ensureascii##1{{\fontencoding{#1}\selectfont##1}}% - \fi}% - \fi}% - \bbl@foreach\@filelist{\bbl@tempb#1\@@}% TODO - \@@ de mas?? + \ifx\bbl@tempb\@empty\else \bbl@xin@{,\cf@encoding,}{,\BabelNonASCII,\BabelNonText,}% \ifin@\else - \edef\ensureascii#1{{% - \noexpand\fontencoding{\cf@encoding}\noexpand\selectfont#1}}% + \edef\bbl@tempc{\cf@encoding}% The default if ascii wins \fi + \edef\ensureascii#1{% + {\noexpand\fontencoding{\bbl@tempc}\noexpand\selectfont#1}}% + \DeclareTextCommandDefault{\TeX}{\ensureascii{\org@TeX}}% + \DeclareTextCommandDefault{\LaTeX}{\ensureascii{\org@LaTeX}}% \fi} % \end{macrocode} % @@ -6415,16 +6471,12 @@ help from Bernd Raichle, for which I am grateful. \ifx\cf@encoding\bbl@t@one \xdef\latinencoding{\bbl@t@one}% \else - \ifx\@fontenc@load@list\@undefined - \@ifl@aded{def}{t1enc}{\xdef\latinencoding{\bbl@t@one}}{}% - \else - \def\@elt#1{,#1,}% - \edef\bbl@tempa{\expandafter\@gobbletwo\@fontenc@load@list}% - \let\@elt\relax - \bbl@xin@{,T1,}\bbl@tempa - \ifin@ - \xdef\latinencoding{\bbl@t@one}% - \fi + \def\@elt#1{,#1,}% + \edef\bbl@tempa{\expandafter\@gobbletwo\@fontenc@load@list}% + \let\@elt\relax + \bbl@xin@{,T1,}\bbl@tempa + \ifin@ + \xdef\latinencoding{\bbl@t@one}% \fi \fi}} % \end{macrocode} @@ -6457,6 +6509,19 @@ help from Bernd Raichle, for which I am grateful. \fi % \end{macrocode} % +% For several functions, we need to execute some code with +% |\selectfont|. With \LaTeX\ 2021-06-01, there is a hook for this +% purpose, but in older versions the \LaTeX{} command is patched (the +% latter solution will be eventually removed). +% +% \begin{macrocode} +\bbl@ifformatlater{2021-06-01}% + {\def\bbl@patchfont#1{\AddToHook{selectfont}{#1}}} + {\def\bbl@patchfont#1{% + \expandafter\bbl@add\csname selectfont \endcsname{#1}% + \expandafter\bbl@toglobal\csname selectfont \endcsname}} +% \end{macrocode} +% % \end{macro} % % \subsection{Basic bidi support} @@ -6494,78 +6559,10 @@ help from Bernd Raichle, for which I am grateful. % shows, vertical typesetting is possible, too. % \end{itemize} % -% As a frist step, add a handler for bidi and digits (and potentially -% other processes) just before \textsf{luaoftload} is applied, which is -% loaded by default by \LaTeX. Just in case, consider the possibility -% it has not been loaded. -% -% \begin{macrocode} -\ifodd\bbl@engine - \def\bbl@activate@preotf{% - \let\bbl@activate@preotf\relax % only once - \directlua{ - Babel = Babel or {} - % - function Babel.pre_otfload_v(head) - if Babel.numbers and Babel.digits_mapped then - head = Babel.numbers(head) - end - if Babel.bidi_enabled then - head = Babel.bidi(head, false, dir) - end - return head - end - % - function Babel.pre_otfload_h(head, gc, sz, pt, dir) - if Babel.numbers and Babel.digits_mapped then - head = Babel.numbers(head) - end - if Babel.bidi_enabled then - head = Babel.bidi(head, false, dir) - end - return head - end - % - luatexbase.add_to_callback('pre_linebreak_filter', - Babel.pre_otfload_v, - 'Babel.pre_otfload_v', - luatexbase.priority_in_callback('pre_linebreak_filter', - 'luaotfload.node_processor') or nil) - % - luatexbase.add_to_callback('hpack_filter', - Babel.pre_otfload_h, - 'Babel.pre_otfload_h', - luatexbase.priority_in_callback('hpack_filter', - 'luaotfload.node_processor') or nil) - }} -\fi -% \end{macrocode} -% -% The basic setup. In luatex, the output is modified at a very low -% level to set the |\bodydir| to the |\pagedir|. -% % \begin{macrocode} \bbl@trace{Loading basic (internal) bidi support} \ifodd\bbl@engine - \ifnum\bbl@bidimode>100 \ifnum\bbl@bidimode<200 - \let\bbl@beforeforeign\leavevmode - \AtEndOfPackage{\EnableBabelHook{babel-bidi}} - \RequirePackage{luatexbase} - \bbl@activate@preotf - \directlua{ - require('babel-data-bidi.lua') - \ifcase\expandafter\@gobbletwo\the\bbl@bidimode\or - require('babel-bidi-basic.lua') - \or - require('babel-bidi-basic-r.lua') - \fi} - % TODO - to locale_props, not as separate attribute - \newattribute\bbl@attr@dir - % TODO. I don't like it, hackish: - \bbl@exp{\output{\bodydir\pagedir\the\output}} - \AtEndOfPackage{\EnableBabelHook{babel-bidi}} - \fi\fi -\else +\else % TODO. Move to txtbabel \ifnum\bbl@bidimode>100 \ifnum\bbl@bidimode<200 \bbl@error {The bidi method 'basic' is available only in\\% @@ -6597,11 +6594,13 @@ help from Bernd Raichle, for which I am grateful. \fi \fi \fi +% TODO? Separate: \ifnum\bbl@bidimode=\@ne \let\bbl@beforeforeign\leavevmode \ifodd\bbl@engine \newattribute\bbl@attr@dir - \bbl@exp{\output{\bodydir\pagedir\the\output}}% + \directlua{ Babel.attr_dir = luatexbase.registernumber'bbl@attr@dir' } + \bbl@exp{\output{\bodydir\pagedir\the\output}} \fi \AtEndOfPackage{% \EnableBabelHook{babel-bidi}% @@ -6660,55 +6659,10 @@ help from Bernd Raichle, for which I am grateful. % \end{macrocode} % % Now the engine-dependent macros. TODO. Must be moved to the engine -% files? +% files. % % \begin{macrocode} \ifodd\bbl@engine % luatex=1 - \chardef\bbl@thetextdir\z@ - \chardef\bbl@thepardir\z@ - \def\bbl@getluadir#1{% - \directlua{ - if tex.#1dir == 'TLT' then - tex.sprint('0') - elseif tex.#1dir == 'TRT' then - tex.sprint('1') - end}} - \def\bbl@setluadir#1#2#3{% 1=text/par.. 2=\textdir.. 3=0 lr/1 rl - \ifcase#3\relax - \ifcase\bbl@getluadir{#1}\relax\else - #2 TLT\relax - \fi - \else - \ifcase\bbl@getluadir{#1}\relax - #2 TRT\relax - \fi - \fi} - \def\bbl@textdir#1{% - \bbl@setluadir{text}\textdir{#1}% - \chardef\bbl@thetextdir#1\relax - \setattribute\bbl@attr@dir{\numexpr\bbl@thepardir*3+#1}} - \def\bbl@pardir#1{% - \bbl@setluadir{par}\pardir{#1}% - \chardef\bbl@thepardir#1\relax} - \def\bbl@bodydir{\bbl@setluadir{body}\bodydir} - \def\bbl@pagedir{\bbl@setluadir{page}\pagedir} - \def\bbl@dirparastext{\pardir\the\textdir\relax}% %%%% - % Sadly, we have to deal with boxes in math with basic. - % Activated every math with the package option bidi=: - \ifnum\bbl@bidimode>\z@ - \def\bbl@mathboxdir{% - \ifcase\bbl@thetextdir\relax - \everyhbox{\bbl@mathboxdir@aux L}% - \else - \everyhbox{\bbl@mathboxdir@aux R}% - \fi} - \def\bbl@mathboxdir@aux#1{% - \@ifnextchar\egroup{}{\textdir T#1T\relax}} - \frozen@everymath\expandafter{% - \expandafter\bbl@mathboxdir\the\frozen@everymath} - \frozen@everydisplay\expandafter{% - \expandafter\bbl@mathboxdir\the\frozen@everydisplay} - \fi \else % pdftex=0, xetex=2 \newcount\bbl@dirlevel \chardef\bbl@thetextdir\z@ @@ -7053,9 +7007,16 @@ help from Bernd Raichle, for which I am grateful. \bbl@ldfinit \let\CurrentOption\bbl@opt@main \ifx\bbl@opt@provide\@nnil - \bbl@exp{\\\babelprovide[import,main]{\bbl@opt@main}} + \bbl@exp{\\\babelprovide[import,main]{\bbl@opt@main}}% \else - \bbl@exp{\\\babelprovide[\bbl@opt@provide,main]{\bbl@opt@main}}% + \bbl@exp{\\\bbl@forkv{\@nameuse{@raw@opt@babel.sty}}}{% + \bbl@xin@{,provide,}{,#1,}% + \ifin@ + \def\bbl@opt@provide{#2}% + \bbl@replace\bbl@opt@provide{;}{,}% + \fi}% + \bbl@exp{% + \\\babelprovide[\bbl@opt@provide,import,main]{\bbl@opt@main}}% \fi \bbl@afterldf{}% \else % case 0,2 @@ -7118,7 +7079,7 @@ help from Bernd Raichle, for which I am grateful. % \end{macrocode} % % The file |babel.def| expects some definitions made in the \LaTeXe{} -% style file. So, In \LaTeX2.09 and Plain{} we must provide at least +% style file. So, in Plain{} we must provide at least % some predefined values as well some tools to set them (even if not % all options are available). There are no package options, and % therefore and alternative mechanism is provided. For the moment, @@ -7154,16 +7115,6 @@ help from Bernd Raichle, for which I am grateful. \fi % \end{macrocode} % -% Exit immediately with 2.09. An error is raised by the sty file, but -% also try to minimize the number of errors. -% -% \begin{macrocode} -\ifx\bbl@trace\@undefined - \let\LdfInit\endinput - \def\ProvidesLanguage#1{\endinput} -\endinput\fi % Same line! -% \end{macrocode} -% % And continue. % % \section{Multiple languages} @@ -7210,7 +7161,7 @@ help from Bernd Raichle, for which I am grateful. % exists. Otherwise raises and error. % % The argument of |\bbl@fixname| has to be a macro name, as it may get -% ``fixed'' if casing (lc/uc) is wrong. It's intented to fix a +% ``fixed'' if casing (lc/uc) is wrong. It's an attempt to fix a % long-standing bug when |\foreignlanguage| and the like appear in % a |\MakeXXXcase|. However, a lowercase form is not imposed to % improve backward compatibility (perhaps you defined a language @@ -7371,8 +7322,9 @@ help from Bernd Raichle, for which I am grateful. \ifx\@undefined\protect\let\protect\relax\fi % \end{macrocode} % -% The following definition is preserved for backwards compatibility. It -% is related to a trick for 2.09. +% The following definition is preserved for backwards compatibility +% (eg, \textsf{arabi}, \textsf{koma}). It is related to a trick for +% 2.09, now discarded. % % \begin{macrocode} \let\xstring\string @@ -7417,7 +7369,15 @@ help from Bernd Raichle, for which I am grateful. % \begin{macrocode} \def\bbl@push@language{% \ifx\languagename\@undefined\else - \xdef\bbl@language@stack{\languagename+\bbl@language@stack}% + \ifx\currentgrouplevel\@undefined + \xdef\bbl@language@stack{\languagename+\bbl@language@stack}% + \else + \ifnum\currentgrouplevel=\z@ + \xdef\bbl@language@stack{\languagename+}% + \else + \xdef\bbl@language@stack{\languagename+\bbl@language@stack}% + \fi + \fi \fi} % \end{macrocode} % @@ -7603,10 +7563,10 @@ help from Bernd Raichle, for which I am grateful. \let\bbl@select@type\z@ \expandafter\bbl@switch\expandafter{\languagename}% \fi}} -\def\babel@aux#1#2{% TODO. See how to avoid undefined nil's +\def\babel@aux#1#2{% \select@language{#1}% - \bbl@foreach\BabelContentsFiles{% - \@writefile{##1}{\babel@toc{#1}{#2}}}}% %% TODO - ok in plain? + \bbl@foreach\BabelContentsFiles{% \relax -> don't assume vertical mode + \@writefile{##1}{\babel@toc{#1}{#2}\relax}}}% TODO - plain? \def\babel@toc#1#2{% \select@language{#1}} % \end{macrocode} @@ -8288,10 +8248,12 @@ help from Bernd Raichle, for which I am grateful. \newcommand\EnableBabelHook[1]{\bbl@csarg\let{hk@#1}\@firstofone} \newcommand\DisableBabelHook[1]{\bbl@csarg\let{hk@#1}\@gobble} \def\bbl@usehooks#1#2{% + \ifx\UseHook\@undefined\else\UseHook{babel/#1}\fi \def\bbl@elth##1{% \bbl@cs{hk@##1}{\bbl@cs{ev@##1@#1@}#2}}% \bbl@cs{ev@#1@}% \ifx\languagename\@undefined\else % Test required for Plain (?) + \ifx\UseHook\@undefined\else\UseHook{babel/#1/\languagename}\fi \def\bbl@elth##1{% \bbl@cs{hk@##1}{\bbl@cl{ev@##1@#1}#2}}% \bbl@cl{ev@#1}% @@ -8311,6 +8273,10 @@ help from Bernd Raichle, for which I am grateful. beforeextras=0,afterextras=0,stopcommands=0,stringprocess=0,% hyphenation=2,initiateactive=3,afterreset=0,foreign=0,foreign*=0,% beforestart=0,languagename=2} +\ifx\NewHook\@undefined\else + \def\bbl@tempa#1=#2\@@{\NewHook{babel/#1}} + \bbl@foreach\bbl@evargs{\bbl@tempa#1\@@} +\fi % \end{macrocode} % % \begin{macro}{\babelensure} @@ -8499,9 +8465,7 @@ help from Bernd Raichle, for which I am grateful. \let\BabelModifiers\relax \let\bbl@screset\relax}% \def\ldf@finish#1{% - \ifx\loadlocalcfg\@undefined\else % For LaTeX 209 - \loadlocalcfg{#1}% - \fi + \loadlocalcfg{#1}% \bbl@afterldf{#1}% \expandafter\main@language\expandafter{#1}% \catcode`\@=\atcatcode \let\atcatcode\relax @@ -8543,10 +8507,12 @@ help from Bernd Raichle, for which I am grateful. % % \begin{macrocode} \def\bbl@beforestart{% + \def\@nolanerr##1{% + \bbl@warning{Undefined language '##1' in aux.\\Reported}}% \bbl@usehooks{beforestart}{}% \global\let\bbl@beforestart\relax} \AtBeginDocument{% - \@nameuse{bbl@beforestart}% + {\@nameuse{bbl@beforestart}}% Group! \if@filesw \providecommand\babel@aux[2]{}% \immediate\write\@mainaux{% @@ -10920,11 +10886,11 @@ help from Bernd Raichle, for which I am grateful. \ifx\bbl@KVP@captions\@nil \let\bbl@KVP@captions\bbl@KVP@import \fi - % == + % == \ifx\bbl@KVP@transforms\@nil\else \bbl@replace\bbl@KVP@transforms{ }{,}% \fi - % Load ini + % == Load ini == \ifcase\bbl@howloaded \bbl@provide@new{#2}% \else @@ -11007,7 +10973,7 @@ help from Bernd Raichle, for which I am grateful. end}% \ifx\bbl@mapselect\@undefined % TODO. almost the same as mapfont \AtBeginDocument{% - \expandafter\bbl@add\csname selectfont \endcsname{{\bbl@mapselect}}% + \bbl@patchfont{{\bbl@mapselect}}% {\selectfont}}% \def\bbl@mapselect{% \let\bbl@mapselect\relax @@ -11033,9 +10999,9 @@ help from Bernd Raichle, for which I am grateful. {See the manual for details.}}}% \bbl@ifunset{bbl@lsys@\languagename}{\bbl@provide@lsys{\languagename}}{}% \bbl@ifunset{bbl@wdir@\languagename}{\bbl@provide@dirs{\languagename}}{}% - \ifx\bbl@mapselect\@undefined % TODO. See onchar + \ifx\bbl@mapselect\@undefined % TODO. See onchar. \AtBeginDocument{% - \expandafter\bbl@add\csname selectfont \endcsname{{\bbl@mapselect}}% + \bbl@patchfont{{\bbl@mapselect}}% {\selectfont}}% \def\bbl@mapselect{% \let\bbl@mapselect\relax @@ -11056,6 +11022,24 @@ help from Bernd Raichle, for which I am grateful. \bbl@csarg\edef{intsp@#2}{\bbl@KVP@intraspace}% \fi \bbl@provide@intraspace + % == Line breaking: CJK quotes == + \ifcase\bbl@engine\or + \bbl@xin@{/c}{/\bbl@cl{lnbrk}}% + \ifin@ + \bbl@ifunset{bbl@quote@\languagename}{}% + {\directlua{ + Babel.locale_props[\the\localeid].cjk_quotes = {} + local cs = 'op' + for c in string.utfvalues(% + [[\csname bbl@quote@\languagename\endcsname]]) do + if Babel.cjk_characters[c].c == 'qu' then + Babel.locale_props[\the\localeid].cjk_quotes[c] = cs + end + cs = ( cs == 'op') and 'cl' or 'op' + end + }}% + \fi + \fi % == Line breaking: justification == \ifx\bbl@KVP@justification\@nil\else \let\bbl@KVP@linebreaking\bbl@KVP@justification @@ -11128,7 +11112,7 @@ help from Bernd Raichle, for which I am grateful. table.pack(string.utfvalue('\bbl@cl{dgnat}')) if not Babel.numbers then function Babel.numbers(head) - local LOCALE = luatexbase.registernumber'bbl@attr@locale' + local LOCALE = Babel.attr_locale local GLYPH = node.id'glyph' local inmath = false for item in node.traverse(head) do @@ -11255,12 +11239,12 @@ help from Bernd Raichle, for which I am grateful. \StartBabelCommands*{#1}{captions}% \bbl@read@ini{\bbl@KVP@captions}2% % Here all letters cat = 11 \EndBabelCommands - \fi - \ifx\bbl@KVP@import\@nil\else - \StartBabelCommands*{#1}{date}% - \bbl@savetoday - \bbl@savedate - \EndBabelCommands + \fi + \ifx\bbl@KVP@import\@nil\else + \StartBabelCommands*{#1}{date}% + \bbl@savetoday + \bbl@savedate + \EndBabelCommands \fi % == hyphenrules (also in new) == \ifx\bbl@lbkflag\@empty @@ -11598,6 +11582,7 @@ help from Bernd Raichle, for which I am grateful. \bbl@exportkey{intsp}{typography.intraspace}{}% \bbl@exportkey{frspc}{typography.frenchspacing}{u}% \bbl@exportkey{chrng}{characters.ranges}{}% + \bbl@exportkey{quote}{characters.delimiters.quotes}{}% \bbl@exportkey{dgnat}{numbers.digits.native}{}% \ifnum#1=\tw@ % only (re)new \bbl@exportkey{rqtex}{identification.require.babel}{}% @@ -11895,7 +11880,10 @@ help from Bernd Raichle, for which I am grateful. % \textbf{Dates} will require some macros for the basic formatting. % They may be redefined by language, so ``semi-public'' names (camel % case) are used. Oddly enough, the CLDR places particles like “de” -% inconsistently in either in the date or in the month name. +% inconsistently in either in the date or in the month name. Note after +% |\bbl@replace| |\toks@| contains the resulting string, which is used +% by |\bbl@replace@finish@iii| (this implicit behavior doesn't seem a +% good idea, but it’s efficient). % % \begin{macrocode} \let\bbl@calendar\@empty @@ -11919,7 +11907,7 @@ help from Bernd Raichle, for which I am grateful. range 0-9999.}% {There is little you can do. Sorry.}% \fi\fi\fi\fi}} -\newcommand\BabelDateyyyy[1]{{\number#1}} % FIXME - add leading 0 +\newcommand\BabelDateyyyy[1]{{\number#1}} % TODO - add leading 0 \def\bbl@replace@finish@iii#1{% \bbl@exp{\def\\#1####1####2####3{\the\toks@}}} \def\bbl@TG@@date{% @@ -11936,8 +11924,6 @@ help from Bernd Raichle, for which I am grateful. \bbl@replace\bbl@toreplace{[y|}{\bbl@datecntr[####1|}% \bbl@replace\bbl@toreplace{[m|}{\bbl@datecntr[####2|}% \bbl@replace\bbl@toreplace{[d|}{\bbl@datecntr[####3|}% -% Note after \bbl@replace \toks@ contains the resulting string. -% TODO - Using this implicit behavior doesn't seem a good idea. \bbl@replace@finish@iii\bbl@toreplace} \def\bbl@datecntr{\expandafter\bbl@xdatecntr\expandafter} \def\bbl@xdatecntr[#1|#2]{\localenumeral{#2}{#1}} @@ -12000,10 +11986,8 @@ help from Bernd Raichle, for which I am grateful. \expandafter\@secondoftwo % to execute right now \fi \AtBeginDocument{% - \expandafter\bbl@add - \csname selectfont \endcsname{\bbl@xenohyph}% - \expandafter\selectlanguage\expandafter{\languagename}% - \expandafter\bbl@toglobal\csname selectfont \endcsname}% + \bbl@patchfont{\bbl@xenohyph}% + \expandafter\selectlanguage\expandafter{\languagename}}% \fi}}% \fi \bbl@csarg\bbl@toglobal{lsys@#1}} @@ -12297,10 +12281,11 @@ help from Bernd Raichle, for which I am grateful. \directlua{ Babel.ignore_pre_char = function(node) return false end }} -% TODO: use babel name, override +% \end{macrocode} % -% As the final task, load the code for lua. +% As the final task, load the code for lua. TODO: use babel name, override % +% \begin{macrocode} \ifx\directlua\@undefined\else \ifx\bbl@luapatterns\@undefined \input luababel.def @@ -12328,23 +12313,10 @@ help from Bernd Raichle, for which I am grateful. % % The following code is meant to be read by ini\TeX\ because it % should instruct \TeX\ to read hyphenation patterns. To this end the -% \texttt{docstrip} option \texttt{patterns} can be used to include +% \texttt{docstrip} option \texttt{patterns} is used to include % this code in the file \file{hyphen.cfg}. Code is written with lower % level macros. % -% To make sure that \LaTeX$\:$2.09 executes the |\@begindocumenthook| -% we would want to alter |\begin{document}|, but as this done too often -% already, we add the new code at the front of |\@preamblecmds|. But we -% can only do that after it has been defined, so we add this piece of -% code to |\dump|. -% -% This new definition starts by adding an instruction to write a -% message on the terminal and in the transcript file to inform the -% user of the preloaded hyphenation patterns. -% -% Then everything is restored to the old situation and the format -% is dumped. -% % \begin{macrocode} <@Make sure ProvidesFile is defined@> \ProvidesFile{hyphen.cfg}[<@date@> <@version@> Babel hyphens] @@ -12353,15 +12325,6 @@ help from Bernd Raichle, for which I am grateful. \def\bbl@date{<@date@>} \ifx\AtBeginDocument\@undefined \def\@empty{} - \let\orig@dump\dump - \def\dump{% - \ifx\@ztryfc\@undefined - \else - \toks0=\expandafter{\@preamblecmds}% - \edef\@preamblecmds{\noexpand\@begindocumenthook\the\toks0}% - \def\@begindocumenthook{}% - \fi - \let\dump\orig@dump\let\orig@dump\@undefined\dump} \fi <@Define core switching macros@> % \end{macrocode} @@ -12744,7 +12707,7 @@ help from Bernd Raichle, for which I am grateful. \ExplSyntaxOn \catcode`\ =10 \def\bbl@loadfontspec{% - \usepackage{fontspec}% + \usepackage{fontspec}% TODO. Apply patch always \expandafter \def\csname msg~text~>~fontspec/language-not-exist\endcsname##1##2##3##4{% Font '\l_fontspec_fontname_tl' is using the\\% @@ -12778,9 +12741,7 @@ help from Bernd Raichle, for which I am grateful. \newcommand\bbl@bblfont[2][]{% 1=features 2=fontname, @font=rm|sf|tt \bbl@ifunset{\bbl@tempb family}% {\bbl@providefam{\bbl@tempb}}% - {\bbl@exp{% - \\\bbl@sreplace\<\bbl@tempb family >% - {\@nameuse{\bbl@tempb default}}{\<\bbl@tempb default>}}}% + {}% % For the default font, just in case: \bbl@ifunset{bbl@lsys@\languagename}{\bbl@provide@lsys{\languagename}}{}% \expandafter\bbl@ifblank\expandafter{\bbl@tempa}% @@ -12794,7 +12755,7 @@ help from Bernd Raichle, for which I am grateful. % \end{macrocode} % % If the family in the previous command does not exist, it must be -% defined. Here is how: +% defined. Here is how: % % \begin{macrocode} \def\bbl@providefam#1{% @@ -12803,12 +12764,15 @@ help from Bernd Raichle, for which I am grateful. \\\bbl@add@list\\\bbl@font@fams{#1}% \\\DeclareRobustCommand\<#1family>{% \\\not@math@alphabet\<#1family>\relax - \\\fontfamily\<#1default>\\\selectfont}% + % \\\prepare@family@series@update{#1}\<#1default>% TODO. Fails + \\\fontfamily\<#1default>% + \\\\UseHooks\\\@undefined\\\\UseHook{#1family}\% + \\\selectfont}% \\\DeclareTextFontCommand{\}{\<#1family>}}} % \end{macrocode} % % The following macro is activated when the hook |babel-fontspec| is -% enabled. But before we define a macro for a warning, which sets a +% enabled. But before, we define a macro for a warning, which sets a % flag to avoid duplicate them. % % \begin{macrocode} @@ -13117,10 +13081,7 @@ help from Bernd Raichle, for which I am grateful. \ifx\AtBeginDocument\@notprerr \expandafter\@secondoftwo % to execute right now \fi - \AtBeginDocument{% - \expandafter\bbl@add - \csname selectfont \endcsname{\bbl@ispacesize}% - \expandafter\bbl@toglobal\csname selectfont \endcsname}% + \AtBeginDocument{\bbl@patchfont{\bbl@xenohyph}}% \fi}% \fi} \ifx\DisableBabelHook\@undefined\endinput\fi @@ -13511,7 +13472,7 @@ help from Bernd Raichle, for which I am grateful. \endgroup \ifx\newattribute\@undefined\else \newattribute\bbl@attr@locale - \directlua{ Babel.attr_locale = luatexbase.registernumber'bbl@attr@locale'} + \directlua{ Babel.attr_locale = luatexbase.registernumber'bbl@attr@locale' } \AddBabelHook{luatex}{beforeextras}{% \setattribute\bbl@attr@locale\localeid} \fi @@ -13733,11 +13694,15 @@ help from Bernd Raichle, for which I am grateful. local lang = item.lang local LOCALE = node.get_attribute(item, - luatexbase.registernumber'bbl@attr@locale') + Babel.attr_locale) local props = Babel.locale_props[LOCALE] local class = Babel.cjk_class[item.char].c + if props.cjk_quotes and props.cjk_quotes[item.char] then + class = props.cjk_quotes[item.char] + end + if class == 'cp' then class = 'cl' end % )] as CL if class == 'id' then class = 'I' end @@ -13856,8 +13821,9 @@ help from Bernd Raichle, for which I am grateful. \gdef\bbl@arabicjust{% \let\bbl@arabicjust\relax \newattribute\bblar@kashida + \directlua{ Babel.attr_kashida = luatexbase.registernumber'bblar@kashida' }% \bblar@kashida=\z@ - \expandafter\bbl@add\csname selectfont \endcsname{{\bbl@parsejalt}}% + \bbl@patchfont{{\bbl@parsejalt}}% \directlua{ Babel.arabic.elong_map = Babel.arabic.elong_map or {} Babel.arabic.elong_map[\the\localeid] = {} @@ -13962,9 +13928,9 @@ function Babel.arabic.justify_hlist(head, line, gc, size, pack) local elong_map = Babel.arabic.elong_map local last_line local GLYPH = node.id'glyph' - local KASHIDA = luatexbase.registernumber'bblar@kashida' - local LOCALE = luatexbase.registernumber'bbl@attr@locale' - + local KASHIDA = Babel.attr_kashida + local LOCALE = Babel.attr_locale + if line == nil then line = {} line.glue_sign = 1 @@ -14157,7 +14123,7 @@ Babel.script_blocks.Kana = Babel.script_blocks.Jpan function Babel.locale_map(head) if not Babel.locale_mapped then return head end - local LOCALE = luatexbase.registernumber'bbl@attr@locale' + local LOCALE = Babel.attr_locale local GLYPH = node.id('glyph') local inmath = false local toloc_save @@ -14263,534 +14229,11 @@ end % % Post-handling hyphenation patterns for non-standard rules, like |ff| % to |ff-f|. There are still some issues with speed (not very slow, but -% still slow). -% -% After declaring the table containing the patterns with their -% replacements, we define some auxiliary functions: |str_to_nodes| -% converts the string returned by a function to a node list, taking the -% node at |base| as a model (font, language, etc.); |fetch_word| -% fetches a series of glyphs and discretionaries, which |pattern| is -% matched against (if there is a match, it is called again before -% trying other patterns, and this is very likely the main bottleneck). -% -% |post_hyphenate_replace| is the callback applied after -% |lang.hyphenate|. This means the automatic hyphenation points are -% known. As empty captures return a byte position (as explained in the -% \luatex{} manual), we must convert it to a utf8 position. With -% |first|, the last byte can be the leading byte in a utf8 sequence, so -% we just remove it and add 1 to the resulting length. With |last| we -% must take into account the capture position points to the next -% character. Here |word_head| points to the starting node of the text -% to be matched. +% still slow). The Lua code is below. % % \begin{macrocode} -\begingroup % TODO - to a lua file -\catcode`\~=12 -\catcode`\#=12 -\catcode`\%=12 -\catcode`\&=14 \directlua{ - Babel.linebreaking.replacements = {} - Babel.linebreaking.replacements[0] = {} &% pre - Babel.linebreaking.replacements[1] = {} &% post - - &% Discretionaries contain strings as nodes - function Babel.str_to_nodes(fn, matches, base) - local n, head, last - if fn == nil then return nil end - for s in string.utfvalues(fn(matches)) do - if base.id == 7 then - base = base.replace - end - n = node.copy(base) - n.char = s - if not head then - head = n - else - last.next = n - end - last = n - end - return head - end - - Babel.fetch_subtext = {} - - Babel.ignore_pre_char = function(node) - return (node.lang == \the\l@nohyphenation) - end - - &% Merging both functions doesn't seen feasible, because there are too - &% many differences. - Babel.fetch_subtext[0] = function(head) - local word_string = '' - local word_nodes = {} - local lang - local item = head - local inmath = false - - while item do - - if item.id == 11 then - inmath = (item.subtype == 0) - end - - if inmath then - &% pass - - elseif item.id == 29 then - local locale = node.get_attribute(item, Babel.attr_locale) - - if lang == locale or lang == nil then - lang = lang or locale - if Babel.ignore_pre_char(item) then - word_string = word_string .. Babel.us_char - else - word_string = word_string .. unicode.utf8.char(item.char) - end - word_nodes[#word_nodes+1] = item - else - break - end - - elseif item.id == 12 and item.subtype == 13 then - word_string = word_string .. ' ' - word_nodes[#word_nodes+1] = item - - &% Ignore leading unrecognized nodes, too. - elseif word_string ~= '' then - word_string = word_string .. Babel.us_char - word_nodes[#word_nodes+1] = item &% Will be ignored - end - - item = item.next - end - - &% Here and above we remove some trailing chars but not the - &% corresponding nodes. But they aren't accessed. - if word_string:sub(-1) == ' ' then - word_string = word_string:sub(1,-2) - end - word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '') - return word_string, word_nodes, item, lang - end - - Babel.fetch_subtext[1] = function(head) - local word_string = '' - local word_nodes = {} - local lang - local item = head - local inmath = false - - while item do - - if item.id == 11 then - inmath = (item.subtype == 0) - end - - if inmath then - &% pass - - elseif item.id == 29 then - if item.lang == lang or lang == nil then - if (item.char ~= 124) and (item.char ~= 61) then &% not =, not | - lang = lang or item.lang - word_string = word_string .. unicode.utf8.char(item.char) - word_nodes[#word_nodes+1] = item - end - else - break - end - - elseif item.id == 7 and item.subtype == 2 then - word_string = word_string .. '=' - word_nodes[#word_nodes+1] = item - - elseif item.id == 7 and item.subtype == 3 then - word_string = word_string .. '|' - word_nodes[#word_nodes+1] = item - - &% (1) Go to next word if nothing was found, and (2) implictly - &% remove leading USs. - elseif word_string == '' then - &% pass - - &% This is the responsible for splitting by words. - elseif (item.id == 12 and item.subtype == 13) then - break - - else - word_string = word_string .. Babel.us_char - word_nodes[#word_nodes+1] = item &% Will be ignored - end - - item = item.next - end - - word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '') - return word_string, word_nodes, item, lang - end - - function Babel.pre_hyphenate_replace(head) - Babel.hyphenate_replace(head, 0) - end - - function Babel.post_hyphenate_replace(head) - Babel.hyphenate_replace(head, 1) - end - - function Babel.debug_hyph(w, wn, sc, first, last, last_match) - local ss = '' - for pp = 1, 40 do - if wn[pp] then - if wn[pp].id == 29 then - ss = ss .. unicode.utf8.char(wn[pp].char) - else - ss = ss .. '{' .. wn[pp].id .. '}' - end - end - end - print('nod', ss) - print('lst_m', - string.rep(' ', unicode.utf8.len( - string.sub(w, 1, last_match))-1) .. '>') - print('str', w) - print('sc', string.rep(' ', sc-1) .. '^') - if first == last then - print('f=l', string.rep(' ', first-1) .. '!') - else - print('f/l', string.rep(' ', first-1) .. '[' .. - string.rep(' ', last-first-1) .. ']') - end - end - - Babel.us_char = string.char(31) - - function Babel.hyphenate_replace(head, mode) - local u = unicode.utf8 - local lbkr = Babel.linebreaking.replacements[mode] - - local word_head = head - - while true do &% for each subtext block - - local w, w_nodes, nw, lang = Babel.fetch_subtext[mode](word_head) - - if Babel.debug then - print() - print((mode == 0) and '@@@@<' or '@@@@>', w) - end - - if nw == nil and w == '' then break end - - if not lang then goto next end - if not lbkr[lang] then goto next end - - &% For each saved (pre|post)hyphenation. TODO. Reconsider how - &% loops are nested. - for k=1, #lbkr[lang] do - local p = lbkr[lang][k].pattern - local r = lbkr[lang][k].replace - - if Babel.debug then - print('*****', p, mode) - end - - &% This variable is set in some cases below to the first *byte* - &% after the match, either as found by u.match (faster) or the - &% computed position based on sc if w has changed. - local last_match = 0 - local step = 0 - - &% For every match. - while true do - if Babel.debug then - print('=====') - end - local new &% used when inserting and removing nodes - - local matches = { u.match(w, p, last_match) } - - if #matches < 2 then break end - - &% Get and remove empty captures (with ()'s, which return a - &% number with the position), and keep actual captures - &% (from (...)), if any, in matches. - local first = table.remove(matches, 1) - local last = table.remove(matches, #matches) - &% Non re-fetched substrings may contain \31, which separates - &% subsubstrings. - if string.find(w:sub(first, last-1), Babel.us_char) then break end - - local save_last = last &% with A()BC()D, points to D - - &% Fix offsets, from bytes to unicode. Explained above. - first = u.len(w:sub(1, first-1)) + 1 - last = u.len(w:sub(1, last-1)) &% now last points to C - - &% This loop stores in n small table the nodes - &% corresponding to the pattern. Used by 'data' to provide a - &% predictable behavior with 'insert' (now w_nodes is modified on - &% the fly), and also access to 'remove'd nodes. - local sc = first-1 &% Used below, too - local data_nodes = {} - - for q = 1, last-first+1 do - data_nodes[q] = w_nodes[sc+q] - end - - &% This loop traverses the matched substring and takes the - &% corresponding action stored in the replacement list. - &% sc = the position in substr nodes / string - &% rc = the replacement table index - local rc = 0 - - while rc < last-first+1 do &% for each replacement - if Babel.debug then - print('.....', rc + 1) - end - sc = sc + 1 - rc = rc + 1 - - if Babel.debug then - Babel.debug_hyph(w, w_nodes, sc, first, last, last_match) - local ss = '' - for itt in node.traverse(head) do - if itt.id == 29 then - ss = ss .. unicode.utf8.char(itt.char) - else - ss = ss .. '{' .. itt.id .. '}' - end - end - print('*****************', ss) - - end - - local crep = r[rc] - local item = w_nodes[sc] - local item_base = item - local placeholder = Babel.us_char - local d - - if crep and crep.data then - item_base = data_nodes[crep.data] - end - - if crep then - step = crep.step or 0 - end - - if crep and next(crep) == nil then &% = {} - last_match = save_last &% Optimization - goto next - - elseif crep == nil or crep.remove then - node.remove(head, item) - table.remove(w_nodes, sc) - w = u.sub(w, 1, sc-1) .. u.sub(w, sc+1) - sc = sc - 1 &% Nothing has been inserted. - last_match = utf8.offset(w, sc+1+step) - goto next - - elseif crep and crep.kashida then &% Experimental - node.set_attribute(item, - luatexbase.registernumber'bblar@kashida', - crep.kashida) - last_match = utf8.offset(w, sc+1+step) - goto next - - elseif crep and crep.string then - local str = crep.string(matches) - if str == '' then &% Gather with nil - node.remove(head, item) - table.remove(w_nodes, sc) - w = u.sub(w, 1, sc-1) .. u.sub(w, sc+1) - sc = sc - 1 &% Nothing has been inserted. - else - local loop_first = true - for s in string.utfvalues(str) do - d = node.copy(item_base) - d.char = s - if loop_first then - loop_first = false - head, new = node.insert_before(head, item, d) - if sc == 1 then - word_head = head - end - w_nodes[sc] = d - w = u.sub(w, 1, sc-1) .. u.char(s) .. u.sub(w, sc+1) - else - sc = sc + 1 - head, new = node.insert_before(head, item, d) - table.insert(w_nodes, sc, new) - w = u.sub(w, 1, sc-1) .. u.char(s) .. u.sub(w, sc) - end - if Babel.debug then - print('.....', 'str') - Babel.debug_hyph(w, w_nodes, sc, first, last, last_match) - end - end &% for - node.remove(head, item) - end &% if '' - last_match = utf8.offset(w, sc+1+step) - goto next - - elseif mode == 1 and crep and (crep.pre or crep.no or crep.post) then - d = node.new(7, 0) &% (disc, discretionary) - d.pre = Babel.str_to_nodes(crep.pre, matches, item_base) - d.post = Babel.str_to_nodes(crep.post, matches, item_base) - d.replace = Babel.str_to_nodes(crep.no, matches, item_base) - d.attr = item_base.attr - if crep.pre == nil then &% TeXbook p96 - d.penalty = crep.penalty or tex.hyphenpenalty - else - d.penalty = crep.penalty or tex.exhyphenpenalty - end - placeholder = '|' - head, new = node.insert_before(head, item, d) - - elseif mode == 0 and crep and (crep.pre or crep.no or crep.post) then - &% ERROR - - elseif crep and crep.penalty then - d = node.new(14, 0) &% (penalty, userpenalty) - d.attr = item_base.attr - d.penalty = crep.penalty - head, new = node.insert_before(head, item, d) - - elseif crep and crep.space then - &% 655360 = 10 pt = 10 * 65536 sp - d = node.new(12, 13) &% (glue, spaceskip) - local quad = font.getfont(item_base.font).size or 655360 - node.setglue(d, crep.space[1] * quad, - crep.space[2] * quad, - crep.space[3] * quad) - if mode == 0 then - placeholder = ' ' - end - head, new = node.insert_before(head, item, d) - - elseif crep and crep.spacefactor then - d = node.new(12, 13) &% (glue, spaceskip) - local base_font = font.getfont(item_base.font) - node.setglue(d, - crep.spacefactor[1] * base_font.parameters['space'], - crep.spacefactor[2] * base_font.parameters['space_stretch'], - crep.spacefactor[3] * base_font.parameters['space_shrink']) - if mode == 0 then - placeholder = ' ' - end - head, new = node.insert_before(head, item, d) - - elseif mode == 0 and crep and crep.space then - &% ERROR - - end &% ie replacement cases - - &% Shared by disc, space and penalty. - if sc == 1 then - word_head = head - end - if crep.insert then - w = u.sub(w, 1, sc-1) .. placeholder .. u.sub(w, sc) - table.insert(w_nodes, sc, new) - last = last + 1 - else - w_nodes[sc] = d - node.remove(head, item) - w = u.sub(w, 1, sc-1) .. placeholder .. u.sub(w, sc+1) - end - - last_match = utf8.offset(w, sc+1+step) - - ::next:: - - end &% for each replacement - - if Babel.debug then - print('.....', '/') - Babel.debug_hyph(w, w_nodes, sc, first, last, last_match) - end - - end &% for match - - end &% for patterns - - ::next:: - word_head = nw - end &% for substring - return head - end - - &% This table stores capture maps, numbered consecutively - Babel.capture_maps = {} - - &% The following functions belong to the next macro - function Babel.capture_func(key, cap) - local ret = "[[" .. cap:gsub('{([0-9])}', "]]..m[%1]..[[") .. "]]" - local cnt - local u = unicode.utf8 - ret, cnt = ret:gsub('{([0-9])|([^|]+)|(.-)}', Babel.capture_func_map) - if cnt == 0 then - ret = u.gsub(ret, '{(%x%x%x%x+)}', - function (n) - return u.char(tonumber(n, 16)) - end) - end - ret = ret:gsub("%[%[%]%]%.%.", '') - ret = ret:gsub("%.%.%[%[%]%]", '') - return key .. [[=function(m) return ]] .. ret .. [[ end]] - end - - function Babel.capt_map(from, mapno) - return Babel.capture_maps[mapno][from] or from - end - - &% Handle the {n|abc|ABC} syntax in captures - function Babel.capture_func_map(capno, from, to) - local u = unicode.utf8 - from = u.gsub(from, '{(%x%x%x%x+)}', - function (n) - return u.char(tonumber(n, 16)) - end) - to = u.gsub(to, '{(%x%x%x%x+)}', - function (n) - return u.char(tonumber(n, 16)) - end) - local froms = {} - for s in string.utfcharacters(from) do - table.insert(froms, s) - end - local cnt = 1 - table.insert(Babel.capture_maps, {}) - local mlen = table.getn(Babel.capture_maps) - for s in string.utfcharacters(to) do - Babel.capture_maps[mlen][froms[cnt]] = s - cnt = cnt + 1 - end - return "]]..Babel.capt_map(m[" .. capno .. "]," .. - (mlen) .. ").." .. "[[" - end - - &% Create/Extend reversed sorted list of kashida weights: - function Babel.capture_kashida(key, wt) - wt = tonumber(wt) - if Babel.kashida_wts then - for p, q in ipairs(Babel.kashida_wts) do - if wt == q then - break - elseif wt > q then - table.insert(Babel.kashida_wts, p, wt) - break - elseif table.getn(Babel.kashida_wts) == p then - table.insert(Babel.kashida_wts, wt) - end - end - else - Babel.kashida_wts = { wt } - end - return 'kashida = ' .. wt - end + Babel.nohyphenation = \the\l@nohyphenation } % \end{macrocode} % @@ -14809,7 +14252,10 @@ end % in macro names (which explains the internal group, too). % % \begin{macrocode} -\catcode`\#=6 +\begingroup +\catcode`\~=12 +\catcode`\%=12 +\catcode`\&=14 \gdef\babelposthyphenation#1#2#3{&% \bbl@activateposthyphen \begingroup @@ -14904,15 +14350,135 @@ end \def\bbl@activateposthyphen{% \let\bbl@activateposthyphen\relax \directlua{ + require('babel-transforms.lua') Babel.linebreaking.add_after(Babel.post_hyphenate_replace) }} \def\bbl@activateprehyphen{% \let\bbl@activateprehyphen\relax \directlua{ + require('babel-transforms.lua') Babel.linebreaking.add_before(Babel.pre_hyphenate_replace) }} % \end{macrocode} % +% \subsection{Bidi} +% +% As a first step, add a handler for bidi and digits (and potentially +% other processes) just before \textsf{luaoftload} is applied, which is +% loaded by default by \LaTeX. Just in case, consider the possibility +% it has not been loaded. +% +% \begin{macrocode} +\def\bbl@activate@preotf{% + \let\bbl@activate@preotf\relax % only once + \directlua{ + Babel = Babel or {} + % + function Babel.pre_otfload_v(head) + if Babel.numbers and Babel.digits_mapped then + head = Babel.numbers(head) + end + if Babel.bidi_enabled then + head = Babel.bidi(head, false, dir) + end + return head + end + % + function Babel.pre_otfload_h(head, gc, sz, pt, dir) + if Babel.numbers and Babel.digits_mapped then + head = Babel.numbers(head) + end + if Babel.bidi_enabled then + head = Babel.bidi(head, false, dir) + end + return head + end + % + luatexbase.add_to_callback('pre_linebreak_filter', + Babel.pre_otfload_v, + 'Babel.pre_otfload_v', + luatexbase.priority_in_callback('pre_linebreak_filter', + 'luaotfload.node_processor') or nil) + % + luatexbase.add_to_callback('hpack_filter', + Babel.pre_otfload_h, + 'Babel.pre_otfload_h', + luatexbase.priority_in_callback('hpack_filter', + 'luaotfload.node_processor') or nil) + }} +% \end{macrocode} +% +% The basic setup. The output is modified at a very low level to set +% the |\bodydir| to the |\pagedir|. Sadly, we have to deal with boxes +% in math with basic, so the |\bbl@mathboxdir| hack is activated every +% math with the package option bidi=. +% +% \begin{macrocode} +\ifnum\bbl@bidimode>100 \ifnum\bbl@bidimode<200 + \let\bbl@beforeforeign\leavevmode + \AtEndOfPackage{\EnableBabelHook{babel-bidi}} + \RequirePackage{luatexbase} + \bbl@activate@preotf + \directlua{ + require('babel-data-bidi.lua') + \ifcase\expandafter\@gobbletwo\the\bbl@bidimode\or + require('babel-bidi-basic.lua') + \or + require('babel-bidi-basic-r.lua') + \fi} + % TODO - to locale_props, not as separate attribute + \newattribute\bbl@attr@dir + \directlua{ Babel.attr_dir = luatexbase.registernumber'bbl@attr@dir' } + % TODO. I don't like it, hackish: + \bbl@exp{\output{\bodydir\pagedir\the\output}} + \AtEndOfPackage{\EnableBabelHook{babel-bidi}} +\fi\fi +\chardef\bbl@thetextdir\z@ +\chardef\bbl@thepardir\z@ +\def\bbl@getluadir#1{% + \directlua{ + if tex.#1dir == 'TLT' then + tex.sprint('0') + elseif tex.#1dir == 'TRT' then + tex.sprint('1') + end}} +\def\bbl@setluadir#1#2#3{% 1=text/par.. 2=\textdir.. 3=0 lr/1 rl + \ifcase#3\relax + \ifcase\bbl@getluadir{#1}\relax\else + #2 TLT\relax + \fi + \else + \ifcase\bbl@getluadir{#1}\relax + #2 TRT\relax + \fi + \fi} +\def\bbl@textdir#1{% + \bbl@setluadir{text}\textdir{#1}% + \chardef\bbl@thetextdir#1\relax + \setattribute\bbl@attr@dir{\numexpr\bbl@thepardir*3+#1}} +\def\bbl@pardir#1{% + \bbl@setluadir{par}\pardir{#1}% + \chardef\bbl@thepardir#1\relax} +\def\bbl@bodydir{\bbl@setluadir{body}\bodydir} +\def\bbl@pagedir{\bbl@setluadir{page}\pagedir} +\def\bbl@dirparastext{\pardir\the\textdir\relax}% %%%% +% +\ifnum\bbl@bidimode>\z@ + \def\bbl@mathboxdir{% + \ifcase\bbl@thetextdir\relax + \everyhbox{\bbl@mathboxdir@aux L}% + \else + \everyhbox{\bbl@mathboxdir@aux R}% + \fi} + \def\bbl@mathboxdir@aux#1{% + \@ifnextchar\egroup{}{\textdir T#1T\relax}} + \frozen@everymath\expandafter{% + \expandafter\bbl@mathboxdir\the\frozen@everymath} + \frozen@everydisplay\expandafter{% + \expandafter\bbl@mathboxdir\the\frozen@everydisplay} +\fi +% \end{macrocode} +% % \subsection{Layout} % % Unlike \xetex{}, \luatex{} requires only minimal changes for @@ -15135,7 +14701,508 @@ end % % \end{macrocode} % -% \subsection{Auto bidi with \texttt{basic} and \texttt{basic-r}} +% \subsection{Lua: transforms} +% +% After declaring the table containing the patterns with their +% replacements, we define some auxiliary functions: |str_to_nodes| +% converts the string returned by a function to a node list, taking the +% node at |base| as a model (font, language, etc.); |fetch_word| +% fetches a series of glyphs and discretionaries, which |pattern| is +% matched against (if there is a match, it is called again before +% trying other patterns, and this is very likely the main bottleneck). +% +% |post_hyphenate_replace| is the callback applied after +% |lang.hyphenate|. This means the automatic hyphenation points are +% known. As empty captures return a byte position (as explained in the +% \luatex{} manual), we must convert it to a utf8 position. With +% |first|, the last byte can be the leading byte in a utf8 sequence, so +% we just remove it and add 1 to the resulting length. With |last| we +% must take into account the capture position points to the next +% character. Here |word_head| points to the starting node of the text +% to be matched. +% +% \begin{macrocode} +%<*transforms> +Babel.linebreaking.replacements = {} +Babel.linebreaking.replacements[0] = {} -- pre +Babel.linebreaking.replacements[1] = {} -- post + +-- Discretionaries contain strings as nodes +function Babel.str_to_nodes(fn, matches, base) + local n, head, last + if fn == nil then return nil end + for s in string.utfvalues(fn(matches)) do + if base.id == 7 then + base = base.replace + end + n = node.copy(base) + n.char = s + if not head then + head = n + else + last.next = n + end + last = n + end + return head +end + +Babel.fetch_subtext = {} + +Babel.ignore_pre_char = function(node) + return (node.lang == Babel.nohyphenation) +end + +-- Merging both functions doesn't seen feasible, because there are too +-- many differences. +Babel.fetch_subtext[0] = function(head) + local word_string = '' + local word_nodes = {} + local lang + local item = head + local inmath = false + + while item do + + if item.id == 11 then + inmath = (item.subtype == 0) + end + + if inmath then + -- pass + + elseif item.id == 29 then + local locale = node.get_attribute(item, Babel.attr_locale) + + if lang == locale or lang == nil then + lang = lang or locale + if Babel.ignore_pre_char(item) then + word_string = word_string .. Babel.us_char + else + word_string = word_string .. unicode.utf8.char(item.char) + end + word_nodes[#word_nodes+1] = item + else + break + end + + elseif item.id == 12 and item.subtype == 13 then + word_string = word_string .. ' ' + word_nodes[#word_nodes+1] = item + + -- Ignore leading unrecognized nodes, too. + elseif word_string ~= '' then + word_string = word_string .. Babel.us_char + word_nodes[#word_nodes+1] = item -- Will be ignored + end + + item = item.next + end + + -- Here and above we remove some trailing chars but not the + -- corresponding nodes. But they aren't accessed. + if word_string:sub(-1) == ' ' then + word_string = word_string:sub(1,-2) + end + word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '') + return word_string, word_nodes, item, lang +end + +Babel.fetch_subtext[1] = function(head) + local word_string = '' + local word_nodes = {} + local lang + local item = head + local inmath = false + + while item do + + if item.id == 11 then + inmath = (item.subtype == 0) + end + + if inmath then + -- pass + + elseif item.id == 29 then + if item.lang == lang or lang == nil then + if (item.char ~= 124) and (item.char ~= 61) then -- not =, not | + lang = lang or item.lang + word_string = word_string .. unicode.utf8.char(item.char) + word_nodes[#word_nodes+1] = item + end + else + break + end + + elseif item.id == 7 and item.subtype == 2 then + word_string = word_string .. '=' + word_nodes[#word_nodes+1] = item + + elseif item.id == 7 and item.subtype == 3 then + word_string = word_string .. '|' + word_nodes[#word_nodes+1] = item + + -- (1) Go to next word if nothing was found, and (2) implicitly + -- remove leading USs. + elseif word_string == '' then + -- pass + + -- This is the responsible for splitting by words. + elseif (item.id == 12 and item.subtype == 13) then + break + + else + word_string = word_string .. Babel.us_char + word_nodes[#word_nodes+1] = item -- Will be ignored + end + + item = item.next + end + + word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '') + return word_string, word_nodes, item, lang +end + +function Babel.pre_hyphenate_replace(head) + Babel.hyphenate_replace(head, 0) +end + +function Babel.post_hyphenate_replace(head) + Babel.hyphenate_replace(head, 1) +end + +Babel.us_char = string.char(31) + +function Babel.hyphenate_replace(head, mode) + local u = unicode.utf8 + local lbkr = Babel.linebreaking.replacements[mode] + + local word_head = head + + while true do -- for each subtext block + + local w, w_nodes, nw, lang = Babel.fetch_subtext[mode](word_head) + + if Babel.debug then + print() + print((mode == 0) and '@@@@<' or '@@@@>', w) + end + + if nw == nil and w == '' then break end + + if not lang then goto next end + if not lbkr[lang] then goto next end + + -- For each saved (pre|post)hyphenation. TODO. Reconsider how + -- loops are nested. + for k=1, #lbkr[lang] do + local p = lbkr[lang][k].pattern + local r = lbkr[lang][k].replace + + if Babel.debug then + print('*****', p, mode) + end + + -- This variable is set in some cases below to the first *byte* + -- after the match, either as found by u.match (faster) or the + -- computed position based on sc if w has changed. + local last_match = 0 + local step = 0 + + -- For every match. + while true do + if Babel.debug then + print('=====') + end + local new -- used when inserting and removing nodes + + local matches = { u.match(w, p, last_match) } + + if #matches < 2 then break end + + -- Get and remove empty captures (with ()'s, which return a + -- number with the position), and keep actual captures + -- (from (...)), if any, in matches. + local first = table.remove(matches, 1) + local last = table.remove(matches, #matches) + -- Non re-fetched substrings may contain \31, which separates + -- subsubstrings. + if string.find(w:sub(first, last-1), Babel.us_char) then break end + + local save_last = last -- with A()BC()D, points to D + + -- Fix offsets, from bytes to unicode. Explained above. + first = u.len(w:sub(1, first-1)) + 1 + last = u.len(w:sub(1, last-1)) -- now last points to C + + -- This loop stores in n small table the nodes + -- corresponding to the pattern. Used by 'data' to provide a + -- predictable behavior with 'insert' (now w_nodes is modified on + -- the fly), and also access to 'remove'd nodes. + local sc = first-1 -- Used below, too + local data_nodes = {} + + for q = 1, last-first+1 do + data_nodes[q] = w_nodes[sc+q] + end + + -- This loop traverses the matched substring and takes the + -- corresponding action stored in the replacement list. + -- sc = the position in substr nodes / string + -- rc = the replacement table index + local rc = 0 + + while rc < last-first+1 do -- for each replacement + if Babel.debug then + print('.....', rc + 1) + end + sc = sc + 1 + rc = rc + 1 + + if Babel.debug then + Babel.debug_hyph(w, w_nodes, sc, first, last, last_match) + local ss = '' + for itt in node.traverse(head) do + if itt.id == 29 then + ss = ss .. unicode.utf8.char(itt.char) + else + ss = ss .. '{' .. itt.id .. '}' + end + end + print('*****************', ss) + + end + + local crep = r[rc] + local item = w_nodes[sc] + local item_base = item + local placeholder = Babel.us_char + local d + + if crep and crep.data then + item_base = data_nodes[crep.data] + end + + if crep then + step = crep.step or 0 + end + + if crep and next(crep) == nil then -- = {} + last_match = save_last -- Optimization + goto next + + elseif crep == nil or crep.remove then + node.remove(head, item) + table.remove(w_nodes, sc) + w = u.sub(w, 1, sc-1) .. u.sub(w, sc+1) + sc = sc - 1 -- Nothing has been inserted. + last_match = utf8.offset(w, sc+1+step) + goto next + + elseif crep and crep.kashida then -- Experimental + node.set_attribute(item, + Babel.attr_kashida, + crep.kashida) + last_match = utf8.offset(w, sc+1+step) + goto next + + elseif crep and crep.string then + local str = crep.string(matches) + if str == '' then -- Gather with nil + node.remove(head, item) + table.remove(w_nodes, sc) + w = u.sub(w, 1, sc-1) .. u.sub(w, sc+1) + sc = sc - 1 -- Nothing has been inserted. + else + local loop_first = true + for s in string.utfvalues(str) do + d = node.copy(item_base) + d.char = s + if loop_first then + loop_first = false + head, new = node.insert_before(head, item, d) + if sc == 1 then + word_head = head + end + w_nodes[sc] = d + w = u.sub(w, 1, sc-1) .. u.char(s) .. u.sub(w, sc+1) + else + sc = sc + 1 + head, new = node.insert_before(head, item, d) + table.insert(w_nodes, sc, new) + w = u.sub(w, 1, sc-1) .. u.char(s) .. u.sub(w, sc) + end + if Babel.debug then + print('.....', 'str') + Babel.debug_hyph(w, w_nodes, sc, first, last, last_match) + end + end -- for + node.remove(head, item) + end -- if '' + last_match = utf8.offset(w, sc+1+step) + goto next + + elseif mode == 1 and crep and (crep.pre or crep.no or crep.post) then + d = node.new(7, 0) -- (disc, discretionary) + d.pre = Babel.str_to_nodes(crep.pre, matches, item_base) + d.post = Babel.str_to_nodes(crep.post, matches, item_base) + d.replace = Babel.str_to_nodes(crep.no, matches, item_base) + d.attr = item_base.attr + if crep.pre == nil then -- TeXbook p96 + d.penalty = crep.penalty or tex.hyphenpenalty + else + d.penalty = crep.penalty or tex.exhyphenpenalty + end + placeholder = '|' + head, new = node.insert_before(head, item, d) + + elseif mode == 0 and crep and (crep.pre or crep.no or crep.post) then + -- ERROR + + elseif crep and crep.penalty then + d = node.new(14, 0) -- (penalty, userpenalty) + d.attr = item_base.attr + d.penalty = crep.penalty + head, new = node.insert_before(head, item, d) + + elseif crep and crep.space then + -- 655360 = 10 pt = 10 * 65536 sp + d = node.new(12, 13) -- (glue, spaceskip) + local quad = font.getfont(item_base.font).size or 655360 + node.setglue(d, crep.space[1] * quad, + crep.space[2] * quad, + crep.space[3] * quad) + if mode == 0 then + placeholder = ' ' + end + head, new = node.insert_before(head, item, d) + + elseif crep and crep.spacefactor then + d = node.new(12, 13) -- (glue, spaceskip) + local base_font = font.getfont(item_base.font) + node.setglue(d, + crep.spacefactor[1] * base_font.parameters['space'], + crep.spacefactor[2] * base_font.parameters['space_stretch'], + crep.spacefactor[3] * base_font.parameters['space_shrink']) + if mode == 0 then + placeholder = ' ' + end + head, new = node.insert_before(head, item, d) + + elseif mode == 0 and crep and crep.space then + -- ERROR + + end -- ie replacement cases + + -- Shared by disc, space and penalty. + if sc == 1 then + word_head = head + end + if crep.insert then + w = u.sub(w, 1, sc-1) .. placeholder .. u.sub(w, sc) + table.insert(w_nodes, sc, new) + last = last + 1 + else + w_nodes[sc] = d + node.remove(head, item) + w = u.sub(w, 1, sc-1) .. placeholder .. u.sub(w, sc+1) + end + + last_match = utf8.offset(w, sc+1+step) + + ::next:: + + end -- for each replacement + + if Babel.debug then + print('.....', '/') + Babel.debug_hyph(w, w_nodes, sc, first, last, last_match) + end + + end -- for match + + end -- for patterns + + ::next:: + word_head = nw + end -- for substring + return head +end + +-- This table stores capture maps, numbered consecutively +Babel.capture_maps = {} + +-- The following functions belong to the next macro +function Babel.capture_func(key, cap) + local ret = "[[" .. cap:gsub('{([0-9])}', "]]..m[%1]..[[") .. "]]" + local cnt + local u = unicode.utf8 + ret, cnt = ret:gsub('{([0-9])|([^|]+)|(.-)}', Babel.capture_func_map) + if cnt == 0 then + ret = u.gsub(ret, '{(%x%x%x%x+)}', + function (n) + return u.char(tonumber(n, 16)) + end) + end + ret = ret:gsub("%[%[%]%]%.%.", '') + ret = ret:gsub("%.%.%[%[%]%]", '') + return key .. [[=function(m) return ]] .. ret .. [[ end]] +end + +function Babel.capt_map(from, mapno) + return Babel.capture_maps[mapno][from] or from +end + +-- Handle the {n|abc|ABC} syntax in captures +function Babel.capture_func_map(capno, from, to) + local u = unicode.utf8 + from = u.gsub(from, '{(%x%x%x%x+)}', + function (n) + return u.char(tonumber(n, 16)) + end) + to = u.gsub(to, '{(%x%x%x%x+)}', + function (n) + return u.char(tonumber(n, 16)) + end) + local froms = {} + for s in string.utfcharacters(from) do + table.insert(froms, s) + end + local cnt = 1 + table.insert(Babel.capture_maps, {}) + local mlen = table.getn(Babel.capture_maps) + for s in string.utfcharacters(to) do + Babel.capture_maps[mlen][froms[cnt]] = s + cnt = cnt + 1 + end + return "]]..Babel.capt_map(m[" .. capno .. "]," .. + (mlen) .. ").." .. "[[" +end + +-- Create/Extend reversed sorted list of kashida weights: +function Babel.capture_kashida(key, wt) + wt = tonumber(wt) + if Babel.kashida_wts then + for p, q in ipairs(Babel.kashida_wts) do + if wt == q then + break + elseif wt > q then + table.insert(Babel.kashida_wts, p, wt) + break + elseif table.getn(Babel.kashida_wts) == p then + table.insert(Babel.kashida_wts, wt) + end + end + else + Babel.kashida_wts = { wt } + end + return 'kashida = ' .. wt +end +% +% \end{macrocode} +% +% \subsection{Lua: Auto bidi with \texttt{basic} and \texttt{basic-r}} % % The file \textsf{babel-data-bidi.lua} currently only contains data. It is % a large and boring file and it is not shown here (see the generated file), @@ -21371,7 +21438,7 @@ function Babel.bidi(head, ispar) if new_dir then attr_dir = 0 for at in node.traverse(item.attr) do - if at.number == luatexbase.registernumber'bbl@attr@dir' then + if at.number == Babel.attr_dir then attr_dir = at.value % 3 end end @@ -21625,7 +21692,7 @@ function Babel.bidi(head, ispar, hdir) local has_en = false local first_et = nil - local ATDIR = luatexbase.registernumber'bbl@attr@dir' + local ATDIR = Babel.attr_dir local save_outer local temp = node.get_attribute(head, ATDIR) @@ -22645,7 +22712,7 @@ Babel.cjk_breaks = { \fi % \end{macrocode} % -% To prevent wasting two counters in \LaTeX$\:$2.09 (because +% To prevent wasting two counters in \LaTeX\ (because % counters with the same name are allocated later by it) we reset % the counter that holds the next free counter (|\count10|). % diff --git a/macros/latex/required/babel/base/babel.ins b/macros/latex/required/babel/base/babel.ins index feea6e0da7..de98307cae 100644 --- a/macros/latex/required/babel/base/babel.ins +++ b/macros/latex/required/babel/base/babel.ins @@ -26,7 +26,7 @@ %% and covered by LPPL is defined by the unpacking scripts (with %% extension .ins) which are part of the distribution. %% -\def\filedate{2021/06/28} +\def\filedate{2021/07/19} \def\batchfile{babel.ins} \input docstrip.tex @@ -221,6 +221,7 @@ \def\MetaPrefix{--} \usepreamble\luapreamble \nopostamble +\generate{\file{babel-transforms.lua}{\from{babel.dtx}{transforms}}} \generate{\file{babel-data-bidi.lua}{\from{babel.dtx}{bididata}}} \generate{\file{babel-data-cjk.lua}{\from{babel.dtx}{cjkdata}}} \generate{\file{babel-bidi-basic-r.lua}{\from{babel.dtx}{basic-r}}} diff --git a/macros/latex/required/babel/base/babel.pdf b/macros/latex/required/babel/base/babel.pdf index 8e3aefc944..4ae4792afd 100644 Binary files a/macros/latex/required/babel/base/babel.pdf and b/macros/latex/required/babel/base/babel.pdf differ diff --git a/macros/latex/required/babel/base/bbcompat.dtx b/macros/latex/required/babel/base/bbcompat.dtx index 7500615da1..ef63539302 100644 --- a/macros/latex/required/babel/base/bbcompat.dtx +++ b/macros/latex/required/babel/base/bbcompat.dtx @@ -30,7 +30,7 @@ % % \iffalse %<*dtx> -\ProvidesFile{bbcompat.dtx}[2021/06/28 v3.61] +\ProvidesFile{bbcompat.dtx}[2021/07/19 v3.62] % % %% File 'bbcompat.dtx' -- cgit v1.2.3