From 3e16d31a5065dda14ecce076a9fcee58a29edece Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Wed, 16 Dec 2020 21:31:40 +0000 Subject: babel (16dec20) git-svn-id: svn://tug.org/texlive/trunk@57153 c570f23f-e606-0410-a88d-b1316a301751 --- Master/texmf-dist/doc/latex/babel/README.md | 109 ++-- Master/texmf-dist/doc/latex/babel/babel.pdf | Bin 814930 -> 819029 bytes Master/texmf-dist/source/latex/babel/babel.dtx | 600 +++++++++++---------- Master/texmf-dist/source/latex/babel/babel.ins | 2 +- Master/texmf-dist/source/latex/babel/bbcompat.dtx | 2 +- Master/texmf-dist/source/latex/babel/locale.zip | Bin 777803 -> 777806 bytes Master/texmf-dist/tex/generic/babel/babel.def | 16 +- Master/texmf-dist/tex/generic/babel/babel.sty | 27 +- Master/texmf-dist/tex/generic/babel/hyphen.cfg | 6 +- .../tex/generic/babel/locale/sq/babel-sq.ini | 6 +- Master/texmf-dist/tex/generic/babel/luababel.def | 431 ++++++++------- Master/texmf-dist/tex/generic/babel/nil.ldf | 2 +- 12 files changed, 626 insertions(+), 575 deletions(-) diff --git a/Master/texmf-dist/doc/latex/babel/README.md b/Master/texmf-dist/doc/latex/babel/README.md index 39be13e7dbd..30352f225da 100644 --- a/Master/texmf-dist/doc/latex/babel/README.md +++ b/Master/texmf-dist/doc/latex/babel/README.md @@ -1,4 +1,4 @@ -## Babel 3.51 +## Babel 3.52 This package manages culturally-determined typographical (and other) rules, and hyphenation patterns for a wide range of languages. Many @@ -7,9 +7,9 @@ lualatex out of the box. A few even work with plain formats. The latest stable version is available on . -Changes in version 3.51 are described in: +Changes in version 3.52 are described in: -https://github.com/latex3/babel/wiki/What's-new-in-babel-3.51 +https://github.com/latex3/babel/wiki/What's-new-in-babel-3.52 Included is a set of ini files for about 250 languages. @@ -43,64 +43,79 @@ respective authors. ### Latest changes ``` +3.52 2020-12-16 + * WARNING. If you need the style for Afrikaans you must update + babel-dutch. + * Improved \babelposthyphenation and \babelprehyphenation: + - Ordinary spaces can be matched in the latter. + - Keys 'insert' and 'penalty'. + - Some bugs fixed. + * Fixes: + - A couple of issues with \localeinfo and \getlocaleproperty + (#102, #105). + - Save size overflow with many \selectlanguage's (#109). + - Albanian: fix a typo in contents name (#104) + - Missing ‘Unused global option(s)’ warning (#110) + 3.51 2020-10-27 - - Common interface to (re)define captions (with + * Common interface to (re)define captions (with \setlocalecaption). - - frenchspacing filled in many ini files. - - A new internal macro (for ldf files) named \babel@texpdf, to + * frenchspacing filled in many ini files. + * A new internal macro (for ldf files) named \babel@texpdf, to ease interoperativity with hyperref in shorthands. - - Fixes: + * Fixes: - Error when loading a language on the fly in tabular (#97). - 'hyphenrules' raised an error with 'base' option (#59). - Better handling of autoloaded languages (eg, catcodes). - An error was raised with CJK and a null font (#99). - language.tag.bcp47 and tag.ini in \localeinfo didn't work (#102). +https://github.com/latex3/babel/wiki/What's-new-in-babel-3.52 3.50 2020-10-06 - - Fixes: + * Fixes: - main= stopped working in some cases (#96). - Footnotes were not \long with layout=footnotes. https://github.com/latex3/babel/wiki/What's-new-in-babel-3.50 3.49 2020-10-03 - - BUG. This version has a severe bug with main= (fixed in 3.50). - - Allow standard LaTeX syntax to recognize locales based on ini + * BUG. This version has a severe bug with main= (fixed in 3.50). + * Allow standard LaTeX syntax to recognize locales based on ini files in the most common cases. - - frenchspacing is set with ini files. - - Fixes: + * frenchspacing is set with ini files. + * Fixes: - layout.lists=off didn't work (#94) https://github.com/latex3/babel/wiki/What's-new-in-babel-3.49 3.48 2020-09-01 - - New (tentative) tools to customize some labels (like chapters + * New (tentative) tools to customize some labels (like chapters in CJK and Hungarian). - - Warning instead of error if neither 0 nor ZERO WIDTH SPACE is + * Warning instead of error if neither 0 nor ZERO WIDTH SPACE is found in some Southeast Asian languages. - - Fixes: + * Fixes: - \babelshorthand didn't work with some dialects (#91). - \selectlanguage and otherlanguage raised an error inside tabular. https://github.com/latex3/babel/wiki/What's-new-in-babel-3.48 3.47 2020-07-13 - - Fixes: + * Fixes: - Spurious spaces when autoloading locales (bug introduced in 3.46 when fixing #80). - In dtx files, autoloading printed some spurious text (because % is set to ignore, #87). - - New - \getlocaleproperty*, which doesn't raise an error. - - Basic ini+tex templates for about 500 languages in the GitHub + * New - \getlocaleproperty*, which doesn't raise an error. + * Basic ini+tex templates for about 500 languages in the GitHub repository. https://github.com/latex3/babel/wiki/What's-new-in-babel-3.47 3.46 2020-07-06 - - Languages can now be optionally selected with their BCP 47 + * Languages can now be optionally selected with their BCP 47 tags. - - \BabelEnsureInfo now loads the basic data for the requested + * \BabelEnsureInfo now loads the basic data for the requested languages immediately (formerly it was done at the end of the preamble). - - Fixes: + * Fixes: - Getting rid of the hyphen in XeTeX was not working as expected. A new procedure has been devised. - tag.bcp47 now contains what its name promises, instead of @@ -111,15 +126,15 @@ https://github.com/latex3/babel/wiki/What's-new-in-babel-3.47 https://github.com/latex3/babel/wiki/What's-new-in-babel-3.46 3.45 2020-06-10 - - Minor fixes in Hindi, Ancient Greek, Macedonian. - - Improvements in Kurdish, Marathi. - - Alternative date formats with \localedate[calendar=..., + * Minor fixes in Hindi, Ancient Greek, Macedonian. + * Improvements in Kurdish, Marathi. + * Alternative date formats with \localedate[calendar=..., variant=...]{y}{m}{d} (note: no calendar computations, just strings). - - \today is (again) case-aware. - - \localenumeral{digits}{..} and \localecounter{digits}{cntr} + * \today is (again) case-aware. + * \localenumeral{digits}{..} and \localecounter{digits}{cntr} for native digits. - - Fixes + * Fixes - Improved handling of math with \babelposthyphenation and \babelprehyphenation (it was ignoring too much). - An error related to \bbl@foreign@x could be raised in some @@ -127,52 +142,12 @@ https://github.com/latex3/babel/wiki/What's-new-in-babel-3.46 See https://github.com/latex3/babel/wiki/What's-new-in-babel-3.45 3.44 2020-05-13 - - WARNING. If you need the Portuguese style you must update - babel-portuges. - - \localedate, to print date with the current locale. - - [|] syntax in dates, to represent a value with any counter - defined in the ini files. - - Amharic: line breaking (modern and traditional), counters. - - Improvements in French, Portuguese. - - Optional argument in \foreignlanguage and otherlanguage*, - to switch date and/or captions. - - Preliminary code for \babelprehyphenation. - - Fixes: - - \babelcharproperty{..}{linebreak]{..} raised an error. - - \babelposthyphenation raised an error if the last char in - the pattern was ]. - - \babelposthyphenation is now deativated in math. https://github.com/latex3/babel/wiki/What's-new-in-babel-3.44 3.43 2020-04-28 - - Autoloading based on the BCP47 codes, with basic lookup. - - Now only a few essential commands are loaded in the format. - - Improvements in Assamese, Luxembourgish, Marathi, Armenian, - Bengali, and Finnish. - - First steps in the removal of switch.def and plain.def. - - Fixes: - - Babel reset the \sfcode of » to 1000. It should be 0. - - \guillemotleft and \guillemotleft renamed to \guillemetleft - and \guillemetleft (#63). - - A couple of bugs related to autoloading. https://github.com/latex3/babel/wiki/What's-new-in-babel-3.43 3.42 2020-03-22 - - \getlocaleproperty, to get the value of a field from the loaded - ini files. - - Added ini files for Syriac, Coptic and Church Slavic. Improved - some others (Armenian, Sanskrit). - - Added a new field tag.bcp47.likely to many files. - - Improved line breaking with xetex. - - Fixes: - - Remove extra space with languages loaded on the fly (bug - introduced in 3.41). - - Line breaking was no always correct with languages loaded on - the fly. - - Avoid a harmless warning about redefining a protected command - (#52). - - Locale was not switched correctly with COMBINING characters - (#54). See https://github.com/latex3/babel/wiki/What's-new-in-babel-3.42 3.41 2020-02-27 diff --git a/Master/texmf-dist/doc/latex/babel/babel.pdf b/Master/texmf-dist/doc/latex/babel/babel.pdf index 0ed7a10c7bf..a9462340013 100644 Binary files a/Master/texmf-dist/doc/latex/babel/babel.pdf and b/Master/texmf-dist/doc/latex/babel/babel.pdf differ diff --git a/Master/texmf-dist/source/latex/babel/babel.dtx b/Master/texmf-dist/source/latex/babel/babel.dtx index b6b77a34e06..ee0f127c4e1 100644 --- a/Master/texmf-dist/source/latex/babel/babel.dtx +++ b/Master/texmf-dist/source/latex/babel/babel.dtx @@ -31,7 +31,7 @@ % % \iffalse %<*filedriver> -\ProvidesFile{babel.dtx}[2020/10/27 v3.51 The Babel package] +\ProvidesFile{babel.dtx}[2020/12/16 v3.52 The Babel package] \documentclass{ltxdoc} \GetFileInfo{babel.dtx} \usepackage{fontspec} @@ -420,6 +420,14 @@ Package babel Warning: No hyphenation patterns were preloaded for This is not currently done by \babel{} and you must set it by hand. \end{note} +\begin{note} + Although it has been customary to recommend placing |\title|, + |\author| and other elements printed by |\maketitle| after + |\begin{document}|, mainly because of shorthands, it is advisable to + keep them in the preamble. Currently there is no real need to use + shorthands in those macros. +\end{note} + \subsection{Multilingual documents} In multilingual documents, just use a list of the required languages as @@ -2310,7 +2318,7 @@ redefined behave with the ‘new way’ described in the following note. \end{verbatim} (In this particular case, instead of the |captions| group you may need to modify the |captions.licr| one.) - + \item The ‘old way’, still valid for many languages, to redefine a caption is the following: \begin{verbatim} @@ -3005,30 +3013,64 @@ example, if the first capture reads |([ΐΰ])|, the replacement could be \verb+{1|ΐΰ|ίύ}+, which maps \textit{ΐ} to \textit{ί}, and \textit{ΰ} to \textit{ύ}, so that the diaeresis is removed. -This feature is activated with the first |\babelposthyphenation|. +This feature is activated with the first |\babelposthyphenation| or +|\babelprehyphenation|. See the \href{https://github.com/latex3/babel/wiki}{\babel\ wiki} for a -more detailed description and some examples. It also describes an -additional replacement type with the key |string|. +more detailed description and some examples. It also describes a few +additional replacement types (|string|, |penalty|). + +Although the main purpose of this command is non-standard hyphenation, +it may actually be used for other transformations (after hyphenation is +applied, so you must take discretionaries into account). + +You are limited to substitutions as done by \textsf{lua}, although a +future implementation may alternatively accept \textsf{lpeg}. + +\Describe{\babelprehyphenation}{\marg{locale-name}% + \marg{lua-pattern}\marg{replacement}} + +\New{3.44-3-52} This command is not strictly about hyphenation, but +it is include here because it is a clear counterpart of +|\babelposthyphenation|. It is similar to the latter, but (as its name +implies) applied before hyphenation. There are other differences: (1) +the first argument is the locale instead the name of hyphenation +patterns; (2) in the search patterns |=| has no special meaning, while +\verb+|+ stands for an ordinary space; (3) in the replacement, +discretionaries are not accepted. + +It handles glyphs and spaces (but you can not insert spaces). + +Performance is still somewhat poor in some cases, but it is fast in the +most the typical ones. + +This feature is activated with the first |\babelposthyphenation| or +|\babelprehyphenation|. \begin{example} - Although the main purpose of this command is non-standard - hyphenation, it may actually be used for other transformations (after - hyphenation is applied, so you must take discretionaries into - account). For example, you can use the |string| replacement to - replace a character (or series of them) by another character (or - series of them). Thus, to enter \textit{ž} as |zh| and \textit{š} as - |sh| in a newly created locale for transliterated Russian: + You can replace a character (or series of them) by another character + (or series of them). Thus, to enter \textit{ž} as |zh| and \textit{š} + as |sh| in a newly created locale for transliterated Russian: \begin{verbatim} \babelprovide[hyphenrules=+]{russian-latin} % Create locale -\babelposthyphenation{russian-latin}{([sz])h} % Create rule +\babelprehyphenation{russian-latin}{([sz])h} % Create rule { - { string = {1|sz|šž} }, + string = {1|sz|šž}, remove } \end{verbatim} - In other words, it is a quite general tool. (A counterpart - |\babelprehyphenation| is on the way.) +\end{example} + +\begin{example} + The following rule prevent the word “a” from being at the + end of a line: +\begin{verbatim} +\babelprehyphenation{english}{|a|} + {}, {}, % Keep first space and a + {insert, penalty = 10000}, % Insert penalty + {} % Keep last space +} +\end{verbatim} \end{example} \subsection{Selection based on BCP 47 tags} @@ -3387,7 +3429,7 @@ bidirectional (with both R and L paragraphs) documents in \luatex{}. \xetex{} (which will not support a similar option in the short term). It patches an internal command, so it might be ignored by some packages and classes (or even raise an error). \New{3.18}. - + \item[graphics] modifies the |picture| environment so that the whole figure is L but the text is R. It \textit{does not} work with the standard |picture|, and \textit{pict2e} is required if you want sloped @@ -3866,26 +3908,6 @@ labels, both with the relation to captions (chapters, part), and how counters are used to define them. It is still somewhat tentative because it is far from trivial -- see the wiki for further details. -\medskip -\textbf{\string\babelprehyphenation} - -\New{3.44} Note it is tentative, but the current behavior for glyphs -should be correct. - -It is similar to |\babelposthyphenation|, but (as its name implies) -applied before hyphenation. There are other differences: (1) the first -argument is the locale instead the name of hyphenation patterns; (2) in -the search patterns |=| has no special meaning (\verb+|+ is still -reserved, but currently unused); (3) in the replacement, -discretionaries are not accepted, only remove, {}, and string = ... - -Currently it handles glyphs, not discretionaries or spaces (in -particular, it will not catch the hyphen and you can't insert or remove -spaces). Also, you are limited to substitutions as done by lua, -although a future implementation may alternatively accept lpeg. - -Performance is still somewhat poor. - \section{Loading languages with \file{language.dat}} \TeX{} and most engines based on it (pdf\TeX, \xetex, $\epsilon$-\TeX, @@ -4815,8 +4837,8 @@ help from Bernd Raichle, for which I am grateful. % \section{Tools} % % \begin{macrocode} -%<> -%<> +%<> +%<> % \end{macrocode} % % \textbf{Do not use the following macros in \texttt{ldf} files. They @@ -5226,9 +5248,17 @@ help from Bernd Raichle, for which I am grateful. \ProvidesPackage{babel}[<@date@> <@version@> The Babel package] \@ifpackagewith{babel}{debug} {\providecommand\bbl@trace[1]{\message{^^J[ #1 ]}}% - \let\bbl@debug\@firstofone} + \let\bbl@debug\@firstofone + \ifx\directlua\@undefined\else + \directlua{ Babel = Babel or {} + Babel.debug = true }% + \fi} {\providecommand\bbl@trace[1]{}% - \let\bbl@debug\@gobble} + \let\bbl@debug\@gobble + \ifx\directlua\@undefined\else + \directlua{ Babel = Babel or {} + Babel.debug = false }% + \fi} <@Basic macros@> % Temporarily repeat here the code for errors \def\bbl@error#1#2{% @@ -6686,7 +6716,6 @@ help from Bernd Raichle, for which I am grateful. \IfFileExists{\CurrentOption.ldf}% {\bbl@load@language{\CurrentOption}}% {#1\bbl@load@language{#2}#3}} -\DeclareOption{afrikaans}{\bbl@try@load@lang{}{dutch}{}} \DeclareOption{hebrew}{% \input{rlbabel.def}% \bbl@load@language{hebrew}} @@ -6738,16 +6767,16 @@ help from Bernd Raichle, for which I am grateful. % \begin{macrocode} \let\bbl@tempc\relax \bbl@foreach\bbl@language@opts{% - \ifcase\bbl@iniflag + \ifcase\bbl@iniflag % Default \bbl@ifunset{ds@#1}% {\DeclareOption{#1}{\bbl@load@language{#1}}}% {}% - \or + \or % provide=* \@gobble % case 2 same as 1 - \or + \or % provide+=* \bbl@ifunset{ds@#1}% {\IfFileExists{#1.ldf}{}% - {\IfFileExists{babel-#1.tex}{}{\DeclareOption{#1}{}}}}% + {\IfFileExists{babel-#1.tex}{}{\@namedef{ds@#1}{}}}}% {}% \bbl@ifunset{ds@#1}% {\def\bbl@tempc{#1}% @@ -6760,7 +6789,7 @@ help from Bernd Raichle, for which I am grateful. \bbl@load@language{#1}% \fi}}% {}% - \or + \or % provide*=* \def\bbl@tempc{#1}% \bbl@ifunset{ds@#1}% {\DeclareOption{#1}{% @@ -6771,18 +6800,18 @@ help from Bernd Raichle, for which I am grateful. \fi} % \end{macrocode} % -% Now, we make sure an option is explicitly declared for any -% language set as global option, by checking if an |ldf| -% exists. The previous step was, in fact, somewhat redundant, but -% that way we minimize accessing the file system just to see if the -% option could be a language. +% Now, we make sure an option is explicitly declared for any language +% set as global option, by checking if an |ldf| exists. The previous +% step was, in fact, somewhat redundant, but that way we minimize +% accessing the file system just to see if the option could be a +% language. % % \begin{macrocode} \let\bbl@tempb\@nnil \bbl@foreach\@classoptionslist{% \bbl@ifunset{ds@#1}% {\IfFileExists{#1.ldf}{}% - {\IfFileExists{babel-#1.tex}{}{\DeclareOption{#1}{}}}}% + {\IfFileExists{babel-#1.tex}{}{\@namedef{ds@#1}{}}}}% {}% \bbl@ifunset{ds@#1}% {\def\bbl@tempb{#1}% @@ -7475,7 +7504,7 @@ help from Bernd Raichle, for which I am grateful. \csname\languagename @bbl@hyphenmap\endcsname \fi \fi - \global\let\bbl@hymapsel\@cclv + \let\bbl@hymapsel\@cclv % hyphenation - select patterns \bbl@patterns{#1}% % hyphenation - allow stretching with babelnohyphens @@ -8049,17 +8078,17 @@ help from Bernd Raichle, for which I am grateful. \def\bbl@tempa##1,#3=##2,##3\@empty{\def\bbl@tempb{##2}}% \expandafter\bbl@tempa\bbl@evargs,#3=,\@empty \bbl@ifunset{bbl@ev@#2@#3@#1}% - {\bbl@csarg\bbl@add{ev@#3@#1}{\bbl@elt{#2}}}% + {\bbl@csarg\bbl@add{ev@#3@#1}{\bbl@elth{#2}}}% {\bbl@csarg\let{ev@#2@#3@#1}\relax}% \bbl@csarg\newcommand{ev@#2@#3@#1}[\bbl@tempb]} \newcommand\EnableBabelHook[1]{\bbl@csarg\let{hk@#1}\@firstofone} \newcommand\DisableBabelHook[1]{\bbl@csarg\let{hk@#1}\@gobble} \def\bbl@usehooks#1#2{% - \def\bbl@elt##1{% + \def\bbl@elth##1{% \bbl@cs{hk@##1}{\bbl@cs{ev@##1@#1@}#2}}% \bbl@cs{ev@#1@}% \ifx\languagename\@undefined\else % Test required for Plain (?) - \def\bbl@elt##1{% + \def\bbl@elth##1{% \bbl@cs{hk@##1}{\bbl@cl{ev@##1@#1}#2}}% \bbl@cl{ev@#1}% \fi} @@ -11085,7 +11114,7 @@ help from Bernd Raichle, for which I am grateful. {Fix the name or reinstall babel.}% \else \catcode`\[=12 \catcode`\]=12 \catcode`\==12 \catcode`\&=12 - \catcode`\;=12 \catcode`\|=12 \catcode`\%=14 + \catcode`\;=12 \catcode`\|=12 \catcode`\%=14 \catcode`\-=12 \bbl@info{Importing \ifcase#2 \or font and identification \or basic \fi data for \languagename\\% @@ -13050,7 +13079,7 @@ help from Bernd Raichle, for which I am grateful. \endinput\fi % Here stops reading code for hyphen.cfg % The following is read the 2nd time it's loaded -\begingroup +\begingroup % TODO - to a lua file \catcode`\%=12 \catcode`\'=12 \catcode`\"=12 @@ -13204,12 +13233,10 @@ help from Bernd Raichle, for which I am grateful. % \textit{In progress.} Replace regular (ie, implicit) discretionaries % by spaceskips, based on the previous glyph (which I think makes % sense, because the hyphen and the previous char go always together). -% Other discretionaries are not touched. -% -% For the moment, only 3 SA languages are activated by default (see -% Unicode UAX 14). +% Other discretionaries are not touched. See Unicode UAX 14. % % \begin{macrocode} +% TODO - to a lua file \directlua{ Babel = Babel or {} Babel.linebreaking = Babel.linebreaking or {} @@ -13262,7 +13289,7 @@ help from Bernd Raichle, for which I am grateful. function Babel.sea_disc_to_space (head) local sea_ranges = Babel.sea_ranges local last_char = nil - local quad = 655360 ^^ 10 pt = 655360 = 10 * 65536 + local quad = 655360 ^% 10 pt = 655360 = 10 * 65536 for item in node.traverse(head) do local i = item.id if i == node.id'glyph' then @@ -13272,16 +13299,16 @@ help from Bernd Raichle, for which I am grateful. quad = font.getfont(last_char.font).size for lg, rg in pairs(sea_ranges) do if last_char.char > rg[1] and last_char.char < rg[2] then - lg = lg:sub(1, 4) ^^ Remove trailing number of, eg, Cyrl1 + lg = lg:sub(1, 4) ^% Remove trailing number of, eg, Cyrl1 local intraspace = Babel.intraspaces[lg] local intrapenalty = Babel.intrapenalties[lg] local n if intrapenalty ~= 0 then - n = node.new(14, 0) ^^ penalty + n = node.new(14, 0) ^% penalty n.penalty = intrapenalty node.insert_before(head, item, n) end - n = node.new(12, 13) ^^ (glue, spaceskip) + n = node.new(12, 13) ^% (glue, spaceskip) node.setglue(n, intraspace.b * quad, intraspace.p * quad, intraspace.m * quad) @@ -13456,6 +13483,7 @@ help from Bernd Raichle, for which I am grateful. % skipped and discretionaries are handled in a special way. % % \begin{macrocode} +% TODO - to a lua file \directlua{ Babel.script_blocks = { ['Arab'] = {{0x0600, 0x06FF}, {0x08A0, 0x08FF}, {0x0750, 0x077F}, @@ -13640,14 +13668,17 @@ end % to be matched. % % \begin{macrocode} -\begingroup +\begingroup % TODO - to a lua file +\catcode`\~=12 \catcode`\#=12 \catcode`\%=12 \catcode`\&=14 \directlua{ - Babel.linebreaking.post_replacements = {} - Babel.linebreaking.pre_replacements = {} + Babel.linebreaking.replacements = {} + Babel.linebreaking.replacements[0] = {} &% pre + Babel.linebreaking.replacements[1] = {} &% post + &% Discretionaries contain strings as nodes function Babel.str_to_nodes(fn, matches, base) local n, head, last if fn == nil then return nil end @@ -13667,7 +13698,11 @@ end return head end - function Babel.fetch_word(head, funct) + Babel.fetch_subtext = {} + + &% Merging both functions doesn't seen feasible, because there are too + &% many differences. + Babel.fetch_subtext[0] = function(head) local word_string = '' local word_nodes = {} local lang @@ -13676,149 +13711,49 @@ end while item do - if item.id == 29 - and not(item.char == 124) &% ie, not | - and not(item.char == 61) &% ie, not = - and not inmath - and (item.lang == lang or lang == nil) then - lang = lang or item.lang - word_string = word_string .. unicode.utf8.char(item.char) - word_nodes[#word_nodes+1] = item + if item.id == 11 then + inmath = (item.subtype == 0) + end - elseif item.id == 7 and item.subtype == 2 and not inmath then - word_string = word_string .. '=' - word_nodes[#word_nodes+1] = item + if inmath then + &% pass - elseif item.id == 7 and item.subtype == 3 and not inmath then - word_string = word_string .. '|' - word_nodes[#word_nodes+1] = item + elseif item.id == 29 then + local locale = node.get_attribute(item, Babel.attr_locale) - elseif item.id == 11 and item.subtype == 0 then - inmath = true + if lang == locale or lang == nil then + if (item.char ~= 124) then &% ie, not | = space + lang = lang or locale + word_string = word_string .. unicode.utf8.char(item.char) + word_nodes[#word_nodes+1] = item + end + else + break + end - elseif word_string == '' then - &% pass + elseif item.id == 12 and item.subtype == 13 then + word_string = word_string .. '|' + word_nodes[#word_nodes+1] = item - else - return word_string, word_nodes, item, lang + &% Ignore leading unrecognized nodes, too. + elseif word_string ~= '' then + word_string = word_string .. Babel.us_char + word_nodes[#word_nodes+1] = item &% Will be ignored end item = item.next end - end - - function Babel.post_hyphenate_replace(head) - local u = unicode.utf8 - local lbkr = Babel.linebreaking.post_replacements - local word_head = head - - while true do - local w, wn, nw, lang = Babel.fetch_word(word_head) - if not lang then return head end - - if not lbkr[lang] then - break - end - - for k=1, #lbkr[lang] do - local p = lbkr[lang][k].pattern - local r = lbkr[lang][k].replace - - while true do - local matches = { u.match(w, p) } - if #matches < 2 then break end - - local first = table.remove(matches, 1) - local last = table.remove(matches, #matches) - - &% Fix offsets, from bytes to unicode. - first = u.len(w:sub(1, first-1)) + 1 - last = u.len(w:sub(1, last-1)) - - local new &% used when inserting and removing nodes - local changed = 0 - - &% This loop traverses the replace list and takes the - &% corresponding actions - for q = first, last do - local crep = r[q-first+1] - local char_node = wn[q] - local char_base = char_node - - if crep and crep.data then - char_base = wn[crep.data+first-1] - end - - if crep == {} then - break - elseif crep == nil then - changed = changed + 1 - node.remove(head, char_node) - elseif crep and (crep.pre or crep.no or crep.post) then - changed = changed + 1 - d = node.new(7, 0) &% (disc, discretionary) - d.pre = Babel.str_to_nodes(crep.pre, matches, char_base) - d.post = Babel.str_to_nodes(crep.post, matches, char_base) - d.replace = Babel.str_to_nodes(crep.no, matches, char_base) - d.attr = char_base.attr - if crep.pre == nil then &% TeXbook p96 - d.penalty = crep.penalty or tex.hyphenpenalty - else - d.penalty = crep.penalty or tex.exhyphenpenalty - end - head, new = node.insert_before(head, char_node, d) - node.remove(head, char_node) - if q == 1 then - word_head = new - end - elseif crep and crep.string then - changed = changed + 1 - local str = crep.string(matches) - if str == '' then - if q == 1 then - word_head = char_node.next - end - head, new = node.remove(head, char_node) - elseif char_node.id == 29 and u.len(str) == 1 then - char_node.char = string.utfvalue(str) - else - local n - for s in string.utfvalues(str) do - if char_node.id == 7 then - log('Automatic hyphens cannot be replaced, just removed.') - else - n = node.copy(char_base) - end - n.char = s - if q == 1 then - head, new = node.insert_before(head, char_node, n) - word_head = new - else - node.insert_before(head, char_node, n) - end - end - - node.remove(head, char_node) - end &% string length - end &% if char and char.string - end &% for char in match - if changed > 20 then - texio.write('Too many changes. Ignoring the rest.') - elseif changed > 0 then - w, wn, nw = Babel.fetch_word(word_head) - end - end &% for match - end &% for patterns - word_head = nw - end &% for words - return head + &% Here and above we remove some trailing chars but not the + &% corresponding nodes. But they aren't accessed. + if word_string:sub(-1) == '|' then + word_string = word_string:sub(1,-2) + end + word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '') + return word_string, word_nodes, item, lang end - &%%% - &% Preliminary code for \babelprehyphenation - &% TODO. Copypaste pattern. Merge with fetch_word - function Babel.fetch_subtext(head, funct) + Babel.fetch_subtext[1] = function(head) local word_string = '' local word_nodes = {} local lang @@ -13827,145 +13762,256 @@ end while item do - if item.id == 29 then - local locale = node.get_attribute(item, Babel.attr_locale) + if item.id == 11 then + inmath = (item.subtype == 0) + end - if not(item.char == 124) &% ie, not | = space - and not inmath - and (locale == lang or lang == nil) then - lang = lang or locale - word_string = word_string .. unicode.utf8.char(item.char) - word_nodes[#word_nodes+1] = item - end + if inmath then + &% pass - if item == node.tail(head) then - item = nil - return word_string, word_nodes, item, lang + elseif item.id == 29 then + if item.lang == lang or lang == nil then + if (item.char ~= 124) and (item.char ~= 61) then &% not =, not | + lang = lang or item.lang + word_string = word_string .. unicode.utf8.char(item.char) + word_nodes[#word_nodes+1] = item + end + else + break end - elseif item.id == 12 and item.subtype == 13 and not inmath then - word_string = word_string .. '|' + elseif item.id == 7 and item.subtype == 2 then + word_string = word_string .. '=' word_nodes[#word_nodes+1] = item - if item == node.tail(head) then - item = nil - return word_string, word_nodes, item, lang - end - - elseif item.id == 11 and item.subtype == 0 then - inmath = true + elseif item.id == 7 and item.subtype == 3 then + word_string = word_string .. '|' + word_nodes[#word_nodes+1] = item + &% (1) Go to next word if nothing was found, and (2) implictly + &% remove leading USs. elseif word_string == '' then &% pass - else - return word_string, word_nodes, item, lang + &% This is the responsible for splitting by words. + elseif (item.id == 12 and item.subtype == 13) then + break + + else + word_string = word_string .. Babel.us_char + word_nodes[#word_nodes+1] = item &% Will be ignored end item = item.next end + + word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '') + return word_string, word_nodes, item, lang end - &% TODO. Copypaste pattern. Merge with pre_hyphenate_replace function Babel.pre_hyphenate_replace(head) + Babel.hyphenate_replace(head, 0) + end + + function Babel.post_hyphenate_replace(head) + Babel.hyphenate_replace(head, 1) + end + + Babel.us_char = string.char(31) + + function Babel.hyphenate_replace(head, mode) local u = unicode.utf8 - local lbkr = Babel.linebreaking.pre_replacements + local lbkr = Babel.linebreaking.replacements[mode] + local word_head = head - while true do - local w, wn, nw, lang = Babel.fetch_subtext(word_head) - if not lang then return head end + while true do &% for each subtext block - if not lbkr[lang] then - break + local w, wn, nw, lang = Babel.fetch_subtext[mode](word_head) + + if Babel.debug then + print() + print('@@@@@', w, nw) end + if nw == nil and w == '' then break end + + if not lang then goto next end + if not lbkr[lang] then goto next end + + &% For each saved (pre|post)hyphenation. TODO. Reconsider how + &% loops are nested. for k=1, #lbkr[lang] do local p = lbkr[lang][k].pattern local r = lbkr[lang][k].replace + if Babel.debug then + print('=====', p, mode) + end + + &% This variable is set in some cases below to the first *byte* + &% after the match, either as found by u.match (faster) or the + &% computed position based on sc if w has changed. + local last_match = 0 + + &% For every match. while true do - local matches = { u.match(w, p) } + if Babel.debug then + print('-----') + end + local new &% used when inserting and removing nodes + local refetch = false + + local matches = { u.match(w, p, last_match) } if #matches < 2 then break end + &% Get and remove empty captures (with ()'s, which return a + &% number with the position), and keep actual captures + &% (from (...)), if any, in matches. local first = table.remove(matches, 1) - local last = table.remove(matches, #matches) + local last = table.remove(matches, #matches) + &% Non re-fetched substrings may contain \31, which separates + &% subsubstrings. + if string.find(w:sub(first, last-1), Babel.us_char) then break end + + local save_last = last &% with A()BC()D, points to D - &% Fix offsets, from bytes to unicode. + &% Fix offsets, from bytes to unicode. Explained above. first = u.len(w:sub(1, first-1)) + 1 - last = u.len(w:sub(1, last-1)) + last = u.len(w:sub(1, last-1)) &% now last points to C - local new &% used when inserting and removing nodes - local changed = 0 + if Babel.debug then + print(p) + print('', 'sc', 'first', 'last', 'last_m', 'w') + end - &% This loop traverses the replace list and takes the - &% corresponding actions - for q = first, last do - local crep = r[q-first+1] - local char_node = wn[q] + &% This loop traverses the matched substring and takes the + &% corresponding action stored in the replacement list. + &% sc = the position in substr nodes / string + &% rc = the replacement table index + local sc = first-1 + local rc = 0 + while rc < last-first+1 do &% for each replacement + if Babel.debug then + print('.....') + end + sc = sc + 1 + rc = rc + 1 + local crep = r[rc] + local char_node = wn[sc] local char_base = char_node + local end_replacement = false if crep and crep.data then char_base = wn[crep.data+first-1] end - if crep == {} then - break - elseif crep == nil then - changed = changed + 1 + if Babel.debug then + print('*', sc, first, last, last_match, w) + end + + if crep and next(crep) == nil then &% {} + last_match = save_last + + elseif crep == nil then &% remove node.remove(head, char_node) + table.remove(wn, sc) + w = u.sub(w, 1, sc-1) .. u.sub(w, sc+1) + last_match = utf8.offset(w, sc) + sc = sc - 1 &% Nothing has been inserted + + elseif mode == 1 and crep and (crep.pre or crep.no or crep.post) then + local d = node.new(7, 0) &% (disc, discretionary) + d.pre = Babel.str_to_nodes(crep.pre, matches, char_base) + d.post = Babel.str_to_nodes(crep.post, matches, char_base) + d.replace = Babel.str_to_nodes(crep.no, matches, char_base) + d.attr = char_base.attr + if crep.pre == nil then &% TeXbook p96 + d.penalty = crep.penalty or tex.hyphenpenalty + else + d.penalty = crep.penalty or tex.exhyphenpenalty + end + head, new = node.insert_before(head, char_node, d) + end_replacement = true + + elseif crep and crep.penalty then + local d = node.new(14, 0) &% (penalty, userpenalty) + d.attr = char_base.attr + d.penalty = crep.penalty + head, new = node.insert_before(head, char_node, d) + end_replacement = true + elseif crep and crep.string then - changed = changed + 1 local str = crep.string(matches) - if str == '' then - if q == 1 then + if str == '' then &% Gather with nil + refetch = true + if sc == 1 then word_head = char_node.next end head, new = node.remove(head, char_node) elseif char_node.id == 29 and u.len(str) == 1 then char_node.char = string.utfvalue(str) - else + w = u.sub(w, 1, sc-1) .. str .. u.sub(w, sc+1) + last_match = utf8.offset(w, sc+1) + else + refetch = true local n for s in string.utfvalues(str) do if char_node.id == 7 then - log('Automatic hyphens cannot be replaced, just removed.') + &% TODO. Remove this limitation. + texio.write_nl('Automatic hyphens cannot be replaced, just removed.') else n = node.copy(char_base) end n.char = s - if q == 1 then - head, new = node.insert_before(head, char_node, n) + if sc == 1 then + head, new = node.insert_before(head, char_node, n) word_head = new else - node.insert_before(head, char_node, n) + node.insert_before(head, char_node, n) end end - node.remove(head, char_node) end &% string length - end &% if char and char.string - end &% for char in match - if changed > 20 then - texio.write('Too many changes. Ignoring the rest.') - elseif changed > 0 then - &% For one-to-one can we modifiy directly the - &% values without re-fetching? Very likely. - w, wn, nw = Babel.fetch_subtext(word_head) + end &% if char and char.string (ie replacement cases) + + &% Shared by disc and penalty. + if end_replacement then + if sc == 1 then + word_head = new + end + if crep.insert then + last_match = save_last + else + node.remove(head, char_node) + w = u.sub(w, 1, sc-1) .. Babel.us_char .. u.sub(w, sc+1) + last_match = utf8.offset(w, sc) + end + end + end &% for each replacement + + if Babel.debug then + print('/', sc, first, last, last_match, w) end + &% TODO. refetch will be eventually unnecesary. + if refetch then + w, wn, nw, lang = Babel.fetch_subtext[mode](word_head) + end + end &% for match end &% for patterns + + ::next:: word_head = nw - end &% for words + end &% for substring return head end - &%%% end of preliminary code for \babelprehyphenation - - &% The following functions belong to the next macro &% This table stores capture maps, numbered consecutively Babel.capture_maps = {} + &% The following functions belong to the next macro function Babel.capture_func(key, cap) local ret = "[[" .. cap:gsub('{([0-9])}', "]]..m[%1]..[[") .. "]]" ret = ret:gsub('{([0-9])|([^|]+)|(.-)}', Babel.capture_func_map) @@ -14022,15 +14068,16 @@ end \bbl@ifsamestring{##1}{remove}&% {\bbl@add@list\babeltempb{nil}}&% {\directlua{ - local rep = [[##1]] + local rep = [[##1]] + rep = rep:gsub('^%s*(insert)%s*,', 'insert = true, ') rep = rep:gsub( '(no)%s*=%s*([^%s,]*)', Babel.capture_func) rep = rep:gsub( '(pre)%s*=%s*([^%s,]*)', Babel.capture_func) rep = rep:gsub( '(post)%s*=%s*([^%s,]*)', Babel.capture_func) - rep = rep:gsub('(string)%s*=%s*([^%s,]*)', Babel.capture_func) + rep = rep:gsub('(string)%s*=%s*([^%s,]*)', Babel.capture_func) tex.print([[\string\babeltempa{{]] .. rep .. [[}}]]) }}}&% \directlua{ - local lbkr = Babel.linebreaking.post_replacements + local lbkr = Babel.linebreaking.replacements[1] local u = unicode.utf8 &% Convert pattern: local patt = string.gsub([==[#2]==], '%s', '') @@ -14039,7 +14086,6 @@ end end patt = string.gsub(patt, '%(%)%^', '^()') patt = string.gsub(patt, '%$%(%)', '()$') - texio.write('***************' .. patt) patt = u.gsub(patt, '{(.)}', function (n) return '%' .. (tonumber(n) and (tonumber(n)+1) or n) @@ -14049,7 +14095,7 @@ end { pattern = patt, replace = { \babeltempb } }) }&% \endgroup} -% TODO. Working !!! Copypaste pattern. +% TODO. Copypaste pattern. \gdef\babelprehyphenation#1#2#3{&% \bbl@activateprehyphen \begingroup @@ -14060,17 +14106,20 @@ end {\bbl@add@list\babeltempb{nil}}&% {\directlua{ local rep = [[##1]] + rep = rep:gsub('^%s*(insert)%s*,', 'insert = true, ') rep = rep:gsub('(string)%s*=%s*([^%s,]*)', Babel.capture_func) tex.print([[\string\babeltempa{{]] .. rep .. [[}}]]) }}}&% \directlua{ - local lbkr = Babel.linebreaking.pre_replacements + local lbkr = Babel.linebreaking.replacements[0] local u = unicode.utf8 &% Convert pattern: local patt = string.gsub([==[#2]==], '%s', '') if not u.find(patt, '()', nil, true) then patt = '()' .. patt .. '()' - end + end + &% patt = string.gsub(patt, '%(%)%^', '^()') + &% patt = string.gsub(patt, '([^%%])%$%(%)', '%1()$') patt = u.gsub(patt, '{(.)}', function (n) return '%' .. (tonumber(n) and (tonumber(n)+1) or n) @@ -14086,7 +14135,6 @@ end \directlua{ Babel.linebreaking.add_after(Babel.post_hyphenate_replace) }} -% TODO. Working !!! \def\bbl@activateprehyphen{% \let\bbl@activateprehyphen\relax \directlua{ diff --git a/Master/texmf-dist/source/latex/babel/babel.ins b/Master/texmf-dist/source/latex/babel/babel.ins index 3bdf670a2d7..b362ca94e91 100644 --- a/Master/texmf-dist/source/latex/babel/babel.ins +++ b/Master/texmf-dist/source/latex/babel/babel.ins @@ -26,7 +26,7 @@ %% and covered by LPPL is defined by the unpacking scripts (with %% extension .ins) which are part of the distribution. %% -\def\filedate{2020/10/27} +\def\filedate{2020/12/16} \def\batchfile{babel.ins} \input docstrip.tex diff --git a/Master/texmf-dist/source/latex/babel/bbcompat.dtx b/Master/texmf-dist/source/latex/babel/bbcompat.dtx index 1aabf3a55d7..fb1977f094a 100644 --- a/Master/texmf-dist/source/latex/babel/bbcompat.dtx +++ b/Master/texmf-dist/source/latex/babel/bbcompat.dtx @@ -30,7 +30,7 @@ % % \iffalse %<*dtx> -\ProvidesFile{bbcompat.dtx}[2020/10/27 v3.51] +\ProvidesFile{bbcompat.dtx}[2020/12/16 v3.52] % % %% File 'bbcompat.dtx' diff --git a/Master/texmf-dist/source/latex/babel/locale.zip b/Master/texmf-dist/source/latex/babel/locale.zip index 375cc887587..e90a725a782 100644 Binary files a/Master/texmf-dist/source/latex/babel/locale.zip and b/Master/texmf-dist/source/latex/babel/locale.zip differ diff --git a/Master/texmf-dist/tex/generic/babel/babel.def b/Master/texmf-dist/tex/generic/babel/babel.def index 5b22adf11eb..4ad65905c01 100644 --- a/Master/texmf-dist/tex/generic/babel/babel.def +++ b/Master/texmf-dist/tex/generic/babel/babel.def @@ -39,7 +39,7 @@ \wlog{File: #1 #4 #3 <#2>}% \let\ProvidesFile\@undefined} \fi -\ProvidesFile{babel.def}[2020/10/27 3.51 Babel common definitions] +\ProvidesFile{babel.def}[2020/12/16 3.52 Babel common definitions] \ifx\AtBeginDocument\@undefined % TODO. change test. % == Code for plain == \def\@empty{} @@ -397,8 +397,8 @@ \fi \countdef\last@language=19 % TODO. why? remove? \def\addlanguage{\csname newlanguage\endcsname} -\def\bbl@version{3.51} -\def\bbl@date{2020/10/27} +\def\bbl@version{3.52} +\def\bbl@date{2020/12/16} \def\adddialect#1#2{% \global\chardef#1#2\relax \bbl@usehooks{adddialect}{{#1}{#2}}% @@ -671,7 +671,7 @@ \csname\languagename @bbl@hyphenmap\endcsname \fi \fi - \global\let\bbl@hymapsel\@cclv + \let\bbl@hymapsel\@cclv % hyphenation - select patterns \bbl@patterns{#1}% % hyphenation - allow stretching with babelnohyphens @@ -1121,17 +1121,17 @@ \def\bbl@tempa##1,#3=##2,##3\@empty{\def\bbl@tempb{##2}}% \expandafter\bbl@tempa\bbl@evargs,#3=,\@empty \bbl@ifunset{bbl@ev@#2@#3@#1}% - {\bbl@csarg\bbl@add{ev@#3@#1}{\bbl@elt{#2}}}% + {\bbl@csarg\bbl@add{ev@#3@#1}{\bbl@elth{#2}}}% {\bbl@csarg\let{ev@#2@#3@#1}\relax}% \bbl@csarg\newcommand{ev@#2@#3@#1}[\bbl@tempb]} \newcommand\EnableBabelHook[1]{\bbl@csarg\let{hk@#1}\@firstofone} \newcommand\DisableBabelHook[1]{\bbl@csarg\let{hk@#1}\@gobble} \def\bbl@usehooks#1#2{% - \def\bbl@elt##1{% + \def\bbl@elth##1{% \bbl@cs{hk@##1}{\bbl@cs{ev@##1@#1@}#2}}% \bbl@cs{ev@#1@}% \ifx\languagename\@undefined\else % Test required for Plain (?) - \def\bbl@elt##1{% + \def\bbl@elth##1{% \bbl@cs{hk@##1}{\bbl@cl{ev@##1@#1}#2}}% \bbl@cl{ev@#1}% \fi} @@ -2780,7 +2780,7 @@ {Fix the name or reinstall babel.}% \else \catcode`\[=12 \catcode`\]=12 \catcode`\==12 \catcode`\&=12 - \catcode`\;=12 \catcode`\|=12 \catcode`\%=14 + \catcode`\;=12 \catcode`\|=12 \catcode`\%=14 \catcode`\-=12 \bbl@info{Importing \ifcase#2 \or font and identification \or basic \fi data for \languagename\\% diff --git a/Master/texmf-dist/tex/generic/babel/babel.sty b/Master/texmf-dist/tex/generic/babel/babel.sty index d163aecce59..0b745bc02d4 100644 --- a/Master/texmf-dist/tex/generic/babel/babel.sty +++ b/Master/texmf-dist/tex/generic/babel/babel.sty @@ -33,12 +33,20 @@ %% \NeedsTeXFormat{LaTeX2e}[2005/12/01] -\ProvidesPackage{babel}[2020/10/27 3.51 The Babel package] +\ProvidesPackage{babel}[2020/12/16 3.52 The Babel package] \@ifpackagewith{babel}{debug} {\providecommand\bbl@trace[1]{\message{^^J[ #1 ]}}% - \let\bbl@debug\@firstofone} + \let\bbl@debug\@firstofone + \ifx\directlua\@undefined\else + \directlua{ Babel = Babel or {} + Babel.debug = true }% + \fi} {\providecommand\bbl@trace[1]{}% - \let\bbl@debug\@gobble} + \let\bbl@debug\@gobble + \ifx\directlua\@undefined\else + \directlua{ Babel = Babel or {} + Babel.debug = false }% + \fi} \bbl@trace{Basic macros} \def\bbl@stripslash{\expandafter\@gobble\string} \def\bbl@add#1#2{% @@ -1014,7 +1022,6 @@ \IfFileExists{\CurrentOption.ldf}% {\bbl@load@language{\CurrentOption}}% {#1\bbl@load@language{#2}#3}} -\DeclareOption{afrikaans}{\bbl@try@load@lang{}{dutch}{}} \DeclareOption{hebrew}{% \input{rlbabel.def}% \bbl@load@language{hebrew}} @@ -1044,16 +1051,16 @@ \fi \let\bbl@tempc\relax \bbl@foreach\bbl@language@opts{% - \ifcase\bbl@iniflag + \ifcase\bbl@iniflag % Default \bbl@ifunset{ds@#1}% {\DeclareOption{#1}{\bbl@load@language{#1}}}% {}% - \or + \or % provide=* \@gobble % case 2 same as 1 - \or + \or % provide+=* \bbl@ifunset{ds@#1}% {\IfFileExists{#1.ldf}{}% - {\IfFileExists{babel-#1.tex}{}{\DeclareOption{#1}{}}}}% + {\IfFileExists{babel-#1.tex}{}{\@namedef{ds@#1}{}}}}% {}% \bbl@ifunset{ds@#1}% {\def\bbl@tempc{#1}% @@ -1066,7 +1073,7 @@ \bbl@load@language{#1}% \fi}}% {}% - \or + \or % provide*=* \def\bbl@tempc{#1}% \bbl@ifunset{ds@#1}% {\DeclareOption{#1}{% @@ -1079,7 +1086,7 @@ \bbl@foreach\@classoptionslist{% \bbl@ifunset{ds@#1}% {\IfFileExists{#1.ldf}{}% - {\IfFileExists{babel-#1.tex}{}{\DeclareOption{#1}{}}}}% + {\IfFileExists{babel-#1.tex}{}{\@namedef{ds@#1}{}}}}% {}% \bbl@ifunset{ds@#1}% {\def\bbl@tempb{#1}% diff --git a/Master/texmf-dist/tex/generic/babel/hyphen.cfg b/Master/texmf-dist/tex/generic/babel/hyphen.cfg index 04e33b599e8..7803766a21a 100644 --- a/Master/texmf-dist/tex/generic/babel/hyphen.cfg +++ b/Master/texmf-dist/tex/generic/babel/hyphen.cfg @@ -37,10 +37,10 @@ \wlog{File: #1 #4 #3 <#2>}% \let\ProvidesFile\@undefined} \fi -\ProvidesFile{hyphen.cfg}[2020/10/27 3.51 Babel hyphens] +\ProvidesFile{hyphen.cfg}[2020/12/16 3.52 Babel hyphens] \xdef\bbl@format{\jobname} -\def\bbl@version{3.51} -\def\bbl@date{2020/10/27} +\def\bbl@version{3.52} +\def\bbl@date{2020/12/16} \ifx\AtBeginDocument\@undefined \def\@empty{} \let\orig@dump\dump diff --git a/Master/texmf-dist/tex/generic/babel/locale/sq/babel-sq.ini b/Master/texmf-dist/tex/generic/babel/locale/sq/babel-sq.ini index 1ae73931ca5..3b8c892185c 100644 --- a/Master/texmf-dist/tex/generic/babel/locale/sq/babel-sq.ini +++ b/Master/texmf-dist/tex/generic/babel/locale/sq/babel-sq.ini @@ -11,8 +11,8 @@ [identification] charset = utf8 -version = 1.2 -date = 2020-06-30 +version = 1.3 +date = 2020-11-25 name.local = shqip name.english = Albanian name.babel = albanian @@ -35,7 +35,7 @@ abstract = Përmbledhja bib = Bibliografia chapter = Kapitulli appendix = Shtesa -contents = Përmbajta +contents = Përmbajtja listfigure = Figurat listtable = Tabelat index = Indeksi diff --git a/Master/texmf-dist/tex/generic/babel/luababel.def b/Master/texmf-dist/tex/generic/babel/luababel.def index 840b6369e76..d843a2e3e55 100644 --- a/Master/texmf-dist/tex/generic/babel/luababel.def +++ b/Master/texmf-dist/tex/generic/babel/luababel.def @@ -195,7 +195,7 @@ \endinput\fi % Here stops reading code for hyphen.cfg % The following is read the 2nd time it's loaded -\begingroup +\begingroup % TODO - to a lua file \catcode`\%=12 \catcode`\'=12 \catcode`\"=12 @@ -380,7 +380,7 @@ function Babel.sea_disc_to_space (head) local sea_ranges = Babel.sea_ranges local last_char = nil - local quad = 655360 ^^ 10 pt = 655360 = 10 * 65536 + local quad = 655360 ^% 10 pt = 655360 = 10 * 65536 for item in node.traverse(head) do local i = item.id if i == node.id'glyph' then @@ -390,16 +390,16 @@ quad = font.getfont(last_char.font).size for lg, rg in pairs(sea_ranges) do if last_char.char > rg[1] and last_char.char < rg[2] then - lg = lg:sub(1, 4) ^^ Remove trailing number of, eg, Cyrl1 + lg = lg:sub(1, 4) ^% Remove trailing number of, eg, Cyrl1 local intraspace = Babel.intraspaces[lg] local intrapenalty = Babel.intrapenalties[lg] local n if intrapenalty ~= 0 then - n = node.new(14, 0) ^^ penalty + n = node.new(14, 0) ^% penalty n.penalty = intrapenalty node.insert_before(head, item, n) end - n = node.new(12, 13) ^^ (glue, spaceskip) + n = node.new(12, 13) ^% (glue, spaceskip) node.setglue(n, intraspace.b * quad, intraspace.p * quad, intraspace.m * quad) @@ -896,14 +896,17 @@ end Babel.chr_to_loc[\the\count@] = \bbl@ifblank{#1}{-1000}{\the\bbl@cs{id@@#1}}\space }} -\begingroup +\begingroup % TODO - to a lua file +\catcode`\~=12 \catcode`\#=12 \catcode`\%=12 \catcode`\&=14 \directlua{ - Babel.linebreaking.post_replacements = {} - Babel.linebreaking.pre_replacements = {} + Babel.linebreaking.replacements = {} + Babel.linebreaking.replacements[0] = {} &% pre + Babel.linebreaking.replacements[1] = {} &% post + &% Discretionaries contain strings as nodes function Babel.str_to_nodes(fn, matches, base) local n, head, last if fn == nil then return nil end @@ -923,7 +926,11 @@ end return head end - function Babel.fetch_word(head, funct) + Babel.fetch_subtext = {} + + &% Merging both functions doesn't seen feasible, because there are too + &% many differences. + Babel.fetch_subtext[0] = function(head) local word_string = '' local word_nodes = {} local lang @@ -932,149 +939,49 @@ end while item do - if item.id == 29 - and not(item.char == 124) &% ie, not | - and not(item.char == 61) &% ie, not = - and not inmath - and (item.lang == lang or lang == nil) then - lang = lang or item.lang - word_string = word_string .. unicode.utf8.char(item.char) - word_nodes[#word_nodes+1] = item + if item.id == 11 then + inmath = (item.subtype == 0) + end - elseif item.id == 7 and item.subtype == 2 and not inmath then - word_string = word_string .. '=' - word_nodes[#word_nodes+1] = item + if inmath then + &% pass - elseif item.id == 7 and item.subtype == 3 and not inmath then - word_string = word_string .. '|' - word_nodes[#word_nodes+1] = item + elseif item.id == 29 then + local locale = node.get_attribute(item, Babel.attr_locale) - elseif item.id == 11 and item.subtype == 0 then - inmath = true + if lang == locale or lang == nil then + if (item.char ~= 124) then &% ie, not | = space + lang = lang or locale + word_string = word_string .. unicode.utf8.char(item.char) + word_nodes[#word_nodes+1] = item + end + else + break + end - elseif word_string == '' then - &% pass + elseif item.id == 12 and item.subtype == 13 then + word_string = word_string .. '|' + word_nodes[#word_nodes+1] = item - else - return word_string, word_nodes, item, lang + &% Ignore leading unrecognized nodes, too. + elseif word_string ~= '' then + word_string = word_string .. Babel.us_char + word_nodes[#word_nodes+1] = item &% Will be ignored end item = item.next end - end - - function Babel.post_hyphenate_replace(head) - local u = unicode.utf8 - local lbkr = Babel.linebreaking.post_replacements - local word_head = head - - while true do - local w, wn, nw, lang = Babel.fetch_word(word_head) - if not lang then return head end - - if not lbkr[lang] then - break - end - - for k=1, #lbkr[lang] do - local p = lbkr[lang][k].pattern - local r = lbkr[lang][k].replace - - while true do - local matches = { u.match(w, p) } - if #matches < 2 then break end - - local first = table.remove(matches, 1) - local last = table.remove(matches, #matches) - - &% Fix offsets, from bytes to unicode. - first = u.len(w:sub(1, first-1)) + 1 - last = u.len(w:sub(1, last-1)) - local new &% used when inserting and removing nodes - local changed = 0 - - &% This loop traverses the replace list and takes the - &% corresponding actions - for q = first, last do - local crep = r[q-first+1] - local char_node = wn[q] - local char_base = char_node - - if crep and crep.data then - char_base = wn[crep.data+first-1] - end - - if crep == {} then - break - elseif crep == nil then - changed = changed + 1 - node.remove(head, char_node) - elseif crep and (crep.pre or crep.no or crep.post) then - changed = changed + 1 - d = node.new(7, 0) &% (disc, discretionary) - d.pre = Babel.str_to_nodes(crep.pre, matches, char_base) - d.post = Babel.str_to_nodes(crep.post, matches, char_base) - d.replace = Babel.str_to_nodes(crep.no, matches, char_base) - d.attr = char_base.attr - if crep.pre == nil then &% TeXbook p96 - d.penalty = crep.penalty or tex.hyphenpenalty - else - d.penalty = crep.penalty or tex.exhyphenpenalty - end - head, new = node.insert_before(head, char_node, d) - node.remove(head, char_node) - if q == 1 then - word_head = new - end - elseif crep and crep.string then - changed = changed + 1 - local str = crep.string(matches) - if str == '' then - if q == 1 then - word_head = char_node.next - end - head, new = node.remove(head, char_node) - elseif char_node.id == 29 and u.len(str) == 1 then - char_node.char = string.utfvalue(str) - else - local n - for s in string.utfvalues(str) do - if char_node.id == 7 then - log('Automatic hyphens cannot be replaced, just removed.') - else - n = node.copy(char_base) - end - n.char = s - if q == 1 then - head, new = node.insert_before(head, char_node, n) - word_head = new - else - node.insert_before(head, char_node, n) - end - end - - node.remove(head, char_node) - end &% string length - end &% if char and char.string - end &% for char in match - if changed > 20 then - texio.write('Too many changes. Ignoring the rest.') - elseif changed > 0 then - w, wn, nw = Babel.fetch_word(word_head) - end - - end &% for match - end &% for patterns - word_head = nw - end &% for words - return head + &% Here and above we remove some trailing chars but not the + &% corresponding nodes. But they aren't accessed. + if word_string:sub(-1) == '|' then + word_string = word_string:sub(1,-2) + end + word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '') + return word_string, word_nodes, item, lang end - &%%% - &% Preliminary code for \babelprehyphenation - &% TODO. Copypaste pattern. Merge with fetch_word - function Babel.fetch_subtext(head, funct) + Babel.fetch_subtext[1] = function(head) local word_string = '' local word_nodes = {} local lang @@ -1083,145 +990,256 @@ end while item do - if item.id == 29 then - local locale = node.get_attribute(item, Babel.attr_locale) + if item.id == 11 then + inmath = (item.subtype == 0) + end - if not(item.char == 124) &% ie, not | = space - and not inmath - and (locale == lang or lang == nil) then - lang = lang or locale - word_string = word_string .. unicode.utf8.char(item.char) - word_nodes[#word_nodes+1] = item - end + if inmath then + &% pass - if item == node.tail(head) then - item = nil - return word_string, word_nodes, item, lang + elseif item.id == 29 then + if item.lang == lang or lang == nil then + if (item.char ~= 124) and (item.char ~= 61) then &% not =, not | + lang = lang or item.lang + word_string = word_string .. unicode.utf8.char(item.char) + word_nodes[#word_nodes+1] = item + end + else + break end - elseif item.id == 12 and item.subtype == 13 and not inmath then - word_string = word_string .. '|' + elseif item.id == 7 and item.subtype == 2 then + word_string = word_string .. '=' word_nodes[#word_nodes+1] = item - if item == node.tail(head) then - item = nil - return word_string, word_nodes, item, lang - end - - elseif item.id == 11 and item.subtype == 0 then - inmath = true + elseif item.id == 7 and item.subtype == 3 then + word_string = word_string .. '|' + word_nodes[#word_nodes+1] = item + &% (1) Go to next word if nothing was found, and (2) implictly + &% remove leading USs. elseif word_string == '' then &% pass + &% This is the responsible for splitting by words. + elseif (item.id == 12 and item.subtype == 13) then + break + else - return word_string, word_nodes, item, lang + word_string = word_string .. Babel.us_char + word_nodes[#word_nodes+1] = item &% Will be ignored end item = item.next end + + word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '') + return word_string, word_nodes, item, lang end - &% TODO. Copypaste pattern. Merge with pre_hyphenate_replace function Babel.pre_hyphenate_replace(head) + Babel.hyphenate_replace(head, 0) + end + + function Babel.post_hyphenate_replace(head) + Babel.hyphenate_replace(head, 1) + end + + Babel.us_char = string.char(31) + + function Babel.hyphenate_replace(head, mode) local u = unicode.utf8 - local lbkr = Babel.linebreaking.pre_replacements + local lbkr = Babel.linebreaking.replacements[mode] + local word_head = head - while true do - local w, wn, nw, lang = Babel.fetch_subtext(word_head) - if not lang then return head end + while true do &% for each subtext block - if not lbkr[lang] then - break + local w, wn, nw, lang = Babel.fetch_subtext[mode](word_head) + + if Babel.debug then + print() + print('@@@@@', w, nw) end + if nw == nil and w == '' then break end + + if not lang then goto next end + if not lbkr[lang] then goto next end + + &% For each saved (pre|post)hyphenation. TODO. Reconsider how + &% loops are nested. for k=1, #lbkr[lang] do local p = lbkr[lang][k].pattern local r = lbkr[lang][k].replace + if Babel.debug then + print('=====', p, mode) + end + + &% This variable is set in some cases below to the first *byte* + &% after the match, either as found by u.match (faster) or the + &% computed position based on sc if w has changed. + local last_match = 0 + + &% For every match. while true do - local matches = { u.match(w, p) } + if Babel.debug then + print('-----') + end + local new &% used when inserting and removing nodes + local refetch = false + + local matches = { u.match(w, p, last_match) } if #matches < 2 then break end + &% Get and remove empty captures (with ()'s, which return a + &% number with the position), and keep actual captures + &% (from (...)), if any, in matches. local first = table.remove(matches, 1) - local last = table.remove(matches, #matches) + local last = table.remove(matches, #matches) + &% Non re-fetched substrings may contain \31, which separates + &% subsubstrings. + if string.find(w:sub(first, last-1), Babel.us_char) then break end - &% Fix offsets, from bytes to unicode. + local save_last = last &% with A()BC()D, points to D + + &% Fix offsets, from bytes to unicode. Explained above. first = u.len(w:sub(1, first-1)) + 1 - last = u.len(w:sub(1, last-1)) + last = u.len(w:sub(1, last-1)) &% now last points to C - local new &% used when inserting and removing nodes - local changed = 0 + if Babel.debug then + print(p) + print('', 'sc', 'first', 'last', 'last_m', 'w') + end - &% This loop traverses the replace list and takes the - &% corresponding actions - for q = first, last do - local crep = r[q-first+1] - local char_node = wn[q] + &% This loop traverses the matched substring and takes the + &% corresponding action stored in the replacement list. + &% sc = the position in substr nodes / string + &% rc = the replacement table index + local sc = first-1 + local rc = 0 + while rc < last-first+1 do &% for each replacement + if Babel.debug then + print('.....') + end + sc = sc + 1 + rc = rc + 1 + local crep = r[rc] + local char_node = wn[sc] local char_base = char_node + local end_replacement = false if crep and crep.data then char_base = wn[crep.data+first-1] end - if crep == {} then - break - elseif crep == nil then - changed = changed + 1 + if Babel.debug then + print('*', sc, first, last, last_match, w) + end + + if crep and next(crep) == nil then &% {} + last_match = save_last + + elseif crep == nil then &% remove node.remove(head, char_node) + table.remove(wn, sc) + w = u.sub(w, 1, sc-1) .. u.sub(w, sc+1) + last_match = utf8.offset(w, sc) + sc = sc - 1 &% Nothing has been inserted + + elseif mode == 1 and crep and (crep.pre or crep.no or crep.post) then + local d = node.new(7, 0) &% (disc, discretionary) + d.pre = Babel.str_to_nodes(crep.pre, matches, char_base) + d.post = Babel.str_to_nodes(crep.post, matches, char_base) + d.replace = Babel.str_to_nodes(crep.no, matches, char_base) + d.attr = char_base.attr + if crep.pre == nil then &% TeXbook p96 + d.penalty = crep.penalty or tex.hyphenpenalty + else + d.penalty = crep.penalty or tex.exhyphenpenalty + end + head, new = node.insert_before(head, char_node, d) + end_replacement = true + + elseif crep and crep.penalty then + local d = node.new(14, 0) &% (penalty, userpenalty) + d.attr = char_base.attr + d.penalty = crep.penalty + head, new = node.insert_before(head, char_node, d) + end_replacement = true + elseif crep and crep.string then - changed = changed + 1 local str = crep.string(matches) - if str == '' then - if q == 1 then + if str == '' then &% Gather with nil + refetch = true + if sc == 1 then word_head = char_node.next end head, new = node.remove(head, char_node) elseif char_node.id == 29 and u.len(str) == 1 then char_node.char = string.utfvalue(str) + w = u.sub(w, 1, sc-1) .. str .. u.sub(w, sc+1) + last_match = utf8.offset(w, sc+1) else + refetch = true local n for s in string.utfvalues(str) do if char_node.id == 7 then - log('Automatic hyphens cannot be replaced, just removed.') + &% TODO. Remove this limitation. + texio.write_nl('Automatic hyphens cannot be replaced, just removed.') else n = node.copy(char_base) end n.char = s - if q == 1 then + if sc == 1 then head, new = node.insert_before(head, char_node, n) word_head = new else node.insert_before(head, char_node, n) end end - node.remove(head, char_node) end &% string length - end &% if char and char.string - end &% for char in match - if changed > 20 then - texio.write('Too many changes. Ignoring the rest.') - elseif changed > 0 then - &% For one-to-one can we modifiy directly the - &% values without re-fetching? Very likely. - w, wn, nw = Babel.fetch_subtext(word_head) + end &% if char and char.string (ie replacement cases) + + &% Shared by disc and penalty. + if end_replacement then + if sc == 1 then + word_head = new + end + if crep.insert then + last_match = save_last + else + node.remove(head, char_node) + w = u.sub(w, 1, sc-1) .. Babel.us_char .. u.sub(w, sc+1) + last_match = utf8.offset(w, sc) + end + end + end &% for each replacement + + if Babel.debug then + print('/', sc, first, last, last_match, w) + end + + &% TODO. refetch will be eventually unnecesary. + if refetch then + w, wn, nw, lang = Babel.fetch_subtext[mode](word_head) end end &% for match end &% for patterns + + ::next:: word_head = nw - end &% for words + end &% for substring return head end - &%%% end of preliminary code for \babelprehyphenation - - &% The following functions belong to the next macro &% This table stores capture maps, numbered consecutively Babel.capture_maps = {} + &% The following functions belong to the next macro function Babel.capture_func(key, cap) local ret = "[[" .. cap:gsub('{([0-9])}', "]]..m[%1]..[[") .. "]]" ret = ret:gsub('{([0-9])|([^|]+)|(.-)}', Babel.capture_func_map) @@ -1262,6 +1280,7 @@ end {\bbl@add@list\babeltempb{nil}}&% {\directlua{ local rep = [[##1]] + rep = rep:gsub('^%s*(insert)%s*,', 'insert = true, ') rep = rep:gsub( '(no)%s*=%s*([^%s,]*)', Babel.capture_func) rep = rep:gsub( '(pre)%s*=%s*([^%s,]*)', Babel.capture_func) rep = rep:gsub( '(post)%s*=%s*([^%s,]*)', Babel.capture_func) @@ -1269,7 +1288,7 @@ end tex.print([[\string\babeltempa{{]] .. rep .. [[}}]]) }}}&% \directlua{ - local lbkr = Babel.linebreaking.post_replacements + local lbkr = Babel.linebreaking.replacements[1] local u = unicode.utf8 &% Convert pattern: local patt = string.gsub([==[#2]==], '%s', '') @@ -1278,7 +1297,6 @@ end end patt = string.gsub(patt, '%(%)%^', '^()') patt = string.gsub(patt, '%$%(%)', '()$') - texio.write('***************' .. patt) patt = u.gsub(patt, '{(.)}', function (n) return '%' .. (tonumber(n) and (tonumber(n)+1) or n) @@ -1298,17 +1316,20 @@ end {\bbl@add@list\babeltempb{nil}}&% {\directlua{ local rep = [[##1]] + rep = rep:gsub('^%s*(insert)%s*,', 'insert = true, ') rep = rep:gsub('(string)%s*=%s*([^%s,]*)', Babel.capture_func) tex.print([[\string\babeltempa{{]] .. rep .. [[}}]]) }}}&% \directlua{ - local lbkr = Babel.linebreaking.pre_replacements + local lbkr = Babel.linebreaking.replacements[0] local u = unicode.utf8 &% Convert pattern: local patt = string.gsub([==[#2]==], '%s', '') if not u.find(patt, '()', nil, true) then patt = '()' .. patt .. '()' end + &% patt = string.gsub(patt, '%(%)%^', '^()') + &% patt = string.gsub(patt, '([^%%])%$%(%)', '%1()$') patt = u.gsub(patt, '{(.)}', function (n) return '%' .. (tonumber(n) and (tonumber(n)+1) or n) diff --git a/Master/texmf-dist/tex/generic/babel/nil.ldf b/Master/texmf-dist/tex/generic/babel/nil.ldf index 5f60b05e125..8884be460cb 100644 --- a/Master/texmf-dist/tex/generic/babel/nil.ldf +++ b/Master/texmf-dist/tex/generic/babel/nil.ldf @@ -32,7 +32,7 @@ %% extension |.ins|) which are part of the distribution. %% -\ProvidesLanguage{nil}[2020/10/27 3.51 Nil language] +\ProvidesLanguage{nil}[2020/12/16 3.52 Nil language] \LdfInit{nil}{datenil} \ifx\l@nil\@undefined \newlanguage\l@nil -- cgit v1.2.3