diff options
Diffstat (limited to 'Master/texmf-dist/source/latex')
| -rw-r--r-- | Master/texmf-dist/source/latex/babel/babel.dtx | 600 | ||||
| -rw-r--r-- | Master/texmf-dist/source/latex/babel/babel.ins | 2 | ||||
| -rw-r--r-- | Master/texmf-dist/source/latex/babel/bbcompat.dtx | 2 | ||||
| -rw-r--r-- | Master/texmf-dist/source/latex/babel/locale.zip | bin | 777803 -> 777806 bytes |
4 files changed, 326 insertions, 278 deletions
diff --git a/Master/texmf-dist/source/latex/babel/babel.dtx b/Master/texmf-dist/source/latex/babel/babel.dtx index b6b77a34e06..ee0f127c4e1 100644 --- a/Master/texmf-dist/source/latex/babel/babel.dtx +++ b/Master/texmf-dist/source/latex/babel/babel.dtx @@ -31,7 +31,7 @@ % % \iffalse %<*filedriver> -\ProvidesFile{babel.dtx}[2020/10/27 v3.51 The Babel package] +\ProvidesFile{babel.dtx}[2020/12/16 v3.52 The Babel package] \documentclass{ltxdoc} \GetFileInfo{babel.dtx} \usepackage{fontspec} @@ -420,6 +420,14 @@ Package babel Warning: No hyphenation patterns were preloaded for This is not currently done by \babel{} and you must set it by hand. \end{note} +\begin{note} + Although it has been customary to recommend placing |\title|, + |\author| and other elements printed by |\maketitle| after + |\begin{document}|, mainly because of shorthands, it is advisable to + keep them in the preamble. Currently there is no real need to use + shorthands in those macros. +\end{note} + \subsection{Multilingual documents} In multilingual documents, just use a list of the required languages as @@ -2310,7 +2318,7 @@ redefined behave with the ‘new way’ described in the following note. \end{verbatim} (In this particular case, instead of the |captions| group you may need to modify the |captions.licr| one.) - + \item The ‘old way’, still valid for many languages, to redefine a caption is the following: \begin{verbatim} @@ -3005,30 +3013,64 @@ example, if the first capture reads |([ΐΰ])|, the replacement could be \verb+{1|ΐΰ|ίύ}+, which maps \textit{ΐ} to \textit{ί}, and \textit{ΰ} to \textit{ύ}, so that the diaeresis is removed. -This feature is activated with the first |\babelposthyphenation|. +This feature is activated with the first |\babelposthyphenation| or +|\babelprehyphenation|. See the \href{https://github.com/latex3/babel/wiki}{\babel\ wiki} for a -more detailed description and some examples. It also describes an -additional replacement type with the key |string|. +more detailed description and some examples. It also describes a few +additional replacement types (|string|, |penalty|). + +Although the main purpose of this command is non-standard hyphenation, +it may actually be used for other transformations (after hyphenation is +applied, so you must take discretionaries into account). + +You are limited to substitutions as done by \textsf{lua}, although a +future implementation may alternatively accept \textsf{lpeg}. + +\Describe{\babelprehyphenation}{\marg{locale-name}% + \marg{lua-pattern}\marg{replacement}} + +\New{3.44-3-52} This command is not strictly about hyphenation, but +it is include here because it is a clear counterpart of +|\babelposthyphenation|. It is similar to the latter, but (as its name +implies) applied before hyphenation. There are other differences: (1) +the first argument is the locale instead the name of hyphenation +patterns; (2) in the search patterns |=| has no special meaning, while +\verb+|+ stands for an ordinary space; (3) in the replacement, +discretionaries are not accepted. + +It handles glyphs and spaces (but you can not insert spaces). + +Performance is still somewhat poor in some cases, but it is fast in the +most the typical ones. + +This feature is activated with the first |\babelposthyphenation| or +|\babelprehyphenation|. \begin{example} - Although the main purpose of this command is non-standard - hyphenation, it may actually be used for other transformations (after - hyphenation is applied, so you must take discretionaries into - account). For example, you can use the |string| replacement to - replace a character (or series of them) by another character (or - series of them). Thus, to enter \textit{ž} as |zh| and \textit{š} as - |sh| in a newly created locale for transliterated Russian: + You can replace a character (or series of them) by another character + (or series of them). Thus, to enter \textit{ž} as |zh| and \textit{š} + as |sh| in a newly created locale for transliterated Russian: \begin{verbatim} \babelprovide[hyphenrules=+]{russian-latin} % Create locale -\babelposthyphenation{russian-latin}{([sz])h} % Create rule +\babelprehyphenation{russian-latin}{([sz])h} % Create rule { - { string = {1|sz|šž} }, + string = {1|sz|šž}, remove } \end{verbatim} - In other words, it is a quite general tool. (A counterpart - |\babelprehyphenation| is on the way.) +\end{example} + +\begin{example} + The following rule prevent the word “a” from being at the + end of a line: +\begin{verbatim} +\babelprehyphenation{english}{|a|} + {}, {}, % Keep first space and a + {insert, penalty = 10000}, % Insert penalty + {} % Keep last space +} +\end{verbatim} \end{example} \subsection{Selection based on BCP 47 tags} @@ -3387,7 +3429,7 @@ bidirectional (with both R and L paragraphs) documents in \luatex{}. \xetex{} (which will not support a similar option in the short term). It patches an internal command, so it might be ignored by some packages and classes (or even raise an error). \New{3.18}. - + \item[graphics] modifies the |picture| environment so that the whole figure is L but the text is R. It \textit{does not} work with the standard |picture|, and \textit{pict2e} is required if you want sloped @@ -3866,26 +3908,6 @@ labels, both with the relation to captions (chapters, part), and how counters are used to define them. It is still somewhat tentative because it is far from trivial -- see the wiki for further details. -\medskip -\textbf{\string\babelprehyphenation} - -\New{3.44} Note it is tentative, but the current behavior for glyphs -should be correct. - -It is similar to |\babelposthyphenation|, but (as its name implies) -applied before hyphenation. There are other differences: (1) the first -argument is the locale instead the name of hyphenation patterns; (2) in -the search patterns |=| has no special meaning (\verb+|+ is still -reserved, but currently unused); (3) in the replacement, -discretionaries are not accepted, only remove, {}, and string = ... - -Currently it handles glyphs, not discretionaries or spaces (in -particular, it will not catch the hyphen and you can't insert or remove -spaces). Also, you are limited to substitutions as done by lua, -although a future implementation may alternatively accept lpeg. - -Performance is still somewhat poor. - \section{Loading languages with \file{language.dat}} \TeX{} and most engines based on it (pdf\TeX, \xetex, $\epsilon$-\TeX, @@ -4815,8 +4837,8 @@ help from Bernd Raichle, for which I am grateful. % \section{Tools} % % \begin{macrocode} -%<<version=3.51>> -%<<date=2020/10/27>> +%<<version=3.52>> +%<<date=2020/12/16>> % \end{macrocode} % % \textbf{Do not use the following macros in \texttt{ldf} files. They @@ -5226,9 +5248,17 @@ help from Bernd Raichle, for which I am grateful. \ProvidesPackage{babel}[<@date@> <@version@> The Babel package] \@ifpackagewith{babel}{debug} {\providecommand\bbl@trace[1]{\message{^^J[ #1 ]}}% - \let\bbl@debug\@firstofone} + \let\bbl@debug\@firstofone + \ifx\directlua\@undefined\else + \directlua{ Babel = Babel or {} + Babel.debug = true }% + \fi} {\providecommand\bbl@trace[1]{}% - \let\bbl@debug\@gobble} + \let\bbl@debug\@gobble + \ifx\directlua\@undefined\else + \directlua{ Babel = Babel or {} + Babel.debug = false }% + \fi} <@Basic macros@> % Temporarily repeat here the code for errors \def\bbl@error#1#2{% @@ -6686,7 +6716,6 @@ help from Bernd Raichle, for which I am grateful. \IfFileExists{\CurrentOption.ldf}% {\bbl@load@language{\CurrentOption}}% {#1\bbl@load@language{#2}#3}} -\DeclareOption{afrikaans}{\bbl@try@load@lang{}{dutch}{}} \DeclareOption{hebrew}{% \input{rlbabel.def}% \bbl@load@language{hebrew}} @@ -6738,16 +6767,16 @@ help from Bernd Raichle, for which I am grateful. % \begin{macrocode} \let\bbl@tempc\relax \bbl@foreach\bbl@language@opts{% - \ifcase\bbl@iniflag + \ifcase\bbl@iniflag % Default \bbl@ifunset{ds@#1}% {\DeclareOption{#1}{\bbl@load@language{#1}}}% {}% - \or + \or % provide=* \@gobble % case 2 same as 1 - \or + \or % provide+=* \bbl@ifunset{ds@#1}% {\IfFileExists{#1.ldf}{}% - {\IfFileExists{babel-#1.tex}{}{\DeclareOption{#1}{}}}}% + {\IfFileExists{babel-#1.tex}{}{\@namedef{ds@#1}{}}}}% {}% \bbl@ifunset{ds@#1}% {\def\bbl@tempc{#1}% @@ -6760,7 +6789,7 @@ help from Bernd Raichle, for which I am grateful. \bbl@load@language{#1}% \fi}}% {}% - \or + \or % provide*=* \def\bbl@tempc{#1}% \bbl@ifunset{ds@#1}% {\DeclareOption{#1}{% @@ -6771,18 +6800,18 @@ help from Bernd Raichle, for which I am grateful. \fi} % \end{macrocode} % -% Now, we make sure an option is explicitly declared for any -% language set as global option, by checking if an |ldf| -% exists. The previous step was, in fact, somewhat redundant, but -% that way we minimize accessing the file system just to see if the -% option could be a language. +% Now, we make sure an option is explicitly declared for any language +% set as global option, by checking if an |ldf| exists. The previous +% step was, in fact, somewhat redundant, but that way we minimize +% accessing the file system just to see if the option could be a +% language. % % \begin{macrocode} \let\bbl@tempb\@nnil \bbl@foreach\@classoptionslist{% \bbl@ifunset{ds@#1}% {\IfFileExists{#1.ldf}{}% - {\IfFileExists{babel-#1.tex}{}{\DeclareOption{#1}{}}}}% + {\IfFileExists{babel-#1.tex}{}{\@namedef{ds@#1}{}}}}% {}% \bbl@ifunset{ds@#1}% {\def\bbl@tempb{#1}% @@ -7475,7 +7504,7 @@ help from Bernd Raichle, for which I am grateful. \csname\languagename @bbl@hyphenmap\endcsname \fi \fi - \global\let\bbl@hymapsel\@cclv + \let\bbl@hymapsel\@cclv % hyphenation - select patterns \bbl@patterns{#1}% % hyphenation - allow stretching with babelnohyphens @@ -8049,17 +8078,17 @@ help from Bernd Raichle, for which I am grateful. \def\bbl@tempa##1,#3=##2,##3\@empty{\def\bbl@tempb{##2}}% \expandafter\bbl@tempa\bbl@evargs,#3=,\@empty \bbl@ifunset{bbl@ev@#2@#3@#1}% - {\bbl@csarg\bbl@add{ev@#3@#1}{\bbl@elt{#2}}}% + {\bbl@csarg\bbl@add{ev@#3@#1}{\bbl@elth{#2}}}% {\bbl@csarg\let{ev@#2@#3@#1}\relax}% \bbl@csarg\newcommand{ev@#2@#3@#1}[\bbl@tempb]} \newcommand\EnableBabelHook[1]{\bbl@csarg\let{hk@#1}\@firstofone} \newcommand\DisableBabelHook[1]{\bbl@csarg\let{hk@#1}\@gobble} \def\bbl@usehooks#1#2{% - \def\bbl@elt##1{% + \def\bbl@elth##1{% \bbl@cs{hk@##1}{\bbl@cs{ev@##1@#1@}#2}}% \bbl@cs{ev@#1@}% \ifx\languagename\@undefined\else % Test required for Plain (?) - \def\bbl@elt##1{% + \def\bbl@elth##1{% \bbl@cs{hk@##1}{\bbl@cl{ev@##1@#1}#2}}% \bbl@cl{ev@#1}% \fi} @@ -11085,7 +11114,7 @@ help from Bernd Raichle, for which I am grateful. {Fix the name or reinstall babel.}% \else \catcode`\[=12 \catcode`\]=12 \catcode`\==12 \catcode`\&=12 - \catcode`\;=12 \catcode`\|=12 \catcode`\%=14 + \catcode`\;=12 \catcode`\|=12 \catcode`\%=14 \catcode`\-=12 \bbl@info{Importing \ifcase#2 \or font and identification \or basic \fi data for \languagename\\% @@ -13050,7 +13079,7 @@ help from Bernd Raichle, for which I am grateful. \endinput\fi % Here stops reading code for hyphen.cfg % The following is read the 2nd time it's loaded -\begingroup +\begingroup % TODO - to a lua file \catcode`\%=12 \catcode`\'=12 \catcode`\"=12 @@ -13204,12 +13233,10 @@ help from Bernd Raichle, for which I am grateful. % \textit{In progress.} Replace regular (ie, implicit) discretionaries % by spaceskips, based on the previous glyph (which I think makes % sense, because the hyphen and the previous char go always together). -% Other discretionaries are not touched. -% -% For the moment, only 3 SA languages are activated by default (see -% Unicode UAX 14). +% Other discretionaries are not touched. See Unicode UAX 14. % % \begin{macrocode} +% TODO - to a lua file \directlua{ Babel = Babel or {} Babel.linebreaking = Babel.linebreaking or {} @@ -13262,7 +13289,7 @@ help from Bernd Raichle, for which I am grateful. function Babel.sea_disc_to_space (head) local sea_ranges = Babel.sea_ranges local last_char = nil - local quad = 655360 ^^ 10 pt = 655360 = 10 * 65536 + local quad = 655360 ^% 10 pt = 655360 = 10 * 65536 for item in node.traverse(head) do local i = item.id if i == node.id'glyph' then @@ -13272,16 +13299,16 @@ help from Bernd Raichle, for which I am grateful. quad = font.getfont(last_char.font).size for lg, rg in pairs(sea_ranges) do if last_char.char > rg[1] and last_char.char < rg[2] then - lg = lg:sub(1, 4) ^^ Remove trailing number of, eg, Cyrl1 + lg = lg:sub(1, 4) ^% Remove trailing number of, eg, Cyrl1 local intraspace = Babel.intraspaces[lg] local intrapenalty = Babel.intrapenalties[lg] local n if intrapenalty ~= 0 then - n = node.new(14, 0) ^^ penalty + n = node.new(14, 0) ^% penalty n.penalty = intrapenalty node.insert_before(head, item, n) end - n = node.new(12, 13) ^^ (glue, spaceskip) + n = node.new(12, 13) ^% (glue, spaceskip) node.setglue(n, intraspace.b * quad, intraspace.p * quad, intraspace.m * quad) @@ -13456,6 +13483,7 @@ help from Bernd Raichle, for which I am grateful. % skipped and discretionaries are handled in a special way. % % \begin{macrocode} +% TODO - to a lua file \directlua{ Babel.script_blocks = { ['Arab'] = {{0x0600, 0x06FF}, {0x08A0, 0x08FF}, {0x0750, 0x077F}, @@ -13640,14 +13668,17 @@ end % to be matched. % % \begin{macrocode} -\begingroup +\begingroup % TODO - to a lua file +\catcode`\~=12 \catcode`\#=12 \catcode`\%=12 \catcode`\&=14 \directlua{ - Babel.linebreaking.post_replacements = {} - Babel.linebreaking.pre_replacements = {} + Babel.linebreaking.replacements = {} + Babel.linebreaking.replacements[0] = {} &% pre + Babel.linebreaking.replacements[1] = {} &% post + &% Discretionaries contain strings as nodes function Babel.str_to_nodes(fn, matches, base) local n, head, last if fn == nil then return nil end @@ -13667,7 +13698,11 @@ end return head end - function Babel.fetch_word(head, funct) + Babel.fetch_subtext = {} + + &% Merging both functions doesn't seen feasible, because there are too + &% many differences. + Babel.fetch_subtext[0] = function(head) local word_string = '' local word_nodes = {} local lang @@ -13676,149 +13711,49 @@ end while item do - if item.id == 29 - and not(item.char == 124) &% ie, not | - and not(item.char == 61) &% ie, not = - and not inmath - and (item.lang == lang or lang == nil) then - lang = lang or item.lang - word_string = word_string .. unicode.utf8.char(item.char) - word_nodes[#word_nodes+1] = item + if item.id == 11 then + inmath = (item.subtype == 0) + end - elseif item.id == 7 and item.subtype == 2 and not inmath then - word_string = word_string .. '=' - word_nodes[#word_nodes+1] = item + if inmath then + &% pass - elseif item.id == 7 and item.subtype == 3 and not inmath then - word_string = word_string .. '|' - word_nodes[#word_nodes+1] = item + elseif item.id == 29 then + local locale = node.get_attribute(item, Babel.attr_locale) - elseif item.id == 11 and item.subtype == 0 then - inmath = true + if lang == locale or lang == nil then + if (item.char ~= 124) then &% ie, not | = space + lang = lang or locale + word_string = word_string .. unicode.utf8.char(item.char) + word_nodes[#word_nodes+1] = item + end + else + break + end - elseif word_string == '' then - &% pass + elseif item.id == 12 and item.subtype == 13 then + word_string = word_string .. '|' + word_nodes[#word_nodes+1] = item - else - return word_string, word_nodes, item, lang + &% Ignore leading unrecognized nodes, too. + elseif word_string ~= '' then + word_string = word_string .. Babel.us_char + word_nodes[#word_nodes+1] = item &% Will be ignored end item = item.next end - end - - function Babel.post_hyphenate_replace(head) - local u = unicode.utf8 - local lbkr = Babel.linebreaking.post_replacements - local word_head = head - - while true do - local w, wn, nw, lang = Babel.fetch_word(word_head) - if not lang then return head end - - if not lbkr[lang] then - break - end - - for k=1, #lbkr[lang] do - local p = lbkr[lang][k].pattern - local r = lbkr[lang][k].replace - - while true do - local matches = { u.match(w, p) } - if #matches < 2 then break end - - local first = table.remove(matches, 1) - local last = table.remove(matches, #matches) - - &% Fix offsets, from bytes to unicode. - first = u.len(w:sub(1, first-1)) + 1 - last = u.len(w:sub(1, last-1)) - - local new &% used when inserting and removing nodes - local changed = 0 - - &% This loop traverses the replace list and takes the - &% corresponding actions - for q = first, last do - local crep = r[q-first+1] - local char_node = wn[q] - local char_base = char_node - - if crep and crep.data then - char_base = wn[crep.data+first-1] - end - - if crep == {} then - break - elseif crep == nil then - changed = changed + 1 - node.remove(head, char_node) - elseif crep and (crep.pre or crep.no or crep.post) then - changed = changed + 1 - d = node.new(7, 0) &% (disc, discretionary) - d.pre = Babel.str_to_nodes(crep.pre, matches, char_base) - d.post = Babel.str_to_nodes(crep.post, matches, char_base) - d.replace = Babel.str_to_nodes(crep.no, matches, char_base) - d.attr = char_base.attr - if crep.pre == nil then &% TeXbook p96 - d.penalty = crep.penalty or tex.hyphenpenalty - else - d.penalty = crep.penalty or tex.exhyphenpenalty - end - head, new = node.insert_before(head, char_node, d) - node.remove(head, char_node) - if q == 1 then - word_head = new - end - elseif crep and crep.string then - changed = changed + 1 - local str = crep.string(matches) - if str == '' then - if q == 1 then - word_head = char_node.next - end - head, new = node.remove(head, char_node) - elseif char_node.id == 29 and u.len(str) == 1 then - char_node.char = string.utfvalue(str) - else - local n - for s in string.utfvalues(str) do - if char_node.id == 7 then - log('Automatic hyphens cannot be replaced, just removed.') - else - n = node.copy(char_base) - end - n.char = s - if q == 1 then - head, new = node.insert_before(head, char_node, n) - word_head = new - else - node.insert_before(head, char_node, n) - end - end - - node.remove(head, char_node) - end &% string length - end &% if char and char.string - end &% for char in match - if changed > 20 then - texio.write('Too many changes. Ignoring the rest.') - elseif changed > 0 then - w, wn, nw = Babel.fetch_word(word_head) - end - end &% for match - end &% for patterns - word_head = nw - end &% for words - return head + &% Here and above we remove some trailing chars but not the + &% corresponding nodes. But they aren't accessed. + if word_string:sub(-1) == '|' then + word_string = word_string:sub(1,-2) + end + word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '') + return word_string, word_nodes, item, lang end - &%%% - &% Preliminary code for \babelprehyphenation - &% TODO. Copypaste pattern. Merge with fetch_word - function Babel.fetch_subtext(head, funct) + Babel.fetch_subtext[1] = function(head) local word_string = '' local word_nodes = {} local lang @@ -13827,145 +13762,256 @@ end while item do - if item.id == 29 then - local locale = node.get_attribute(item, Babel.attr_locale) + if item.id == 11 then + inmath = (item.subtype == 0) + end - if not(item.char == 124) &% ie, not | = space - and not inmath - and (locale == lang or lang == nil) then - lang = lang or locale - word_string = word_string .. unicode.utf8.char(item.char) - word_nodes[#word_nodes+1] = item - end + if inmath then + &% pass - if item == node.tail(head) then - item = nil - return word_string, word_nodes, item, lang + elseif item.id == 29 then + if item.lang == lang or lang == nil then + if (item.char ~= 124) and (item.char ~= 61) then &% not =, not | + lang = lang or item.lang + word_string = word_string .. unicode.utf8.char(item.char) + word_nodes[#word_nodes+1] = item + end + else + break end - elseif item.id == 12 and item.subtype == 13 and not inmath then - word_string = word_string .. '|' + elseif item.id == 7 and item.subtype == 2 then + word_string = word_string .. '=' word_nodes[#word_nodes+1] = item - if item == node.tail(head) then - item = nil - return word_string, word_nodes, item, lang - end - - elseif item.id == 11 and item.subtype == 0 then - inmath = true + elseif item.id == 7 and item.subtype == 3 then + word_string = word_string .. '|' + word_nodes[#word_nodes+1] = item + &% (1) Go to next word if nothing was found, and (2) implictly + &% remove leading USs. elseif word_string == '' then &% pass - else - return word_string, word_nodes, item, lang + &% This is the responsible for splitting by words. + elseif (item.id == 12 and item.subtype == 13) then + break + + else + word_string = word_string .. Babel.us_char + word_nodes[#word_nodes+1] = item &% Will be ignored end item = item.next end + + word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '') + return word_string, word_nodes, item, lang end - &% TODO. Copypaste pattern. Merge with pre_hyphenate_replace function Babel.pre_hyphenate_replace(head) + Babel.hyphenate_replace(head, 0) + end + + function Babel.post_hyphenate_replace(head) + Babel.hyphenate_replace(head, 1) + end + + Babel.us_char = string.char(31) + + function Babel.hyphenate_replace(head, mode) local u = unicode.utf8 - local lbkr = Babel.linebreaking.pre_replacements + local lbkr = Babel.linebreaking.replacements[mode] + local word_head = head - while true do - local w, wn, nw, lang = Babel.fetch_subtext(word_head) - if not lang then return head end + while true do &% for each subtext block - if not lbkr[lang] then - break + local w, wn, nw, lang = Babel.fetch_subtext[mode](word_head) + + if Babel.debug then + print() + print('@@@@@', w, nw) end + if nw == nil and w == '' then break end + + if not lang then goto next end + if not lbkr[lang] then goto next end + + &% For each saved (pre|post)hyphenation. TODO. Reconsider how + &% loops are nested. for k=1, #lbkr[lang] do local p = lbkr[lang][k].pattern local r = lbkr[lang][k].replace + if Babel.debug then + print('=====', p, mode) + end + + &% This variable is set in some cases below to the first *byte* + &% after the match, either as found by u.match (faster) or the + &% computed position based on sc if w has changed. + local last_match = 0 + + &% For every match. while true do - local matches = { u.match(w, p) } + if Babel.debug then + print('-----') + end + local new &% used when inserting and removing nodes + local refetch = false + + local matches = { u.match(w, p, last_match) } if #matches < 2 then break end + &% Get and remove empty captures (with ()'s, which return a + &% number with the position), and keep actual captures + &% (from (...)), if any, in matches. local first = table.remove(matches, 1) - local last = table.remove(matches, #matches) + local last = table.remove(matches, #matches) + &% Non re-fetched substrings may contain \31, which separates + &% subsubstrings. + if string.find(w:sub(first, last-1), Babel.us_char) then break end + + local save_last = last &% with A()BC()D, points to D - &% Fix offsets, from bytes to unicode. + &% Fix offsets, from bytes to unicode. Explained above. first = u.len(w:sub(1, first-1)) + 1 - last = u.len(w:sub(1, last-1)) + last = u.len(w:sub(1, last-1)) &% now last points to C - local new &% used when inserting and removing nodes - local changed = 0 + if Babel.debug then + print(p) + print('', 'sc', 'first', 'last', 'last_m', 'w') + end - &% This loop traverses the replace list and takes the - &% corresponding actions - for q = first, last do - local crep = r[q-first+1] - local char_node = wn[q] + &% This loop traverses the matched substring and takes the + &% corresponding action stored in the replacement list. + &% sc = the position in substr nodes / string + &% rc = the replacement table index + local sc = first-1 + local rc = 0 + while rc < last-first+1 do &% for each replacement + if Babel.debug then + print('.....') + end + sc = sc + 1 + rc = rc + 1 + local crep = r[rc] + local char_node = wn[sc] local char_base = char_node + local end_replacement = false if crep and crep.data then char_base = wn[crep.data+first-1] end - if crep == {} then - break - elseif crep == nil then - changed = changed + 1 + if Babel.debug then + print('*', sc, first, last, last_match, w) + end + + if crep and next(crep) == nil then &% {} + last_match = save_last + + elseif crep == nil then &% remove node.remove(head, char_node) + table.remove(wn, sc) + w = u.sub(w, 1, sc-1) .. u.sub(w, sc+1) + last_match = utf8.offset(w, sc) + sc = sc - 1 &% Nothing has been inserted + + elseif mode == 1 and crep and (crep.pre or crep.no or crep.post) then + local d = node.new(7, 0) &% (disc, discretionary) + d.pre = Babel.str_to_nodes(crep.pre, matches, char_base) + d.post = Babel.str_to_nodes(crep.post, matches, char_base) + d.replace = Babel.str_to_nodes(crep.no, matches, char_base) + d.attr = char_base.attr + if crep.pre == nil then &% TeXbook p96 + d.penalty = crep.penalty or tex.hyphenpenalty + else + d.penalty = crep.penalty or tex.exhyphenpenalty + end + head, new = node.insert_before(head, char_node, d) + end_replacement = true + + elseif crep and crep.penalty then + local d = node.new(14, 0) &% (penalty, userpenalty) + d.attr = char_base.attr + d.penalty = crep.penalty + head, new = node.insert_before(head, char_node, d) + end_replacement = true + elseif crep and crep.string then - changed = changed + 1 local str = crep.string(matches) - if str == '' then - if q == 1 then + if str == '' then &% Gather with nil + refetch = true + if sc == 1 then word_head = char_node.next end head, new = node.remove(head, char_node) elseif char_node.id == 29 and u.len(str) == 1 then char_node.char = string.utfvalue(str) - else + w = u.sub(w, 1, sc-1) .. str .. u.sub(w, sc+1) + last_match = utf8.offset(w, sc+1) + else + refetch = true local n for s in string.utfvalues(str) do if char_node.id == 7 then - log('Automatic hyphens cannot be replaced, just removed.') + &% TODO. Remove this limitation. + texio.write_nl('Automatic hyphens cannot be replaced, just removed.') else n = node.copy(char_base) end n.char = s - if q == 1 then - head, new = node.insert_before(head, char_node, n) + if sc == 1 then + head, new = node.insert_before(head, char_node, n) word_head = new else - node.insert_before(head, char_node, n) + node.insert_before(head, char_node, n) end end - node.remove(head, char_node) end &% string length - end &% if char and char.string - end &% for char in match - if changed > 20 then - texio.write('Too many changes. Ignoring the rest.') - elseif changed > 0 then - &% For one-to-one can we modifiy directly the - &% values without re-fetching? Very likely. - w, wn, nw = Babel.fetch_subtext(word_head) + end &% if char and char.string (ie replacement cases) + + &% Shared by disc and penalty. + if end_replacement then + if sc == 1 then + word_head = new + end + if crep.insert then + last_match = save_last + else + node.remove(head, char_node) + w = u.sub(w, 1, sc-1) .. Babel.us_char .. u.sub(w, sc+1) + last_match = utf8.offset(w, sc) + end + end + end &% for each replacement + + if Babel.debug then + print('/', sc, first, last, last_match, w) end + &% TODO. refetch will be eventually unnecesary. + if refetch then + w, wn, nw, lang = Babel.fetch_subtext[mode](word_head) + end + end &% for match end &% for patterns + + ::next:: word_head = nw - end &% for words + end &% for substring return head end - &%%% end of preliminary code for \babelprehyphenation - - &% The following functions belong to the next macro &% This table stores capture maps, numbered consecutively Babel.capture_maps = {} + &% The following functions belong to the next macro function Babel.capture_func(key, cap) local ret = "[[" .. cap:gsub('{([0-9])}', "]]..m[%1]..[[") .. "]]" ret = ret:gsub('{([0-9])|([^|]+)|(.-)}', Babel.capture_func_map) @@ -14022,15 +14068,16 @@ end \bbl@ifsamestring{##1}{remove}&% {\bbl@add@list\babeltempb{nil}}&% {\directlua{ - local rep = [[##1]] + local rep = [[##1]] + rep = rep:gsub('^%s*(insert)%s*,', 'insert = true, ') rep = rep:gsub( '(no)%s*=%s*([^%s,]*)', Babel.capture_func) rep = rep:gsub( '(pre)%s*=%s*([^%s,]*)', Babel.capture_func) rep = rep:gsub( '(post)%s*=%s*([^%s,]*)', Babel.capture_func) - rep = rep:gsub('(string)%s*=%s*([^%s,]*)', Babel.capture_func) + rep = rep:gsub('(string)%s*=%s*([^%s,]*)', Babel.capture_func) tex.print([[\string\babeltempa{{]] .. rep .. [[}}]]) }}}&% \directlua{ - local lbkr = Babel.linebreaking.post_replacements + local lbkr = Babel.linebreaking.replacements[1] local u = unicode.utf8 &% Convert pattern: local patt = string.gsub([==[#2]==], '%s', '') @@ -14039,7 +14086,6 @@ end end patt = string.gsub(patt, '%(%)%^', '^()') patt = string.gsub(patt, '%$%(%)', '()$') - texio.write('***************' .. patt) patt = u.gsub(patt, '{(.)}', function (n) return '%' .. (tonumber(n) and (tonumber(n)+1) or n) @@ -14049,7 +14095,7 @@ end { pattern = patt, replace = { \babeltempb } }) }&% \endgroup} -% TODO. Working !!! Copypaste pattern. +% TODO. Copypaste pattern. \gdef\babelprehyphenation#1#2#3{&% \bbl@activateprehyphen \begingroup @@ -14060,17 +14106,20 @@ end {\bbl@add@list\babeltempb{nil}}&% {\directlua{ local rep = [[##1]] + rep = rep:gsub('^%s*(insert)%s*,', 'insert = true, ') rep = rep:gsub('(string)%s*=%s*([^%s,]*)', Babel.capture_func) tex.print([[\string\babeltempa{{]] .. rep .. [[}}]]) }}}&% \directlua{ - local lbkr = Babel.linebreaking.pre_replacements + local lbkr = Babel.linebreaking.replacements[0] local u = unicode.utf8 &% Convert pattern: local patt = string.gsub([==[#2]==], '%s', '') if not u.find(patt, '()', nil, true) then patt = '()' .. patt .. '()' - end + end + &% patt = string.gsub(patt, '%(%)%^', '^()') + &% patt = string.gsub(patt, '([^%%])%$%(%)', '%1()$') patt = u.gsub(patt, '{(.)}', function (n) return '%' .. (tonumber(n) and (tonumber(n)+1) or n) @@ -14086,7 +14135,6 @@ end \directlua{ Babel.linebreaking.add_after(Babel.post_hyphenate_replace) }} -% TODO. Working !!! \def\bbl@activateprehyphen{% \let\bbl@activateprehyphen\relax \directlua{ diff --git a/Master/texmf-dist/source/latex/babel/babel.ins b/Master/texmf-dist/source/latex/babel/babel.ins index 3bdf670a2d7..b362ca94e91 100644 --- a/Master/texmf-dist/source/latex/babel/babel.ins +++ b/Master/texmf-dist/source/latex/babel/babel.ins @@ -26,7 +26,7 @@ %% and covered by LPPL is defined by the unpacking scripts (with %% extension .ins) which are part of the distribution. %% -\def\filedate{2020/10/27} +\def\filedate{2020/12/16} \def\batchfile{babel.ins} \input docstrip.tex diff --git a/Master/texmf-dist/source/latex/babel/bbcompat.dtx b/Master/texmf-dist/source/latex/babel/bbcompat.dtx index 1aabf3a55d7..fb1977f094a 100644 --- a/Master/texmf-dist/source/latex/babel/bbcompat.dtx +++ b/Master/texmf-dist/source/latex/babel/bbcompat.dtx @@ -30,7 +30,7 @@ % % \iffalse %<*dtx> -\ProvidesFile{bbcompat.dtx}[2020/10/27 v3.51] +\ProvidesFile{bbcompat.dtx}[2020/12/16 v3.52] %</dtx> % %% File 'bbcompat.dtx' diff --git a/Master/texmf-dist/source/latex/babel/locale.zip b/Master/texmf-dist/source/latex/babel/locale.zip Binary files differindex 375cc887587..e90a725a782 100644 --- a/Master/texmf-dist/source/latex/babel/locale.zip +++ b/Master/texmf-dist/source/latex/babel/locale.zip |