From 87979956545645d8547439694c3ae1e8dd270992 Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Mon, 19 Jul 2021 20:10:36 +0000 Subject: babel (19jul21) git-svn-id: svn://tug.org/texlive/trunk@59990 c570f23f-e606-0410-a88d-b1316a301751 --- .../tex/generic/babel/babel-bidi-basic-r.lua | 2 +- .../tex/generic/babel/babel-bidi-basic.lua | 2 +- .../tex/generic/babel/babel-transforms.lua | 511 ++++++++++++++++ Master/texmf-dist/tex/generic/babel/babel.def | 96 ++-- Master/texmf-dist/tex/generic/babel/babel.sty | 197 ++----- Master/texmf-dist/tex/generic/babel/hyphen.cfg | 17 +- Master/texmf-dist/tex/generic/babel/luababel.def | 639 ++++----------------- Master/texmf-dist/tex/generic/babel/nil.ldf | 2 +- Master/texmf-dist/tex/generic/babel/xebabel.def | 16 +- 9 files changed, 763 insertions(+), 719 deletions(-) create mode 100644 Master/texmf-dist/tex/generic/babel/babel-transforms.lua (limited to 'Master/texmf-dist/tex/generic') diff --git a/Master/texmf-dist/tex/generic/babel/babel-bidi-basic-r.lua b/Master/texmf-dist/tex/generic/babel/babel-bidi-basic-r.lua index ba2f05c10ea..43328442bee 100644 --- a/Master/texmf-dist/tex/generic/babel/babel-bidi-basic-r.lua +++ b/Master/texmf-dist/tex/generic/babel/babel-bidi-basic-r.lua @@ -100,7 +100,7 @@ function Babel.bidi(head, ispar) if new_dir then attr_dir = 0 for at in node.traverse(item.attr) do - if at.number == luatexbase.registernumber'bbl@attr@dir' then + if at.number == Babel.attr_dir then attr_dir = at.value % 3 end end diff --git a/Master/texmf-dist/tex/generic/babel/babel-bidi-basic.lua b/Master/texmf-dist/tex/generic/babel/babel-bidi-basic.lua index cd41ee15ad4..27467e83fd4 100644 --- a/Master/texmf-dist/tex/generic/babel/babel-bidi-basic.lua +++ b/Master/texmf-dist/tex/generic/babel/babel-bidi-basic.lua @@ -106,7 +106,7 @@ function Babel.bidi(head, ispar, hdir) local has_en = false local first_et = nil - local ATDIR = luatexbase.registernumber'bbl@attr@dir' + local ATDIR = Babel.attr_dir local save_outer local temp = node.get_attribute(head, ATDIR) diff --git a/Master/texmf-dist/tex/generic/babel/babel-transforms.lua b/Master/texmf-dist/tex/generic/babel/babel-transforms.lua new file mode 100644 index 00000000000..ab6e7bc9daf --- /dev/null +++ b/Master/texmf-dist/tex/generic/babel/babel-transforms.lua @@ -0,0 +1,511 @@ +-- +-- This is file `babel-transforms.lua', +-- generated with the docstrip utility. +-- +-- The original source files were: +-- +-- babel.dtx (with options: `transforms') +-- +-- +-- Copyright (C) 2012-2021 Javier Bezos and Johannes L. Braams. +-- Copyright (C) 1989-2012 Johannes L. Braams and +-- any individual authors listed elsewhere in this file. +-- All rights reserved. +-- +-- +-- This file is part of the Babel system. +-- -------------------------------------- +-- +-- It may be distributed and/or modified under the +-- conditions of the LaTeX Project Public License, either version 1.3 +-- of this license or (at your option) any later version. +-- The latest version of this license is in +-- http://www.latex-project.org/lppl.txt +-- and version 1.3 or later is part of all distributions of LaTeX +-- version 2003/12/01 or later. +-- +-- This work has the LPPL maintenance status "maintained". +-- +-- The Current Maintainer of this work is Javier Bezos. +-- +-- The list of derived (unpacked) files belonging to the distribution +-- and covered by LPPL is defined by the unpacking scripts (with +-- extension |.ins|) which are part of the distribution. +-- + +Babel.linebreaking.replacements = {} +Babel.linebreaking.replacements[0] = {} -- pre +Babel.linebreaking.replacements[1] = {} -- post + +-- Discretionaries contain strings as nodes +function Babel.str_to_nodes(fn, matches, base) + local n, head, last + if fn == nil then return nil end + for s in string.utfvalues(fn(matches)) do + if base.id == 7 then + base = base.replace + end + n = node.copy(base) + n.char = s + if not head then + head = n + else + last.next = n + end + last = n + end + return head +end + +Babel.fetch_subtext = {} + +Babel.ignore_pre_char = function(node) + return (node.lang == Babel.nohyphenation) +end + +-- Merging both functions doesn't seen feasible, because there are too +-- many differences. +Babel.fetch_subtext[0] = function(head) + local word_string = '' + local word_nodes = {} + local lang + local item = head + local inmath = false + + while item do + + if item.id == 11 then + inmath = (item.subtype == 0) + end + + if inmath then + -- pass + + elseif item.id == 29 then + local locale = node.get_attribute(item, Babel.attr_locale) + + if lang == locale or lang == nil then + lang = lang or locale + if Babel.ignore_pre_char(item) then + word_string = word_string .. Babel.us_char + else + word_string = word_string .. unicode.utf8.char(item.char) + end + word_nodes[#word_nodes+1] = item + else + break + end + + elseif item.id == 12 and item.subtype == 13 then + word_string = word_string .. ' ' + word_nodes[#word_nodes+1] = item + + -- Ignore leading unrecognized nodes, too. + elseif word_string ~= '' then + word_string = word_string .. Babel.us_char + word_nodes[#word_nodes+1] = item -- Will be ignored + end + + item = item.next + end + + -- Here and above we remove some trailing chars but not the + -- corresponding nodes. But they aren't accessed. + if word_string:sub(-1) == ' ' then + word_string = word_string:sub(1,-2) + end + word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '') + return word_string, word_nodes, item, lang +end + +Babel.fetch_subtext[1] = function(head) + local word_string = '' + local word_nodes = {} + local lang + local item = head + local inmath = false + + while item do + + if item.id == 11 then + inmath = (item.subtype == 0) + end + + if inmath then + -- pass + + elseif item.id == 29 then + if item.lang == lang or lang == nil then + if (item.char ~= 124) and (item.char ~= 61) then -- not =, not | + lang = lang or item.lang + word_string = word_string .. unicode.utf8.char(item.char) + word_nodes[#word_nodes+1] = item + end + else + break + end + + elseif item.id == 7 and item.subtype == 2 then + word_string = word_string .. '=' + word_nodes[#word_nodes+1] = item + + elseif item.id == 7 and item.subtype == 3 then + word_string = word_string .. '|' + word_nodes[#word_nodes+1] = item + + -- (1) Go to next word if nothing was found, and (2) implicitly + -- remove leading USs. + elseif word_string == '' then + -- pass + + -- This is the responsible for splitting by words. + elseif (item.id == 12 and item.subtype == 13) then + break + + else + word_string = word_string .. Babel.us_char + word_nodes[#word_nodes+1] = item -- Will be ignored + end + + item = item.next + end + + word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '') + return word_string, word_nodes, item, lang +end + +function Babel.pre_hyphenate_replace(head) + Babel.hyphenate_replace(head, 0) +end + +function Babel.post_hyphenate_replace(head) + Babel.hyphenate_replace(head, 1) +end + +Babel.us_char = string.char(31) + +function Babel.hyphenate_replace(head, mode) + local u = unicode.utf8 + local lbkr = Babel.linebreaking.replacements[mode] + + local word_head = head + + while true do -- for each subtext block + + local w, w_nodes, nw, lang = Babel.fetch_subtext[mode](word_head) + + if Babel.debug then + print() + print((mode == 0) and '@@@@<' or '@@@@>', w) + end + + if nw == nil and w == '' then break end + + if not lang then goto next end + if not lbkr[lang] then goto next end + + -- For each saved (pre|post)hyphenation. TODO. Reconsider how + -- loops are nested. + for k=1, #lbkr[lang] do + local p = lbkr[lang][k].pattern + local r = lbkr[lang][k].replace + + if Babel.debug then + print('*****', p, mode) + end + + -- This variable is set in some cases below to the first *byte* + -- after the match, either as found by u.match (faster) or the + -- computed position based on sc if w has changed. + local last_match = 0 + local step = 0 + + -- For every match. + while true do + if Babel.debug then + print('=====') + end + local new -- used when inserting and removing nodes + + local matches = { u.match(w, p, last_match) } + + if #matches < 2 then break end + + -- Get and remove empty captures (with ()'s, which return a + -- number with the position), and keep actual captures + -- (from (...)), if any, in matches. + local first = table.remove(matches, 1) + local last = table.remove(matches, #matches) + -- Non re-fetched substrings may contain \31, which separates + -- subsubstrings. + if string.find(w:sub(first, last-1), Babel.us_char) then break end + + local save_last = last -- with A()BC()D, points to D + + -- Fix offsets, from bytes to unicode. Explained above. + first = u.len(w:sub(1, first-1)) + 1 + last = u.len(w:sub(1, last-1)) -- now last points to C + + -- This loop stores in n small table the nodes + -- corresponding to the pattern. Used by 'data' to provide a + -- predictable behavior with 'insert' (now w_nodes is modified on + -- the fly), and also access to 'remove'd nodes. + local sc = first-1 -- Used below, too + local data_nodes = {} + + for q = 1, last-first+1 do + data_nodes[q] = w_nodes[sc+q] + end + + -- This loop traverses the matched substring and takes the + -- corresponding action stored in the replacement list. + -- sc = the position in substr nodes / string + -- rc = the replacement table index + local rc = 0 + + while rc < last-first+1 do -- for each replacement + if Babel.debug then + print('.....', rc + 1) + end + sc = sc + 1 + rc = rc + 1 + + if Babel.debug then + Babel.debug_hyph(w, w_nodes, sc, first, last, last_match) + local ss = '' + for itt in node.traverse(head) do + if itt.id == 29 then + ss = ss .. unicode.utf8.char(itt.char) + else + ss = ss .. '{' .. itt.id .. '}' + end + end + print('*****************', ss) + + end + + local crep = r[rc] + local item = w_nodes[sc] + local item_base = item + local placeholder = Babel.us_char + local d + + if crep and crep.data then + item_base = data_nodes[crep.data] + end + + if crep then + step = crep.step or 0 + end + + if crep and next(crep) == nil then -- = {} + last_match = save_last -- Optimization + goto next + + elseif crep == nil or crep.remove then + node.remove(head, item) + table.remove(w_nodes, sc) + w = u.sub(w, 1, sc-1) .. u.sub(w, sc+1) + sc = sc - 1 -- Nothing has been inserted. + last_match = utf8.offset(w, sc+1+step) + goto next + + elseif crep and crep.kashida then -- Experimental + node.set_attribute(item, + Babel.attr_kashida, + crep.kashida) + last_match = utf8.offset(w, sc+1+step) + goto next + + elseif crep and crep.string then + local str = crep.string(matches) + if str == '' then -- Gather with nil + node.remove(head, item) + table.remove(w_nodes, sc) + w = u.sub(w, 1, sc-1) .. u.sub(w, sc+1) + sc = sc - 1 -- Nothing has been inserted. + else + local loop_first = true + for s in string.utfvalues(str) do + d = node.copy(item_base) + d.char = s + if loop_first then + loop_first = false + head, new = node.insert_before(head, item, d) + if sc == 1 then + word_head = head + end + w_nodes[sc] = d + w = u.sub(w, 1, sc-1) .. u.char(s) .. u.sub(w, sc+1) + else + sc = sc + 1 + head, new = node.insert_before(head, item, d) + table.insert(w_nodes, sc, new) + w = u.sub(w, 1, sc-1) .. u.char(s) .. u.sub(w, sc) + end + if Babel.debug then + print('.....', 'str') + Babel.debug_hyph(w, w_nodes, sc, first, last, last_match) + end + end -- for + node.remove(head, item) + end -- if '' + last_match = utf8.offset(w, sc+1+step) + goto next + + elseif mode == 1 and crep and (crep.pre or crep.no or crep.post) then + d = node.new(7, 0) -- (disc, discretionary) + d.pre = Babel.str_to_nodes(crep.pre, matches, item_base) + d.post = Babel.str_to_nodes(crep.post, matches, item_base) + d.replace = Babel.str_to_nodes(crep.no, matches, item_base) + d.attr = item_base.attr + if crep.pre == nil then -- TeXbook p96 + d.penalty = crep.penalty or tex.hyphenpenalty + else + d.penalty = crep.penalty or tex.exhyphenpenalty + end + placeholder = '|' + head, new = node.insert_before(head, item, d) + + elseif mode == 0 and crep and (crep.pre or crep.no or crep.post) then + -- ERROR + + elseif crep and crep.penalty then + d = node.new(14, 0) -- (penalty, userpenalty) + d.attr = item_base.attr + d.penalty = crep.penalty + head, new = node.insert_before(head, item, d) + + elseif crep and crep.space then + -- 655360 = 10 pt = 10 * 65536 sp + d = node.new(12, 13) -- (glue, spaceskip) + local quad = font.getfont(item_base.font).size or 655360 + node.setglue(d, crep.space[1] * quad, + crep.space[2] * quad, + crep.space[3] * quad) + if mode == 0 then + placeholder = ' ' + end + head, new = node.insert_before(head, item, d) + + elseif crep and crep.spacefactor then + d = node.new(12, 13) -- (glue, spaceskip) + local base_font = font.getfont(item_base.font) + node.setglue(d, + crep.spacefactor[1] * base_font.parameters['space'], + crep.spacefactor[2] * base_font.parameters['space_stretch'], + crep.spacefactor[3] * base_font.parameters['space_shrink']) + if mode == 0 then + placeholder = ' ' + end + head, new = node.insert_before(head, item, d) + + elseif mode == 0 and crep and crep.space then + -- ERROR + + end -- ie replacement cases + + -- Shared by disc, space and penalty. + if sc == 1 then + word_head = head + end + if crep.insert then + w = u.sub(w, 1, sc-1) .. placeholder .. u.sub(w, sc) + table.insert(w_nodes, sc, new) + last = last + 1 + else + w_nodes[sc] = d + node.remove(head, item) + w = u.sub(w, 1, sc-1) .. placeholder .. u.sub(w, sc+1) + end + + last_match = utf8.offset(w, sc+1+step) + + ::next:: + + end -- for each replacement + + if Babel.debug then + print('.....', '/') + Babel.debug_hyph(w, w_nodes, sc, first, last, last_match) + end + + end -- for match + + end -- for patterns + + ::next:: + word_head = nw + end -- for substring + return head +end + +-- This table stores capture maps, numbered consecutively +Babel.capture_maps = {} + +-- The following functions belong to the next macro +function Babel.capture_func(key, cap) + local ret = "[[" .. cap:gsub('{([0-9])}', "]]..m[%1]..[[") .. "]]" + local cnt + local u = unicode.utf8 + ret, cnt = ret:gsub('{([0-9])|([^|]+)|(.-)}', Babel.capture_func_map) + if cnt == 0 then + ret = u.gsub(ret, '{(%x%x%x%x+)}', + function (n) + return u.char(tonumber(n, 16)) + end) + end + ret = ret:gsub("%[%[%]%]%.%.", '') + ret = ret:gsub("%.%.%[%[%]%]", '') + return key .. [[=function(m) return ]] .. ret .. [[ end]] +end + +function Babel.capt_map(from, mapno) + return Babel.capture_maps[mapno][from] or from +end + +-- Handle the {n|abc|ABC} syntax in captures +function Babel.capture_func_map(capno, from, to) + local u = unicode.utf8 + from = u.gsub(from, '{(%x%x%x%x+)}', + function (n) + return u.char(tonumber(n, 16)) + end) + to = u.gsub(to, '{(%x%x%x%x+)}', + function (n) + return u.char(tonumber(n, 16)) + end) + local froms = {} + for s in string.utfcharacters(from) do + table.insert(froms, s) + end + local cnt = 1 + table.insert(Babel.capture_maps, {}) + local mlen = table.getn(Babel.capture_maps) + for s in string.utfcharacters(to) do + Babel.capture_maps[mlen][froms[cnt]] = s + cnt = cnt + 1 + end + return "]]..Babel.capt_map(m[" .. capno .. "]," .. + (mlen) .. ").." .. "[[" +end + +-- Create/Extend reversed sorted list of kashida weights: +function Babel.capture_kashida(key, wt) + wt = tonumber(wt) + if Babel.kashida_wts then + for p, q in ipairs(Babel.kashida_wts) do + if wt == q then + break + elseif wt > q then + table.insert(Babel.kashida_wts, p, wt) + break + elseif table.getn(Babel.kashida_wts) == p then + table.insert(Babel.kashida_wts, wt) + end + end + else + Babel.kashida_wts = { wt } + end + return 'kashida = ' .. wt +end diff --git a/Master/texmf-dist/tex/generic/babel/babel.def b/Master/texmf-dist/tex/generic/babel/babel.def index 9797455fe42..8d7db041887 100644 --- a/Master/texmf-dist/tex/generic/babel/babel.def +++ b/Master/texmf-dist/tex/generic/babel/babel.def @@ -40,7 +40,7 @@ \wlog{File: #1 #4 #3 <#2>}% \let\ProvidesFile\@undefined} \fi -\ProvidesFile{babel.def}[2021/06/28 3.61 Babel common definitions] +\ProvidesFile{babel.def}[2021/07/19 3.62 Babel common definitions] \ifx\AtBeginDocument\@undefined % TODO. change test. % == Code for plain == \def\@empty{} @@ -389,17 +389,13 @@ \expandafter\newif\csname ifbbl@single\endcsname \chardef\bbl@bidimode\z@ \fi -\ifx\bbl@trace\@undefined - \let\LdfInit\endinput - \def\ProvidesLanguage#1{\endinput} -\endinput\fi % Same line! \ifx\language\@undefined \csname newcount\endcsname\language \fi -\countdef\last@language=19 % TODO. why? remove? +\countdef\last@language=19 \def\addlanguage{\csname newlanguage\endcsname} -\def\bbl@version{3.61} -\def\bbl@date{2021/06/28} +\def\bbl@version{3.62} +\def\bbl@date{2021/07/19} \def\adddialect#1#2{% \global\chardef#1#2\relax \bbl@usehooks{adddialect}{{#1}{#2}}% @@ -522,7 +518,15 @@ \def\bbl@language@stack{} \def\bbl@push@language{% \ifx\languagename\@undefined\else - \xdef\bbl@language@stack{\languagename+\bbl@language@stack}% + \ifx\currentgrouplevel\@undefined + \xdef\bbl@language@stack{\languagename+\bbl@language@stack}% + \else + \ifnum\currentgrouplevel=\z@ + \xdef\bbl@language@stack{\languagename+}% + \else + \xdef\bbl@language@stack{\languagename+\bbl@language@stack}% + \fi + \fi \fi} \def\bbl@pop@lang#1+#2\@@{% \edef\languagename{#1}% @@ -631,10 +635,10 @@ \let\bbl@select@type\z@ \expandafter\bbl@switch\expandafter{\languagename}% \fi}} -\def\babel@aux#1#2{% TODO. See how to avoid undefined nil's +\def\babel@aux#1#2{% \select@language{#1}% - \bbl@foreach\BabelContentsFiles{% - \@writefile{##1}{\babel@toc{#1}{#2}}}}% %% TODO - ok in plain? + \bbl@foreach\BabelContentsFiles{% \relax -> don't assume vertical mode + \@writefile{##1}{\babel@toc{#1}{#2}\relax}}}% TODO - plain? \def\babel@toc#1#2{% \select@language{#1}} \newif\ifbbl@usedategroup @@ -1094,6 +1098,11 @@ \else \expandafter\@firstofone \fi} +\ifx\IfFormatAtLeastTF\@undefined + \def\bbl@ifformatlater{\@ifl@t@r\fmtversion} +\else + \let\bbl@ifformatlater\IfFormatAtLeastTF +\fi \def\bbl@extras@wrap#1#2#3{% 1:in-test, 2:before, 3:after \toks@\expandafter\expandafter\expandafter{% \csname extras\languagename\endcsname}% @@ -1171,10 +1180,12 @@ \newcommand\EnableBabelHook[1]{\bbl@csarg\let{hk@#1}\@firstofone} \newcommand\DisableBabelHook[1]{\bbl@csarg\let{hk@#1}\@gobble} \def\bbl@usehooks#1#2{% + \ifx\UseHook\@undefined\else\UseHook{babel/#1}\fi \def\bbl@elth##1{% \bbl@cs{hk@##1}{\bbl@cs{ev@##1@#1@}#2}}% \bbl@cs{ev@#1@}% \ifx\languagename\@undefined\else % Test required for Plain (?) + \ifx\UseHook\@undefined\else\UseHook{babel/#1/\languagename}\fi \def\bbl@elth##1{% \bbl@cs{hk@##1}{\bbl@cl{ev@##1@#1}#2}}% \bbl@cl{ev@#1}% @@ -1185,6 +1196,10 @@ beforeextras=0,afterextras=0,stopcommands=0,stringprocess=0,% hyphenation=2,initiateactive=3,afterreset=0,foreign=0,foreign*=0,% beforestart=0,languagename=2} +\ifx\NewHook\@undefined\else + \def\bbl@tempa#1=#2\@@{\NewHook{babel/#1}} + \bbl@foreach\bbl@evargs{\bbl@tempa#1\@@} +\fi \bbl@trace{Defining babelensure} \newcommand\babelensure[2][]{% TODO - revise test files \AddBabelHook{babel-ensure}{afterextras}{% @@ -1287,9 +1302,7 @@ \let\BabelModifiers\relax \let\bbl@screset\relax}% \def\ldf@finish#1{% - \ifx\loadlocalcfg\@undefined\else % For LaTeX 209 - \loadlocalcfg{#1}% - \fi + \loadlocalcfg{#1}% \bbl@afterldf{#1}% \expandafter\main@language\expandafter{#1}% \catcode`\@=\atcatcode \let\atcatcode\relax @@ -1303,10 +1316,12 @@ \bbl@id@assign \bbl@patterns{\languagename}} \def\bbl@beforestart{% + \def\@nolanerr##1{% + \bbl@warning{Undefined language '##1' in aux.\\Reported}}% \bbl@usehooks{beforestart}{}% \global\let\bbl@beforestart\relax} \AtBeginDocument{% - \@nameuse{bbl@beforestart}% + {\@nameuse{bbl@beforestart}}% Group! \if@filesw \providecommand\babel@aux[2]{}% \immediate\write\@mainaux{% @@ -2459,7 +2474,7 @@ \ifx\bbl@KVP@transforms\@nil\else \bbl@replace\bbl@KVP@transforms{ }{,}% \fi - % Load ini + % == Load ini == \ifcase\bbl@howloaded \bbl@provide@new{#2}% \else @@ -2542,7 +2557,7 @@ end}% \ifx\bbl@mapselect\@undefined % TODO. almost the same as mapfont \AtBeginDocument{% - \expandafter\bbl@add\csname selectfont \endcsname{{\bbl@mapselect}}% + \bbl@patchfont{{\bbl@mapselect}}% {\selectfont}}% \def\bbl@mapselect{% \let\bbl@mapselect\relax @@ -2568,9 +2583,9 @@ {See the manual for details.}}}% \bbl@ifunset{bbl@lsys@\languagename}{\bbl@provide@lsys{\languagename}}{}% \bbl@ifunset{bbl@wdir@\languagename}{\bbl@provide@dirs{\languagename}}{}% - \ifx\bbl@mapselect\@undefined % TODO. See onchar + \ifx\bbl@mapselect\@undefined % TODO. See onchar. \AtBeginDocument{% - \expandafter\bbl@add\csname selectfont \endcsname{{\bbl@mapselect}}% + \bbl@patchfont{{\bbl@mapselect}}% {\selectfont}}% \def\bbl@mapselect{% \let\bbl@mapselect\relax @@ -2591,6 +2606,24 @@ \bbl@csarg\edef{intsp@#2}{\bbl@KVP@intraspace}% \fi \bbl@provide@intraspace + % == Line breaking: CJK quotes == + \ifcase\bbl@engine\or + \bbl@xin@{/c}{/\bbl@cl{lnbrk}}% + \ifin@ + \bbl@ifunset{bbl@quote@\languagename}{}% + {\directlua{ + Babel.locale_props[\the\localeid].cjk_quotes = {} + local cs = 'op' + for c in string.utfvalues(% + [[\csname bbl@quote@\languagename\endcsname]]) do + if Babel.cjk_characters[c].c == 'qu' then + Babel.locale_props[\the\localeid].cjk_quotes[c] = cs + end + cs = ( cs == 'op') and 'cl' or 'op' + end + }}% + \fi + \fi % == Line breaking: justification == \ifx\bbl@KVP@justification\@nil\else \let\bbl@KVP@linebreaking\bbl@KVP@justification @@ -2663,7 +2696,7 @@ table.pack(string.utfvalue('\bbl@cl{dgnat}')) if not Babel.numbers then function Babel.numbers(head) - local LOCALE = luatexbase.registernumber'bbl@attr@locale' + local LOCALE = Babel.attr_locale local GLYPH = node.id'glyph' local inmath = false for item in node.traverse(head) do @@ -2782,12 +2815,12 @@ \StartBabelCommands*{#1}{captions}% \bbl@read@ini{\bbl@KVP@captions}2% % Here all letters cat = 11 \EndBabelCommands - \fi - \ifx\bbl@KVP@import\@nil\else - \StartBabelCommands*{#1}{date}% - \bbl@savetoday - \bbl@savedate - \EndBabelCommands + \fi + \ifx\bbl@KVP@import\@nil\else + \StartBabelCommands*{#1}{date}% + \bbl@savetoday + \bbl@savedate + \EndBabelCommands \fi % == hyphenrules (also in new) == \ifx\bbl@lbkflag\@empty @@ -3046,6 +3079,7 @@ \bbl@exportkey{intsp}{typography.intraspace}{}% \bbl@exportkey{frspc}{typography.frenchspacing}{u}% \bbl@exportkey{chrng}{characters.ranges}{}% + \bbl@exportkey{quote}{characters.delimiters.quotes}{}% \bbl@exportkey{dgnat}{numbers.digits.native}{}% \ifnum#1=\tw@ % only (re)new \bbl@exportkey{rqtex}{identification.require.babel}{}% @@ -3304,7 +3338,7 @@ range 0-9999.}% {There is little you can do. Sorry.}% \fi\fi\fi\fi}} -\newcommand\BabelDateyyyy[1]{{\number#1}} % FIXME - add leading 0 +\newcommand\BabelDateyyyy[1]{{\number#1}} % TODO - add leading 0 \def\bbl@replace@finish@iii#1{% \bbl@exp{\def\\#1####1####2####3{\the\toks@}}} \def\bbl@TG@@date{% @@ -3372,10 +3406,8 @@ \expandafter\@secondoftwo % to execute right now \fi \AtBeginDocument{% - \expandafter\bbl@add - \csname selectfont \endcsname{\bbl@xenohyph}% - \expandafter\selectlanguage\expandafter{\languagename}% - \expandafter\bbl@toglobal\csname selectfont \endcsname}% + \bbl@patchfont{\bbl@xenohyph}% + \expandafter\selectlanguage\expandafter{\languagename}}% \fi}}% \fi \bbl@csarg\bbl@toglobal{lsys@#1}} diff --git a/Master/texmf-dist/tex/generic/babel/babel.sty b/Master/texmf-dist/tex/generic/babel/babel.sty index bdf21cb3358..4378752b16e 100644 --- a/Master/texmf-dist/tex/generic/babel/babel.sty +++ b/Master/texmf-dist/tex/generic/babel/babel.sty @@ -34,13 +34,14 @@ %% \NeedsTeXFormat{LaTeX2e}[2005/12/01] -\ProvidesPackage{babel}[2021/06/28 3.61 The Babel package] +\ProvidesPackage{babel}[2021/07/19 3.62 The Babel package] \@ifpackagewith{babel}{debug} {\providecommand\bbl@trace[1]{\message{^^J[ #1 ]}}% \let\bbl@debug\@firstofone \ifx\directlua\@undefined\else \directlua{ Babel = Babel or {} Babel.debug = true }% + \input{babel-debug.tex}% \fi} {\providecommand\bbl@trace[1]{}% \let\bbl@debug\@gobble @@ -217,6 +218,11 @@ \else \expandafter\@firstofone \fi} +\ifx\IfFormatAtLeastTF\@undefined + \def\bbl@ifformatlater{\@ifl@t@r\fmtversion} +\else + \let\bbl@ifformatlater\IfFormatAtLeastTF +\fi \def\bbl@extras@wrap#1#2#3{% 1:in-test, 2:before, 3:after \toks@\expandafter\expandafter\expandafter{% \csname extras\languagename\endcsname}% @@ -419,9 +425,6 @@ \ProcessOptions* \ifx\bbl@opt@provide\@nnil\else % Tests. Ignore. \chardef\bbl@iniflag\@ne - \bbl@replace\bbl@opt@provide{;}{,} - \bbl@add\bbl@opt@provide{,import} - \show\bbl@opt@provide \fi \bbl@trace{Conditional loading of shorthands} \def\bbl@sh@string#1{% @@ -665,40 +668,37 @@ } \@onlypreamble\substitutefontfamily \bbl@trace{Encoding and fonts} -\newcommand\BabelNonASCII{LGR,X2,OT2,OT3,OT6,LHE,LWN,LMA,LMC,LMS,LMU,PU,PD1} +\newcommand\BabelNonASCII{LGR,X2,OT2,OT3,OT6,LHE,LWN,LMA,LMC,LMS,LMU} \newcommand\BabelNonText{TS1,T3,TS3} \let\org@TeX\TeX \let\org@LaTeX\LaTeX \let\ensureascii\@firstofone \AtBeginDocument{% - \in@false - \bbl@foreach\BabelNonASCII{% is there a text non-ascii enc? - \ifin@\else - \lowercase{\bbl@xin@{,#1enc.def,}{,\@filelist,}}% + \def\@elt#1{,#1,}% + \edef\bbl@tempa{\expandafter\@gobbletwo\@fontenc@load@list}% + \let\@elt\relax + \let\bbl@tempb\@empty + \def\bbl@tempc{OT1}% + \bbl@foreach\BabelNonASCII{% LGR loaded in a non-standard way + \bbl@ifunset{T@#1}{}{\def\bbl@tempb{#1}}}% + \bbl@foreach\bbl@tempa{% + \bbl@xin@{#1}{\BabelNonASCII}% + \ifin@ + \def\bbl@tempb{#1}% Store last non-ascii + \else\bbl@xin@{#1}{\BabelNonText}% Pass + \ifin@\else + \def\bbl@tempc{#1}% Store last ascii + \fi \fi}% - \ifin@ % if a text non-ascii has been loaded - \def\ensureascii#1{{\fontencoding{OT1}\selectfont#1}}% - \DeclareTextCommandDefault{\TeX}{\org@TeX}% - \DeclareTextCommandDefault{\LaTeX}{\org@LaTeX}% - \def\bbl@tempb#1\@@{\uppercase{\bbl@tempc#1}ENC.DEF\@empty\@@}% - \def\bbl@tempc#1ENC.DEF#2\@@{% - \ifx\@empty#2\else - \bbl@ifunset{T@#1}% - {}% - {\bbl@xin@{,#1,}{,\BabelNonASCII,\BabelNonText,}% - \ifin@ - \DeclareTextCommand{\TeX}{#1}{\ensureascii{\org@TeX}}% - \DeclareTextCommand{\LaTeX}{#1}{\ensureascii{\org@LaTeX}}% - \else - \def\ensureascii##1{{\fontencoding{#1}\selectfont##1}}% - \fi}% - \fi}% - \bbl@foreach\@filelist{\bbl@tempb#1\@@}% TODO - \@@ de mas?? + \ifx\bbl@tempb\@empty\else \bbl@xin@{,\cf@encoding,}{,\BabelNonASCII,\BabelNonText,}% \ifin@\else - \edef\ensureascii#1{{% - \noexpand\fontencoding{\cf@encoding}\noexpand\selectfont#1}}% + \edef\bbl@tempc{\cf@encoding}% The default if ascii wins \fi + \edef\ensureascii#1{% + {\noexpand\fontencoding{\bbl@tempc}\noexpand\selectfont#1}}% + \DeclareTextCommandDefault{\TeX}{\ensureascii{\org@TeX}}% + \DeclareTextCommandDefault{\LaTeX}{\ensureascii{\org@LaTeX}}% \fi} \AtEndOfPackage{\edef\latinencoding{\cf@encoding}} \AtBeginDocument{% @@ -713,16 +713,12 @@ \ifx\cf@encoding\bbl@t@one \xdef\latinencoding{\bbl@t@one}% \else - \ifx\@fontenc@load@list\@undefined - \@ifl@aded{def}{t1enc}{\xdef\latinencoding{\bbl@t@one}}{}% - \else - \def\@elt#1{,#1,}% - \edef\bbl@tempa{\expandafter\@gobbletwo\@fontenc@load@list}% - \let\@elt\relax - \bbl@xin@{,T1,}\bbl@tempa - \ifin@ - \xdef\latinencoding{\bbl@t@one}% - \fi + \def\@elt#1{,#1,}% + \edef\bbl@tempa{\expandafter\@gobbletwo\@fontenc@load@list}% + \let\@elt\relax + \bbl@xin@{,T1,}\bbl@tempa + \ifin@ + \xdef\latinencoding{\bbl@t@one}% \fi \fi}} \DeclareRobustCommand{\latintext}{% @@ -733,66 +729,14 @@ \else \DeclareTextFontCommand{\textlatin}{\latintext} \fi -\ifodd\bbl@engine - \def\bbl@activate@preotf{% - \let\bbl@activate@preotf\relax % only once - \directlua{ - Babel = Babel or {} - % - function Babel.pre_otfload_v(head) - if Babel.numbers and Babel.digits_mapped then - head = Babel.numbers(head) - end - if Babel.bidi_enabled then - head = Babel.bidi(head, false, dir) - end - return head - end - % - function Babel.pre_otfload_h(head, gc, sz, pt, dir) - if Babel.numbers and Babel.digits_mapped then - head = Babel.numbers(head) - end - if Babel.bidi_enabled then - head = Babel.bidi(head, false, dir) - end - return head - end - % - luatexbase.add_to_callback('pre_linebreak_filter', - Babel.pre_otfload_v, - 'Babel.pre_otfload_v', - luatexbase.priority_in_callback('pre_linebreak_filter', - 'luaotfload.node_processor') or nil) - % - luatexbase.add_to_callback('hpack_filter', - Babel.pre_otfload_h, - 'Babel.pre_otfload_h', - luatexbase.priority_in_callback('hpack_filter', - 'luaotfload.node_processor') or nil) - }} -\fi +\bbl@ifformatlater{2021-06-01}% + {\def\bbl@patchfont#1{\AddToHook{selectfont}{#1}}} + {\def\bbl@patchfont#1{% + \expandafter\bbl@add\csname selectfont \endcsname{#1}% + \expandafter\bbl@toglobal\csname selectfont \endcsname}} \bbl@trace{Loading basic (internal) bidi support} \ifodd\bbl@engine - \ifnum\bbl@bidimode>100 \ifnum\bbl@bidimode<200 - \let\bbl@beforeforeign\leavevmode - \AtEndOfPackage{\EnableBabelHook{babel-bidi}} - \RequirePackage{luatexbase} - \bbl@activate@preotf - \directlua{ - require('babel-data-bidi.lua') - \ifcase\expandafter\@gobbletwo\the\bbl@bidimode\or - require('babel-bidi-basic.lua') - \or - require('babel-bidi-basic-r.lua') - \fi} - % TODO - to locale_props, not as separate attribute - \newattribute\bbl@attr@dir - % TODO. I don't like it, hackish: - \bbl@exp{\output{\bodydir\pagedir\the\output}} - \AtEndOfPackage{\EnableBabelHook{babel-bidi}} - \fi\fi -\else +\else % TODO. Move to txtbabel \ifnum\bbl@bidimode>100 \ifnum\bbl@bidimode<200 \bbl@error {The bidi method 'basic' is available only in\\% @@ -828,7 +772,8 @@ \let\bbl@beforeforeign\leavevmode \ifodd\bbl@engine \newattribute\bbl@attr@dir - \bbl@exp{\output{\bodydir\pagedir\the\output}}% + \directlua{ Babel.attr_dir = luatexbase.registernumber'bbl@attr@dir' } + \bbl@exp{\output{\bodydir\pagedir\the\output}} \fi \AtEndOfPackage{% \EnableBabelHook{babel-bidi}% @@ -878,51 +823,6 @@ \AddBabelHook{babel-bidi}{afterextras}{\bbl@switchdir} \DisableBabelHook{babel-bidi} \ifodd\bbl@engine % luatex=1 - \chardef\bbl@thetextdir\z@ - \chardef\bbl@thepardir\z@ - \def\bbl@getluadir#1{% - \directlua{ - if tex.#1dir == 'TLT' then - tex.sprint('0') - elseif tex.#1dir == 'TRT' then - tex.sprint('1') - end}} - \def\bbl@setluadir#1#2#3{% 1=text/par.. 2=\textdir.. 3=0 lr/1 rl - \ifcase#3\relax - \ifcase\bbl@getluadir{#1}\relax\else - #2 TLT\relax - \fi - \else - \ifcase\bbl@getluadir{#1}\relax - #2 TRT\relax - \fi - \fi} - \def\bbl@textdir#1{% - \bbl@setluadir{text}\textdir{#1}% - \chardef\bbl@thetextdir#1\relax - \setattribute\bbl@attr@dir{\numexpr\bbl@thepardir*3+#1}} - \def\bbl@pardir#1{% - \bbl@setluadir{par}\pardir{#1}% - \chardef\bbl@thepardir#1\relax} - \def\bbl@bodydir{\bbl@setluadir{body}\bodydir} - \def\bbl@pagedir{\bbl@setluadir{page}\pagedir} - \def\bbl@dirparastext{\pardir\the\textdir\relax}% %%%% - % Sadly, we have to deal with boxes in math with basic. - % Activated every math with the package option bidi=: - \ifnum\bbl@bidimode>\z@ - \def\bbl@mathboxdir{% - \ifcase\bbl@thetextdir\relax - \everyhbox{\bbl@mathboxdir@aux L}% - \else - \everyhbox{\bbl@mathboxdir@aux R}% - \fi} - \def\bbl@mathboxdir@aux#1{% - \@ifnextchar\egroup{}{\textdir T#1T\relax}} - \frozen@everymath\expandafter{% - \expandafter\bbl@mathboxdir\the\frozen@everymath} - \frozen@everydisplay\expandafter{% - \expandafter\bbl@mathboxdir\the\frozen@everydisplay} - \fi \else % pdftex=0, xetex=2 \newcount\bbl@dirlevel \chardef\bbl@thetextdir\z@ @@ -1158,9 +1058,16 @@ \bbl@ldfinit \let\CurrentOption\bbl@opt@main \ifx\bbl@opt@provide\@nnil - \bbl@exp{\\\babelprovide[import,main]{\bbl@opt@main}} + \bbl@exp{\\\babelprovide[import,main]{\bbl@opt@main}}% \else - \bbl@exp{\\\babelprovide[\bbl@opt@provide,main]{\bbl@opt@main}}% + \bbl@exp{\\\bbl@forkv{\@nameuse{@raw@opt@babel.sty}}}{% + \bbl@xin@{,provide,}{,#1,}% + \ifin@ + \def\bbl@opt@provide{#2}% + \bbl@replace\bbl@opt@provide{;}{,}% + \fi}% + \bbl@exp{% + \\\babelprovide[\bbl@opt@provide,import,main]{\bbl@opt@main}}% \fi \bbl@afterldf{}% \else % case 0,2 diff --git a/Master/texmf-dist/tex/generic/babel/hyphen.cfg b/Master/texmf-dist/tex/generic/babel/hyphen.cfg index 96965b6a086..699456b7718 100644 --- a/Master/texmf-dist/tex/generic/babel/hyphen.cfg +++ b/Master/texmf-dist/tex/generic/babel/hyphen.cfg @@ -38,26 +38,17 @@ \wlog{File: #1 #4 #3 <#2>}% \let\ProvidesFile\@undefined} \fi -\ProvidesFile{hyphen.cfg}[2021/06/28 3.61 Babel hyphens] +\ProvidesFile{hyphen.cfg}[2021/07/19 3.62 Babel hyphens] \xdef\bbl@format{\jobname} -\def\bbl@version{3.61} -\def\bbl@date{2021/06/28} +\def\bbl@version{3.62} +\def\bbl@date{2021/07/19} \ifx\AtBeginDocument\@undefined \def\@empty{} - \let\orig@dump\dump - \def\dump{% - \ifx\@ztryfc\@undefined - \else - \toks0=\expandafter{\@preamblecmds}% - \edef\@preamblecmds{\noexpand\@begindocumenthook\the\toks0}% - \def\@begindocumenthook{}% - \fi - \let\dump\orig@dump\let\orig@dump\@undefined\dump} \fi \ifx\language\@undefined \csname newcount\endcsname\language \fi -\countdef\last@language=19 % TODO. why? remove? +\countdef\last@language=19 \def\addlanguage{\csname newlanguage\endcsname} \def\process@line#1#2 #3 #4 {% \ifx=#1% diff --git a/Master/texmf-dist/tex/generic/babel/luababel.def b/Master/texmf-dist/tex/generic/babel/luababel.def index ca9aef3ac7e..348289cd67c 100644 --- a/Master/texmf-dist/tex/generic/babel/luababel.def +++ b/Master/texmf-dist/tex/generic/babel/luababel.def @@ -252,7 +252,7 @@ \endgroup \ifx\newattribute\@undefined\else \newattribute\bbl@attr@locale - \directlua{ Babel.attr_locale = luatexbase.registernumber'bbl@attr@locale'} + \directlua{ Babel.attr_locale = luatexbase.registernumber'bbl@attr@locale' } \AddBabelHook{luatex}{beforeextras}{% \setattribute\bbl@attr@locale\localeid} \fi @@ -433,11 +433,15 @@ local lang = item.lang local LOCALE = node.get_attribute(item, - luatexbase.registernumber'bbl@attr@locale') + Babel.attr_locale) local props = Babel.locale_props[LOCALE] local class = Babel.cjk_class[item.char].c + if props.cjk_quotes and props.cjk_quotes[item.char] then + class = props.cjk_quotes[item.char] + end + if class == 'cp' then class = 'cl' end % )] as CL if class == 'id' then class = 'I' end @@ -551,8 +555,9 @@ \gdef\bbl@arabicjust{% \let\bbl@arabicjust\relax \newattribute\bblar@kashida + \directlua{ Babel.attr_kashida = luatexbase.registernumber'bblar@kashida' }% \bblar@kashida=\z@ - \expandafter\bbl@add\csname selectfont \endcsname{{\bbl@parsejalt}}% + \bbl@patchfont{{\bbl@parsejalt}}% \directlua{ Babel.arabic.elong_map = Babel.arabic.elong_map or {} Babel.arabic.elong_map[\the\localeid] = {} @@ -651,8 +656,8 @@ function Babel.arabic.justify_hlist(head, line, gc, size, pack) local elong_map = Babel.arabic.elong_map local last_line local GLYPH = node.id'glyph' - local KASHIDA = luatexbase.registernumber'bblar@kashida' - local LOCALE = luatexbase.registernumber'bbl@attr@locale' + local KASHIDA = Babel.attr_kashida + local LOCALE = Babel.attr_locale if line == nil then line = {} @@ -770,7 +775,7 @@ end \ExplSyntaxOn \catcode`\ =10 \def\bbl@loadfontspec{% - \usepackage{fontspec}% + \usepackage{fontspec}% TODO. Apply patch always \expandafter \def\csname msg~text~>~fontspec/language-not-exist\endcsname##1##2##3##4{% Font '\l_fontspec_fontname_tl' is using the\\% @@ -804,9 +809,7 @@ end \newcommand\bbl@bblfont[2][]{% 1=features 2=fontname, @font=rm|sf|tt \bbl@ifunset{\bbl@tempb family}% {\bbl@providefam{\bbl@tempb}}% - {\bbl@exp{% - \\\bbl@sreplace\<\bbl@tempb family >% - {\@nameuse{\bbl@tempb default}}{\<\bbl@tempb default>}}}% + {}% % For the default font, just in case: \bbl@ifunset{bbl@lsys@\languagename}{\bbl@provide@lsys{\languagename}}{}% \expandafter\bbl@ifblank\expandafter{\bbl@tempa}% @@ -823,7 +826,10 @@ end \\\bbl@add@list\\\bbl@font@fams{#1}% \\\DeclareRobustCommand\<#1family>{% \\\not@math@alphabet\<#1family>\relax - \\\fontfamily\<#1default>\\\selectfont}% + % \\\prepare@family@series@update{#1}\<#1default>% TODO. Fails + \\\fontfamily\<#1default>% + \\\\UseHooks\\\@undefined\\\\UseHook{#1family}\% + \\\selectfont}% \\\DeclareTextFontCommand{\}{\<#1family>}}} \def\bbl@nostdfont#1{% \bbl@ifunset{bbl@WFF@\f@family}% @@ -1029,7 +1035,7 @@ Babel.script_blocks.Kana = Babel.script_blocks.Jpan function Babel.locale_map(head) if not Babel.locale_mapped then return head end - local LOCALE = luatexbase.registernumber'bbl@attr@locale' + local LOCALE = Babel.attr_locale local GLYPH = node.id('glyph') local inmath = false local toloc_save @@ -1125,515 +1131,13 @@ end Babel.chr_to_loc[\the\count@] = \bbl@ifblank{#1}{-1000}{\the\bbl@cs{id@@#1}}\space }} -\begingroup % TODO - to a lua file +\directlua{ + Babel.nohyphenation = \the\l@nohyphenation +} +\begingroup \catcode`\~=12 -\catcode`\#=12 \catcode`\%=12 \catcode`\&=14 -\directlua{ - Babel.linebreaking.replacements = {} - Babel.linebreaking.replacements[0] = {} &% pre - Babel.linebreaking.replacements[1] = {} &% post - - &% Discretionaries contain strings as nodes - function Babel.str_to_nodes(fn, matches, base) - local n, head, last - if fn == nil then return nil end - for s in string.utfvalues(fn(matches)) do - if base.id == 7 then - base = base.replace - end - n = node.copy(base) - n.char = s - if not head then - head = n - else - last.next = n - end - last = n - end - return head - end - - Babel.fetch_subtext = {} - - Babel.ignore_pre_char = function(node) - return (node.lang == \the\l@nohyphenation) - end - - &% Merging both functions doesn't seen feasible, because there are too - &% many differences. - Babel.fetch_subtext[0] = function(head) - local word_string = '' - local word_nodes = {} - local lang - local item = head - local inmath = false - - while item do - - if item.id == 11 then - inmath = (item.subtype == 0) - end - - if inmath then - &% pass - - elseif item.id == 29 then - local locale = node.get_attribute(item, Babel.attr_locale) - - if lang == locale or lang == nil then - lang = lang or locale - if Babel.ignore_pre_char(item) then - word_string = word_string .. Babel.us_char - else - word_string = word_string .. unicode.utf8.char(item.char) - end - word_nodes[#word_nodes+1] = item - else - break - end - - elseif item.id == 12 and item.subtype == 13 then - word_string = word_string .. ' ' - word_nodes[#word_nodes+1] = item - - &% Ignore leading unrecognized nodes, too. - elseif word_string ~= '' then - word_string = word_string .. Babel.us_char - word_nodes[#word_nodes+1] = item &% Will be ignored - end - - item = item.next - end - - &% Here and above we remove some trailing chars but not the - &% corresponding nodes. But they aren't accessed. - if word_string:sub(-1) == ' ' then - word_string = word_string:sub(1,-2) - end - word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '') - return word_string, word_nodes, item, lang - end - - Babel.fetch_subtext[1] = function(head) - local word_string = '' - local word_nodes = {} - local lang - local item = head - local inmath = false - - while item do - - if item.id == 11 then - inmath = (item.subtype == 0) - end - - if inmath then - &% pass - - elseif item.id == 29 then - if item.lang == lang or lang == nil then - if (item.char ~= 124) and (item.char ~= 61) then &% not =, not | - lang = lang or item.lang - word_string = word_string .. unicode.utf8.char(item.char) - word_nodes[#word_nodes+1] = item - end - else - break - end - - elseif item.id == 7 and item.subtype == 2 then - word_string = word_string .. '=' - word_nodes[#word_nodes+1] = item - - elseif item.id == 7 and item.subtype == 3 then - word_string = word_string .. '|' - word_nodes[#word_nodes+1] = item - - &% (1) Go to next word if nothing was found, and (2) implictly - &% remove leading USs. - elseif word_string == '' then - &% pass - - &% This is the responsible for splitting by words. - elseif (item.id == 12 and item.subtype == 13) then - break - - else - word_string = word_string .. Babel.us_char - word_nodes[#word_nodes+1] = item &% Will be ignored - end - - item = item.next - end - - word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '') - return word_string, word_nodes, item, lang - end - - function Babel.pre_hyphenate_replace(head) - Babel.hyphenate_replace(head, 0) - end - - function Babel.post_hyphenate_replace(head) - Babel.hyphenate_replace(head, 1) - end - - function Babel.debug_hyph(w, wn, sc, first, last, last_match) - local ss = '' - for pp = 1, 40 do - if wn[pp] then - if wn[pp].id == 29 then - ss = ss .. unicode.utf8.char(wn[pp].char) - else - ss = ss .. '{' .. wn[pp].id .. '}' - end - end - end - print('nod', ss) - print('lst_m', - string.rep(' ', unicode.utf8.len( - string.sub(w, 1, last_match))-1) .. '>') - print('str', w) - print('sc', string.rep(' ', sc-1) .. '^') - if first == last then - print('f=l', string.rep(' ', first-1) .. '!') - else - print('f/l', string.rep(' ', first-1) .. '[' .. - string.rep(' ', last-first-1) .. ']') - end - end - - Babel.us_char = string.char(31) - - function Babel.hyphenate_replace(head, mode) - local u = unicode.utf8 - local lbkr = Babel.linebreaking.replacements[mode] - - local word_head = head - - while true do &% for each subtext block - - local w, w_nodes, nw, lang = Babel.fetch_subtext[mode](word_head) - - if Babel.debug then - print() - print((mode == 0) and '@@@@<' or '@@@@>', w) - end - - if nw == nil and w == '' then break end - - if not lang then goto next end - if not lbkr[lang] then goto next end - - &% For each saved (pre|post)hyphenation. TODO. Reconsider how - &% loops are nested. - for k=1, #lbkr[lang] do - local p = lbkr[lang][k].pattern - local r = lbkr[lang][k].replace - - if Babel.debug then - print('*****', p, mode) - end - - &% This variable is set in some cases below to the first *byte* - &% after the match, either as found by u.match (faster) or the - &% computed position based on sc if w has changed. - local last_match = 0 - local step = 0 - - &% For every match. - while true do - if Babel.debug then - print('=====') - end - local new &% used when inserting and removing nodes - - local matches = { u.match(w, p, last_match) } - - if #matches < 2 then break end - - &% Get and remove empty captures (with ()'s, which return a - &% number with the position), and keep actual captures - &% (from (...)), if any, in matches. - local first = table.remove(matches, 1) - local last = table.remove(matches, #matches) - &% Non re-fetched substrings may contain \31, which separates - &% subsubstrings. - if string.find(w:sub(first, last-1), Babel.us_char) then break end - - local save_last = last &% with A()BC()D, points to D - - &% Fix offsets, from bytes to unicode. Explained above. - first = u.len(w:sub(1, first-1)) + 1 - last = u.len(w:sub(1, last-1)) &% now last points to C - - &% This loop stores in n small table the nodes - &% corresponding to the pattern. Used by 'data' to provide a - &% predictable behavior with 'insert' (now w_nodes is modified on - &% the fly), and also access to 'remove'd nodes. - local sc = first-1 &% Used below, too - local data_nodes = {} - - for q = 1, last-first+1 do - data_nodes[q] = w_nodes[sc+q] - end - - &% This loop traverses the matched substring and takes the - &% corresponding action stored in the replacement list. - &% sc = the position in substr nodes / string - &% rc = the replacement table index - local rc = 0 - - while rc < last-first+1 do &% for each replacement - if Babel.debug then - print('.....', rc + 1) - end - sc = sc + 1 - rc = rc + 1 - - if Babel.debug then - Babel.debug_hyph(w, w_nodes, sc, first, last, last_match) - local ss = '' - for itt in node.traverse(head) do - if itt.id == 29 then - ss = ss .. unicode.utf8.char(itt.char) - else - ss = ss .. '{' .. itt.id .. '}' - end - end - print('*****************', ss) - - end - - local crep = r[rc] - local item = w_nodes[sc] - local item_base = item - local placeholder = Babel.us_char - local d - - if crep and crep.data then - item_base = data_nodes[crep.data] - end - - if crep then - step = crep.step or 0 - end - - if crep and next(crep) == nil then &% = {} - last_match = save_last &% Optimization - goto next - - elseif crep == nil or crep.remove then - node.remove(head, item) - table.remove(w_nodes, sc) - w = u.sub(w, 1, sc-1) .. u.sub(w, sc+1) - sc = sc - 1 &% Nothing has been inserted. - last_match = utf8.offset(w, sc+1+step) - goto next - - elseif crep and crep.kashida then &% Experimental - node.set_attribute(item, - luatexbase.registernumber'bblar@kashida', - crep.kashida) - last_match = utf8.offset(w, sc+1+step) - goto next - - elseif crep and crep.string then - local str = crep.string(matches) - if str == '' then &% Gather with nil - node.remove(head, item) - table.remove(w_nodes, sc) - w = u.sub(w, 1, sc-1) .. u.sub(w, sc+1) - sc = sc - 1 &% Nothing has been inserted. - else - local loop_first = true - for s in string.utfvalues(str) do - d = node.copy(item_base) - d.char = s - if loop_first then - loop_first = false - head, new = node.insert_before(head, item, d) - if sc == 1 then - word_head = head - end - w_nodes[sc] = d - w = u.sub(w, 1, sc-1) .. u.char(s) .. u.sub(w, sc+1) - else - sc = sc + 1 - head, new = node.insert_before(head, item, d) - table.insert(w_nodes, sc, new) - w = u.sub(w, 1, sc-1) .. u.char(s) .. u.sub(w, sc) - end - if Babel.debug then - print('.....', 'str') - Babel.debug_hyph(w, w_nodes, sc, first, last, last_match) - end - end &% for - node.remove(head, item) - end &% if '' - last_match = utf8.offset(w, sc+1+step) - goto next - - elseif mode == 1 and crep and (crep.pre or crep.no or crep.post) then - d = node.new(7, 0) &% (disc, discretionary) - d.pre = Babel.str_to_nodes(crep.pre, matches, item_base) - d.post = Babel.str_to_nodes(crep.post, matches, item_base) - d.replace = Babel.str_to_nodes(crep.no, matches, item_base) - d.attr = item_base.attr - if crep.pre == nil then &% TeXbook p96 - d.penalty = crep.penalty or tex.hyphenpenalty - else - d.penalty = crep.penalty or tex.exhyphenpenalty - end - placeholder = '|' - head, new = node.insert_before(head, item, d) - - elseif mode == 0 and crep and (crep.pre or crep.no or crep.post) then - &% ERROR - - elseif crep and crep.penalty then - d = node.new(14, 0) &% (penalty, userpenalty) - d.attr = item_base.attr - d.penalty = crep.penalty - head, new = node.insert_before(head, item, d) - - elseif crep and crep.space then - &% 655360 = 10 pt = 10 * 65536 sp - d = node.new(12, 13) &% (glue, spaceskip) - local quad = font.getfont(item_base.font).size or 655360 - node.setglue(d, crep.space[1] * quad, - crep.space[2] * quad, - crep.space[3] * quad) - if mode == 0 then - placeholder = ' ' - end - head, new = node.insert_before(head, item, d) - - elseif crep and crep.spacefactor then - d = node.new(12, 13) &% (glue, spaceskip) - local base_font = font.getfont(item_base.font) - node.setglue(d, - crep.spacefactor[1] * base_font.parameters['space'], - crep.spacefactor[2] * base_font.parameters['space_stretch'], - crep.spacefactor[3] * base_font.parameters['space_shrink']) - if mode == 0 then - placeholder = ' ' - end - head, new = node.insert_before(head, item, d) - - elseif mode == 0 and crep and crep.space then - &% ERROR - - end &% ie replacement cases - - &% Shared by disc, space and penalty. - if sc == 1 then - word_head = head - end - if crep.insert then - w = u.sub(w, 1, sc-1) .. placeholder .. u.sub(w, sc) - table.insert(w_nodes, sc, new) - last = last + 1 - else - w_nodes[sc] = d - node.remove(head, item) - w = u.sub(w, 1, sc-1) .. placeholder .. u.sub(w, sc+1) - end - - last_match = utf8.offset(w, sc+1+step) - - ::next:: - - end &% for each replacement - - if Babel.debug then - print('.....', '/') - Babel.debug_hyph(w, w_nodes, sc, first, last, last_match) - end - - end &% for match - - end &% for patterns - - ::next:: - word_head = nw - end &% for substring - return head - end - - &% This table stores capture maps, numbered consecutively - Babel.capture_maps = {} - - &% The following functions belong to the next macro - function Babel.capture_func(key, cap) - local ret = "[[" .. cap:gsub('{([0-9])}', "]]..m[%1]..[[") .. "]]" - local cnt - local u = unicode.utf8 - ret, cnt = ret:gsub('{([0-9])|([^|]+)|(.-)}', Babel.capture_func_map) - if cnt == 0 then - ret = u.gsub(ret, '{(%x%x%x%x+)}', - function (n) - return u.char(tonumber(n, 16)) - end) - end - ret = ret:gsub("%[%[%]%]%.%.", '') - ret = ret:gsub("%.%.%[%[%]%]", '') - return key .. [[=function(m) return ]] .. ret .. [[ end]] - end - - function Babel.capt_map(from, mapno) - return Babel.capture_maps[mapno][from] or from - end - - &% Handle the {n|abc|ABC} syntax in captures - function Babel.capture_func_map(capno, from, to) - local u = unicode.utf8 - from = u.gsub(from, '{(%x%x%x%x+)}', - function (n) - return u.char(tonumber(n, 16)) - end) - to = u.gsub(to, '{(%x%x%x%x+)}', - function (n) - return u.char(tonumber(n, 16)) - end) - local froms = {} - for s in string.utfcharacters(from) do - table.insert(froms, s) - end - local cnt = 1 - table.insert(Babel.capture_maps, {}) - local mlen = table.getn(Babel.capture_maps) - for s in string.utfcharacters(to) do - Babel.capture_maps[mlen][froms[cnt]] = s - cnt = cnt + 1 - end - return "]]..Babel.capt_map(m[" .. capno .. "]," .. - (mlen) .. ").." .. "[[" - end - - &% Create/Extend reversed sorted list of kashida weights: - function Babel.capture_kashida(key, wt) - wt = tonumber(wt) - if Babel.kashida_wts then - for p, q in ipairs(Babel.kashida_wts) do - if wt == q then - break - elseif wt > q then - table.insert(Babel.kashida_wts, p, wt) - break - elseif table.getn(Babel.kashida_wts) == p then - table.insert(Babel.kashida_wts, wt) - end - end - else - Babel.kashida_wts = { wt } - end - return 'kashida = ' .. wt - end -} -\catcode`\#=6 \gdef\babelposthyphenation#1#2#3{&% \bbl@activateposthyphen \begingroup @@ -1727,13 +1231,114 @@ end \def\bbl@activateposthyphen{% \let\bbl@activateposthyphen\relax \directlua{ + require('babel-transforms.lua') Babel.linebreaking.add_after(Babel.post_hyphenate_replace) }} \def\bbl@activateprehyphen{% \let\bbl@activateprehyphen\relax \directlua{ + require('babel-transforms.lua') Babel.linebreaking.add_before(Babel.pre_hyphenate_replace) }} +\def\bbl@activate@preotf{% + \let\bbl@activate@preotf\relax % only once + \directlua{ + Babel = Babel or {} + % + function Babel.pre_otfload_v(head) + if Babel.numbers and Babel.digits_mapped then + head = Babel.numbers(head) + end + if Babel.bidi_enabled then + head = Babel.bidi(head, false, dir) + end + return head + end + % + function Babel.pre_otfload_h(head, gc, sz, pt, dir) + if Babel.numbers and Babel.digits_mapped then + head = Babel.numbers(head) + end + if Babel.bidi_enabled then + head = Babel.bidi(head, false, dir) + end + return head + end + % + luatexbase.add_to_callback('pre_linebreak_filter', + Babel.pre_otfload_v, + 'Babel.pre_otfload_v', + luatexbase.priority_in_callback('pre_linebreak_filter', + 'luaotfload.node_processor') or nil) + % + luatexbase.add_to_callback('hpack_filter', + Babel.pre_otfload_h, + 'Babel.pre_otfload_h', + luatexbase.priority_in_callback('hpack_filter', + 'luaotfload.node_processor') or nil) + }} +\ifnum\bbl@bidimode>100 \ifnum\bbl@bidimode<200 + \let\bbl@beforeforeign\leavevmode + \AtEndOfPackage{\EnableBabelHook{babel-bidi}} + \RequirePackage{luatexbase} + \bbl@activate@preotf + \directlua{ + require('babel-data-bidi.lua') + \ifcase\expandafter\@gobbletwo\the\bbl@bidimode\or + require('babel-bidi-basic.lua') + \or + require('babel-bidi-basic-r.lua') + \fi} + % TODO - to locale_props, not as separate attribute + \newattribute\bbl@attr@dir + \directlua{ Babel.attr_dir = luatexbase.registernumber'bbl@attr@dir' } + % TODO. I don't like it, hackish: + \bbl@exp{\output{\bodydir\pagedir\the\output}} + \AtEndOfPackage{\EnableBabelHook{babel-bidi}} +\fi\fi +\chardef\bbl@thetextdir\z@ +\chardef\bbl@thepardir\z@ +\def\bbl@getluadir#1{% + \directlua{ + if tex.#1dir == 'TLT' then + tex.sprint('0') + elseif tex.#1dir == 'TRT' then + tex.sprint('1') + end}} +\def\bbl@setluadir#1#2#3{% 1=text/par.. 2=\textdir.. 3=0 lr/1 rl + \ifcase#3\relax + \ifcase\bbl@getluadir{#1}\relax\else + #2 TLT\relax + \fi + \else + \ifcase\bbl@getluadir{#1}\relax + #2 TRT\relax + \fi + \fi} +\def\bbl@textdir#1{% + \bbl@setluadir{text}\textdir{#1}% + \chardef\bbl@thetextdir#1\relax + \setattribute\bbl@attr@dir{\numexpr\bbl@thepardir*3+#1}} +\def\bbl@pardir#1{% + \bbl@setluadir{par}\pardir{#1}% + \chardef\bbl@thepardir#1\relax} +\def\bbl@bodydir{\bbl@setluadir{body}\bodydir} +\def\bbl@pagedir{\bbl@setluadir{page}\pagedir} +\def\bbl@dirparastext{\pardir\the\textdir\relax}% %%%% +\ifnum\bbl@bidimode>\z@ + \def\bbl@mathboxdir{% + \ifcase\bbl@thetextdir\relax + \everyhbox{\bbl@mathboxdir@aux L}% + \else + \everyhbox{\bbl@mathboxdir@aux R}% + \fi} + \def\bbl@mathboxdir@aux#1{% + \@ifnextchar\egroup{}{\textdir T#1T\relax}} + \frozen@everymath\expandafter{% + \expandafter\bbl@mathboxdir\the\frozen@everymath} + \frozen@everydisplay\expandafter{% + \expandafter\bbl@mathboxdir\the\frozen@everydisplay} +\fi \bbl@trace{Redefinitions for bidi layout} \ifx\@eqnnum\@undefined\else \ifx\bbl@attr@dir\@undefined\else diff --git a/Master/texmf-dist/tex/generic/babel/nil.ldf b/Master/texmf-dist/tex/generic/babel/nil.ldf index cd65a825af3..f68ebb5286b 100644 --- a/Master/texmf-dist/tex/generic/babel/nil.ldf +++ b/Master/texmf-dist/tex/generic/babel/nil.ldf @@ -33,7 +33,7 @@ %% extension |.ins|) which are part of the distribution. %% -\ProvidesLanguage{nil}[2021/06/28 3.61 Nil language] +\ProvidesLanguage{nil}[2021/07/19 3.62 Nil language] \LdfInit{nil}{datenil} \ifx\l@nil\@undefined \newlanguage\l@nil diff --git a/Master/texmf-dist/tex/generic/babel/xebabel.def b/Master/texmf-dist/tex/generic/babel/xebabel.def index 531a79be27f..3c2bebce680 100644 --- a/Master/texmf-dist/tex/generic/babel/xebabel.def +++ b/Master/texmf-dist/tex/generic/babel/xebabel.def @@ -87,10 +87,7 @@ \ifx\AtBeginDocument\@notprerr \expandafter\@secondoftwo % to execute right now \fi - \AtBeginDocument{% - \expandafter\bbl@add - \csname selectfont \endcsname{\bbl@ispacesize}% - \expandafter\bbl@toglobal\csname selectfont \endcsname}% + \AtBeginDocument{\bbl@patchfont{\bbl@xenohyph}}% \fi}% \fi} \ifx\DisableBabelHook\@undefined\endinput\fi @@ -102,7 +99,7 @@ \ExplSyntaxOn \catcode`\ =10 \def\bbl@loadfontspec{% - \usepackage{fontspec}% + \usepackage{fontspec}% TODO. Apply patch always \expandafter \def\csname msg~text~>~fontspec/language-not-exist\endcsname##1##2##3##4{% Font '\l_fontspec_fontname_tl' is using the\\% @@ -136,9 +133,7 @@ \newcommand\bbl@bblfont[2][]{% 1=features 2=fontname, @font=rm|sf|tt \bbl@ifunset{\bbl@tempb family}% {\bbl@providefam{\bbl@tempb}}% - {\bbl@exp{% - \\\bbl@sreplace\<\bbl@tempb family >% - {\@nameuse{\bbl@tempb default}}{\<\bbl@tempb default>}}}% + {}% % For the default font, just in case: \bbl@ifunset{bbl@lsys@\languagename}{\bbl@provide@lsys{\languagename}}{}% \expandafter\bbl@ifblank\expandafter{\bbl@tempa}% @@ -155,7 +150,10 @@ \\\bbl@add@list\\\bbl@font@fams{#1}% \\\DeclareRobustCommand\<#1family>{% \\\not@math@alphabet\<#1family>\relax - \\\fontfamily\<#1default>\\\selectfont}% + % \\\prepare@family@series@update{#1}\<#1default>% TODO. Fails + \\\fontfamily\<#1default>% + \\\\UseHooks\\\@undefined\\\\UseHook{#1family}\% + \\\selectfont}% \\\DeclareTextFontCommand{\}{\<#1family>}}} \def\bbl@nostdfont#1{% \bbl@ifunset{bbl@WFF@\f@family}% -- cgit v1.2.3