------------------------ ----- char-ini.lua ----- ------------------------ -- local template_a = "\\startextendcatcodetable{%s}\\chardef\\l=11\\chardef\\a=13\\let\\c\\catcode%s\\let\\a\\undefined\\let\\l\\undefined\\let\\c\\undefined\\stopextendcatcodetable" -- local template_b = "\\chardef\\l=11\\chardef\\a=13\\let\\c\\catcode%s\\let\\a\\undefined\\let\\l\\undefined\\let\\c\\undefined" -- -- function characters.define(tobelettered, tobeactivated) -- catcodetables -- local lettered, activated, l, a = { }, { }, 0, 0 -- for u, chr in next, data do -- -- we can use a macro instead of direct settings -- local fallback = chr.fallback -- if fallback then -- -- texprint(format("{\\catcode %s=13\\unexpanded\\gdef %s{\\checkedchar{%s}{%s}}}",u,utfchar(u),u,fallback)) -- texsprint("{\\catcode",u,"=13\\unexpanded\\gdef ",utfchar(u),"{\\checkedchar{",u,"}{",fallback,"}}}") -- no texprint -- a = a + 1 -- activated[a] = "\\c"..u.."\\a" -- else -- local contextname = chr.contextname -- local category = chr.category -- if contextname then -- if is_character[category] then -- -- by this time, we're still in normal catcode mode -- -- subtle: not "\\",contextname but "\\"..contextname -- if chr.unicodeslot < 128 then -- texprint(ctxcatcodes,format("\\chardef\\%s=%s",contextname,u)) -- else -- texprint(ctxcatcodes,format("\\let\\%s=%s",contextname,utfchar(u))) -- if is_letter[category] then -- l = l + 1 -- lettered[l] = "\\c"..u.."\\l" -- end -- end -- elseif is_command[category] then -- -- this might change: contextcommand ipv contextname -- -- texprint(format("{\\catcode %s=13\\unexpanded\\gdef %s{\\%s}}",u,utfchar(u),contextname)) -- texsprint("{\\catcode",u,"=13\\unexpanded\\gdef ",utfchar(u),"{\\"..contextname,"}}") -- no texprint -- a = a + 1 -- activated[a] = "\\c"..u.."\\a" -- end -- elseif is_letter[category] then -- if u >= 128 and u <= 65536 then -- catch private mess -- l = l + 1 -- lettered[l] = "\\c"..u.."\\l" -- end -- end -- end -- local range = chr.range -- if range then -- l = l + 1 -- lettered[l] = format('\\dofastrecurse{"%05X}{"%05X}{1}{\\c\\fastrecursecounter\\l}',range.first,range.last) -- end -- end -- -- if false then -- l = l + 1 -- lettered[l] = "\\c"..0x200C.."\\l" -- non-joiner -- l = l + 1 -- lettered[l] = "\\c"..0x200D.."\\l" -- joiner -- -- fi -- if tobelettered then -- lettered = concat(lettered) -- if true then -- texsprint(ctxcatcodes,format(template_b,lettered)) -- global -- else -- for l=1,#tobelettered do -- texsprint(ctxcatcodes,format(template_a,tobelettered[l],lettered)) -- end -- end -- end -- if tobeactivated then -- activated = concat(activated) -- for a=1,#tobeactivated do -- texsprint(ctxcatcodes,format(template_a,tobeactivated[a],activated)) -- end -- end -- end -- -- function characters.setcodes() -- for code, chr in next, data do -- local cc = chr.category -- if cc == 'll' or cc == 'lu' or cc == 'lt' then -- local lc, uc = chr.lccode, chr.uccode -- if not lc then chr.lccode, lc = code, code end -- if not uc then chr.uccode, uc = code, code end -- texsprint(ctxcatcodes,format("\\setcclcuc{%i}{%i}{%i}",code,lc,uc)) -- end -- if cc == "lu" then -- texprint(ctxcatcodes,"\\sfcode ",code,"999 ") -- end -- if cc == "lo" then -- local range = chr.range -- if range then -- texsprint(ctxcatcodes,format('\\dofastrecurse{"%05X}{"%05X}{1}{\\setcclcucself\\fastrecursecounter}',range.first,range.last)) -- end -- end -- end -- end -- --[[ldx-- --
The next variant has lazy token collecting, on a 140 page mk.tex this saves -- about .25 seconds, which is understandable because we have no graphemes and -- not collecting tokens is not only faster but also saves garbage collecting. --
-- --ldx]]-- -- -- function utffilters.collapse(str,filename) -- we can make high a seperate pass (never needed with collapse) -- if skippable[filesuffix(filename)] then -- return str -- -- elseif find(filename,"^virtual://") then -- -- return str -- -- else -- -- -- print("\n"..filename) -- end -- if str and str ~= "" then -- local nstr = #str -- if nstr > 1 then -- if initialize then -- saves a call -- initialize() -- end -- local tokens, t, first, done, n = { }, 0, false, false, 0 -- for second in utfcharacters(str) do -- if done then -- if first then -- if second == " " then -- t = t + 1 -- tokens[t] = first -- first = second -- else -- -- local crs = high[second] -- -- if crs then -- -- t = t + 1 -- -- tokens[t] = first -- -- first = crs -- -- else -- local cgf = graphemes[first] -- if cgf and cgf[second] then -- first = cgf[second] -- else -- t = t + 1 -- tokens[t] = first -- first = second -- end -- -- end -- end -- elseif second == " " then -- first = second -- else -- -- local crs = high[second] -- -- if crs then -- -- first = crs -- -- else -- first = second -- -- end -- end -- elseif second == " " then -- first = nil -- n = n + 1 -- else -- -- local crs = high[second] -- -- if crs then -- -- for s in utfcharacters(str) do -- -- if n == 1 then -- -- break -- -- else -- -- t = t + 1 -- -- tokens[t] = s -- -- n = n - 1 -- -- end -- -- end -- -- if first then -- -- t = t + 1 -- -- tokens[t] = first -- -- end -- -- first = crs -- -- done = true -- -- else -- local cgf = graphemes[first] -- if cgf and cgf[second] then -- for s in utfcharacters(str) do -- if n == 1 then -- break -- else -- t = t + 1 -- tokens[t] = s -- n = n - 1 -- end -- end -- first = cgf[second] -- done = true -- else -- first = second -- n = n + 1 -- end -- -- end -- end -- end -- if done then -- if first then -- t = t + 1 -- tokens[t] = first -- end -- return concat(tokens) -- seldom called -- end -- elseif nstr > 0 then -- return high[str] or str -- this will go from here -- end -- end -- return str -- end -- function utffilters.decompose(str) -- if str and str ~= "" then -- local nstr = #str -- if nstr > 1 then -- -- if initialize then -- saves a call -- -- initialize() -- -- end -- local tokens, t, done, n = { }, 0, false, 0 -- for s in utfcharacters(str) do -- local dec = decomposed[s] -- if dec then -- if not done then -- if n > 0 then -- for s in utfcharacters(str) do -- if n == 0 then -- break -- else -- t = t + 1 -- tokens[t] = s -- n = n - 1 -- end -- end -- end -- done = true -- end -- t = t + 1 -- tokens[t] = dec -- elseif done then -- t = t + 1 -- tokens[t] = s -- else -- n = n + 1 -- end -- end -- if done then -- return concat(tokens) -- seldom called -- end -- end -- end -- return str -- end -- local replacer = nil -- local finder = nil -- -- function utffilters.decompose(str) -- 3 to 4 times faster than the above -- if not replacer then -- if initialize then -- initialize() -- end -- local tree = utfchartabletopattern(decomposed) -- finder = lpeg.finder(tree,false,true) -- replacer = lpeg.replacer(tree,decomposed,false,true) -- end -- if str and str ~= "" and #str > 1 and lpegmatch(finder,str) then -- return lpegmatch(replacer,str) -- end -- return str -- end