diff options
Diffstat (limited to 'Master/texmf-dist/tex/context/base/regi-ini.lua')
-rw-r--r-- | Master/texmf-dist/tex/context/base/regi-ini.lua | 181 |
1 files changed, 157 insertions, 24 deletions
diff --git a/Master/texmf-dist/tex/context/base/regi-ini.lua b/Master/texmf-dist/tex/context/base/regi-ini.lua index ec6f812ccb1..d5d278b1649 100644 --- a/Master/texmf-dist/tex/context/base/regi-ini.lua +++ b/Master/texmf-dist/tex/context/base/regi-ini.lua @@ -12,11 +12,15 @@ if not modules then modules = { } end modules ['regi-ini'] = { runtime.</p> --ldx]]-- -local utfchar, utfgsub = utf.char, utf.gsub -local char, gsub, format = string.char, string.gsub, string.format -local next = next -local insert, remove = table.insert, table.remove +local commands, context = commands, context +local utfchar = utf.char +local P, Cs, lpegmatch = lpeg.P, lpeg.Cs, lpeg.match +local char, gsub, format, gmatch, byte, match = string.char, string.gsub, string.format, string.gmatch, string.byte, string.match +local next = next +local insert, remove, fastcopy = table.insert, table.remove, table.fastcopy +local concat = table.concat +local totable = string.totable local allocate = utilities.storage.allocate local sequencers = utilities.sequencers @@ -107,10 +111,10 @@ local function loadregime(mapping,regime) for eightbit, unicode in next, data do vector[char(eightbit)] = utfchar(unicode) end - report_loading("vector '%s' is loaded",regime) + report_loading("vector %a is loaded",regime) else vector = false - report_loading("vector '%s' is unknown",regime) + report_loading("vector %a is unknown",regime) end mapping[regime] = vector return vector @@ -125,8 +129,8 @@ local function loadreverse(t,k) return t end -setmetatableindex(mapping, loadregime) -setmetatableindex(backmapping, loadreverse) +setmetatableindex(mapping, loadregime) +setmetatableindex(backmapping,loadreverse) local function translate(line,regime) if line and #line > 0 then @@ -138,17 +142,51 @@ local function translate(line,regime) return line end +-- local remappers = { } +-- +-- local function toregime(vector,str,default) -- toregime('8859-1',"abcde Ä","?") +-- local t = backmapping[vector] +-- local remapper = remappers[vector] +-- if not remapper then +-- remapper = utf.remapper(t) +-- remappers[t] = remapper +-- end +-- local m = getmetatable(t) +-- setmetatableindex(t, function(t,k) +-- local v = default or "?" +-- t[k] = v +-- return v +-- end) +-- str = remapper(str) +-- setmetatable(t,m) +-- return str +-- end +-- +-- -- much faster (but only matters when we have > 10K calls + +local cache = { } -- if really needed we can copy vectors and hash defaults + +setmetatableindex(cache, function(t,k) + local v = { remappers = { } } + t[k] = v + return v +end) + local function toregime(vector,str,default) -- toregime('8859-1',"abcde Ä","?") - local t = backmapping[vector] - local m = getmetatable(t) - setmetatableindex(t, function(t,k) - local v = default or "?" - t[k] = v - return v - end) - str = utfgsub(str,".",t) - setmetatable(t,m) - return str + local d = default or "?" + local c = cache[vector].remappers + local r = c[d] + if not r then + local t = fastcopy(backmapping[vector]) + setmetatableindex(t, function(t,k) + local v = d + t[k] = v + return v + end) + r = utf.remapper(t) + c[d] = r + end + return r(str) end local function disable() @@ -186,22 +224,25 @@ function regimes.process(str,filename,currentline,noflines,coding) return str end -function regimes.push() +local function push() level = level + 1 if trace_translating then - report_translating("pushing level: %s",level) + report_translating("pushing level %s",level) end end -function regimes.pop() +local function pop() if level > 0 then if trace_translating then - report_translating("popping level: %s",level) + report_translating("popping level %s",level) end level = level - 1 end end +regimes.push = push +regimes.pop = pop + sequencers.prependaction(textlineactions,"system","regimes.process") sequencers.disableaction(textlineactions,"regimes.process") @@ -210,6 +251,9 @@ sequencers.disableaction(textlineactions,"regimes.process") commands.enableregime = enable commands.disableregime = disable +commands.pushregime = push +commands.popregime = pop + function commands.currentregime() context(currentregime) end @@ -219,7 +263,7 @@ local stack = { } function commands.startregime(regime) insert(stack,currentregime) if trace_translating then - report_translating("start: '%s'",regime) + report_translating("start using %a",regime) end enable(regime) end @@ -228,12 +272,101 @@ function commands.stopregime() if #stack > 0 then local regime = remove(stack) if trace_translating then - report_translating("stop: '%s'",regime) + report_translating("stop using %a",regime) end enable(regime) end end +-- Next we provide some hacks. Unfortunately we run into crappy encoded +-- (read : mixed) encoded xml files that have these ë ä ö ü sequences +-- instead of ë ä ö ü + +local patterns = { } + +-- function regimes.cleanup(regime,str) +-- local p = patterns[regime] +-- if p == nil then +-- regime = regime and synonyms[regime] or regime or currentregime +-- local vector = regime ~= "utf" and mapping[regime] +-- if vector then +-- local list = { } +-- for k, uchar in next, vector do +-- local stream = totable(uchar) +-- for i=1,#stream do +-- stream[i] = vector[stream[i]] +-- end +-- list[concat(stream)] = uchar +-- end +-- p = lpeg.append(list,nil,true) +-- p = Cs((p+1)^0) +-- -- lpeg.print(p) -- size 1604 +-- else +-- p = false +-- end +-- patterns[vector] = p +-- end +-- return p and lpegmatch(p,str) or str +-- end +-- +-- twice as fast and much less lpeg bytecode + +function regimes.cleanup(regime,str) + local p = patterns[regime] + if p == nil then + regime = regime and synonyms[regime] or regime or currentregime + local vector = regime ~= "utf" and mapping[regime] + if vector then + local utfchars = { } + local firsts = { } + for k, uchar in next, vector do + local stream = { } + local split = totable(uchar) + local nofsplits = #split + if nofsplits > 1 then + local first + for i=1,nofsplits do + local u = vector[split[i]] + if not first then + first = firsts[u] + if not first then + first = { } + firsts[u] = first + end + end + stream[i] = u + end + local nofstream = #stream + if nofstream > 1 then + first[#first+1] = concat(stream,2,nofstream) + utfchars[concat(stream)] = uchar + end + end + end + p = P(false) + for k, v in next, firsts do + local q = P(false) + for i=1,#v do + q = q + P(v[i]) + end + p = p + P(k) * q + end + p = Cs(((p+1)/utfchars)^1) + -- lpeg.print(p) -- size: 1042 + else + p = false + end + patterns[regime] = p + end + return p and lpegmatch(p,str) or str +end + +-- local map = require("regi-cp1252") +-- local old = [[test ë ä ö ü crap]] +-- local new = correctencoding(map,old) +-- +-- print(old,new) + -- obsolete: -- -- function regimes.setsynonym(synonym,target) |