summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/context/base/regi-ini.lua
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/tex/context/base/regi-ini.lua')
-rw-r--r--Master/texmf-dist/tex/context/base/regi-ini.lua181
1 files changed, 157 insertions, 24 deletions
diff --git a/Master/texmf-dist/tex/context/base/regi-ini.lua b/Master/texmf-dist/tex/context/base/regi-ini.lua
index ec6f812ccb1..d5d278b1649 100644
--- a/Master/texmf-dist/tex/context/base/regi-ini.lua
+++ b/Master/texmf-dist/tex/context/base/regi-ini.lua
@@ -12,11 +12,15 @@ if not modules then modules = { } end modules ['regi-ini'] = {
runtime.</p>
--ldx]]--
-local utfchar, utfgsub = utf.char, utf.gsub
-local char, gsub, format = string.char, string.gsub, string.format
-local next = next
-local insert, remove = table.insert, table.remove
+local commands, context = commands, context
+local utfchar = utf.char
+local P, Cs, lpegmatch = lpeg.P, lpeg.Cs, lpeg.match
+local char, gsub, format, gmatch, byte, match = string.char, string.gsub, string.format, string.gmatch, string.byte, string.match
+local next = next
+local insert, remove, fastcopy = table.insert, table.remove, table.fastcopy
+local concat = table.concat
+local totable = string.totable
local allocate = utilities.storage.allocate
local sequencers = utilities.sequencers
@@ -107,10 +111,10 @@ local function loadregime(mapping,regime)
for eightbit, unicode in next, data do
vector[char(eightbit)] = utfchar(unicode)
end
- report_loading("vector '%s' is loaded",regime)
+ report_loading("vector %a is loaded",regime)
else
vector = false
- report_loading("vector '%s' is unknown",regime)
+ report_loading("vector %a is unknown",regime)
end
mapping[regime] = vector
return vector
@@ -125,8 +129,8 @@ local function loadreverse(t,k)
return t
end
-setmetatableindex(mapping, loadregime)
-setmetatableindex(backmapping, loadreverse)
+setmetatableindex(mapping, loadregime)
+setmetatableindex(backmapping,loadreverse)
local function translate(line,regime)
if line and #line > 0 then
@@ -138,17 +142,51 @@ local function translate(line,regime)
return line
end
+-- local remappers = { }
+--
+-- local function toregime(vector,str,default) -- toregime('8859-1',"abcde Ä","?")
+-- local t = backmapping[vector]
+-- local remapper = remappers[vector]
+-- if not remapper then
+-- remapper = utf.remapper(t)
+-- remappers[t] = remapper
+-- end
+-- local m = getmetatable(t)
+-- setmetatableindex(t, function(t,k)
+-- local v = default or "?"
+-- t[k] = v
+-- return v
+-- end)
+-- str = remapper(str)
+-- setmetatable(t,m)
+-- return str
+-- end
+--
+-- -- much faster (but only matters when we have > 10K calls
+
+local cache = { } -- if really needed we can copy vectors and hash defaults
+
+setmetatableindex(cache, function(t,k)
+ local v = { remappers = { } }
+ t[k] = v
+ return v
+end)
+
local function toregime(vector,str,default) -- toregime('8859-1',"abcde Ä","?")
- local t = backmapping[vector]
- local m = getmetatable(t)
- setmetatableindex(t, function(t,k)
- local v = default or "?"
- t[k] = v
- return v
- end)
- str = utfgsub(str,".",t)
- setmetatable(t,m)
- return str
+ local d = default or "?"
+ local c = cache[vector].remappers
+ local r = c[d]
+ if not r then
+ local t = fastcopy(backmapping[vector])
+ setmetatableindex(t, function(t,k)
+ local v = d
+ t[k] = v
+ return v
+ end)
+ r = utf.remapper(t)
+ c[d] = r
+ end
+ return r(str)
end
local function disable()
@@ -186,22 +224,25 @@ function regimes.process(str,filename,currentline,noflines,coding)
return str
end
-function regimes.push()
+local function push()
level = level + 1
if trace_translating then
- report_translating("pushing level: %s",level)
+ report_translating("pushing level %s",level)
end
end
-function regimes.pop()
+local function pop()
if level > 0 then
if trace_translating then
- report_translating("popping level: %s",level)
+ report_translating("popping level %s",level)
end
level = level - 1
end
end
+regimes.push = push
+regimes.pop = pop
+
sequencers.prependaction(textlineactions,"system","regimes.process")
sequencers.disableaction(textlineactions,"regimes.process")
@@ -210,6 +251,9 @@ sequencers.disableaction(textlineactions,"regimes.process")
commands.enableregime = enable
commands.disableregime = disable
+commands.pushregime = push
+commands.popregime = pop
+
function commands.currentregime()
context(currentregime)
end
@@ -219,7 +263,7 @@ local stack = { }
function commands.startregime(regime)
insert(stack,currentregime)
if trace_translating then
- report_translating("start: '%s'",regime)
+ report_translating("start using %a",regime)
end
enable(regime)
end
@@ -228,12 +272,101 @@ function commands.stopregime()
if #stack > 0 then
local regime = remove(stack)
if trace_translating then
- report_translating("stop: '%s'",regime)
+ report_translating("stop using %a",regime)
end
enable(regime)
end
end
+-- Next we provide some hacks. Unfortunately we run into crappy encoded
+-- (read : mixed) encoded xml files that have these ë ä ö ü sequences
+-- instead of ë ä ö ü
+
+local patterns = { }
+
+-- function regimes.cleanup(regime,str)
+-- local p = patterns[regime]
+-- if p == nil then
+-- regime = regime and synonyms[regime] or regime or currentregime
+-- local vector = regime ~= "utf" and mapping[regime]
+-- if vector then
+-- local list = { }
+-- for k, uchar in next, vector do
+-- local stream = totable(uchar)
+-- for i=1,#stream do
+-- stream[i] = vector[stream[i]]
+-- end
+-- list[concat(stream)] = uchar
+-- end
+-- p = lpeg.append(list,nil,true)
+-- p = Cs((p+1)^0)
+-- -- lpeg.print(p) -- size 1604
+-- else
+-- p = false
+-- end
+-- patterns[vector] = p
+-- end
+-- return p and lpegmatch(p,str) or str
+-- end
+--
+-- twice as fast and much less lpeg bytecode
+
+function regimes.cleanup(regime,str)
+ local p = patterns[regime]
+ if p == nil then
+ regime = regime and synonyms[regime] or regime or currentregime
+ local vector = regime ~= "utf" and mapping[regime]
+ if vector then
+ local utfchars = { }
+ local firsts = { }
+ for k, uchar in next, vector do
+ local stream = { }
+ local split = totable(uchar)
+ local nofsplits = #split
+ if nofsplits > 1 then
+ local first
+ for i=1,nofsplits do
+ local u = vector[split[i]]
+ if not first then
+ first = firsts[u]
+ if not first then
+ first = { }
+ firsts[u] = first
+ end
+ end
+ stream[i] = u
+ end
+ local nofstream = #stream
+ if nofstream > 1 then
+ first[#first+1] = concat(stream,2,nofstream)
+ utfchars[concat(stream)] = uchar
+ end
+ end
+ end
+ p = P(false)
+ for k, v in next, firsts do
+ local q = P(false)
+ for i=1,#v do
+ q = q + P(v[i])
+ end
+ p = p + P(k) * q
+ end
+ p = Cs(((p+1)/utfchars)^1)
+ -- lpeg.print(p) -- size: 1042
+ else
+ p = false
+ end
+ patterns[regime] = p
+ end
+ return p and lpegmatch(p,str) or str
+end
+
+-- local map = require("regi-cp1252")
+-- local old = [[test ë ä ö ü crap]]
+-- local new = correctencoding(map,old)
+--
+-- print(old,new)
+
-- obsolete:
--
-- function regimes.setsynonym(synonym,target)