summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/context/base/regi-ini.lua
blob: 37a88fd5f967eaede451c2b398c5e830a1405268 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
if not modules then modules = { } end modules ['regi-ini'] = {
    version   = 1.001,
    comment   = "companion to regi-ini.mkiv",
    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
    copyright = "PRAGMA ADE / ConTeXt Development Team",
    license   = "see context related readme files"
}

--[[ldx--
<p>Regimes take care of converting the input characters into
<l n='utf'/> sequences. The conversion tables are loaded at
runtime.</p>
--ldx]]--

local commands, context = commands, context

local utfchar = utf.char
local P, Cs, Cc, lpegmatch = lpeg.P, lpeg.Cs, lpeg.Cc, lpeg.match
local char, gsub, format, gmatch, byte, match = string.char, string.gsub, string.format, string.gmatch, string.byte, string.match
local next = next
local insert, remove, fastcopy = table.insert, table.remove, table.fastcopy
local concat = table.concat
local totable = string.totable

local allocate          = utilities.storage.allocate
local sequencers        = utilities.sequencers
local textlineactions   = resolvers.openers.helpers.textlineactions
local setmetatableindex = table.setmetatableindex

local implement         = interfaces.implement
local setmacro          = interfaces.setmacro

--[[ldx--
<p>We will hook regime handling code into the input methods.</p>
--ldx]]--

local trace_translating = false  trackers.register("regimes.translating", function(v) trace_translating = v end)

local report_loading     = logs.reporter("regimes","loading")
local report_translating = logs.reporter("regimes","translating")

regimes        = regimes or { }
local regimes  = regimes

local mapping  = allocate {
    utf = false
}

local backmapping = allocate {
}

-- regimes.mapping  = mapping

local synonyms = { -- backward compatibility list

    ["windows-1250"] = "cp1250",
    ["windows-1251"] = "cp1251",
    ["windows-1252"] = "cp1252",
    ["windows-1253"] = "cp1253",
    ["windows-1254"] = "cp1254",
    ["windows-1255"] = "cp1255",
    ["windows-1256"] = "cp1256",
    ["windows-1257"] = "cp1257",
    ["windows-1258"] = "cp1258",

    ["il1"]          = "8859-1",
    ["il2"]          = "8859-2",
    ["il3"]          = "8859-3",
    ["il4"]          = "8859-4",
    ["il5"]          = "8859-9",
    ["il6"]          = "8859-10",
    ["il7"]          = "8859-13",
    ["il8"]          = "8859-14",
    ["il9"]          = "8859-15",
    ["il10"]         = "8859-16",

    ["iso-8859-1"]   = "8859-1",
    ["iso-8859-2"]   = "8859-2",
    ["iso-8859-3"]   = "8859-3",
    ["iso-8859-4"]   = "8859-4",
    ["iso-8859-9"]   = "8859-9",
    ["iso-8859-10"]  = "8859-10",
    ["iso-8859-13"]  = "8859-13",
    ["iso-8859-14"]  = "8859-14",
    ["iso-8859-15"]  = "8859-15",
    ["iso-8859-16"]  = "8859-16",

    ["latin1"]       = "8859-1",
    ["latin2"]       = "8859-2",
    ["latin3"]       = "8859-3",
    ["latin4"]       = "8859-4",
    ["latin5"]       = "8859-9",
    ["latin6"]       = "8859-10",
    ["latin7"]       = "8859-13",
    ["latin8"]       = "8859-14",
    ["latin9"]       = "8859-15",
    ["latin10"]      = "8859-16",

    ["utf-8"]        = "utf",
    ["utf8"]         = "utf",
    [""]             = "utf",

    ["windows"]      = "cp1252",

    ["pdf"]          = "pdfdoc",

}

local currentregime = "utf"

local function loadregime(mapping,regime)
    local name = resolvers.findfile(format("regi-%s.lua",regime)) or ""
    local data = name ~= "" and dofile(name)
    if data then
        vector = { }
        for eightbit, unicode in next, data do
            vector[char(eightbit)] = utfchar(unicode)
        end
        report_loading("vector %a is loaded",regime)
    else
        vector = false
        report_loading("vector %a is unknown",regime)
    end
    mapping[regime] = vector
    return vector
end

local function loadreverse(t,k)
    local t = { }
    for k, v in next, mapping[k] do
        t[v] = k
    end
    backmapping[k] = t
    return t
end

setmetatableindex(mapping,    loadregime)
setmetatableindex(backmapping,loadreverse)

local function fromregime(regime,line)
    if line and #line > 0 then
        local map = mapping[regime and synonyms[regime] or regime or currentregime]
        if map then
            line = gsub(line,".",map)
        end
    end
    return line
end

-- local remappers = { }
--
-- local function toregime(vector,str,default) -- toregime('8859-1',"abcde Ä","?")
--     local t = backmapping[vector]
--     local remapper = remappers[vector]
--     if not remapper then
--         remapper = utf.remapper(t)
--         remappers[t] = remapper
--     end
--     local m = getmetatable(t)
--     setmetatableindex(t, function(t,k)
--         local v = default or "?"
--         t[k] = v
--         return v
--     end)
--     str = remapper(str)
--     setmetatable(t,m)
--     return str
-- end
--
-- -- much faster (but only matters when we have > 10K calls

local cache = { } -- if really needed we can copy vectors and hash defaults

setmetatableindex(cache, function(t,k)
    local v = { remappers = { } }
    t[k] = v
    return v
end)

local function toregime(vector,str,default) -- toregime('8859-1',"abcde Ä","?")
    local d = default or "?"
    local c = cache[vector].remappers
    local r = c[d]
    if not r then
        local t = fastcopy(backmapping[vector])
     -- r = utf.remapper(t) -- not good for defaults here
        local pattern = Cs((lpeg.utfchartabletopattern(t)/t + lpeg.patterns.utf8character/d + P(1)/d)^0)
        r = function(str)
            if not str or str == "" then
                return ""
            else
                return lpegmatch(pattern,str)
            end
        end
        c[d]  = r
    end
    return r(str)
end

local function disable()
    currentregime = "utf"
    sequencers.disableaction(textlineactions,"regimes.process")
    return currentregime
end

local function enable(regime)
    regime = synonyms[regime] or regime
    if mapping[regime] == false then
        disable()
    else
        currentregime = regime
        sequencers.enableaction(textlineactions,"regimes.process")
    end
    return currentregime
end

regimes.toregime   = toregime
regimes.fromregime = fromregime
regimes.translate  = function(str,regime) return fromregime(regime,str) end
regimes.enable     = enable
regimes.disable    = disable

-- The following function can be used when we want to make sure that
-- utf gets passed unharmed. This is needed for modules.

local level = 0

function regimes.process(str,filename,currentline,noflines,coding)
    if level == 0 and coding ~= "utf-8" then
        str = fromregime(currentregime,str)
        if trace_translating then
            report_translating("utf: %s",str)
        end
    end
    return str
end

local function push()
    level = level + 1
    if trace_translating then
        report_translating("pushing level %s",level)
    end
end

local function pop()
    if level > 0 then
        if trace_translating then
            report_translating("popping level %s",level)
        end
        level = level - 1
    end
end

regimes.push = push
regimes.pop  = pop

if sequencers then

    sequencers.prependaction(textlineactions,"system","regimes.process")
    sequencers.disableaction(textlineactions,"regimes.process")

end

-- interface:

implement {
    name      = "enableregime",
    arguments = "string",
    actions   = function(regime) setmacro("currentregime",enable(regime)) end
}

implement {
    name      = "disableregime",
    actions   = function() setmacro("currentregime",disable()) end
}

implement {
    name      = "pushregime",
    actions   = push
}

implement {
    name      = "popregime",
    actions   = pop
}

local stack = { }

implement {
    name      = "startregime",
    arguments = "string",
    actions   = function(regime)
        insert(stack,currentregime)
        if trace_translating then
            report_translating("start using %a",regime)
        end
        setmacro("currentregime",enable(regime))
    end
}

implement {
    name      = "stopregime",
    actions   = function()
        if #stack > 0 then
            local regime = remove(stack)
            if trace_translating then
                report_translating("stop using %a",regime)
            end
            setmacro("currentregime",enable(regime))
        end
    end
}

-- Next we provide some hacks. Unfortunately we run into crappy encoded
-- (read : mixed) encoded xml files that have these ë ä ö ü sequences
-- instead of ë ä ö ü

local patterns = { }

-- function regimes.cleanup(regime,str)
--     local p = patterns[regime]
--     if p == nil then
--         regime = regime and synonyms[regime] or regime or currentregime
--         local vector = regime ~= "utf" and mapping[regime]
--         if vector then
--             local list = { }
--             for k, uchar in next, vector do
--                 local stream = totable(uchar)
--                 for i=1,#stream do
--                     stream[i] = vector[stream[i]]
--                 end
--                 list[concat(stream)] = uchar
--             end
--             p = lpeg.append(list,nil,true)
--             p = Cs((p+1)^0)
--          -- lpeg.print(p) -- size 1604
--         else
--             p = false
--         end
--         patterns[vector] = p
--     end
--     return p and lpegmatch(p,str) or str
-- end
--
-- twice as fast and much less lpeg bytecode

-- function regimes.cleanup(regime,str)
--     if not str or str == "" then
--         return str
--     end
--     local p = patterns[regime]
--     if p == nil then
--         regime = regime and synonyms[regime] or regime or currentregime
--         local vector = regime ~= "utf" and regime ~= "utf-8" and mapping[regime]
--         if vector then
--             local utfchars = { }
--             local firsts = { }
--             for k, uchar in next, vector do
--                 local stream = { }
--                 local split = totable(uchar)
--                 local nofsplits = #split
--                 if nofsplits > 1 then
--                     local first
--                     for i=1,nofsplits do
--                         local u = vector[split[i]]
--                         if not first then
--                             first = firsts[u]
--                             if not first then
--                                 first = { }
--                                 firsts[u] = first
--                             end
--                         end
--                         stream[i] = u
--                     end
--                     local nofstream = #stream
--                     if nofstream > 1 then
--                         first[#first+1] = concat(stream,2,nofstream)
--                         utfchars[concat(stream)] = uchar
--                     end
--                 end
--             end
--             p = P(false)
--             for k, v in next, firsts do
--                 local q = P(false)
--                 for i=1,#v do
--                     q = q + P(v[i])
--                 end
--                 p = p + P(k) * q
--             end
--             p = Cs(((p+1)/utfchars)^1)
--          -- lpeg.print(p) -- size: 1042
--         else
--             p = false
--         end
--         patterns[regime] = p
--     end
--     return p and lpegmatch(p,str) or str
-- end
--
-- 5 times faster:

function regimes.cleanup(regime,str)
    if not str or str == "" then
        return str
    end
    local p = patterns[regime]
    if p == nil then
        regime = regime and synonyms[regime] or regime or currentregime
        local vector = regime ~= "utf" and regime ~= "utf-8" and mapping[regime]
        if vector then
            local mapping = { }
            for k, v in next, vector do
                local split = totable(v)
                for i=1,#split do
                    split[i] = utfchar(byte(split[i]))
                end
                split = concat(split)
                if v ~= split then
                    mapping[split] = v
                end
            end
            p = Cs((lpeg.utfchartabletopattern(mapping)/mapping+P(1))^0)
        else
            p = false
        end
        patterns[regime] = p
    end
    return p and lpegmatch(p,str) or str
end

-- local old = [[test ë ä ö ü crap]]
-- local new = regimes.cleanup("cp1252",old)
-- report_translating("%s -> %s",old,new)
-- local old = "Pozn" .. char(0xE1) .. "mky"
-- local new = fromregime("cp1250",old)
-- report_translating("%s -> %s",old,new)