summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/context/base/toks-ini.lua
blob: 0f0c016f8e9db2962f65d52105923e5ab9473924 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
if not modules then modules = { } end modules ['toks-ini'] = {
    version   = 1.001,
    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
    copyright = "PRAGMA ADE / ConTeXt Development Team",
    license   = "see context related readme files"
}

local context, commands = context, commands
local utfbyte, utfchar, utfvalues = utf.byte, utf.char, utf.values
local format, gsub = string.format, string.gsub

--[[ldx--
<p>This code is experimental and needs a cleanup. The visualizers will move to
a module.</p>
--ldx]]--

-- 1 = command, 2 = modifier (char), 3 = controlsequence id
--
-- callback.register('token_filter', token.get_next)
--
-- token.get_next()
-- token.expand()
-- token.create()
-- token.csname_id()
-- token.csname_name(v)
-- token.command_id()
-- token.command_name(v)
-- token.is_expandable()
-- token.is_activechar()
-- token.lookup(v)

-- actually, we can use token registers to store tokens

local token, tex = token, tex

local createtoken   = token.create
local csname_id     = token.csname_id
local command_id    = token.command_id
local command_name  = token.command_name
local get_next      = token.get_next
local expand        = token.expand
local is_activechar = token.is_activechar
local csname_name   = token.csname_name

tokens        = tokens or { }
local tokens  = tokens

tokens.vbox   = createtoken("vbox")
tokens.hbox   = createtoken("hbox")
tokens.vtop   = createtoken("vtop")
tokens.bgroup = createtoken(utfbyte("{"), 1)
tokens.egroup = createtoken(utfbyte("}"), 2)

tokens.letter = function(chr) return createtoken(utfbyte(chr), 11) end
tokens.other  = function(chr) return createtoken(utfbyte(chr), 12) end

tokens.letters = function(str)
    local t, n = { }, 0
    for chr in utfvalues(str) do
        n = n + 1
        t[n] = createtoken(chr, 11)
    end
    return t
end

tokens.collectors     = tokens.collectors or { }
local collectors      = tokens.collectors

collectors.data       = collectors.data or { }
local collectordata   = collectors.data

collectors.registered = collectors.registered or { }
local registered      = collectors.registered

local function printlist(data)
    callbacks.push('token_filter', function ()
       callbacks.pop('token_filter') -- tricky but the nil assignment helps
       return data
    end)
end

tex.printlist = printlist -- will change to another namespace

function collectors.flush(tag)
    printlist(collectordata[tag])
end

function collectors.test(tag)
    printlist(collectordata[tag])
end

function collectors.register(name)
    registered[csname_id(name)] = name
end

local call   = command_id("call")
local letter = command_id("letter")
local other  = command_id("other_char")

function collectors.install(tag,end_cs)
    local data, d = { }, 0
    collectordata[tag] = data
    local endcs = csname_id(end_cs)
    while true do
        local t = get_next()
        local a, b = t[1], t[3]
        if b == endcs then
            context["end_cs"]()
            return
        elseif a == call and registered[b] then
            expand()
        else
            d = d + 1
            data[d] = t
        end
    end
end

function collectors.handle(tag,handle,flush)
    collectordata[tag] = handle(collectordata[tag])
    if flush then
        collectors.flush(tag)
    end
end

local show_methods      = { }
collectors.show_methods = show_methods

function collectors.show(tag, method)
    if type(tag) == "table" then
        show_methods[method or 'a'](tag)
    else
        show_methods[method or 'a'](collectordata[tag])
    end
end

function collectors.defaultwords(t,str)
    local n = #t
    n = n + 1
    t[n] = tokens.bgroup
    n = n + 1
    t[n] = createtoken("red")
    for i=1,#str do
        n = n + 1
        t[n] = tokens.other('*')
    end
    n = n + 1
    t[n] = tokens.egroup
end

function collectors.dowithwords(tag,handle)
    local t, w, tn, wn = { }, { }, 0, 0
    handle = handle or collectors.defaultwords
    local tagdata = collectordata[tag]
    for k=1,#tagdata do
        local v = tagdata[k]
        if v[1] == letter then
            wn = wn + 1
            w[wn] = v[2]
        else
            if wn > 0 then
                handle(t,w)
                wn = 0
            end
            tn = tn + 1
            t[tn] = v
        end
    end
    if wn > 0 then
        handle(t,w)
    end
    collectordata[tag] = t
end

local function showtoken(t)
    if t then
        local cmd, chr, id, cs, name = t[1], t[2], t[3], nil, command_name(t) or ""
        if cmd == letter or cmd == other then
            return format("%s-> %s -> %s", name, chr, utfchar(chr))
        elseif id > 0 then
            cs = csname_name(t) or nil
            if cs then
                return format("%s-> %s", name, cs)
            elseif tonumber(chr) < 0 then
                return format("%s-> %s", name, id)
            else
                return format("%s-> (%s,%s)", name, chr, id)
            end
        else
            return format("%s", name)
        end
    else
        return "no node"
    end
end

collectors.showtoken = showtoken

function collectors.trace()
    local t = get_next()
    logs.report("tokenlist",showtoken(t))
    return t
end

-- these might move to a runtime module

show_methods.a = function(data) -- no need to store the table, just pass directly
    local function row(one,two,three,four,five)
        context.NC() context(one)
        context.NC() context(two)
        context.NC() context(three)
        context.NC() context(four)
        context.NC() context(five)
        context.NC() context.NR()
    end
    context.starttabulate { "|T|Tr|cT|Tr|T|" }
    row("cmd","chr","","id","name")
    context.HL()
    for _,v in next, data do
        local cmd, chr, id, cs, sym = v[1], v[2], v[3], "", ""
        local name = gsub(command_name(v) or "","_","\\_")
        if id > 0 then
            cs = csname_name(v) or ""
            if cs ~= "" then cs = "\\string " .. cs end
        else
            id = ""
        end
        if cmd == letter or cmd == other then
            sym = "\\char " .. chr
        end
        if tonumber(chr) < 0 then
            row(name,"",sym,id,cs)
        else
            row(name,chr,sym,id,cs)
        end
    end
    context.stoptabulate()
end

local function show_b_c(data,swap) -- no need to store the table, just pass directly
    local function row(one,two,three)
        context.NC() context(one)
        context.NC() context(two)
        context.NC() context(three)
        context.NC() context.NR()
    end
    if swap then
        context.starttabulate { "|Tl|Tl|Tr|" }
    else
        context.starttabulate { "|Tl|Tr|Tl|" }
    end
    row("cmd","chr","name")
    context.HL()
    for _,v in next, data do
        local cmd, chr, id, cs, sym = v[1], v[2], v[3], "", ""
        local name = gsub(command_name(v) or "","_","\\_")
        if id > 0 then
            cs = csname_name(v) or ""
        end
        if cmd == letter or cmd == other then
            sym = "\\char " .. chr
        elseif cs == "" then
            -- okay
        elseif is_activechar(v) then
            sym = "\\string " .. cs
        else
            sym = "\\string\\" .. cs
        end
        if swap then
            row(name,sym,chr)
        elseif tonumber(chr) < 0 then
            row(name,"",sym)
        else
            row(name,chr,sym)
        end
    end
    context.stoptabulate()
end

-- Even more experimental ...

show_methods.b = function(data) show_b_c(data,false) end
show_methods.c = function(data) show_b_c(data,true ) end

local remapper      = { }  -- namespace
collectors.remapper = remapper

local remapperdata  = { }  -- user mappings
remapper.data       = remapperdata

function remapper.store(tag,class,key)
    local s = remapperdata[class]
    if not s then
        s = { }
        remapperdata[class] = s
    end
    s[key] = collectordata[tag]
    collectordata[tag] = nil
end

function remapper.convert(tag,toks)
    local data = remapperdata[tag]
    local leftbracket, rightbracket = utfbyte('['), utfbyte(']')
    local skipping = 0
    -- todo: math
    if data then
        local t, n = { }, 0
        for s=1,#toks do
            local tok = toks[s]
            local one, two = tok[1], tok[2]
            if one == 11 or one == 12 then
                if two == leftbracket then
                    skipping = skipping + 1
                    n = n + 1 ; t[n] = tok
                elseif two == rightbracket then
                    skipping = skipping - 1
                    n = n + 1 ; t[n] = tok
                elseif skipping == 0 then
                    local new = data[two]
                    if new then
                        if #new > 1 then
                            for n=1,#new do
                                n = n + 1 ; t[n] = new[n]
                            end
                        else
                            n = n + 1 ; t[n] = new[1]
                        end
                    else
                        n = n + 1 ; t[n] = tok
                    end
                else
                    n = n + 1 ; t[n] = tok
                end
            else
                n = n + 1 ; t[n] = tok
            end
        end
        return t
    else
        return toks
    end
end