diff options
author | Taco Hoekwater <taco@elvenkind.com> | 2011-06-01 08:54:21 +0000 |
---|---|---|
committer | Taco Hoekwater <taco@elvenkind.com> | 2011-06-01 08:54:21 +0000 |
commit | d7ccb42582f85acf30568913610ccf4d602023fb (patch) | |
tree | 7292e3545a420676878e7451b68892d360c62cb6 /Master/texmf-dist/tex/context/base/toks-ini.lua | |
parent | 2d62a6fe9b80def59c392268022f1f9a2d6e358f (diff) |
commit context 2011.05.18
git-svn-id: svn://tug.org/texlive/trunk@22719 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/tex/context/base/toks-ini.lua')
-rw-r--r-- | Master/texmf-dist/tex/context/base/toks-ini.lua | 294 |
1 files changed, 156 insertions, 138 deletions
diff --git a/Master/texmf-dist/tex/context/base/toks-ini.lua b/Master/texmf-dist/tex/context/base/toks-ini.lua index ec60402d857..f8b5b94396d 100644 --- a/Master/texmf-dist/tex/context/base/toks-ini.lua +++ b/Master/texmf-dist/tex/context/base/toks-ini.lua @@ -6,9 +6,8 @@ if not modules then modules = { } end modules ['toks-ini'] = { } local utf = unicode.utf8 -local format, gsub, texsprint = string.format, string.gsub, tex.sprint - -local ctxcatcodes = tex.ctxcatcodes +local utfbyte, utfchar = utf.byte, utf.char +local format, gsub = string.format, string.gsub --[[ldx-- <p>This code is experimental and needs a cleanup. The visualizers will move to @@ -32,151 +31,156 @@ a module.</p> -- actually, we can use token registers to store tokens -tokens = tokens or { } +local token, tex = token, tex + +local texsprint = tex.sprint + +local createtoken = token.create +local csname_id = token.csname_id +local command_id = token.command_id +local command_name = token.command_name +local get_next = token.get_next +local expand = token.expand +local is_activechar = token.is_activechar +local csname_name = token.csname_name -tokens.vbox = token.create("vbox") -tokens.hbox = token.create("hbox") -tokens.vtop = token.create("vtop") -tokens.bgroup = token.create(utf.byte("{"), 1) -tokens.egroup = token.create(utf.byte("}"), 2) +tokens = tokens or { } +local tokens = tokens -tokens.letter = function(chr) return token.create(utf.byte(chr), 11) end -tokens.other = function(chr) return token.create(utf.byte(chr), 12) end +tokens.vbox = createtoken("vbox") +tokens.hbox = createtoken("hbox") +tokens.vtop = createtoken("vtop") +tokens.bgroup = createtoken(utfbyte("{"), 1) +tokens.egroup = createtoken(utfbyte("}"), 2) + +tokens.letter = function(chr) return createtoken(utfbyte(chr), 11) end +tokens.other = function(chr) return createtoken(utfbyte(chr), 12) end tokens.letters = function(str) - local t = { } + local t, n = { }, 0 for chr in string.utfvalues(str) do - t[#t+1] = token.create(chr, 11) + n = n + 1 + t[n] = createtoken(chr, 11) end return t end -collectors = collectors or { } -collectors.data = collectors.data or { } +tokens.collectors = tokens.collectors or { } +local collectors = tokens.collectors + +collectors.data = collectors.data or { } +local collectordata = collectors.data -function tex.printlist(data) +collectors.registered = collectors.registered or { } +local registered = collectors.registered + +local function printlist(data) callbacks.push('token_filter', function () callbacks.pop('token_filter') -- tricky but the nil assignment helps return data end) end +tex.printlist = printlist + function collectors.flush(tag) - tex.printlist(collectors.data[tag]) + printlist(collectordata[tag]) end function collectors.test(tag) - tex.printlist(collectors.data[tag]) + printlist(collectordata[tag]) end -collectors.registered = { } - function collectors.register(name) - collectors.registered[token.csname_id(name)] = name + registered[csname_id(name)] = name end ---~ function collectors.install(tag,end_cs) ---~ collectors.data[tag] = { } ---~ local data = collectors.data[tag] ---~ local call = token.command_id("call") ---~ local relax = token.command_id("relax") ---~ local endcs = token.csname_id(end_cs) ---~ local expand = collectors.registered ---~ local get = token.get_next -- so no callback! ---~ while true do ---~ local t = get() ---~ local a, b = t[1], t[3] ---~ if a == relax and b == endcs then ---~ return ---~ elseif a == call and expand[b] then ---~ token.expand() ---~ else ---~ data[#data+1] = t ---~ end ---~ end ---~ end +local call = command_id("call") +local letter = command_id("letter") +local other = command_id("other_char") function collectors.install(tag,end_cs) - collectors.data[tag] = { } - local data = collectors.data[tag] - local call = token.command_id("call") - local endcs = token.csname_id(end_cs) - local expand = collectors.registered - local get = token.get_next + local data, d = { }, 0 + collectordata[tag] = data + local endcs = csname_id(end_cs) while true do - local t = get() + local t = get_next() local a, b = t[1], t[3] if b == endcs then - tex.print('\\' ..end_cs) + texsprint('\\' ..end_cs) -- to be checked, can be context[end_cs]() return - elseif a == call and expand[b] then - token.expand() + elseif a == call and registered[b] then + expand() else - data[#data+1] = t + d = d + 1 + data[d] = t end end end function collectors.handle(tag,handle,flush) - collectors.data[tag] = handle(collectors.data[tag]) + collectordata[tag] = handle(collectordata[tag]) if flush then collectors.flush(tag) end end -collectors.show_methods = { } +local show_methods = { } +collectors.show_methods = show_methods function collectors.show(tag, method) if type(tag) == "table" then - collectors.show_methods[method or 'a'](tag) + show_methods[method or 'a'](tag) else - collectors.show_methods[method or 'a'](collectors.data[tag]) + show_methods[method or 'a'](collectordata[tag]) end end -commands = commands or { } - -commands.letter = token.command_id("letter") -commands.other = token.command_id("other_char") - -function collectors.default_words(t,str) - t[#t+1] = tokens.bgroup - t[#t+1] = token.create("red") +function collectors.defaultwords(t,str) + local n = #t + n = n + 1 + t[n] = tokens.bgroup + n = n + 1 + t[n] = createtoken("red") for i=1,#str do - t[#t+1] = tokens.other('*') + n = n + 1 + t[n] = tokens.other('*') end - t[#t+1] = tokens.egroup + n = n + 1 + t[n] = tokens.egroup end -function collectors.with_words(tag,handle) - local t, w = { }, { } - handle = handle or collectors.default_words - local tagdata = collectors.data[tag] +function collectors.dowithwords(tag,handle) + local t, w, tn, wn = { }, { }, 0, 0 + handle = handle or collectors.defaultwords + local tagdata = collectordata[tag] for k=1,#tagdata do local v = tagdata[k] - if v[1] == commands.letter then - w[#w+1] = v[2] + if v[1] == letter then + wn = wn + 1 + w[wn] = v[2] else - if #w > 0 then + if wn > 0 then handle(t,w) - w = { } + wn = 0 end - t[#t+1] = v + tn = tn + 1 + t[tn] = v end end - if #w > 0 then + if wn > 0 then handle(t,w) end - collectors.data[tag] = t + collectordata[tag] = t end -function collectors.show_token(t) +local function showtoken(t) if t then - local cmd, chr, id, cs, name = t[1], t[2], t[3], nil, token.command_name(t) or "" - if cmd == commands.letter or cmd == commands.other then - return format("%s-> %s -> %s", name, chr, utf.char(chr)) + local cmd, chr, id, cs, name = t[1], t[2], t[3], nil, command_name(t) or "" + if cmd == letter or cmd == other then + return format("%s-> %s -> %s", name, chr, utfchar(chr)) elseif id > 0 then - cs = token.csname_name(t) or nil + cs = csname_name(t) or nil if cs then return format("%s-> %s", name, cs) elseif tonumber(chr) < 0 then @@ -192,131 +196,145 @@ function collectors.show_token(t) end end +collectors.showtoken = showtoken + function collectors.trace() - local t = token.get_next() - texio.write_nl(collectors.show_token(t)) + local t = get_next() + texio.write_nl(showtoken(t)) return t end -collectors.show_methods.a = function(data) -- no need to store the table, just pass directly - local template = "\\NC %s\\NC %s\\NC %s\\NC %s\\NC %s\\NC\\NR " - texsprint(ctxcatcodes, "\\starttabulate[|T|Tr|cT|Tr|T|]") - texsprint(ctxcatcodes, format(template,"cmd","chr","","id","name")) - texsprint(ctxcatcodes, "\\HL") +-- these might move to a runtime module + +show_methods.a = function(data) -- no need to store the table, just pass directly + local function row(one,two,three,four,five) + context.NC() context(one) + context.NC() context(two) + context.NC() context(three) + context.NC() context(four) + context.NC() context(five) + context.NC() context.NR() + end + context.starttabulate { "|T|Tr|cT|Tr|T|" } + row("cmd","chr","","id","name") + context.HL() for _,v in next, data do local cmd, chr, id, cs, sym = v[1], v[2], v[3], "", "" - local name = gsub(token.command_name(v) or "","_","\\_") + local name = gsub(command_name(v) or "","_","\\_") if id > 0 then - cs = token.csname_name(v) or "" + cs = csname_name(v) or "" if cs ~= "" then cs = "\\string " .. cs end else id = "" end - if cmd == commands.letter or cmd == commands.other then + if cmd == letter or cmd == other then sym = "\\char " .. chr end if tonumber(chr) < 0 then - texsprint(ctxcatcodes, format(template, name, "", sym, id, cs)) + row(name,"",sym,id,cs) else - texsprint(ctxcatcodes, format(template, name, chr, sym, id, cs)) + row(name,chr,sym,id,cs) end end - texsprint(ctxcatcodes, "\\stoptabulate") + context.stoptabulate() end -collectors.show_methods.b_c = function(data,swap) -- no need to store the table, just pass directly - local template = "\\NC %s\\NC %s\\NC %s\\NC\\NR" +local function show_b_c(data,swap) -- no need to store the table, just pass directly + local function row(one,two,three) + context.NC() context(one) + context.NC() context(two) + context.NC() context(three) + context.NC() context.NR() + end if swap then - texsprint(ctxcatcodes, "\\starttabulate[|Tl|Tl|Tr|]") + context.starttabulate { "|Tl|Tl|Tr|" } else - texsprint(ctxcatcodes, "\\starttabulate[|Tl|Tr|Tl|]") + context.starttabulate { "|Tl|Tr|Tl|" } end - texsprint(ctxcatcodes, format(template,"cmd","chr","name")) - texsprint(ctxcatcodes, "\\HL") + row("cmd","chr","name") + context.HL() for _,v in next, data do local cmd, chr, id, cs, sym = v[1], v[2], v[3], "", "" - local name = gsub(token.command_name(v) or "","_","\\_") + local name = gsub(command_name(v) or "","_","\\_") if id > 0 then - cs = token.csname_name(v) or "" + cs = csname_name(v) or "" end - if cmd == commands.letter or cmd == commands.other then + if cmd == letter or cmd == other then sym = "\\char " .. chr - elseif cs ~= "" then - if token.is_activechar(v) then - sym = "\\string " .. cs - else - sym = "\\string\\" .. cs - end + elseif cs == "" then + -- okay + elseif is_activechar(v) then + sym = "\\string " .. cs + else + sym = "\\string\\" .. cs end if swap then - texsprint(ctxcatcodes, format(template, name, sym, chr)) + row(name,sym,chr) elseif tonumber(chr) < 0 then - texsprint(ctxcatcodes, format(template, name, "", sym)) + row(name,"",sym) else - texsprint(ctxcatcodes, format(template, name, chr, sym)) + row(name,chr,sym) end end - texsprint(ctxcatcodes, "\\stoptabulate") + context.stoptabulate() end -- Even more experimental ... -collectors.show_methods.b = function(tag) collectors.show_methods.b_c(tag,false) end -collectors.show_methods.c = function(tag) collectors.show_methods.b_c(tag,true ) end +show_methods.b = function(data) show_b_c(data,false) end +show_methods.c = function(data) show_b_c(data,true ) end -collectors.remapper = { - -- namespace -} +local remapper = { } -- namespace +collectors.remapper = remapper -collectors.remapper.data = { - -- user mappings -} +local remapperdata = { } -- user mappings +remapper.data = remapperdata -function collectors.remapper.store(tag,class,key) - local s = collectors.remapper.data[class] +function remapper.store(tag,class,key) + local s = remapperdata[class] if not s then s = { } - collectors.remapper.data[class] = s + remapperdata[class] = s end - s[key] = collectors.data[tag] - collectors.data[tag] = nil + s[key] = collectordata[tag] + collectordata[tag] = nil end -function collectors.remapper.convert(tag,toks) - local data = collectors.remapper.data[tag] - local leftbracket, rightbracket = utf.byte('['), utf.byte(']') +function remapper.convert(tag,toks) + local data = remapperdata[tag] + local leftbracket, rightbracket = utfbyte('['), utfbyte(']') local skipping = 0 -- todo: math if data then - local t = { } + local t, n = { }, 0 for s=1,#toks do local tok = toks[s] local one, two = tok[1], tok[2] if one == 11 or one == 12 then if two == leftbracket then skipping = skipping + 1 - t[#t+1] = tok + n = n + 1 ; t[n] = tok elseif two == rightbracket then skipping = skipping - 1 - t[#t+1] = tok + n = n + 1 ; t[n] = tok elseif skipping == 0 then local new = data[two] if new then if #new > 1 then for n=1,#new do - t[#t+1] = new[n] + n = n + 1 ; t[n] = new[n] end else - t[#t+1] = new[1] + n = n + 1 ; t[n] = new[1] end else - t[#t+1] = tok + n = n + 1 ; t[n] = tok end else - t[#t+1] = tok + n = n + 1 ; t[n] = tok end else - t[#t+1] = tok + n = n + 1 ; t[n] = tok end end return t |