diff options
Diffstat (limited to 'Master/texmf-dist/tex/context/base/mkxl/toks-scn.lmt')
-rw-r--r-- | Master/texmf-dist/tex/context/base/mkxl/toks-scn.lmt | 582 |
1 files changed, 582 insertions, 0 deletions
diff --git a/Master/texmf-dist/tex/context/base/mkxl/toks-scn.lmt b/Master/texmf-dist/tex/context/base/mkxl/toks-scn.lmt new file mode 100644 index 00000000000..73eedbba789 --- /dev/null +++ b/Master/texmf-dist/tex/context/base/mkxl/toks-scn.lmt @@ -0,0 +1,582 @@ +if not modules then modules = { } end modules ['toks-scn'] = { + version = 1.001, + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- Writing this kind of code (and completing the newtoken code base) is fun. I did +-- so with the brilliant film music from The Girl with the Dragon Tattoo running in a +-- loop in the background (three cd's by Trent Reznor and Atticus Ross). An alien +-- feeling helps with alien code. + +-- todo: more \let's at the tex end + +local type, next, tostring, tonumber = type, next, tostring, tonumber + +local formatters = string.formatters +local concat = table.concat + +local scanners = tokens.scanners +local tokenbits = tokens.bits + +local scanstring = scanners.string +local scanargument = scanners.argument +local scandelimited = scanners.delimited +local scanverbatim = scanners.verbatim +local scantokenlist = scanners.tokenlist +local scantoks = scanners.toks +local scaninteger = scanners.integer +local scancardinal = scanners.cardinal +local scannumber = scanners.number +local scankeyword = scanners.keyword +local scankeywordcs = scanners.keywordcs +local scanword = scanners.word +local scanletters = scanners.letters +local scankey = scanners.key +local scancode = scanners.code +local scanboolean = scanners.boolean +local scandimen = scanners.dimen +local scanglue = scanners.glue +local scangluevalues = scanners.gluevalues +local scangluespec = scanners.gluespec +local scancsname = scanners.csname +local scanintegerargument = scanners.integerargument +local scandimenargument = scanners.dimenargument + +local todimen = number.todimen +local toboolean = toboolean + +local lpegmatch = lpeg.match +local p_unquoted = lpeg.Cs(lpeg.patterns.unquoted) + +local trace_compile = false trackers.register("tokens.compile", function(v) trace_compile = v end) +local report_compile = logs.reporter("tokens","compile") +local report_scan = logs.reporter("tokens","scan") + +local open = tokenbits.open +local close = tokenbits.close + +local function scanopen() + while true do + local c = scancode(open) + if c == 123 then + return true + -- elseif c ~= 32 then + elseif not c then + return + end + end +end + +local function scanclose() + while true do + local c = scancode(close) + if c == 125 then + return true + -- elseif c ~= 32 then + elseif not c then + return + end + end +end + +scanners.scanopen = scanopen +scanners.scanclose = scanclose + +local function scanlist() + local wrapped = scanopen() + local list = { } + local size = 0 + while true do + local entry = scanstring() + if entry then + size = size + 1 + list[size] = entry + else + break + end + end + if wrapped then + scanclose() + end + return list +end + +local function scanconditional() + local kw = scanword() + if kw == "true" then + return true + end + if kw == "false" then + return false + end + local c = scaninteger() + if c then + return c == 0 -- with a conditional 0=true + end + return nil +end + +local function scantable(t,data) + if not data then + data = { } + end + if t then + local wrapped = scanopen() + while true do + local key = scanword(true) + if key then + local get = t[key] + if get then + data[key] = get() + else + -- catch all we can get + end + else + break + end + end + if wrapped then + scanclose() + end + end + return data +end + +function tokens.constant(s) + if type(s) == "string" then + return "'" .. s .. "'" + else + return s + end +end + +scanners.list = scanlist +scanners.table = scantable +scanners.conditional = scanconditional + +function scanners.whd() + local width, height, depth + while true do + if scankeyword("width") then + width = scandimen() + elseif scankeyword("height") then + height = scandimen() + elseif scankeyword("depth") then + depth = scandimen() + else + break + end + end + if width or height or depth then + return width or 0, height or 0, depth or 0 + else + -- we inherit + end +end + +local l = utf.byte("[") +local r = utf.byte("]") + +local function scanbracketed() + local s = scandelimited(l, r) + if s then + return s + else + local readstate = status.getreadstate() + report_scan("missing argument in line %i of %a", readstate.linenumber, readstate.filename) + return "" + end +end + +local function scanoptional() + return scandelimited(l, r) or "" +end + +local function scanbracketedasis() + return scandelimited(l, r, false) +end + +local function scanargumentasis() + return scanargument(false) +end + +scanners.bracketed = scanbracketed +scanners.optional = scanoptional +scanners.bracketedasis = scanbracketedasis +scanners.argumentasis = scanargumentasis + +local shortcuts = { + tokens = tokens, + bits = tokenbits, + open = open, + close = close, + scanners = scanners, + scanstring = scanstring, + scanargument = scanargument, + scanverbatim = scanverbatim, + scantokenlist = scantokenlist, + scantoks = scantoks, + scaninteger = scaninteger, + scancardinal = scancardinal, + scannumber = scannumber, + scantable = scantable, -- not directly useable + scankeyword = scankeyword, + scankeywordcs = scankeywordcs, + scanword = scanword, + scanletters = scanletters, + -- scankey = scankey, + scancode = scancode, + scanboolean = scanboolean, + scanglue = scanglue, -- list + scangluespec = scangluespec, + scangluevalues = scangluevalues, + scandimen = scandimen, + scandimension = scandimen, + scanbox = scanners.box, + scanhbox = scanners.hbox, + scanvbox = scanners.vbox, + scanvtop = scanners.vtop, + scanconditional = scanconditional, + scanopen = scanopen, + scanclose = scanclose, + scanlist = scanlist, + scancsname = scancsname, + scandelimited = scandelimited, -- not directly useable + scanbracketed = scanbracketed, + scanoptional = scanoptional, + scanbracketedasis = scanbracketedasis, + scanargumentasis = scanargumentasis, + scanintegerargument = scanintegerargument, + scandimenargument = scandimenargument, + todimen = todimen, + tonumber = tonumber, + tostring = tostring, + toboolean = toboolean, + inspect = inspect, + report = report_scan, +} + +tokens.shortcuts = shortcuts + +local load = load +local dump = string.dump + +local function loadstripped(code) + return load(code,nil,nil,shortcuts) + -- return load(dump(load(code),true),nil,nil,shortcuts) +end + +tokens.converters = { + tonumber = "tonumber", + tostring = "tostring", + toboolean = "toboolean", + todimen = "todimen", + toglue = "todimen", +} + +-- We could just pickup a keyword but then we really need to make sure that no number +-- follows it when that is the assignment and adding an optional = defeats the gain +-- in speed. Currently we have sources with no spaces (\startcontextdefinitioncode +-- ...) so it fails there. +-- +-- Another drawback is that we then need to use { } instead of ending with \relax (as +-- we can do now) but that is no big deal. It's just that I then need to check the TeX +-- end. More pain than gain and a bit risky too. Using scanletters works better, but +-- the gain is only some 10 percent but if we don't have keywords with numbers it might +-- make sense in the end, some day. + +local f_if = formatters[ " if scankeywordcs('%s') then data['%s'] = scan%s()"] +local f_elseif = formatters[" elseif scankeywordcs('%s') then data['%s'] = scan%s()"] + +----- f_if_x = formatters[ " if not data['%s'] and scankeywordcs('%s') then data['%s'] = scan%s()"] +----- f_elseif_x = formatters[" elseif not data['%s'] and scankeywordcs('%s') then data['%s'] = scan%s()"] + +----- f_if = formatters[" local key = scanletters() if key == '' then break elseif key == '%s' then data['%s'] = scan%s()"] +----- f_elseif = formatters[" elseif key == '%s' then data['%s'] = scan%s()"] + +local f_local = formatters["local scan%s = scanners.%s"] +local f_scan = formatters["scan%s()"] +local f_shortcut = formatters["local %s = scanners.converters.%s"] + +local f_if_c = formatters[ " if scankeywordcs('%s') then data['%s'] = %s(scan%s())"] +local f_elseif_c = formatters[" elseif scankeywordcs('%s') then data['%s'] = %s(scan%s())"] +local f_scan_c = formatters["%s(scan%s())"] + +-- see above + +----- f_if_c = formatters[" local key = scanletters() if key == '' then break elseif key == '%s' then data['%s'] = %s(scan%s())"] +----- f_elseif_c = formatters[" elseif k == '%s' then data['%s'] = %s(scan%s())"] + +local f_any = formatters[" else local key = scanword(true) if key then data[key] = scan%s() else break end end"] +local f_any_c = formatters[" else local key = scanword(true) if key then data[key] = %s(scan%s()) else break end end"] +local s_done = " else break end" + +local f_any_all = formatters[" local key = scanword(true) if key then data[key] = scan%s() else break end"] +local f_any_all_c= formatters[" local key = scanword(true) if key then data[key] = %s(scan%s()) else break end"] + +local f_table = formatters["%\nt\nreturn function()\n local data = { }\n%s\n return %s\nend\n"] +local f_sequence = formatters["%\nt\n%\nt\n%\nt\nreturn function()\n return %s\nend\n"] +local f_singular = formatters["%\nt\n%\nt\n\nreturn function(%s)\n return %s\nend\n"] +local f_simple = formatters["%\nt\nreturn function()\n return %s\nend\n"] +local f_string = formatters["%q"] +local f_action_f = formatters["action%s(%s)"] +local f_action_s = formatters["local action%s = tokens._action[%s]"] +local f_nested = formatters["local function scan%s()\n local data = { }\n%s\n return data\nend\n"] + +local f_check = formatters[ [[ + local wrapped = scanopen() + while true do + ]] .. "%\nt\n" .. [[ + %s + end + if wrapped then + scanclose() + end +]] ] + +-- using these shortcuts saves temporary small tables (okay, it looks uglier) + +local presets = { + ["1 string" ] = { "string" }, + ["2 strings"] = { "string", "string" }, + ["3 strings"] = { "string", "string", "string" }, + ["4 strings"] = { "string", "string", "string", "string" }, + ["5 strings"] = { "string", "string", "string", "string", "string" }, + ["6 strings"] = { "string", "string", "string", "string", "string", "string" }, + ["7 strings"] = { "string", "string", "string", "string", "string", "string", "string" }, + ["8 strings"] = { "string", "string", "string", "string", "string", "string", "string", "string" }, + + ["1 argument" ] = { "argument" }, + ["2 arguments"] = { "argument", "argument" }, + ["3 arguments"] = { "argument", "argument", "argument" }, + ["4 arguments"] = { "argument", "argument", "argument", "argument" }, +} + +tokens.presets = presets + +function tokens.compile(specification) + local f = { } + local n = 0 + local c = { } + local t = specification.arguments or specification + local a = specification.actions or nil + if type(a) == "function" then + a = { a } + end + local code + local args + local function compile(t,nested) + local done = s_done + local r = { } + local m = 0 + for i=1,#t do + local ti = t[i] + if ti == "*" and i == 1 then + done = f_any_all("string") + else + local t1 = ti[1] + local t2 = ti[2] or "string" + if type(t2) == "table" then + n = n + 1 + f[n] = compile(t2,n) + t2 = n + end + local t3 = ti[3] + if type(t3) == "function" then + -- todo: also create shortcut + elseif t3 then + c[t3] = f_shortcut(t3,t3) + if t1 == "*" then + if i == 1 then + done = f_any_all_c(t3,t2) + break + else + done = f_any_c(t3,t2) + end + else + m = m + 1 + r[m] = (m > 1 and f_elseif_c or f_if_c)(t1,t1,t3,t2) + end + else + if t1 == "*" then + if i == 1 then + done = f_any_all(t2) + break + else + done = f_any(t2) + end + else + m = m + 1 + r[m] = (m > 1 and f_elseif or f_if )(t1,t1,t2) + -- r[m] = (m > 1 and f_elseif_x or f_if_x)(t1,t1,t1,t2) + end + end + end + end + local c = f_check(r,done) + if nested then + return f_nested(nested,c) + else + return c + end + end + local p = t and presets[t] -- already done in implement + if p then + t = p + end + local tt = type(t) + if tt == "string" then + if a then + local s = lpegmatch(p_unquoted,t) + if s and t ~= s then + code = t + else + code = f_scan(t) + end + tokens._action = a + for i=1,#a do + code = f_action_f(i,code) + n = n + 1 + f[n] = f_action_s(i,i) + end + code = f_simple(f,code) + else + return scanners[t] + end + elseif tt ~= "table" then + return + elseif #t == 1 then + local ti = t[1] + if type(ti) == "table" then + ti = compile(ti) + code = "data" + if a then + tokens._action = a + for i=1,#a do + code = f_action_f(i,code) + n = n + 1 + f[n] = f_action_s(i,i) + end + end + code = f_table(f,ti,code) + elseif a then + code = f_scan(ti) + tokens._action = a + for i=1,#a do + code = f_action_f(i,code) + n = n + 1 + f[n] = f_action_s(i,i) + end + code = f_simple(f,code) + else + return scanners[ti] + end + elseif #t == 0 then + if specification.usage == "value" then + code = "b" + args = "_,b" + else + code = "" + args = "" + end + if a then + tokens._action = a + for i=1,#a do + code = f_action_f(i,code) + n = n + 1 + f[n] = f_action_s(i,i) + end + end + code = f_singular(c,f,args,code) + else + local r = { } + local p = { } + local m = 0 + for i=1,#t do + local ti = t[i] + local tt = type(ti) + if tt == "table" then + if ti[1] == "_constant_" then + local v = ti[2] + if type(v) == "string" then + r[i] = f_string(v) + else + r[i] = tostring(v) + end + else + m = m + 1 + p[m] = compile(ti,100+m) + r[i] = f_scan(100+m) + end + elseif tt == "number" then + r[i] = tostring(ti) + elseif tt == "boolean" then + r[i] = tostring(ti) + else + local s = lpegmatch(p_unquoted,ti) + if s and ti ~= s then + r[i] = ti -- a string, given as "'foo'" or '"foo"' + elseif scanners[ti] then + r[i] = f_scan(ti) + else + report_compile("unknown scanner %a",ti) + r[i] = ti + end + end + end + code = concat(r,",") + if a then + tokens._action = a + for i=1,#a do + code = f_action_f(i,code) + n = n + 1 + f[n] = f_action_s(i,i) + end + end + code = f_sequence(c,f,p,code) + end + if not code then + return + end + if trace_compile then + report_compile("code: %s",code) + end + local code, message = loadstripped(code) + if code then + code = code() -- sets action + else + report_compile("error in code: %s",code) + report_compile("error message: %s",message) + end + if a then + tokens._action = nil + end + if code then + return code + end +end + +-- local fetch = tokens.compile { +-- "string", +-- "string", +-- { +-- { "data", "string" }, +-- { "tab", "string" }, +-- { "method", "string" }, +-- { "foo", { +-- { "method", "integer" }, +-- { "compact", "number" }, +-- { "nature" }, +-- { "*" }, -- any key +-- } }, +-- { "compact", "string", "tonumber" }, +-- { "nature", "boolean" }, +-- { "escape", "string" }, +-- { "escape" }, +-- }, +-- "boolean", +-- } +-- +-- os.exit() |