diff options
author | Karl Berry <karl@freefriends.org> | 2020-03-05 00:48:46 +0000 |
---|---|---|
committer | Karl Berry <karl@freefriends.org> | 2020-03-05 00:48:46 +0000 |
commit | f210bce174e1f2f05305ab03e88e120a1cbfc4da (patch) | |
tree | 5c4e2ad096b5c745e859516ac3196fa0864292d5 /Master/texmf-dist/scripts | |
parent | 35fd641a3546acc0c62e0aa7f134888e36da30d4 (diff) |
context (from cont-tmf.zip of Feb 17 16:00, size 116339406)
git-svn-id: svn://tug.org/texlive/trunk@54086 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/scripts')
24 files changed, 11711 insertions, 6321 deletions
diff --git a/Master/texmf-dist/scripts/context/lua/mtx-base.lua b/Master/texmf-dist/scripts/context/lua/mtx-base.lua index bcc9aeb602b..eab82c9005e 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-base.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-base.lua @@ -76,14 +76,6 @@ elseif environment.arguments["find-path"] then resolvers.load() local path = resolvers.findpath(environment.files[1], fileformat) print(path) -- quite basic, wil become function in logs -elseif environment.arguments["run"] then - resolvers.load("nofiles") -- ! no need for loading databases - trackers.enable("resolvers.locating") - environment.run_format(environment.files[1] or "",environment.files[2] or "",environment.files[3] or "") -elseif environment.arguments["fmt"] then - resolvers.load("nofiles") -- ! no need for loading databases - trackers.enable("resolvers.locating") - environment.run_format(environment.arguments["fmt"], environment.files[1] or "",environment.files[2] or "") elseif environment.arguments["expand-braces"] then resolvers.load("nofiles") resolvers.dowithfilesandreport(resolvers.expandbraces, environment.files) diff --git a/Master/texmf-dist/scripts/context/lua/mtx-cache.lua b/Master/texmf-dist/scripts/context/lua/mtx-cache.lua index 56d3df188d2..4f378ff0a69 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-cache.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-cache.lua @@ -12,12 +12,12 @@ local helpinfo = [[ <metadata> <entry name="name">mtx-cache</entry> <entry name="detail">ConTeXt & MetaTeX Cache Management</entry> - <entry name="version">0.10</entry> + <entry name="version">1.01</entry> </metadata> <flags> <category name="basic"> <subcategory> - <flag name="purge"><short>remove not used files</short></flag> + <flag name="make"><short>generate databases and formats</short></flag> <flag name="erase"><short>completely remove cache</short></flag> <flag name="list"><short>show cache</short></flag> </subcategory> @@ -48,97 +48,86 @@ scripts.cache = scripts.cache or { } local function collect(path) local all = dir.glob(path .. "/**/*") - local tmas, tmcs, rest = { }, { }, { } + local ext = table.setmetatableindex("table") for i=1,#all do local name = all[i] - local suffix = filesuffix(name) - if suffix == "tma" then - tmas[#tmas+1] = name - elseif suffix == "tmc" then - tmcs[#tmcs+1] = name - else - rest[#rest+1] = name - end + local list = ext[filesuffix(name)] + list[#list+1] = name end - return tmas, tmcs, rest, all + return ext end -local function list(banner,path,tmas,tmcs,rest) +local function list(banner,path,ext) + local total = 0 report("%s: %s",banner,path) report() - report("tma : %4i",#tmas) - report("tmc : %4i",#tmcs) - report("rest : %4i",#rest) - report("total : %4i",#tmas+#tmcs+#rest) + for k, v in table.sortedhash(ext) do + total = total + #v + report("%-6s : %4i",k,#v) + end + report() + report("total : %4i",total) report() end -local function purge(banner,path,list,all) +local function erase(banner,path,list) report("%s: %s",banner,path) report() - local fonts = environment.argument("fonts") - local n = 0 - for i=1,#list do - local filename = list[i] - if find(filename,"luatex%-cache") then -- safeguard - if fonts and not find(filename,"fonts") then - -- skip - elseif all then + for ext, list in table.sortedhash(list) do + local gone = 0 + local kept = 0 + for i=1,#list do + local filename = list[i] + if find(filename,"luatex%-cache") then remove(filename) - n = n + 1 - elseif not fonts or find(filename,"fonts") then - local suffix = filesuffix(filename) - if suffix == "tma" then - local checkname = replacesuffix(filename,"tma","tmc") - if isfile(checkname) then - remove(filename) - n = n + 1 - end + if isfile(filename) then + kept = kept + 1 + else + gone = gone + 1 end end end + report("%-6s : %4i gone, %4i kept",ext,gone,kept) end - report("removed tma files : %i",n) report() - return n end -function scripts.cache.purge() - local writable = caches.getwritablepath() - local tmas, tmcs, rest = collect(writable) - list("writable path",writable,tmas,tmcs,rest) - purge("writable path",writable,tmas) - list("writable path",writable,tmas,tmcs,rest) +function scripts.cache.make() + os.execute("mtxrun --generate") + os.execute("context --make") + os.execute("mtxrun --script font --reload") end function scripts.cache.erase() local writable = caches.getwritablepath() - local tmas, tmcs, rest, all = collect(writable) - list("writable path",writable,tmas,tmcs,rest) - purge("writable path",writable,all,true) - list("writable path",writable,tmas,tmcs,rest) + local groups = collect(writable) + list("writable path",writable,groups) + erase("writable path",writable,groups) + if environment.argument("make") then + scripts.cache.make() + end end function scripts.cache.list() local readables = caches.getreadablepaths() - local writable = caches.getwritablepath() - local tmas, tmcs, rest = collect(writable) - list("writable path",writable,tmas,tmcs,rest) + local writable = caches.getwritablepath() + local groups = collect(writable) + list("writable path",writable,groups) for i=1,#readables do local readable = readables[i] if readable ~= writable then - local tmas, tmcs = collect(readable) - list("readable path",readable,tmas,tmcs,rest) + local groups = collect(readable) + list("readable path",readable,groups) end end end -if environment.argument("purge") then - scripts.cache.purge() -elseif environment.argument("erase") then +if environment.argument("erase") then scripts.cache.erase() elseif environment.argument("list") then scripts.cache.list() +elseif environment.argument("make") then + scripts.cache.make() elseif environment.argument("exporthelp") then application.export(environment.argument("exporthelp"),environment.files[1]) else diff --git a/Master/texmf-dist/scripts/context/lua/mtx-check.lua b/Master/texmf-dist/scripts/context/lua/mtx-check.lua index 16b9cb64a82..90f358cce68 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-check.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-check.lua @@ -184,6 +184,9 @@ local grammars = { tex = contextgrammar, mkii = contextgrammar, mkiv = contextgrammar, + mkvi = contextgrammar, + mkil = contextgrammar, + mkli = contextgrammar, } function validator.check(str,filetype) diff --git a/Master/texmf-dist/scripts/context/lua/mtx-context.lua b/Master/texmf-dist/scripts/context/lua/mtx-context.lua index 49d40dda8c4..4e8c364d57f 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-context.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-context.lua @@ -12,7 +12,7 @@ if not modules then modules = { } end modules['mtx-context'] = { local type, next, tostring, tonumber = type, next, tostring, tonumber local format, gmatch, match, gsub, find = string.format, string.gmatch, string.match, string.gsub, string.find local quote, validstring = string.quote, string.valid -local sort, concat, insert, sortedhash = table.sort, table.concat, table.insert, table.sortedhash +local sort, concat, insert, sortedhash, tohash = table.sort, table.concat, table.insert, table.sortedhash, table.tohash local settings_to_array = utilities.parsers.settings_to_array local appendtable = table.append local lpegpatterns, lpegmatch, Cs, P = lpeg.patterns, lpeg.match, lpeg.Cs, lpeg.P @@ -34,7 +34,7 @@ local formatters = string.formatters local application = logs.application { name = "mtx-context", - banner = "ConTeXt Process Management 1.02", + banner = "ConTeXt Process Management 1.03", -- helpinfo = helpinfo, -- table with { category_a = text_1, category_b = text_2 } or helpstring or xml_blob helpinfo = "mtx-context.xml", } @@ -209,7 +209,11 @@ local persistent_runfiles = { } local special_runfiles = { - "%-mpgraph", "%-mprun", "%-temp%-" + "%-mpgraph", "%-mprun", "%-temp%-", +} + +local extra_runfiles = { + "^l_m_t_x_.-%.pdf$", } local function purge_file(dfile,cfile) @@ -514,35 +518,7 @@ local function result_save_keep(oldbase,newbase) end end --- executing luatex - -local function flags_to_string(flags,prefix) -- context flags get prepended by c: - local t = { } - for k, v in table.sortedhash(flags) do - if prefix then - k = format("c:%s",k) - end - if not v or v == "" or v == '""' then - -- no need to flag false - elseif v == true then - t[#t+1] = format('--%s',k) - elseif type(v) == "string" then - t[#t+1] = format('--%s=%s',k,quote(v)) - else - t[#t+1] = format('--%s=%s',k,tostring(v)) - end - end - return concat(t," ") -end - -local function luatex_command(l_flags,c_flags,filename,engine) - return format('%s %s %s "%s"', - engine or (status and status.luatex_engine) or "luatex", - flags_to_string(l_flags), - flags_to_string(c_flags,true), - filename - ) -end +-- use mtx-plain instead local plain_formats = { ["plain"] = "plain", @@ -563,10 +539,10 @@ local function run_plain(plainformat,filename) local pdfview = getargument("autopdf") or getargument("closepdf") if pdfview then pdf_close(resultname,pdfview) - os.execute(command) + os.execute(command) -- maybe also a proper runner pdf_open(resultname,pdfview) else - os.execute(command) + os.execute(command) -- maybe also a proper runner end end end @@ -599,8 +575,32 @@ local function run_texexec(filename,a_purge,a_purgeall) end end +-- executing luatex + +local function flags_to_string(flags,prefix) + -- context flags get prepended by c: ... this will move to the sbx module + local t = { } + for k, v in table.sortedhash(flags) do + if prefix then + k = format("c:%s",k) + end + if not v or v == "" or v == '""' then + -- no need to flag false + elseif v == true then + t[#t+1] = format('--%s',k) + elseif type(v) == "string" then + t[#t+1] = format('--%s=%s',k,quote(v)) + else + t[#t+1] = format('--%s=%s',k,tostring(v)) + end + end + return concat(t," ") +end + function scripts.context.run(ctxdata,filename) -- + local verbose = false + -- local a_nofile = getargument("nofile") local a_engine = getargument("engine") -- @@ -627,9 +627,10 @@ function scripts.context.run(ctxdata,filename) return end -- - local interface = validstring(getargument("interface")) or "en" + local interface = validstring(getargument("interface")) or "en" local formatname = formatofinterface[interface] or "cont-en" - local formatfile, scriptfile = resolvers.locateformat(formatname) -- regular engine ! + local formatfile, + scriptfile = resolvers.locateformat(formatname) -- regular engine ! if not formatfile or not scriptfile then report("warning: no format found, forcing remake (commandline driven)") scripts.context.make(formatname) @@ -667,17 +668,6 @@ function scripts.context.run(ctxdata,filename) local a_nodates = getargument("nodates") local a_trailerid = getargument("trailerid") local a_nocompression = getargument("nocompression") - - -- the following flag is not officially supported because i cannot forsee - -- side effects (so no bug reports please) .. we provide --sandbox that - -- does similar things but tries to ensure that context works as expected - - -- local a_safer = getargument("safer") - -- - -- if a_safer then - -- report("warning: using the luatex safer options, processing is not guaranteed") - -- end - -- a_batchmode = (a_batchmode and "batchmode") or (a_nonstopmode and "nonstopmode") or (a_scrollmode and "scrollmode") or nil -- @@ -820,8 +810,8 @@ function scripts.context.run(ctxdata,filename) -- ["safer"] = a_safer, -- better use --sandbox -- ["no-mktex"] = true, -- ["file-line-error-style"] = true, - ["fmt"] = formatfile, - ["lua"] = scriptfile, +-- ["fmt"] = formatfile, +-- ["lua"] = scriptfile, ["jobname"] = jobname, ["jithash"] = a_jithash, } @@ -868,15 +858,17 @@ function scripts.context.run(ctxdata,filename) c_flags.noarrange = a_noarrange or a_arrange or nil c_flags.profile = a_profile and (tonumber(a_profile) or 0) or nil -- - local command = luatex_command(l_flags,c_flags,mainfile,a_engine) - -- - report("run %s: %s",currentrun,command) print("") -- cleaner, else continuation on same line --- local returncode, errorstring = os.spawn(command) - local returncode = os.execute(command) + local returncode = environment.run_format( + formatfile, + scriptfile, + mainfile, + flags_to_string(l_flags), + flags_to_string(c_flags,true), + verbose + ) -- todo: remake format when no proper format is found if not returncode then --- report("fatal error: no return code, message: %s",errorstring or "?") report("fatal error: no return code") if resultname then result_save_error(oldbase,newbase) @@ -908,6 +900,15 @@ function scripts.context.run(ctxdata,filename) -- end -- + if environment.arguments["ansilog"] then + local logfile = file.replacesuffix(jobname,"log") + local logdata = io.loaddata(logfile) or "" + if logdata ~= "" then + io.savedata(logfile,(gsub(logdata,"%[.-m",""))) + end + end + -- + -- -- this will go away after we update luatex -- local syncctx = fileaddsuffix(jobname,"syncctx") @@ -922,10 +923,17 @@ function scripts.context.run(ctxdata,filename) c_flags.currentrun = c_flags.currentrun + 1 c_flags.noarrange = nil -- - local command = luatex_command(l_flags,c_flags,mainfile,a_engine) - -- report("arrange run: %s",command) - local returncode, errorstring = os.spawn(command) + -- + local returncode = environment.run_format( + formatfile, + scriptfile, + mainfile, + flags_to_string(l_flags), + flags_to_string(c_flags,true), + verbose + ) + -- if not returncode then report("fatal error: no return code, message: %s",errorstring or "?") os.exit(1) @@ -985,7 +993,7 @@ function scripts.context.run(ctxdata,filename) report() report("making epub file: ",command) report() - os.execute(command) + os.execute(command) -- todo: also a runner end -- if a_timing then @@ -1047,20 +1055,24 @@ function scripts.context.pipe() -- still used? io.savedata(filename,"\\relax") report("entering scrollmode using '%s' with optionfile, end job with \\end",filename) end - local command = luatex_command(l_flags,c_flags,filename) - os.spawn(command) + local returncode = environment.run_format( + formatfile, + scriptfile, + filename, + flags_to_string(l_flags), + flags_to_string(c_flags,true), + verbose + ) if getargument("purge") then scripts.context.purge_job(filename) elseif getargument("purgeall") then scripts.context.purge_job(filename,true) removefile(filename) end + elseif formatname then + report("error, no format found with name: %s, aborting",formatname) else - if formatname then - report("error, no format found with name: %s, aborting",formatname) - else - report("error, no format found (provide formatname or interface)") - end + report("error, no format found (provide formatname or interface)") end end @@ -1231,7 +1243,7 @@ function scripts.context.autoctx() if chunk then ctxname = match(chunk,"<%?context%-directive%s+job%s+ctxfile%s+([^ ]-)%s*?>") end - elseif suffix == "tex" or suffix == "mkiv" then + elseif suffix == "tex" or suffix == "mkiv" or suffix == "mkxl" then local analysis = preamble_analyze(firstfile) ctxname = analysis.ctxfile or analysis.ctx end @@ -1245,66 +1257,27 @@ function scripts.context.autoctx() scripts.context.run(ctxdata) end --- no longer ok as mlib-run misses something: - --- local template = [[ --- \starttext --- \directMPgraphic{%s}{input "%s"} --- \stoptext --- ]] --- --- local loaded = false --- --- function scripts.context.metapost() --- local filename = environment.filenames[1] or "" --- if not loaded then --- dofile(resolvers.findfile("mlib-run.lua")) --- loaded = true --- commands = commands or { } --- commands.writestatus = report -- no longer needed --- end --- local formatname = getargument("format") or "metafun" --- if formatname == "" or type(formatname) == "boolean" then --- formatname = "metafun" --- end --- if getargument("pdf") then --- local basename = removesuffix(filename) --- local resultname = getargument("result") or basename --- local jobname = "mtx-context-metapost" --- local tempname = fileaddsuffix(jobname,"tex") --- io.savedata(tempname,format(template,"metafun",filename)) --- environment.filenames[1] = tempname --- setargument("result",resultname) --- setargument("once",true) --- scripts.context.run() --- scripts.context.purge_job(jobname,true) --- scripts.context.purge_job(resultname,true) --- elseif getargument("svg") then --- metapost.directrun(formatname,filename,"svg") --- else --- metapost.directrun(formatname,filename,"mps") --- end --- end - --- -- - function scripts.context.version() - local name = resolvers.findfile("context.mkiv") - if name ~= "" then - report("main context file: %s",name) - local data = io.loaddata(name) - if data then - local version = match(data,"\\edef\\contextversion{(.-)}") - if version then - report("current version: %s",version) + local list = { "context.mkiv", "context.mkxl" } + for i=1,#list do + local base = list[i] + local name = resolvers.findfile(base) + if name ~= "" then + report("main context file: %s",name) + local data = io.loaddata(name) + if data then + local version = match(data,"\\edef\\contextversion{(.-)}") + if version then + report("current version: %s",version) + else + report("context version: unknown, no timestamp found") + end else - report("context version: unknown, no timestamp found") + report("context version: unknown, load error") end else - report("context version: unknown, load error") + report("main context file: unknown, %a not found",base) end - else - report("main context file: unknown, 'context.mkiv' not found") end end @@ -1340,15 +1313,15 @@ function scripts.context.purge_job(jobname,all,mkiitoo,fulljobname) end function scripts.context.purge(all,pattern,mkiitoo) - local all = all or getargument("all") - local pattern = getargument("pattern") or (pattern and (pattern.."*")) or "*.*" - local files = dir.glob(pattern) - local obsolete = table.tohash(obsolete_results) - local temporary = table.tohash(temporary_runfiles) - local synctex = table.tohash(synctex_runfiles) - local persistent = table.tohash(persistent_runfiles) - local generic = table.tohash(generic_files) - local deleted = { } + local all = all or getargument("all") + local pattern = getargument("pattern") or (pattern and (pattern.."*")) or "*.*" + local files = dir.glob(pattern) + local obsolete = tohash(obsolete_results) + local temporary = tohash(temporary_runfiles) + local synctex = tohash(synctex_runfiles) + local persistent = tohash(persistent_runfiles) + local generic = tohash(generic_files) + local deleted = { } for i=1,#files do local name = files[i] local suffix = filesuffix(name) @@ -1362,6 +1335,11 @@ function scripts.context.purge(all,pattern,mkiitoo) end end end + for i=1,#extra_runfiles do + if find(basename,extra_runfiles[i]) then + deleted[#deleted+1] = purge_file(name) + end + end end if #deleted > 0 then report("purged files: %s", concat(deleted,", ")) @@ -1422,17 +1400,19 @@ local function touchfiles(suffix,kind,path) end end +local tobetouched = tohash { "mkii", "mkiv", "mkvi", "mkxl", "mklx" } + function scripts.context.touch() if getargument("expert") then local touch = getargument("touch") local kind = getargument("kind") local path = getargument("basepath") - if touch == "mkii" or touch == "mkiv" or touch == "mkvi" then -- mkix mkxi + if tobetouched[touch] then -- mkix mkxi ctix ctxi touchfiles(touch,kind,path) else - touchfiles("mkii",kind,path) - touchfiles("mkiv",kind,path) - touchfiles("mkvi",kind,path) + for touch in sortedhash(tobetouched) do + touchfiles(touch,kind,path) + end end else report("touching needs --expert") @@ -1442,7 +1422,8 @@ end -- modules local labels = { "title", "comment", "status" } -local cards = { "*.mkvi", "*.mkiv", "*.mkxi", "*.mkix", "*.tex" } +local cards = { "*.mkiv", "*.mkvi", "*.mkix", "*.mkxi", "*.mkxl", "*.mklx", "*.tex" } +local valid = tohash { "mkiv", "mkvi", "mkix", "mkxi", "mkxl", "mklx", "tex" } function scripts.context.modules(pattern) local list = { } @@ -1468,7 +1449,7 @@ function scripts.context.modules(pattern) if not done[base] then done[base] = true local suffix = filesuffix(base) - if suffix == "tex" or suffix == "mkiv" or suffix == "mkvi" or suffix == "mkix" or suffix == "mkxi" then + if valid[suffix] then local prefix, rest = match(base,"^([xmst])%-(.*)") if prefix then v = resolvers.findfile(base) -- so that files on my dev path are seen @@ -1588,148 +1569,10 @@ function scripts.context.logcategories() scripts.context.run() end --- updating (often one will use mtx-update instead) - function scripts.context.timed(action) statistics.timed(action,true) end -local zipname = "cont-tmf.zip" -local mainzip = "http://www.pragma-ade.com/context/latest/" .. zipname -local validtrees = { "texmf-local", "texmf-context" } -local selfscripts = { "mtxrun.lua" } -- was: { "luatools.lua", "mtxrun.lua" } - -function zip.loaddata(zipfile,filename) -- should be in zip lib - local f = zipfile:open(filename) - if f then - local data = f:read("*a") - f:close() - return data - end - return nil -end - -function scripts.context.update() - local force = getargument("force") - local socket = require("socket") - local http = require("socket.http") - local basepath = resolvers.findfile("context.mkiv") or "" - if basepath == "" then - report("quiting, no 'context.mkiv' found") - return - end - local basetree = basepath.match(basepath,"^(.-)tex/context/base/.*context.mkiv$") or "" - if basetree == "" then - report("quiting, no proper tds structure (%s)",basepath) - return - end - local function is_okay(basetree) - for _, tree in next, validtrees do - local pattern = gsub(tree,"%-","%%-") - if find(basetree,pattern) then - return tree - end - end - return false - end - local okay = is_okay(basetree) - if not okay then - report("quiting, tree '%s' is protected",okay) - return - else - report("updating tree '%s'",okay) - end - if not lfs.chdir(basetree) then - report("quiting, unable to change to '%s'",okay) - return - end - report("fetching '%s'",mainzip) - local latest = http.request(mainzip) - if not latest then - report("context tree '%s' can be updated, use --force",okay) - return - end - io.savedata("cont-tmf.zip",latest) - if false then - -- variant 1 - os.execute("mtxrun --script unzip cont-tmf.zip") - else - -- variant 2 - local zipfile = zip.open(zipname) - if not zipfile then - report("quiting, unable to open '%s'",zipname) - return - end - local newfile = zip.loaddata(zipfile,"tex/context/base/mkiv/context.mkiv") - if not newfile then - report("quiting, unable to open '%s'","context.mkiv") - return - end - local oldfile = io.loaddata(resolvers.findfile("context.mkiv")) or "" - local function versiontonumber(what,str) - local version = match(str,"\\edef\\contextversion{(.-)}") or "" - local year, month, day, hour, minute = match(str,"\\edef\\contextversion{(%d+)%.(%d+)%.(%d+) *(%d+)%:(%d+)}") - if year and minute then - local time = os.time { year=year,month=month,day=day,hour=hour,minute=minute} - report("%s version: %s (%s)",what,version,time) - return time - else - report("%s version: %s (unknown)",what,version) - return nil - end - end - local oldversion = versiontonumber("old",oldfile) - local newversion = versiontonumber("new",newfile) - if not oldversion or not newversion then - report("quiting, version cannot be determined") - return - elseif oldversion == newversion then - report("quiting, your current version is up-to-date") - return - elseif oldversion > newversion then - report("quiting, your current version is newer") - return - end - for k in zipfile:files() do - local filename = k.filename - if find(filename,"/$") then - lfs.mkdir(filename) - else - local data = zip.loaddata(zipfile,filename) - if data then - if force then - io.savedata(filename,data) - end - report(filename) - end - end - end - for _, scriptname in next, selfscripts do - local oldscript = resolvers.findfile(scriptname) or "" - if oldscript ~= "" and is_okay(oldscript) then - local newscript = "./scripts/context/lua/" .. scriptname - local data = io.loaddata(newscript) or "" - if data ~= "" then - report("replacing script '%s' by '%s'",oldscript,newscript) - if force then - io.savedata(oldscript,data) - end - end - else - report("keeping script '%s'",oldscript) - end - end - if force then - scripts.context.make() - end - end - if force then - report("context tree '%s' has been updated",okay) - else - report("context tree '%s' can been updated (use --force)",okay) - end -end - -- getting it done if getargument("timedlog") then @@ -1753,6 +1596,17 @@ end do + local htmlerrorpage = getargument("htmlerrorpage") + if htmlerrorpage == "scite" then + directives.enable("system.showerror=scite") + elseif htmlerrorpage then + directives.enable("system.showerror") + end + +end + +do + local silent = getargument("silent") if type(silent) == "string" then directives.enable(format("logs.blocked={%s}",silent)) @@ -1786,15 +1640,11 @@ elseif getargument("generate") then scripts.context.timed(function() scripts.context.generate() end) elseif getargument("ctx") and not getargument("noctx") then scripts.context.timed(scripts.context.ctx) --- elseif getargument("mp") or getargument("metapost") then --- scripts.context.timed(scripts.context.metapost) elseif getargument("version") then application.identify() scripts.context.version() elseif getargument("touch") then scripts.context.touch() -elseif getargument("update") then - scripts.context.update() elseif getargument("expert") then application.help("expert", "special") elseif getargument("showmodules") or getargument("modules") then diff --git a/Master/texmf-dist/scripts/context/lua/mtx-context.xml b/Master/texmf-dist/scripts/context/lua/mtx-context.xml index 8c21eaa8c5c..0c7038d2680 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-context.xml +++ b/Master/texmf-dist/scripts/context/lua/mtx-context.xml @@ -4,7 +4,7 @@ <metadata> <entry name="name">mtx-context</entry> <entry name="detail">ConTeXt Process Management</entry> - <entry name="version">1.01</entry> + <entry name="version">1.03</entry> <entry name="comment">external helpinfo file</entry> </metadata> <flags> @@ -74,7 +74,10 @@ <short>strip Lua code (only meant for production where no errors are expected)</short> </flag> <flag name="errors" value="list"> - <short>show errors at the end of a run, quit when in list (also when <ref name="--silent"/>)</short> + <short>show errors at the end of a run, quit when in list (also when <ref name="silent"/>)</short> + </flag> + <flag name="htmlerrorpage"> + <short>generate html error page instead (optional: =scite)</short> </flag> <flag name="noconsole"> <short>disable logging to the console (logfile only)</short> @@ -131,12 +134,16 @@ <flag name="nonstopmode"> <short>run without stopping</short> </flag> + </subcategory> + <subcategory> <flag name="nosynctex"> <short>never initializes synctex (for production runs)</short> </flag> <flag name="synctex"> <short>run with synctex enabled (better use \setupsynctex[state=start]</short> </flag> + </subcategory> + <subcategory> <flag name="nodates"> <short>omit runtime dates in pdf file (optional value: a number (this 1970 offset time) or string "YYYY-MM-DD HH:MM")</short> </flag> @@ -146,7 +153,7 @@ <flag name="trailerid"> <short>alternative trailer id (or constant one)</short> </flag> - </subcategory> + </subcategory> <subcategory> <flag name="generate"> <short>generate file database etc. (as luatools does)</short> @@ -172,12 +179,12 @@ <flag name="touch"> <short>update context version number (also provide <ref name="expert"/>, optionally provide <ref name="basepath"/>)</short> </flag> - <flag name="nostatistics"> + <flag name="nostatistics"> <short>omit runtime statistics at the end of the run</short> </flag> - <flag name="update"> + <!-- flag name="update"> <short>update context from website (not to be confused with contextgarden)</short> - </flag> + </flag --> <flag name="profile"> <short>profile job (use: mtxrun <ref name="script"/> profile <ref name="analyze"/>)</short> </flag> @@ -225,6 +232,11 @@ <short>process file in a limited environment</short> </flag> </subcategory> + <subcategory> + <flag name="addbinarypath"> + <short>prepend the (found) binarypath to runners</short> + </flag> + </subcategory> </category> </flags> </application> diff --git a/Master/texmf-dist/scripts/context/lua/mtx-convert.lua b/Master/texmf-dist/scripts/context/lua/mtx-convert.lua index b3e20ea8769..61ce1504072 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-convert.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-convert.lua @@ -90,7 +90,7 @@ function converters.convertgraphic(kind,oldname,newname) local command = converters[kind](oldname,tmpname) report("command: %s",command) io.flush() - os.spawn(command) + os.execute(command) os.remove(newname) os.rename(tmpname,newname) if lfs.attributes(newname,"size") == 0 then diff --git a/Master/texmf-dist/scripts/context/lua/mtx-fonts.lua b/Master/texmf-dist/scripts/context/lua/mtx-fonts.lua index 2e50543c386..fcba696c43c 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-fonts.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-fonts.lua @@ -16,7 +16,7 @@ local lower = string.lower local concat = table.concat local write_nl = (logs and logs.writer) or (texio and texio.write_nl) or print -local otlversion = 3.106 +local otlversion = 3.111 local helpinfo = [[ <?xml version="1.0"?> @@ -268,7 +268,7 @@ local function showfeatures(tag,specification) report() indeed("instances : % t",instancenames) end - local features = fonts.helpers.getfeatures(specification.filename,not getargument("nosave")) + local features, tables = fonts.helpers.getfeatures(specification.filename,not getargument("nosave")) if features then for what, v in table.sortedhash(features) do local data = features[what] @@ -276,7 +276,7 @@ local function showfeatures(tag,specification) report() report("%s features:",what) report() - report("feature script languages") + report(" feature script languages") report() for f,ff in table.sortedhash(data) do local done = false @@ -288,7 +288,7 @@ local function showfeatures(tag,specification) else done = true end - report("% -8s % -8s % -8s",f,s,concat(table.sortedkeys(ss), " ")) -- todo: padd 4 + report(" % -8s % -8s % -8s",f,s,concat(table.sortedkeys(ss), " ")) -- todo: padd 4 end end end @@ -296,6 +296,24 @@ local function showfeatures(tag,specification) else report("no features") end + if tables then + tables = table.tohash(tables) + local methods = { + overlay = (tables.colr or tables.cpal) and { format = "cff/ttf", feature = "color:overlay" } or nil, + bitmap = (tables.cblc or tables.cbdt) and { format = "png", feature = "color:bitmap" } or nil, + outline = (tables.svg ) and { format = "svg", feature = "color:svg" } or nil, + } + if next(methods) then + report() + report("color features:") + report() + report(" method feature formats") + report() + for k, v in table.sortedhash(methods) do + report(" % -8s % -14s %s",k,v.feature,v.format) + end + end + end report() collectgarbage("collect") end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-grep.lua b/Master/texmf-dist/scripts/context/lua/mtx-grep.lua index 9a4237737cf..e4a2a8d2f07 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-grep.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-grep.lua @@ -173,15 +173,17 @@ function scripts.grep.find(pattern, files, offset) local globbed = dir.glob(files[i]) for i=1,#globbed do name = globbed[i] - local data = io.loaddata(name) - if data then - n, m, noffiles = 0, 0, noffiles + 1 - lpegmatch(capture,data) - if count and m > 0 then - nofmatches = nofmatches + m - nofmatchedfiles = nofmatchedfiles + 1 - write_nl(format("%5i %s",m,name)) - io.flush() + if not find(name,"/%.") then + local data = io.loaddata(name) + if data then + n, m, noffiles = 0, 0, noffiles + 1 + lpegmatch(capture,data) + if count and m > 0 then + nofmatches = nofmatches + m + nofmatchedfiles = nofmatchedfiles + 1 + write_nl(format("%5i %s",m,name)) + io.flush() + end end end end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-install.lua b/Master/texmf-dist/scripts/context/lua/mtx-install.lua index 275102e3789..b9b4103531d 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-install.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-install.lua @@ -26,6 +26,7 @@ local helpinfo = [[ <flag name="goodies" value="string"><short>extra binaries (like scite and texworks)</short></flag> <flag name="install"><short>install context</short></flag> <flag name="update"><short>update context</short></flag> + <flag name="erase"><short>wipe the cache</short></flag> <flag name="identify"><short>create list of files</short></flag> </subcategory> </category> @@ -90,34 +91,39 @@ local platforms = { -- ["linux-armhf"] = "linux-armhf", -- - ["freebsd"] = "freebsd", + ["openbsd"] = "openbsd6.6", + ["openbsd-i386"] = "openbsd6.6", + ["openbsd-amd64"] = "openbsd6.6-amd64", -- + ["freebsd"] = "freebsd", + ["freebsd-i386"] = "freebsd", ["freebsd-amd64"] = "freebsd-amd64", -- - ["kfreebsd"] = "kfreebsd-i386", - ["kfreebsd-i386"] = "kfreebsd-i386", - -- - ["kfreebsd-amd64"] = "kfreebsd-amd64", + -- ["kfreebsd"] = "kfreebsd-i386", + -- ["kfreebsd-i386"] = "kfreebsd-i386", + -- ["kfreebsd-amd64"] = "kfreebsd-amd64", -- - ["linux-ppc"] = "linux-ppc", - ["ppc"] = "linux-ppc", + -- ["linux-ppc"] = "linux-ppc", + -- ["ppc"] = "linux-ppc", -- - ["osx"] = "osx-intel", - ["macosx"] = "osx-intel", - ["osx-intel"] = "osx-intel", - ["osxintel"] = "osx-intel", + -- ["osx"] = "osx-intel", + -- ["macosx"] = "osx-intel", + -- ["osx-intel"] = "osx-intel", + -- ["osxintel"] = "osx-intel", -- - ["osx-ppc"] = "osx-ppc", - ["osx-powerpc"] = "osx-ppc", - ["osxppc"] = "osx-ppc", - ["osxpowerpc"] = "osx-ppc", + -- ["osx-ppc"] = "osx-ppc", + -- ["osx-powerpc"] = "osx-ppc", + -- ["osxppc"] = "osx-ppc", + -- ["osxpowerpc"] = "osx-ppc", -- + ["macosx"] = "osx-64", + ["osx"] = "osx-64", ["osx-64"] = "osx-64", -- - ["solaris-intel"] = "solaris-intel", + -- ["solaris-intel"] = "solaris-intel", -- - ["solaris-sparc"] = "solaris-sparc", - ["solaris"] = "solaris-sparc", + -- ["solaris-sparc"] = "solaris-sparc", + -- ["solaris"] = "solaris-sparc", -- ["unknown"] = "unknown", } @@ -127,6 +133,8 @@ function install.identify() -- We have to be in "...../tex" where subdirectories are prefixed with -- "texmf". We strip the "tex/texm*/" from the name in the list. + local hashdata = sha2 and sha2.HASH256 or md5.hex + local function collect(root,tree) local path = root .. "/" .. tree @@ -141,12 +149,12 @@ function install.identify() local total = 0 for i=1,#files do - local name = files[i] - local size = filesize(name) - local base = gsub(name,pattern,"") - local stamp = md5.hex(io.loaddata(name)) - details[i] = { base, size, stamp } - total = total + size + local name = files[i] + local size = filesize(name) + local base = gsub(name,pattern,"") + local stamp = hashdata(io.loaddata(name)) + details[i] = { base, size, stamp } + total = total + size end report("%-20s : %4i files, %3.0f MB",tree,#files,total/(1000*1000)) @@ -164,10 +172,30 @@ function install.identify() end end + savetable("./tex/status.tma",{ + name = "context", + version = "lmtx", + date = os.date("%Y-%m-%d"), + }) + +end + +local function disclaimer() + report("ConTeXt LMTX with LuaMetaTeX is still experimental and when you get a crash this") + report("can be due to a mismatch between Lua bytecode and the engine. In that case you can") + report("try the following:") + report("") + report(" - wipe the texmf-cache directory") + report(" - run: mtxrun --generate") + report(" - run: context --make") + report("") + report("When that doesn't solve the problem, ask on the mailing list (ntg-context@ntg.nl).") end function install.update() + local hashdata = sha2 and sha2.HASH256 or md5.hex + local function validdir(d) local ok = isdir(d) if not ok then @@ -177,7 +205,7 @@ function install.update() return ok end - local function download(what,url,target,total,done) + local function download(what,url,target,total,done,oldhash) local data = fetch(url .. "/" .. target) if data then if total and done then @@ -185,11 +213,15 @@ function install.update() else report("%-8s : %8i : %s",what,#data,target) end - if validdir(dirname(target)) then - savedata(target,data) + if oldhash and oldhash ~= hashdata(data) then + return "different hash value" + elseif not validdir(dirname(target)) then + return "wrong target directory" else - -- message + savedata(target,data) end + else + return "unable to download" end end @@ -338,7 +370,12 @@ function install.update() if action then local size = newhash[2] total = total + size - todo[#todo+1] = { action, target, size } + todo[#todo+1] = { + action = action, + target = target, + size = size, + hash = newhash[3], + } end else report("skipping %s",target) @@ -349,8 +386,22 @@ function install.update() for i=1,count do local entry = todo[i] - download(entry[1],url,entry[2],total,done) - done = done + entry[3] + for i=1,5 do + local target = entry.target + local message = download(entry.action,url,target,total,done,entry.hash) + if message then + if i == 5 then + report("%s, try again later: %s",target) + os.exit() + else + report("%s, trying again: %s",target) + os.sleep(2) + end + else + break + end + end + done = done + entry.size end for oldname, oldhash in sortedhash(hold) do @@ -371,39 +422,57 @@ function install.update() local targetroot = dir.current() - local server = environment.arguments.server or "" - local port = environment.arguments.port or "" - local instance = environment.arguments.instance or "" + local server = environment.arguments.server or "" + local instance = environment.arguments.instance or "" + local osplatform = environment.arguments.platform or nil + local platform = platforms[osplatform or os.platform or ""] - if server == "" then - report("provide server") - return + if (platform == "unknown" or platform == "" or not platform) and osplatform then + -- catches openbsdN.M kind of specifications + platform = osplatform + elseif not osplatform then + osplatform = platform end - local url = "http://" .. server - - if port ~= "" then - url = url .. ":" .. port + if server == "" then + server = "lmtx.contextgarden.net,lmtx.pragma-ade.com,lmtx.pragma-ade.nl,dmz.pragma-ade.nl" + end + if instance == "" then + instance = "install-lmtx" + end + if not platform then + report("unknown platform") + return end - url = url .. "/" + local list = utilities.parsers.settings_to_array(server) + local server = false - if instance ~= "" then - url = url .. instance .. "/" + for i=1,#list do + local host = list[i] + local data, status, detail = fetch("http://" .. host .. "/" .. instance .. "/tex/status.tma") + if status == 200 and type(data) == "string" then + local t = loadstring(data) + if type(t) == "function" then + t = t() + end + if type(t) == "table" and t.name == "context" and t.version == "lmtx" then + server = host + break + end + end end - local osplatform = os.platform - local platform = platforms[osplatform] - - if not platform then - report("unknown platform") + if not server then + report("provide valid server and instance") return end + local url = "http://" .. server .. "/" .. instance .. "/" + local texmfplatform = "texmf-" .. platform report("server : %s",server) - report("port : %s",port == "" and 80 or "80") report("instance : %s",instance) report("platform : %s",osplatform) report("system : %s",ostype) @@ -495,8 +564,13 @@ function install.update() end run("%s --generate",mtxrunbin) + if environment.argument("erase") then + run("%s --script cache --erase",mtxrunbin) + run("%s --generate",mtxrunbin) + end run("%s --make en", contextbin) + -- in calling script: update mtxrun.exe and mtxrun.lua report("") @@ -504,6 +578,8 @@ function install.update() report("%-20s : %4i files with %9i bytes installed",unpack(status[i])) end report("") + disclaimer() + report("") report("update, done") end @@ -518,4 +594,6 @@ elseif environment.argument("exporthelp") then application.export(environment.argument("exporthelp"),environment.files[1]) else application.help() + report("") + disclaimer() end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-modules.lua b/Master/texmf-dist/scripts/context/lua/mtx-modules.lua index 78e1418fd98..60427c08674 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-modules.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-modules.lua @@ -22,8 +22,8 @@ local helpinfo = [[ <flags> <category name="basic"> <subcategory> - <flag name="convert"><short>convert source files (tex, mkii, mkiv, mp) to 'ted' files</short></flag> - <flag name="process"><short>process source files (tex, mkii, mkiv, mp) to 'pdf' files</short></flag> + <flag name="convert"><short>convert source files (tex, mkii, mkiv, mp, etc.) to 'ted' files</short></flag> + <flag name="process"><short>process source files (tex, mkii, mkiv, mp, etc.) to 'pdf' files</short></flag> <flag name="prep"><short>use original name with suffix 'prep' appended</short></flag> <flag name="direct"><short>use old method instead of extra</short></flag> </subcategory> @@ -163,7 +163,12 @@ local function source_to_ted(inpname,outname,filetype) return true end -local suffixes = table.tohash { 'tex', 'mkii', 'mkiv', 'mkvi', 'mp', 'mpii', 'mpiv' } +local suffixes = table.tohash { + "tex", + "mkii", + "mkiv", "mkvi", "mkil", "mkli", + "mp", "mpii", "mpiv", +} function scripts.modules.process(runtex) local processed = { } diff --git a/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua b/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua index 4eb5711178c..37843a6b604 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua @@ -133,6 +133,7 @@ scripts.patterns.list = { -- { "lo", "hyph-lo", "lao" }, { "lt", "hyph-lt", "lithuanian" }, { "lv", "hyph-lv", "latvian" }, + { "mk", "hyph-mk", "macedonian" }, { "ml", "hyph-ml", "malayalam" }, { "mn", "hyph-mn-cyrl", "mongolian, cyrillic script" }, -- { "mr", "hyph-mr", "..." }, @@ -163,8 +164,6 @@ scripts.patterns.list = { -- stripped down from lpeg example: -local utf = unicode.utf8 - function utf.check(str) return lpegmatch(lpegpatterns.validutf8,str) end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-pdf.lua b/Master/texmf-dist/scripts/context/lua/mtx-pdf.lua index 2e73fa841d3..fbb1a3995e5 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-pdf.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-pdf.lua @@ -90,6 +90,7 @@ function scripts.pdf.info(filename) report("%-17s > %s","title", info.Title or unset) report("%-17s > %s","creator", info.Creator or unset) report("%-17s > %s","producer", info.Producer or unset) + report("%-17s > %s","author", info.Author or unset) report("%-17s > %s","creation date", info.CreationDate or unset) report("%-17s > %s","modification date", info.ModDate or unset) diff --git a/Master/texmf-dist/scripts/context/lua/mtx-plain.lua b/Master/texmf-dist/scripts/context/lua/mtx-plain.lua index 72cc48f92ad..e6c790f7c33 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-plain.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-plain.lua @@ -6,11 +6,13 @@ if not modules then modules = { } end modules ['mtx-plain'] = { license = "see context related readme files" } --- future version will use the texmf-cache/generic/formats/<engine> path +-- Future version will use the texmf-cache/generic/formats/<engine> path -- instead because then we can use some more of the generic context -- initializers ... in that case we will also use the regular database -- instead of kpse here, just like with the font database code (as that --- one also works with kpse runtime) +-- one also works with kpse runtime). + +-- Maybe I have to update this one to use more recent ways to run programs. local format = string.format diff --git a/Master/texmf-dist/scripts/context/lua/mtx-scite.lua b/Master/texmf-dist/scripts/context/lua/mtx-scite.lua index ae8c673879f..96020de3dba 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-scite.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-scite.lua @@ -25,7 +25,7 @@ local helpinfo = [[ <subcategory> <flag name="words"><short>convert spell-*.txt into spell-*.lua</short></flag> <flag name="tree"><short>converts a tree into an html tree (--source --target --numbers)</short></flag> - <flag name="file"><short>converts a file into an html tree (--source --target --numbers --lexer)</short></flag> + <flag name="file"><short>converts a file into an html file (--source --target --numbers --lexer)</short></flag> </subcategory> </category> </flags> diff --git a/Master/texmf-dist/scripts/context/lua/mtx-unicode.lua b/Master/texmf-dist/scripts/context/lua/mtx-unicode.lua index fd65766462d..297807889e4 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-unicode.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-unicode.lua @@ -57,11 +57,11 @@ if not modules then modules = { } end modules ['mtx-unicode'] = { -- curl -o unicodedata.txt http://www.unicode.org/Public/UNIDATA/UnicodeData.txt -- curl -o unihan.zip http://www.unicode.org/Public/UNIDATA/Unihan.zip -- --- curl -o emoji-data.txt http://unicode.org/Public/emoji/11.0/emoji-data.txt --- curl -o emoji-sequences.txt http://unicode.org/Public/emoji/11.0/emoji-sequences.txt --- curl -o emoji-variation-sequences.txt http://unicode.org/Public/emoji/11.0/emoji-variation-sequences.txt --- curl -o emoji-zwj-sequences.txt http://unicode.org/Public/emoji/11.0/emoji-zwj-sequences.txt --- curl -o emoji-test.txt http://unicode.org/Public/emoji/11.0/emoji-test.txt +-- curl -o emoji-data.txt http://unicode.org/Public/emoji/12.0/emoji-data.txt +-- curl -o emoji-sequences.txt http://unicode.org/Public/emoji/12.0/emoji-sequences.txt +-- curl -o emoji-variation-sequences.txt http://unicode.org/Public/emoji/12.0/emoji-variation-sequences.txt +-- curl -o emoji-zwj-sequences.txt http://unicode.org/Public/emoji/12.0/emoji-zwj-sequences.txt +-- curl -o emoji-test.txt http://unicode.org/Public/emoji/12.0/emoji-test.txt -- -- todo: -- @@ -526,10 +526,21 @@ function scripts.unicode.load() end end +-- local variants_emoji={ +-- [0xFE0E]="text style", +-- [0xFE0F]="emoji style", +-- } +-- +-- local variants_forms={ +-- [0xFE00]="corner-justified form", +-- [0xFE01]="centered form", +-- } + function scripts.unicode.save(filename) if preamble then local data = table.serialize(characters.data,"characters.data", { hexify = true, noquotes = true }) data = gsub(data,"%{%s+%[0xFE0E%]=\"text style\",%s+%[0xFE0F%]=\"emoji style\",%s+%}","variants_emoji") + data = gsub(data,"%{%s+%[0xFE00%]=\"corner%-justified form\",%s+%[0xFE01%]=\"centered form\",%s+%}","variants_forms") io.savedata(filename,preamble .. data) end end @@ -688,9 +699,12 @@ do local hash = { } + local crap = lpeg.P("e") * lpeg.R("09","..","09")^1 * lpeg.P(" ")^1 + local replace = lpeg.replacer { - ["#"] = "hash", - ["*"] = "asterisk" + [crap] = "", + ["#"] = "hash", + ["*"] = "asterisk", } for i=1,#t do @@ -722,8 +736,10 @@ else scripts.unicode.extras() scripts.unicode.save("char-def-new.lua") scripts.unicode.emoji("char-emj-new.lua") + report("saved file %a","char-def-new.lua") + report("saved file %a (current 12.0, check for updates, see above!)","char-emj-new.lua") else report("nothing to do") end - report("stop working on %a, output char-def-new.lua\n",lfs.currentdir()) + report("stop working on %a\n",lfs.currentdir()) end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-unzip.lua b/Master/texmf-dist/scripts/context/lua/mtx-unzip.lua index 02d9676bcaa..0bc2193863b 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-unzip.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-unzip.lua @@ -8,7 +8,7 @@ if not modules then modules = { } end modules ['mtx-unzip'] = { -- maybe --pattern -local format = string.format +local format, find = string.format, string.find local helpinfo = [[ <?xml version="1.0"?> @@ -22,8 +22,7 @@ local helpinfo = [[ <category name="basic"> <subcategory> <flag name="list"><short>list files in archive</short></flag> - <flag name="junk"><short>flatten unzipped directory structure</short></flag> - <flag name="extract"><short>extract files</short></flag> + <flag name="extract"><short>extract files [--silent --steps]</short></flag> </subcategory> </category> </flags> @@ -41,92 +40,87 @@ local report = application.report scripts = scripts or { } scripts.unzipper = scripts.unzipper or { } -function scripts.unzipper.opened() +local function validfile() local filename = environment.files[1] if filename and filename ~= "" then filename = file.addsuffix(filename,'zip') - local zipfile = zip.open(filename) - if zipfile then - return zipfile + if lfs.isfile(filename) then + return filename + else + report("invalid zip file: %s",filename) end + else + report("no zip file") end - report("no zip file: %s",filename) return false end function scripts.unzipper.list() - local zipfile = scripts.unzipper.opened() - if zipfile then - local n = 0 - for k in zipfile:files() do - if #k.filename > n then n = #k.filename end - end - local files, paths, compressed, uncompressed = 0, 0, 0, 0 - local template_a = "%-"..n.."s" - local template_b = "%-"..n.."s % 9i % 9i" - local template_c = "\n%-"..n.."s % 9i % 9i" - for k in zipfile:files() do - if k.filename:find("/$") then - paths = paths + 1 - print(format(template_a, k.filename)) - else - files = files + 1 - local cs, us = k.compressed_size, k.uncompressed_size - if cs > compressed then - compressed = cs - end - if us > uncompressed then - uncompressed = us + local filename = validfile() + if filename then + local zipfile = utilities.zipfiles.open(filename) + if zipfile then + local list = utilities.zipfiles.list(zipfile) + if list then + local n = 0 + for i=1,#list do + local l = list[i] + if #l.filename > n then + n = #l.filename + end end - print(format(template_b,k.filename,cs,us)) + local files, paths, compressed, uncompressed = 0, 0, 0, 0 + local template_a = "%-" .. n .."s" + local template_b = "%-" .. n .."s % 9i % 9i" + local template_c = "\n%-" .. n .."s % 9i % 9i" + for i=1,#list do + local l = list[i] + local f = l.filename + if find(f,"/$") then + paths = paths + 1 + print(format(template_a, f)) + else + files = files + 1 + local cs = l.compressed + local us = l.uncompressed + if cs > compressed then + compressed = cs + end + if us > uncompressed then + uncompressed = us + end + print(format(template_b,f,cs,us)) + end + end -- check following pattern, n is not enough + print(format(template_c,files .. " files, " .. paths .. " directories",compressed,uncompressed)) end - end -- check following pattern, n is not enough - print(format(template_c,files .. " files, " .. paths .. " directories",compressed,uncompressed)) - end -end - -function zip.loaddata(zipfile,filename) - local f = zipfile:open(filename) - if f then - local data = f:read("*a") - f:close() - return data + utilities.zipfiles.close(zipfile) + else + report("invalid zip file: %s",filename) + end end - return nil end function scripts.unzipper.extract() - local zipfile = scripts.unzipper.opened() - if zipfile then - local junk = environment.arguments["j"] or environment.arguments["junk"] - for k in zipfile:files() do - local filename = k.filename - if filename:find("/$") then - if not junk then - lfs.mkdir(filename) - end - else - local data = zip.loaddata(zipfile,filename) - if data then - if junk then - filename = file.basename(filename) - end - io.savedata(filename,data) - print(filename) - end - end - end + local filename = validfile() + if validfile then + -- todo --junk + local silent = environment.arguments["silent"] + local steps = environment.arguments["steps"] + utilities.zipfiles.unzipdir { + zipname = filename, + path = ".", + verbose = not silent and (steps and "steps" or true), + } end end -if environment.arguments["h"] or environment.arguments["help"] then - application.help() -elseif environment.arguments["l"] or environment.arguments["list"] then - scripts.unzipper.list(zipfile) +if environment.arguments["list"] then + scripts.unzipper.list() +elseif environment.arguments["extract"] then + scripts.unzipper.extract() elseif environment.arguments["exporthelp"] then application.export(environment.arguments["exporthelp"],environment.files[1]) -elseif environment.files[1] then -- implicit --extract - scripts.unzipper.extract(zipfile) else application.help() end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-update.lua b/Master/texmf-dist/scripts/context/lua/mtx-update.lua index d9deb1b36dc..e03012901da 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-update.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-update.lua @@ -98,13 +98,6 @@ update.repositories = { "experimental" } --- more options than just these two are available (no idea why this is here) - -update.versions = { - "current", - "latest" -} - -- list of basic folders that are needed to make a functional distribution update.base = { @@ -164,19 +157,72 @@ update.goodies = { }, } +-- update.platforms = { +-- ["mswin"] = "mswin", +-- ["windows"] = "mswin", +-- ["win32"] = "mswin", +-- ["win"] = "mswin", +-- -- ["mswin"] = "win32", +-- -- ["windows"] = "win32", +-- -- ["win32"] = "win32", +-- -- ["win"] = "win32", +-- -- +-- -- ["mswin-64"] = "mswin-64", +-- -- ["windows-64"] = "mswin-64", +-- -- ["win64"] = "mswin-64", +-- ["mswin-64"] = "win64", +-- ["windows-64"] = "win64", +-- ["win64"] = "win64", +-- -- +-- ["linux"] = "linux", +-- ["linux-32"] = "linux", +-- ["linux32"] = "linux", +-- -- +-- ["linux-64"] = "linux-64", +-- ["linux64"] = "linux-64", +-- -- +-- ["linuxmusl-64"] = "linuxmusl-64", +-- -- +-- ["linux-armhf"] = "linux-armhf", +-- -- +-- ["freebsd"] = "freebsd", +-- -- +-- ["freebsd-amd64"] = "freebsd-amd64", +-- -- +-- ["kfreebsd"] = "kfreebsd-i386", +-- ["kfreebsd-i386"] = "kfreebsd-i386", +-- -- +-- ["kfreebsd-amd64"] = "kfreebsd-amd64", +-- -- +-- ["linux-ppc"] = "linux-ppc", +-- ["ppc"] = "linux-ppc", +-- -- +-- ["osx"] = "osx-intel", +-- ["macosx"] = "osx-intel", +-- ["osx-intel"] = "osx-intel", +-- ["osxintel"] = "osx-intel", +-- -- +-- ["osx-ppc"] = "osx-ppc", +-- ["osx-powerpc"] = "osx-ppc", +-- ["osxppc"] = "osx-ppc", +-- ["osxpowerpc"] = "osx-ppc", +-- -- +-- ["osx-64"] = "osx-64", +-- -- +-- ["solaris-intel"] = "solaris-intel", +-- -- +-- ["solaris-sparc"] = "solaris-sparc", +-- ["solaris"] = "solaris-sparc", +-- -- +-- ["unknown"] = "unknown", +-- } + update.platforms = { ["mswin"] = "mswin", ["windows"] = "mswin", ["win32"] = "mswin", ["win"] = "mswin", - -- ["mswin"] = "win32", - -- ["windows"] = "win32", - -- ["win32"] = "win32", - -- ["win"] = "win32", -- - -- ["mswin-64"] = "mswin-64", - -- ["windows-64"] = "mswin-64", - -- ["win64"] = "mswin-64", ["mswin-64"] = "win64", ["windows-64"] = "win64", ["win64"] = "win64", @@ -192,34 +238,39 @@ update.platforms = { -- ["linux-armhf"] = "linux-armhf", -- - ["freebsd"] = "freebsd", + ["openbsd"] = "openbsd6.5", + ["openbsd-i386"] = "openbsd6.5", + ["openbsd-amd64"] = "openbsd6.5-amd64", -- + ["freebsd"] = "freebsd", + ["freebsd-i386"] = "freebsd", ["freebsd-amd64"] = "freebsd-amd64", -- - ["kfreebsd"] = "kfreebsd-i386", - ["kfreebsd-i386"] = "kfreebsd-i386", - -- - ["kfreebsd-amd64"] = "kfreebsd-amd64", + -- ["kfreebsd"] = "kfreebsd-i386", + -- ["kfreebsd-i386"] = "kfreebsd-i386", + -- ["kfreebsd-amd64"] = "kfreebsd-amd64", -- - ["linux-ppc"] = "linux-ppc", - ["ppc"] = "linux-ppc", + -- ["linux-ppc"] = "linux-ppc", + -- ["ppc"] = "linux-ppc", -- - ["osx"] = "osx-intel", - ["macosx"] = "osx-intel", - ["osx-intel"] = "osx-intel", - ["osxintel"] = "osx-intel", + -- ["osx"] = "osx-intel", + -- ["macosx"] = "osx-intel", + -- ["osx-intel"] = "osx-intel", + -- ["osxintel"] = "osx-intel", -- - ["osx-ppc"] = "osx-ppc", - ["osx-powerpc"] = "osx-ppc", - ["osxppc"] = "osx-ppc", - ["osxpowerpc"] = "osx-ppc", + -- ["osx-ppc"] = "osx-ppc", + -- ["osx-powerpc"] = "osx-ppc", + -- ["osxppc"] = "osx-ppc", + -- ["osxpowerpc"] = "osx-ppc", -- + ["macosx"] = "osx-64", + ["osx"] = "osx-64", ["osx-64"] = "osx-64", -- - ["solaris-intel"] = "solaris-intel", + -- ["solaris-intel"] = "solaris-intel", -- - ["solaris-sparc"] = "solaris-sparc", - ["solaris"] = "solaris-sparc", + -- ["solaris-sparc"] = "solaris-sparc", + -- ["solaris"] = "solaris-sparc", -- ["unknown"] = "unknown", } @@ -626,7 +677,7 @@ if scripts.savestate then local valid = table.tohash(update.repositories) for r in gmatch(environment.argument("repository") or "current","([^, ]+)") do - if valid[r] then states.set("repositories." .. r, true) end + if valid[r] then states.set(" ." .. r, true) end end local valid = update.engines @@ -652,15 +703,33 @@ if scripts.savestate then end end + -- old + local valid = update.platforms for r in gmatch(environment.argument("platform") or os.platform,"([^, ]+)") do if valid[r] then states.set("platforms." .. r, true) end end + -- new + +-- local osplatform = environment.arguments.platform or nil +-- local platform = platforms[osplatform or os.platform or ""] +-- +-- if (platform == "unknown" or platform == "" or not platform) and osplatform then +-- -- catches openbsdN.M kind of specifications +-- platform = osplatform +-- elseif not osplatform then +-- osplatform = platform +-- end +-- states.set("platforms." .. platform, true) end + + -- so far + local valid = table.tohash(update.texformats) for r in gmatch(environment.argument("formats") or "","([^, ]+)") do if valid[r] then states.set("formats." .. r, true) end end + -- local valid = table.tohash(update.mpformats) -- for r in gmatch(environment.argument("formats") or "","([^, ]+)") do -- if valid[r] then states.set("formats." .. r, true) end diff --git a/Master/texmf-dist/scripts/context/lua/mtx-vscode.lua b/Master/texmf-dist/scripts/context/lua/mtx-vscode.lua new file mode 100644 index 00000000000..96cc5f9009b --- /dev/null +++ b/Master/texmf-dist/scripts/context/lua/mtx-vscode.lua @@ -0,0 +1,3179 @@ +if not modules then modules = { } end modules ['mtx-vscode'] = { + version = 1.000, + comment = "this script is experimental", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE", + license = "see context related readme files" +} + +-- todo: folding and comments +-- todo: runners (awaiting global script setup) +-- todo: dark theme + +-- Already for quite a while lexing in ConTeXt is kind of standardized and the way +-- the format evolved also relates to how I see the source. We started with lexing +-- beginning of 1990 with texedit/wdt in Modula2 and went via perltk (texwork), +-- Scite (native), Scite (lpeg) as well as different stages in verbatim. So, as +-- github uses VSCODE I decided to convert the couple of base lexers to the format +-- that this editor likes. It's all about habits and consistency, not about tons of +-- fancy features that I don't need and would not use anyway. +-- +-- I use a lua script to generate the lexer definitions simply because that way the +-- update will be in sync with the updates in the context distribution. +-- +-- code.exe --extensions-dir e:\vscode\extensions --install-extension context +-- +-- In the end all these systems look alike so again we we have these token onto +-- styling mappings. We even have embedded lexers. Actually, when reading the +-- explanations it has become internally more close to what scintilla does with +-- tokens and numbers related to it but then mapped back onto css. +-- +-- Multiline lexing is a pain here, so I just assume that stuff belonging together is +-- on one line (like keys to simple values). I'm not in the mood for ugly multistep +-- parsing now. Here the lpeg lexers win. +-- +-- We can optimize these expressions if needed but it all looks fast enough. Anyway, +-- we do start from the already old lexers that we use in SciTe. The lexing as well +-- as use of colors is kind of consistent and standardized in context and I don't +-- want to change it. The number of colors is not that large and (at least to me) it +-- looks less extreme. We also use a gray background because over time we figured +-- out that this works best (1) for long sessions, and (2) for colors. We have quite +-- some embedding so that is another reason for consistency. +-- +-- I do remember generating plist files many years ago but stopped doing that +-- because I never could check them in practice. We're now kind of back to that. The +-- reason for using a lua script to generate the json file is that it is easier to +-- keep in sync with context and also because then a user can just generate the +-- extension her/himself. +-- +-- There are nice examples of lexer definitions in the vc extensions path. My regexp +-- experiences are somewhat rusted and I don't really have intentions to spend too +-- much time on them. Compared to the lpeg lexers the regexp based ones are often +-- more compact. It's just a different concept. Anyway, I might improve things after +-- I've read more of the specs (it seems like the regexp engine is the one from ruby). + +-- We normally use a light gray background with rather dark colors which at least +-- for us is less tiresome. The problem with dark backgrounds is that one needs to +-- use light colors from pastel palettes. I need to figure out a mapping that works +-- for a dark background so that optionally one can install without color theme. + +-- It is possible to define tasks and even relate them to languages but for some reason +-- it can not be done global but per workspace which makes using vscode no option for +-- me (too many different folders with source code, documentation etc). It's kind of +-- strange because simple runners are provided by many editors. I don't want to program +-- a lot to get such simple things done so, awaiting global tasks I stick to using the +-- terminal. But we're prepared. + +-- Another showstopper is the fact that we cannot disable utf8 for languages (like pdf, +-- which is just bytes). I couldn't figure out how to set it in the extension. + +-- { +-- "window.zoomLevel": 2, +-- "editor.renderWhitespace": "all", +-- "telemetry.enableCrashReporter": false, +-- "telemetry.enableTelemetry": false, +-- "editor.fontFamily": "Dejavu Sans Mono, Consolas, 'Courier New', monospace", +-- "window.autoDetectHighContrast": false, +-- "zenMode.hideLineNumbers": false, +-- "zenMode.centerLayout": false, +-- "zenMode.fullScreen": false, +-- "zenMode.hideTabs": false, +-- "workbench.editor.showIcons": false, +-- "workbench.settings.enableNaturalLanguageSearch": false, +-- "window.enableMenuBarMnemonics": false, +-- "search.location": "panel", +-- "breadcrumbs.enabled": false, +-- "workbench.activityBar.visible": false, +-- "editor.minimap.enabled": false, +-- "workbench.iconTheme": null, +-- "extensions.ignoreRecommendations": true, +-- "editor.renderControlCharacters": true, +-- "terminal.integrated.scrollback": 5000, +-- "workbench.colorTheme": "ConTeXt", +-- "[context.cld]": {}, +-- "terminal.integrated.fontSize": 10, +-- "terminal.integrated.rendererType": "dom", +-- "workbench.colorCustomizations": { +-- "terminal.ansiBlack": "#000000", +-- "terminal.ansiWhite": "#FFFFFF", +-- "terminal.ansiRed": "#7F0000", +-- "terminal.ansiGreen": "#007F00", +-- "terminal.ansiBlue": "#00007F", +-- "terminal.ansiMagenta": "#7F007F", +-- "terminal.ansiCyan": "#007F7F", +-- "terminal.ansiYellow": "#7F7F00", +-- "terminal.ansiBrightBlack": "#000000", +-- "terminal.ansiBrightWhite": "#FFFFFF", +-- "terminal.ansiBrightRed": "#7F0000", +-- "terminal.ansiBrightGreen": "#007F00", +-- "terminal.ansiBrightBlue": "#00007F", +-- "terminal.ansiBrightMagenta": "#7F007F", +-- "terminal.ansiBrightCyan": "#007F7F", +-- "terminal.ansiBrightYellow": "#7F7F00", +-- } +-- } + +-- kind of done: +-- +-- tex mps lua cld bibtex sql bnf(untested) pdf xml json c(pp)(simplified) +-- +-- unlikely to be done (ok, i'm not interested in all this ide stuff anyway): +-- +-- cpp-web tex-web web web-snippets txt +-- +-- still todo: +-- +-- xml: preamble and dtd +-- pdf: nested string (..(..)..) + +local helpinfo = [[ +<?xml version="1.0"?> +<application> + <metadata> + <entry name="name">mtx-vscode</entry> + <entry name="detail">vscode extension generator</entry> + <entry name="version">1.00</entry> + </metadata> + <flags> + <category name="basic"> + <subcategory> + <flag name="generate"><short>generate extension in sync with current version</short></flag> + <flag name="start"><short>start vscode with extension context</short></flag> + </subcategory> + </category> + </flags> + <examples> + <category> + <title>Example</title> + <subcategory> + <example><command>mtxrun --script vscode --generate e:/vscode/extensions</command></example> + <example><command>mtxrun --script vscode --generate</command></example> + <example><command>mtxrun --script vscode --start</command></example> + </subcategory> + </category> + </examples> +</application> +]] + +local application = logs.application { + name = "mtx-vscode", + banner = "vscode extension generator", + helpinfo = helpinfo, +} + +local report = application.report + +require("util-jsn") + +scripts = scripts or { } +scripts.vscode = scripts.vscode or { } + +local readmedata = [[ +These files are generated. You can use these extensions with for instance: + + code.exe --extensions-dir <someplace>/tex/texmf-context/context/data/vscode/extensions --install-extension context + +There are examples of scripts and keybindings too. +]] + +local function locate() + local name = resolvers.findfile("vscode-context.readme") + if name and name ~= "" then + local path = file.dirname(file.dirname(name)) + if lfs.isdir(path) then + return path + end + end +end + +function scripts.vscode.generate(targetpath) + + local targetpath = targetpath or environment.files[1] or locate() + + if not targetpath or targetpath == "" or not lfs.isdir(targetpath) then + report("invalid targetpath %a",targetpath) + return + end + + local contextpath = string.gsub(targetpath,"\\","/") .. "/context" + + dir.makedirs(contextpath) + + if not lfs.chdir(contextpath) then + return + end + + local syntaxpath = contextpath .. "/syntaxes" + local themepath = contextpath .. "/themes" + local taskpath = contextpath .. "/tasks" + local keybindingpath = contextpath .. "/keybindings" + local settingspath = contextpath .. "/settings" + + dir.makedirs(syntaxpath) + dir.makedirs(themepath) + dir.makedirs(taskpath) + dir.makedirs(keybindingpath) + dir.makedirs(settingspath) + + if not lfs.isdir(syntaxpath) then return end + if not lfs.isdir(themepath) then return end + if not lfs.isdir(taskpath) then return end + if not lfs.isdir(keybindingpath) then return end + if not lfs.isdir(settingspath) then return end + + -- The package. + + local languages = { } + local grammars = { } + local themes = { } + local tasks = { } + local keybindings = { } + + local function registerlexer(lexer) + + local category = lexer.category + local contextid = "context." .. category + local scope = "source." .. contextid + + local setupfile = "./settings/context-settings-" .. category .. ".json" + local grammarfile = "./syntaxes/context-syntax-" .. category .. ".json" + + local grammar = utilities.json.tojson { + name = contextid, + scopeName = scope, + version = lexer.version, + repository = lexer.repository, + patterns = lexer.patterns, + } + + local setup = utilities.json.tojson(lexer.setup) + + local suffixes = lexer.suffixes or { } + local extensions = { } + + for i=1,#suffixes do + extensions[i] = "." .. string.gsub(suffixes[i],"%.","") + end + + table.sort(extensions) + + languages[#languages+1] = { + id = contextid, + extensions = #extensions > 0 and extensions or nil, + aliases = { lexer.description }, + configuration = setupfile, + } + + grammars[#grammars+1] = { + language = contextid, + scopeName = "source." .. contextid, + path = grammarfile, + } + + report("saving grammar for %a in %a",category,grammarfile) + report("saving setup for %a in %a",category,setupfile) + + io.savedata(grammarfile, grammar) + io.savedata(setupfile, setup) + + end + + local function registertheme(theme) + + local category = theme.category + local filename = "./themes/" .. category .. ".json" + + themes[#themes+1] = { + label = theme.description, + uiTheme = "vs", + path = filename, + } + + local data = utilities.json.tojson { + ["$schema"] = "vscode://schemas/color-theme", + ["name"] = category, + ["colors"] = theme.colors, + ["tokenColors"] = theme.styles, + } + + report("saving theme %a in %a",category,filename) + + io.savedata(filename,data) + + end + + local function registertask(task) + + local category = task.category + local filename = "./tasks/" .. category .. ".json" + + tasks[#tasks+1] = { + label = task.description, + path = filename, + } + + local data = utilities.json.tojson { + ["name"] = category, + ["tasks"] = task.tasks, + } + + report("saving task %a in %a",category,filename) + io.savedata(filename,data) + + end + + local function registerkeybinding(keybinding) + + local bindings = keybinding.keybindings + + if bindings then + + local category = keybinding.category + local filename = "./keybindings/" .. category .. ".json" + + report("saving keybinding %a in %a",category,filename) + + io.savedata(filename,utilities.json.tojson(bindings)) + + for i=1,#bindings do + keybindings[#keybindings+1] = bindings[i] + end + + end + end + + local function savepackage() + + local packagefile = "package.json" + local whateverfile = "package.nls.json" + local readmefile = "vscode-context.readme" + + local specification = utilities.json.tojson { + name = "context", + displayName = "ConTeXt", + description = "ConTeXt Syntax Highlighting", + publisher = "ConTeXt Development Team", + version = "1.0.0", + engines = { + vscode = "*" + }, + categories = { + "Lexers", + "Syntaxes" + }, + contributes = { + languages = languages, + grammars = grammars, + themes = themes, + tasks = tasks, + keybindings = keybindings, + }, + + } + + report("saving package in %a",packagefile) + + io.savedata(packagefile,specification) + + local whatever = utilities.json.tojson { + displayName = "ConTeXt", + description = "Provides syntax highlighting and bracket matching in ConTeXt files.", + } + + report("saving whatever in %a",whateverfile) + + io.savedata(whateverfile,whatever) + + report("saving readme in %a",readmefile) + + io.savedata(readmefile,readmedata) + + end + + -- themes + + do + + local mycolors = { + red = "#7F0000", + green = "#007F00", + blue = "#00007F", + cyan = "#007F7F", + magenta = "#7F007F", + yellow = "#7F7F00", + orange = "#B07F00", + white = "#FFFFFF", + light = "#CFCFCF", + grey = "#808080", + dark = "#4F4F4F", + black = "#000000", + selection = "#F7F7F7", + logpanel = "#E7E7E7", + textpanel = "#CFCFCF", + linepanel = "#A7A7A7", + tippanel = "#444444", + right = "#0000FF", + wrong = "#FF0000", + } + + local colors = { + ["editor.background"] = mycolors.textpanel, + ["editor.foreground"] = mycolors.black, + ["editorLineNumber.foreground"] = mycolors.black, + ["editorIndentGuide.background"] = mycolors.textpanel, + ["editorBracketMatch.background"] = mycolors.textpanel, + ["editorBracketMatch.border"] = mycolors.orange, + ["editor.lineHighlightBackground"] = mycolors.textpanel, + ["focusBorder"] = mycolors.black, + + ["activityBar.background"] = mycolors.black, + + ["sideBar.background"] = mycolors.linepanel, + ["sideBar.foreground"] = mycolors.black, + ["sideBar.border"] = mycolors.white, + ["sideBarTitle.foreground"] = mycolors.black, + ["sideBarSectionHeader.background"] = mycolors.linepanel, + ["sideBarSectionHeader.foreground"] = mycolors.black, + + ["statusBar.foreground"] = mycolors.black, + ["statusBar.background"] = mycolors.linepanel, + ["statusBar.border"] = mycolors.white, + ["statusBar.noFolderForeground"] = mycolors.black, + ["statusBar.noFolderBackground"] = mycolors.linepanel, + ["statusBar.debuggingForeground"] = mycolors.black, + ["statusBar.debuggingBackground"] = mycolors.linepanel, + + ["notification.background"] = mycolors.black, + } + + local styles = { + + { scope = "context.whitespace", settings = { } }, + { scope = "context.default", settings = { foreground = mycolors.black } }, + { scope = "context.number", settings = { foreground = mycolors.cyan } }, + { scope = "context.comment", settings = { foreground = mycolors.yellow } }, + { scope = "context.keyword", settings = { foreground = mycolors.blue, fontStyle = "bold" } }, + { scope = "context.string", settings = { foreground = mycolors.magenta } }, + { scope = "context.error", settings = { foreground = mycolors.red } }, + { scope = "context.label", settings = { foreground = mycolors.red, fontStyle = "bold" } }, + { scope = "context.nothing", settings = { } }, + { scope = "context.class", settings = { foreground = mycolors.black, fontStyle = "bold" } }, + { scope = "context.function", settings = { foreground = mycolors.black, fontStyle = "bold" } }, + { scope = "context.constant", settings = { foreground = mycolors.cyan, fontStyle = "bold" } }, + { scope = "context.operator", settings = { foreground = mycolors.blue } }, + { scope = "context.regex", settings = { foreground = mycolors.magenta } }, + { scope = "context.preprocessor", settings = { foreground = mycolors.yellow, fontStyle = "bold" } }, + { scope = "context.tag", settings = { foreground = mycolors.cyan } }, + { scope = "context.type", settings = { foreground = mycolors.blue } }, + { scope = "context.variable", settings = { foreground = mycolors.black } }, + { scope = "context.identifier", settings = { } }, + { scope = "context.linenumber", settings = { background = mycolors.linepanel } }, + { scope = "context.bracelight", settings = { foreground = mycolors.orange, fontStyle = "bold" } }, + { scope = "context.bracebad", settings = { foreground = mycolors.orange, fontStyle = "bold" } }, + { scope = "context.controlchar", settings = { } }, + { scope = "context.indentguide", settings = { foreground = mycolors.linepanel, back = colors.white } }, + { scope = "context.calltip", settings = { foreground = mycolors.white, back = colors.tippanel } }, + { scope = "context.invisible", settings = { background = mycolors.orange } }, + { scope = "context.quote", settings = { foreground = mycolors.blue, fontStyle = "bold" } }, + { scope = "context.special", settings = { foreground = mycolors.blue } }, + { scope = "context.extra", settings = { foreground = mycolors.yellow } }, + { scope = "context.embedded", settings = { foreground = mycolors.black, fontStyle = "bold" } }, + { scope = "context.char", settings = { foreground = mycolors.magenta } }, + { scope = "context.reserved", settings = { foreground = mycolors.magenta, fontStyle = "bold" } }, + { scope = "context.definition", settings = { foreground = mycolors.black, fontStyle = "bold" } }, + { scope = "context.okay", settings = { foreground = mycolors.dark } }, + { scope = "context.warning", settings = { foreground = mycolors.orange } }, + { scope = "context.standout", settings = { foreground = mycolors.orange, fontStyle = "bold" } }, + { scope = "context.command", settings = { foreground = mycolors.green, fontStyle = "bold" } }, + { scope = "context.internal", settings = { foreground = mycolors.orange, fontStyle = "bold" } }, + { scope = "context.preamble", settings = { foreground = mycolors.yellow } }, + { scope = "context.grouping", settings = { foreground = mycolors.red } }, + { scope = "context.primitive", settings = { foreground = mycolors.blue, fontStyle = "bold" } }, + { scope = "context.plain", settings = { foreground = mycolors.dark, fontStyle = "bold" } }, + { scope = "context.user", settings = { foreground = mycolors.green } }, + { scope = "context.data", settings = { foreground = mycolors.cyan, fontStyle = "bold" } }, + { scope = "context.text", settings = { foreground = mycolors.black } }, + + { scope = { "emphasis" }, settings = { fontStyle = "italic" } }, + { scope = { "strong" }, settings = { fontStyle = "bold" } }, + + { scope = { "comment" }, settings = { foreground = mycolors.black } }, + { scope = { "string" }, settings = { foreground = mycolors.magenta } }, + + { + scope = { + "constant.numeric", + "constant.language.null", + "variable.language.this", + "support.type.primitive", + "support.function", + "support.variable.dom", + "support.variable.property", + "support.variable.property", + "meta.property-name", + "meta.property-value", + "support.constant.handlebars" + }, + settings = { + foreground = mycolors.cyan, + } + }, + + { + scope = { + "keyword", + "storage.modifier", + "storage.type", + "variable.parameter" + }, + settings = { + foreground = mycolors.blue, + fontStyle = "bold", + } + }, + + { + scope = { + "entity.name.type", + "entity.other.inherited-class", + "meta.function-call", + "entity.other.attribute-name", + "entity.name.function.shell" + }, + settings = { + foreground = mycolors.black, + } + }, + + { + scope = { + "entity.name.tag", + }, + settings = { + foreground = mycolors.black, + } + }, + + } + + registertheme { + category = "context", + description = "ConTeXt", + colors = colors, + styles = styles, + } + + end + + do + + local presentation = { + echo = true, + reveal = "always", + focus = false, + panel = "shared", + showReuseMessage = false, + clear = true, + } + + local tasks = { + { + group = "build", + label = "process tex file", + type = "shell", + command = "context --autogenerate --autopdf ${file}", + windows = { command = "chcp 65001 ; context.exe --autogenerate --autopdf ${file}" }, + }, + { + group = "build", + label = "check tex file", + type = "shell", + command = "mtxrun --autogenerate --script check ${file}", + windows = { command = "chcp 65001 ; mtxrun.exe --autogenerate --script check ${file}" }, + }, + { + group = "build", + label = "identify fonts", + type = "shell", + command = "mtxrun --script fonts --reload --force", + windows = { command = "chcp 65001 ; mtxrun.exe --script fonts --reload --force" }, + }, + { + group = "build", + label = "process lua file", + type = "shell", + command = "mtxrun --script ${file}", + windows = { command = "chcp 65001 ; mtxrun.exe --script ${file}" }, + }, + } + + for i=1,#tasks do + local task = tasks[i] + if not task.windows then + task.windows = { command = "chcp 65001 ; " .. task.command } + end + if not task.presentation then + task.presentation = presentation + end + end + + registertask { + category = "context", + description = "ConTeXt Tasks", + tasks = tasks, + } + + end + + do + + local keybindings = { + { + -- runner = "context --autogenerate --autopdf ${file}", + key = "ctrl-F12", + command = "workbench.action.tasks.runTask", + args = "process tex file", + when = "editorTextFocus && editorLangId == context.tex", + }, + { + -- runner = "mtxrun --autogenerate --script check ${file}", + key = "F12", + command = "workbench.action.tasks.runTask", + args = "check tex file", + when = "editorTextFocus && editorLangId == context.tex", + }, + { + -- runner = "mtxrun --script ${file}", + key = "ctrl-F12", + command = "workbench.action.tasks.runTask", + args = "process lua file", + when = "editorTextFocus && editorLangId == context.cld", + } + } + + registerkeybinding { + category = "context", + description = "ConTeXt Keybindings", + keybindings = keybindings, + } + + end + + -- helpers + + local function loaddefinitions(name) + return table.load(resolvers.findfile(name)) + end + + local escapes = { + ["."] = "\\.", + ["-"] = "\\-", + ["+"] = "\\+", + ["*"] = "\\*", + ['"'] = '\\"', + ["'"] = "\\'", + ['^'] = '\\^', + ['$'] = '\\$', + ["|"] = "\\|", + ["\\"] = "\\\\", + ["["] = "\\[", + ["]"] = "\\]", + ["("] = "\\(", + [")"] = "\\)", + ["%"] = "\\%", + ["!"] = "\\!", + ["&"] = "\\&", + ["?"] = "\\?", + ["~"] = "\\~", + } + + local function sorter(a,b) + return a > b + end + + local function oneof(t) + local result = { } + table.sort(t,sorter) + for i=1,#t do + result[i] = string.gsub(t[i],".",escapes) + end + return table.concat(result,"|") + end + + local function capture(str) + return "(" .. str .. ")" + end + + local function captures(str) + return "\\*(" .. str .. ")\\*" + end + + local function include(str) + return { include = str } + end + + local function configuration(s) + if s then + local pairs = s.pairs + local comments = s.comments + return { + brackets = pairs, + autoClosingPairs = pairs, + surroundingPairs = pairs, + comments = { + lineComment = comments and comments.inline or nil, + blockComment = comments and comments.display or nil, + }, + } + else + return { } + end + end + + -- I need to figure out a decent mapping for dark as the defaults are just + -- not to my taste and we also have a different categorization. + + local mapping = { + ["context.default"] = "text source", + ["context.number"] = "constant.numeric", + ["context.comment"] = "comment", + ["context.keyword"] = "keyword", + ["context.string"] = "string source", + ["context.label"] = "meta.tag", + ["context.constant"] = "support.constant", + ["context.operator"] = "keyword.operator.js", + ["context.identifier"] = "support.variable", + ["context.quote"] = "string", + ["context.special"] = "unset", + ["context.extra"] = "unset", + ["context.embedded"] = "meta.embedded", + ["context.reserved"] = "unset", + ["context.definition"] = "keyword", + ["context.warning"] = "invalid", + ["context.command"] = "unset", + ["context.grouping"] = "unset", + ["context.primitive"] = "keyword", + ["context.plain"] = "unset", + ["context.user"] = "unset", + ["context.data"] = "text source", + ["context.text"] = "text source", + } + + local function styler(namespace) + local done = { } + local style = function(what,where) + if not what or not where then + report() + report("? %-5s %-20s %s",namespace,what or "?",where or "?") + report() + os.exit() + end +-- if mapping then +-- what = mapping[what] or what +-- end + local hash = what .. "." .. where + if done[hash] then + report("- %-5s %-20s %s",namespace,what,where) + else + -- report("+ %-5s %-20s %s",namespace,what,where) + done[hash] = true + end + return hash .. "." .. namespace + end + return style, function(what,where) return { name = style(what, where) } end + end + + local function embedded(name) + return { { include = "source.context." .. name } } + end + + -- The tex lexer. + + do + + local interface_lowlevel = loaddefinitions("scite-context-data-context.lua") + local interface_interfaces = loaddefinitions("scite-context-data-interfaces.lua") + local interface_tex = loaddefinitions("scite-context-data-tex.lua") + + local constants = interface_lowlevel.constants + local helpers = interface_lowlevel.helpers + local interfaces = interface_interfaces.common + local primitives = { } + local overloaded = { } + + for i=1,#helpers do + overloaded[helpers[i]] = true + end + for i=1,#constants do + overloaded[constants[i]] = true + end + + local function add(data) + for k, v in next, data do + if v ~= "/" and v ~= "-" then + if not overloaded[v] then + primitives[#primitives+1] = v + end + v = "normal" .. v + if not overloaded[v] then + primitives[#primitives+1] = v + end + end + end + end + + add(interface_tex.tex) + add(interface_tex.etex) + add(interface_tex.pdftex) + add(interface_tex.aleph) + add(interface_tex.omega) + add(interface_tex.luatex) + add(interface_tex.xetex) + + local luacommands = { + "ctxlua", "ctxcommand", "ctxfunction", + "ctxlatelua", "ctxlatecommand", + "cldcommand", "cldcontext", + "luaexpr", "luascript", "luathread", + "directlua", "latelua", + } + + local luaenvironments = { + "luacode", + "luasetups", "luaparameterset", + "ctxfunction", "ctxfunctiondefinition", + } + + local mpscommands = { + "reusableMPgraphic", "usableMPgraphic", + "uniqueMPgraphic", "uniqueMPpagegraphic", + "useMPgraphic", "reuseMPgraphic", + "MPpositiongraphic", + } + + local mpsenvironments_o = { + "MPpage" + } + + local mpsenvironments_a = { + "MPcode", "useMPgraphic", "reuseMPgraphic", + "MPinclusions", "MPinitializations", "MPdefinitions", "MPextensions", + "MPgraphic", "MPcalculation", + } + + -- clf_a-zA-z_ + + -- btx|xml a-z + -- a-z btx|xml a-z + + -- mp argument {...text} + local function words(list) + table.sort(list,sorter) + return "\\\\(" .. table.concat(list,"|") .. ")" .. "(?=[^a-zA-Z])" + end + + local function bwords(list) + table.sort(list,sorter) + return "(\\\\(" .. table.concat(list,"|") .. "))\\s*(\\{)" + end + + local function ewords() + return "(\\})" + end + + local function environments(list) + table.sort(list,sorter) + last = table.concat(list,"|") + if #list > 1 then + last = "(?:" .. last .. ")" + end + return capture("\\\\start" .. last), capture("\\\\stop" .. last) + end + + local capturedconstants = words(constants) + local capturedprimitives = words(primitives) + local capturedhelpers = words(helpers) + local capturedcommands = words(interfaces) + local capturedmpscommands = words(mpscommands) + + local spaces = "\\s*" + local identifier = "[a-zA-Z\\_@!?\127-\255]+" + + local comment = "%.*$\\n?" + local ifprimitive = "\\\\if[a-zA-Z\\_@!?\127-\255]*" + local csname = "\\\\[a-zA-Z\\_@!?\127-\255]+" + local csprefix = "\\\\(btx|xml)[a-z]+" + local cssuffix = "\\\\[a-z]+(btx|xml)[a-z]*" + local csreserved = "\\\\(\\?\\?|[a-z]\\!)[a-zA-Z\\_@!?\127-\255]+" + + local luaenvironmentopen, + luaenvironmentclose = environments(luaenvironments) + local luacommandopen, + luacommandclose = environments(luacommands) + + local mpsenvironmentopen_o, + mpsenvironmentclose_o = environments(mpsenvironments_o) + local mpsenvironmentopen_a, + mpsenvironmentclose_a = environments(mpsenvironments_a) + + local argumentopen = capture("\\{") + local argumentclose = capture("\\}") + local argumentcontent = capture("[^\\}]*") + + local optionopen = capture("\\[") + local optionclose = capture("\\]") + local optioncontent = capture("[^\\]]*") + + -- not ok yet, todo: equal in settings .. but it would become quite ugly, lpeg wins here + -- so instead we color differently + + local option = "(?:" .. optionopen .. optioncontent .. optionclose .. ")?" + local argument = "(?:" .. argumentopen .. argumentcontent .. argumentclose .. ")?" + + local mpsenvironmentopen_o = mpsenvironmentopen_o .. spaces .. option .. spaces .. option + local mpsenvironmentopen_a = mpsenvironmentopen_a .. spaces .. argument .. spaces .. argument + + local style, styled = styler("tex") + + local capturedgroupings = oneof { + "{", "}", "$" + } + + local capturedextras = oneof { + "~", "%", "^", "&", "_", + "-", "+", "/", + "'", "`", + "\\", "|", + } + + local capturedspecials = oneof { + "(", ")", "[", "]", "<", ">", + "#", "=", '"', + } + + local capturedescaped = "\\\\." + + registerlexer { + + category = "tex", + description = "ConTeXt TEX", + suffixes = { "tex", "mkiv", "mkvi", "mkix", "mkxi", "mkil", "mkxl", "mklx" }, + version = "1.0.0", + + setup = configuration { + pairs = { + { "{", "}" }, + { "[", "]" }, + { "(", ")" }, + }, + comments = { + inline = "%", + }, + }, + + repository = { + + comment = { + name = style("context.comment", "comment"), + match = texcomment, + }, + + constant = { + name = style("context.constant", "commands.constant"), + match = capturedconstants, + }, + + ifprimitive = { + name = style("context.primitive", "commands.if"), + match = ifprimitive, + }, + + primitive = { + name = style("context.primitive", "commands.primitive"), + match = capturedprimitives, + }, + + helper = { + name = style("context.plain", "commands.plain"), + match = capturedhelpers, + }, + + command = { + name = style("context.command", "commands.context"), + match = capturedcommands, + }, + + csname = { + name = style("context.user", "commands.user"), + match = csname, + }, + + escaped = { + name = style("context.command", "commands.escaped"), + match = capturedescaped, + }, + + subsystem_prefix = { + name = style("context.embedded", "subsystem.prefix"), + match = csprefix, + }, + + subsystem_suffix = { + name = style("context.embedded", "subsystem.suffix"), + match = cssuffix, + }, + + grouping = { + name = style("context.grouping", "symbols.groups"), + match = capturedgroupings, + }, + + extra = { + name = style("context.extra", "symbols.extras"), + match = capturedextras, + }, + + special = { + name = style("context.special", "symbols.special"), + match = capturedspecials, + }, + + reserved = { + name = style("context.reserved", "commands.reserved"), + match = csreserved, + }, + + lua_environment = { + ["begin"] = luaenvironmentopen, + ["end"] = luaenvironmentclose, + patterns = embedded("cld"), + beginCaptures = { ["0"] = styled("context.embedded", "lua.environment.open") }, + endCaptures = { ["0"] = styled("context.embedded", "lua.environment.close") }, + }, + + lua_command = { + ["begin"] = luacommandopen, + ["end"] = luacommandclose, + patterns = embedded("cld"), + beginCaptures = { + ["1"] = styled("context.embedded", "lua.command.open"), + ["2"] = styled("context.grouping", "lua.command.open"), + }, + endCaptures = { + ["1"] = styled("context.grouping", "lua.command.close"), + }, + + }, + + metafun_environment_o = { + ["begin"] = mpsenvironmentopen_o, + ["end"] = mpsenvironmentclose_o, + patterns = embedded("mps"), + beginCaptures = { + ["1"] = styled("context.embedded", "metafun.environment.start.o"), + ["2"] = styled("context.embedded", "metafun.environment.open.o.1"), + ["3"] = styled("context.warning", "metafun.environment.content.o.1"), + ["4"] = styled("context.embedded", "metafun.environment.close.o.1"), + ["5"] = styled("context.embedded", "metafun.environment.open.o.2"), + ["6"] = styled("context.warning", "metafun.environment.content.o.2"), + ["7"] = styled("context.embedded", "metafun.environment.close.o.2"), + }, + endCaptures = { + ["0"] = styled("context.embedded", "metafun.environment.stop.o") + }, + }, + + metafun_environment_a = { + ["begin"] = mpsenvironmentopen_a, + ["end"] = mpsenvironmentclose_a, + patterns = embedded("mps"), + beginCaptures = { + ["1"] = styled("context.embedded", "metafun.environment.start.a"), + ["2"] = styled("context.embedded", "metafun.environment.open.a.1"), + ["3"] = styled("context.warning", "metafun.environment.content.a.1"), + ["4"] = styled("context.embedded", "metafun.environment.close.a.1"), + ["5"] = styled("context.embedded", "metafun.environment.open.a.2"), + ["6"] = styled("context.warning", "metafun.environment.content.a.2"), + ["7"] = styled("context.embedded", "metafun.environment.close.a.2"), + }, + endCaptures = { + ["0"] = styled("context.embedded", "metafun.environment.stop.a") + }, + }, + + metafun_command = { + name = style("context.embedded", "metafun.command"), + match = capturedmpscommands, + }, + + }, + + patterns = { + include("#comment"), + include("#constant"), + include("#lua_environment"), + include("#lua_command"), + include("#metafun_environment_o"), + include("#metafun_environment_a"), + include("#metafun_command"), + include("#subsystem_prefix"), + include("#subsystem_suffix"), + include("#ifprimitive"), + include("#helper"), + include("#command"), + include("#primitive"), + include("#reserved"), + include("#csname"), + include("#escaped"), + include("#grouping"), + include("#special"), + include("#extra"), + }, + + } + + end + + -- The metafun lexer. + + do + + local metapostprimitives = { } + local metapostinternals = { } + local metapostshortcuts = { } + local metapostcommands = { } + + local metafuninternals = { } + local metafunshortcuts = { } + local metafuncommands = { } + + local mergedshortcuts = { } + local mergedinternals = { } + + do + + local definitions = loaddefinitions("scite-context-data-metapost.lua") + + if definitions then + metapostprimitives = definitions.primitives or { } + metapostinternals = definitions.internals or { } + metapostshortcuts = definitions.shortcuts or { } + metapostcommands = definitions.commands or { } + end + + local definitions = loaddefinitions("scite-context-data-metafun.lua") + + if definitions then + metafuninternals = definitions.internals or { } + metafunshortcuts = definitions.shortcuts or { } + metafuncommands = definitions.commands or { } + end + + for i=1,#metapostshortcuts do + mergedshortcuts[#mergedshortcuts+1] = metapostshortcuts[i] + end + for i=1,#metafunshortcuts do + mergedshortcuts[#mergedshortcuts+1] = metafunshortcuts[i] + end + + for i=1,#metapostinternals do + mergedinternals[#mergedinternals+1] = metapostinternals[i] + end + for i=1,#metafuninternals do + mergedinternals[#mergedinternals+1] = metafuninternals[i] + end + + + end + + local function words(list) + table.sort(list,sorter) + return "(" .. table.concat(list,"|") .. ")" .. "(?=[^a-zA-Z\\_@!?\127-\255])" + end + + local capturedshortcuts = oneof(mergedshortcuts) + local capturedinternals = words(mergedinternals) + local capturedmetapostcommands = words(metapostcommands) + local capturedmetafuncommands = words(metafuncommands) + local capturedmetapostprimitives = words(metapostprimitives) + + local capturedsuffixes = oneof { + "#@", "@#", "#" + } + local capturedspecials = oneof { + "#@", "@#", "#", + "(", ")", "[", "]", "{", "}", + "<", ">", "=", ":", + '"', + } + local capturedexatras = oneof { + "+-+", "++", + "~", "%", "^", "&", + "_", "-", "+", "*", "/", + "`", "'", "|", "\\", + } + + local spaces = "\\s*" + local mandatespaces = "\\s+" + + local identifier = "[a-zA-Z\\_@!?\127-\255]+" + + local decnumber = "[\\-]?[0-9]+(\\.[0-9]+)?([eE]\\-?[0-9]+)?" + + local comment = "%.*$\\n?" + + local stringopen = "\"" + local stringclose = stringopen + + local qualifier = "[\\.]" + local optionalqualifier = spaces .. qualifier .. spaces + + local capturedstringopen = capture(stringopen) + local capturedstringclose = capture(stringclose) + + local capturedlua = capture("lua") + + local capturedopen = capture("\\(") + local capturedclose = capture("\\)") + + local capturedtexopen = capture("(?:b|verbatim)tex") .. mandatespaces + local capturedtexclose = mandatespaces .. capture("etex") + + local texcommand = "\\[a-zA-Z\\_@!?\127-\255]+" + + local style, styled = styler("mps") + + registerlexer { + + category = "mps", + description = "ConTeXt MetaFun", + suffixes = { "mp", "mpii", "mpiv", "mpxl" }, + version = "1.0.0", + + setup = configuration { + pairs = { + { "{", "}" }, + { "[", "]" }, + { "(", ")" }, + }, + comments = { + inline = "%", + }, + }, + + repository = { + + comment = { + name = style("context.comment", "comment"), + match = comment, + }, + + internal = { + name = style("context.reserved", "internal"), + match = capturedshortcuts, + }, + + shortcut = { + name = style("context.data", "shortcut"), + match = capturedinternals, + }, + + helper = { + name = style("context.command.metafun", "helper"), + match = capturedmetafuncommands, + }, + + plain = { + name = style("context.plain", "plain"), + match = capturedmetapostcommands, + }, + + primitive = { + name = style("context.primitive", "primitive"), + match = capturedmetapostprimitives, + }, + + quoted = { + name = style("context.string", "string.text"), + ["begin"] = stringopen, + ["end"] = stringclose, + beginCaptures = { ["0"] = styled("context.special", "string.open") }, + endCaptures = { ["0"] = styled("context.special", "string.close") }, + }, + + identifier = { + name = style("context.default", "identifier"), + match = identifier, + }, + + suffix = { + name = style("context.number", "suffix"), + match = capturedsuffixes, + }, + + special = { + name = style("context.special", "special"), + match = capturedspecials, + }, + + number = { + name = style("context.number", "number"), + match = decnumber, + }, + + extra = { + name = "context.extra", + match = capturedexatras, + }, + + luacall = { + ["begin"] = capturedlua .. spaces .. capturedopen .. spaces .. capturedstringopen, + ["end"] = capturedstringclose .. spaces .. capturedclose, + patterns = embedded("cld"), + beginCaptures = { + ["1"] = styled("context.embedded", "lua.command"), + ["2"] = styled("context.special", "lua.open"), + ["3"] = styled("context.special", "lua.text.open"), + }, + endCaptures = { + ["1"] = styled("context.special", "lua.text.close"), + ["2"] = styled("context.special", "lua.close"), + }, + }, + + -- default and embedded have the same color but differ in boldness + + luacall_suffixed = { + name = style("context.embedded", "luacall"), + ["begin"] = capturedlua, + ["end"] = "(?!(" .. optionalqualifier .. identifier .. "))", + patterns = { + { + match = qualifier, + -- name = style("context.operator", "luacall.qualifier"), + name = style("context.default", "luacall.qualifier"), + }, + } + }, + + texlike = { -- simplified variant + name = style("context.warning","unexpected.tex"), + match = texcommand, + }, + + texstuff = { + name = style("context.string", "tex"), + ["begin"] = capturedtexopen, + ["end"] = capturedtexclose, + patterns = embedded("tex"), + beginCaptures = { ["1"] = styled("context.primitive", "tex.open") }, + endCaptures = { ["1"] = styled("context.primitive", "tex.close") }, + }, + + }, + + patterns = { + include("#comment"), + include("#internal"), + include("#shortcut"), + include("#luacall_suffixed"), + include("#luacall"), + include("#helper"), + include("#plain"), + include("#primitive"), + include("#texstuff"), + include("#suffix"), + include("#identifier"), + include("#number"), + include("#quoted"), + include("#special"), + include("#texlike"), + include("#extra"), + }, + + } + + end + + -- The lua lexer. + + do + + local function words(list) + table.sort(list,sorter) + return "(" .. table.concat(list,"|") .. ")" .. "(?=[^a-zA-Z])" + end + + local capturedkeywords = words { + "and", "break", "do", "else", "elseif", "end", "false", "for", "function", -- "goto", + "if", "in", "local", "nil", "not", "or", "repeat", "return", "then", "true", + "until", "while", + } + + local capturedbuiltin = words { + "assert", "collectgarbage", "dofile", "error", "getmetatable", + "ipairs", "load", "loadfile", "module", "next", "pairs", + "pcall", "print", "rawequal", "rawget", "rawset", "require", + "setmetatable", "tonumber", "tostring", "type", "unpack", "xpcall", "select", + "string", "table", "coroutine", "debug", "file", "io", "lpeg", "math", "os", "package", "bit32", "utf8", + -- todo: also extra luametatex ones + } + + local capturedconstants = words { + "_G", "_VERSION", "_M", "\\.\\.\\.", "_ENV", + "__add", "__call", "__concat", "__div", "__idiv", "__eq", "__gc", "__index", + "__le", "__lt", "__metatable", "__mode", "__mul", "__newindex", + "__pow", "__sub", "__tostring", "__unm", "__len", + "__pairs", "__ipairs", + "__close", + "NaN", + "<const>", "<toclose>", + } + + local capturedcsnames = words { -- todo: option + "commands", + "context", + -- "ctxcmd", + -- "ctx", + "metafun", + "metapost", + "ctx[A-Za-z_]*", + } + + local capturedoperators = oneof { + "+", "-", "*", "/", "%", "^", + "#", "=", "<", ">", + ";", ":", ",", ".", + "{", "}", "[", "]", "(", ")", + "|", "~", "'" + } + + local spaces = "\\s*" + + local identifier = "[_\\w][_\\w0-9]*" + local qualifier = "[\\.\\:]" + local optionalqualifier = spaces .. "[\\.\\:]*" .. spaces + + local doublequote = "\"" + local singlequote = "\'" + + local doublecontent = "(?:\\\\\"|[^\"])*" + local singlecontent = "(?:\\\\\'|[^\'])*" + + local captureddouble = capture(doublequote) .. capture(doublecontent) .. capture(doublequote) + local capturedsingle = capture(singlequote) .. capture(singlecontent) .. capture(singlequote) + + local longcommentopen = "--\\[\\[" + local longcommentclose = "\\]\\]" + + local longstringopen = "\\[(=*)\\[" + local longstringclose = "\\](\\2)\\]" + + local shortcomment = "--.*$\\n?" + + local hexnumber = "[\\-]?0[xX][A-Fa-f0-9]+(\\.[A-Fa-f0-9]+)?([eEpP]\\-?[A-Fa-f0-9]+)?" + local decnumber = "[\\-]?[0-9]+(\\.[0-9]+)?([eEpP]\\-?[0-9]+)?" + + local capturedidentifier = capture(identifier) + local capturedgotodelimiter = capture("::") + local capturedqualifier = capture(qualifier) + local capturedgoto = capture("goto") + + local style, styled = styler("lua") + + local lualexer = { + + category = "lua", + description = "ConTeXt Lua", + -- suffixes = { "lua", "luc", "cld", "tuc", "luj", "lum", "tma", "lfg", "luv", "lui" }, + version = "1.0.0", + + setup = configuration { + pairs = { + { "(", ")" }, + { "{", "}" }, + { "[", "]" }, + }, + comments = { + inline = "--", + display = { "--[[", "]]" }, + }, + }, + + repository = { + + shortcomment = { + name = style("context.comment", "comment.short"), + match = shortcomment, + }, + + longcomment = { + name = style("context.comment", "comment.long"), + ["begin"] = longcommentopen, + ["end"] = longcommentclose, + }, + + keyword = { + name = style("context.keyword", "reserved.keyword"), + match = capturedkeywords, + }, + + builtin = { + name = style("context.plain", "reserved.builtin"), + match = capturedbuiltin, + }, + + constant = { + name = style("context.data", "reserved.constants"), + match = capturedconstants, + }, + + csname = { + name = style("context.user", "csname"), + ["begin"] = capturedcsnames, + ["end"] = "(?!(" .. optionalqualifier .. identifier .. "))", + patterns = { + { + match = qualifier, + name = style("context.operator", "csname.qualifier") + }, + } + }, + + identifier_keyword = { + match = spaces .. capturedqualifier .. spaces .. capturedkeywords, + captures = { + ["1"] = styled("context.operator", "identifier.keyword"), + ["2"] = styled("context.warning", "identifier.keyword"), + }, + }, + + identifier_valid = { + name = style("context.default", "identifier.valid"), + match = identifier, + }, + + ["goto"] = { + match = capturedgoto .. spaces .. capturedidentifier, + captures = { + ["1"] = styled("context.keyword", "goto.keyword"), + ["2"] = styled("context.grouping", "goto.target"), + } + }, + + label = { + match = capturedgotodelimiter .. capturedidentifier .. capturedgotodelimiter, + captures = { + ["1"] = styled("context.keyword", "label.open"), + ["2"] = styled("context.grouping", "label.target"), + ["3"] = styled("context.keyword", "label.close"), + } + }, + + operator = { + name = style("context.special", "operator"), + match = capturedoperators, + }, + + string_double = { + match = captureddouble, + captures = { + ["1"] = styled("context.special", "doublequoted.open"), + ["2"] = styled("context.string", "doublequoted.text"), + ["3"] = styled("context.special", "doublequoted.close"), + }, + }, + + string_single = { + match = capturedsingle, + captures = { + ["1"] = styled("context.special", "singlequoted.open"), + ["2"] = styled("context.string", "singlequoted.text"), + ["3"] = styled("context.special", "singlequoted.close"), + }, + }, + + string_long = { + name = style("context.string", "long.text"), + ["begin"] = longstringopen, + ["end"] = longstringclose, + beginCaptures = { ["0"] = styled("context.special", "string.long.open") }, + endCaptures = { ["0"] = styled("context.special", "string.long.close") }, + }, + + number_hex = { + name = style("context.number", "hexnumber"), + match = hexnumber, + }, + + number = { + name = style("context.number", "decnumber"), + match = decnumber, + }, + + }, + + patterns = { + include("#keyword"), + include("#buildin"), + include("#constant"), + include("#csname"), + include("#goto"), + include("#number_hex"), + include("#number"), + include("#identifier_keyword"), + include("#identifier_valid"), + include("#longcomment"), + include("#string_long"), + include("#string_double"), + include("#string_single"), + include("#shortcomment"), + include("#label"), + include("#operator"), + }, + + } + + local texstringopen = "\\\\!!bs" + local texstringclose = "\\\\!!es" + local texcommand = "\\\\[A-Za-z\127-\255@\\!\\?_]*" + + local cldlexer = { + + category = "cld", + description = "ConTeXt CLD", + suffixes = { "lua", "luc", "cld", "tuc", "luj", "lum", "tma", "lfg", "luv", "lui" }, + version = lualexer.version, + setup = lualexer.setup, + + repository = { + + texstring = { + name = style("context.string", "texstring.text"), + ["begin"] = texstringopen, + ["end"] = texstringclose, + beginCaptures = { ["0"] = styled("context.special", "texstring.open") }, + endCaptures = { ["0"] = styled("context.special", "texstring.close") }, + }, + + -- texcomment = { + -- -- maybe some day + -- }, + + texcommand = { + name = style("context.warning", "texcommand"), + match = texcommand + }, + + }, + + patterns = { + include("#texstring"), + -- include("#texcomment"), + include("#texcommand"), + }, + + } + + table.merge (cldlexer.repository,lualexer.repository) + table.imerge(cldlexer.patterns, lualexer.patterns) + + registerlexer(lualexer) + registerlexer(cldlexer) + + end + + -- The xml lexer. + + local xmllexer, xmlconfiguration do + + local spaces = "\\s*" + + local namespace = "(?:[-\\w.]+:)?" + local name = "[-\\w.:]+" + + local equal = "=" + + local elementopen = "<" + local elementclose = ">" + local elementopenend = "</" + local elementempty = "/?" + local elementnoclose = "?:([^>]*)" + + local entity = "&.*?;" + + local doublequote = "\"" + local singlequote = "\'" + + local doublecontent = "(?:\\\\\"|[^\"])*" + local singlecontent = "(?:\\\\\'|[^\'])*" + + local captureddouble = capture(doublequote) .. capture(doublecontent) .. capture(doublequote) + local capturedsingle = capture(singlequote) .. capture(singlecontent) .. capture(singlequote) + + local capturednamespace = capture(namespace) + local capturedname = capture(name) + local capturedopen = capture(elementopen) + local capturedclose = capture(elementclose) + local capturedempty = capture(elementempty) + local capturedopenend = capture(elementopenend) + + local cdataopen = "<!\\[CDATA\\[" + local cdataclose = "]]>" + + local commentopen = "<!--" + local commentclose = "-->" + + local processingopen = "<\\?" + local processingclose = "\\?>" + + local instructionopen = processingopen .. name + local instructionclose = processingclose + + local xmlopen = processingopen .. "xml" + local xmlclose = processingclose + + local luaopen = processingopen .. "lua" + local luaclose = processingclose + + local style, styled = styler("xml") + + registerlexer { + + category = "xml", + description = "ConTeXt XML", + suffixes = { + "xml", "xsl", "xsd", "fo", "exa", "rlb", "rlg", "rlv", "rng", + "xfdf", "xslt", "dtd", "lmx", "htm", "html", "xhtml", "ctx", + "export", "svg", "xul", + }, + version = "1.0.0", + + setup = configuration { + comments = { + display = { "<!--", "-->" }, + }, + }, + + repository = { + + attribute_double = { + match = capturednamespace .. capturedname .. spaces .. equal .. spaces .. captureddouble, + captures = { + ["1"] = styled("context.plain", "attribute.double.namespace"), + ["2"] = styled("context.constant", "attribute.double.name"), + ["3"] = styled("context.special", "attribute.double.open"), + ["4"] = styled("context.string", "attribute.double.text"), + ["5"] = styled("context.special", "attribute.double.close"), + }, + }, + + attribute_single = { + match = capturednamespace .. capturedname .. spaces .. equal .. spaces .. capturedsingle, + captures = { + ["1"] = styled("context.plain", "attribute.single.namespace"), + ["2"] = styled("context.constant", "attribute.single.name"), + ["3"] = styled("context.special", "attribute.single.open"), + ["4"] = styled("context.string", "attribute.single.text"), + ["5"] = styled("context.special", "attribute.single.close"), + }, + }, + + attributes = { + patterns = { + include("#attribute_double"), + include("#attribute_single"), + } + }, + + entity = { + name = style("context.constant", "entity"), + match = entity, + }, + + instruction = { + name = style("context.default", "instruction.text"), + ["begin"] = instructionopen, + ["end"] = instructionclose, + beginCaptures = { ["0"] = styled("context.command", "instruction.open") }, + endCaptures = { ["0"] = styled("context.command", "instruction.close") }, + }, + + instruction_xml = { + ["begin"] = xmlopen, + ["end"] = xmlclose, + beginCaptures = { ["0"] = styled("context.command", "instruction.xml.open") }, + endCaptures = { ["0"] = styled("context.command", "instruction.xml.close") }, + patterns = { include("#attributes") } + }, + + instruction_lua = { + ["begin"] = luaopen, + ["end"] = luaclose, + patterns = embedded("cld"), + beginCaptures = { ["0"] = styled("context.command", "instruction.lua.open") }, + endCaptures = { ["0"] = styled("context.command", "instruction.lua.close") }, + }, + + cdata = { + name = style("context.default", "cdata.text"), + ["begin"] = cdataopen, + ["end"] = cdataclose, + beginCaptures = { ["0"] = styled("context.command", "cdata.open") }, + endCaptures = { ["0"] = styled("context.command", "cdata.close") }, + }, + + comment = { + name = style("context.comment", "comment.text"), + ["begin"] = commentopen, + ["end"] = commentclose, + beginCaptures = { ["0"] = styled("context.command", "comment.open") }, + endCaptures = { ["0"] = styled("context.command", "comment.close") }, + }, + + open = { + ["begin"] = capturedopen .. capturednamespace .. capturedname, + ["end"] = capturedempty .. capturedclose, + patterns = { include("#attributes") }, + beginCaptures = { + ["1"] = styled("context.keyword", "open.open"), + ["2"] = styled("context.plain", "open.namespace"), + ["3"] = styled("context.keyword", "open.name"), + }, + endCaptures = { + ["1"] = styled("context.keyword", "open.empty"), + ["2"] = styled("context.keyword", "open.close"), + }, + }, + + close = { + match = capturedopenend .. capturednamespace .. capturedname .. spaces .. capturedclose, + captures = { + ["1"] = styled("context.keyword", "close.open"), + ["2"] = styled("context.plain", "close.namespace"), + ["3"] = styled("context.keyword", "close.name"), + ["4"] = styled("context.keyword", "close.close"), + }, + }, + + element_error = { + name = style("context.error","error"), + match = elementopen .. elementnoclose .. elementclose, + }, + + }, + + patterns = { + -- include("#preamble"), + include("#comment"), + include("#cdata"), + -- include("#doctype"), + include("#instruction_xml"), + include("#instruction_lua"), + include("#instruction"), + include("#close"), + include("#open"), + include("#element_error"), + include("#entity"), + }, + + } + + end + + -- The bibtex lexer. Again we assume the keys to be on the same line as the + -- first snippet of the value. + + do + + local spaces = "\\s*" + local open = "{" + local close = "}" + local hash = "#" + local equal = "=" + local comma = "," + + local doublequote = "\"" + local doublecontent = "(?:\\\\\"|[^\"])*" + + local singlequote = "\'" + local singlecontent = "(?:\\\\\'|[^\'])*" + + local groupopen = "{" + local groupclose = "}" + local groupcontent = "(?:\\\\{|\\\\}|[^\\{\\}])*" + + local shortcut = "@(?:string|String|STRING)" -- enforce consistency + local comment = "@(?:comment|Comment|COMMENT)" -- enforce consistency + + local keyword = "[a-zA-Z0-9\\_@:\\-]+" + + local capturedcomment = spaces .. capture(comment) .. spaces + local capturedshortcut = spaces .. capture(shortcut) .. spaces + local capturedkeyword = spaces .. capture(keyword) .. spaces + local capturedopen = spaces .. capture(open) .. spaces + local capturedclose = spaces .. capture(close) .. spaces + local capturedequal = spaces .. capture(equal) .. spaces + local capturedcomma = spaces .. capture(comma) .. spaces + local capturedhash = spaces .. capture(hash) .. spaces + + local captureddouble = spaces .. capture(doublequote) .. capture(doublecontent) .. capture(doublequote) .. spaces + local capturedsingle = spaces .. capture(singlequote) .. capture(singlecontent) .. capture(singlequote) .. spaces + local capturedgroup = spaces .. capture(groupopen) .. capture(groupcontent) .. capture(groupclose) .. spaces + + local forget = "%.*$\\n?" + + local style, styled = styler("bibtex") + + registerlexer { + + category = "bibtex", + description = "ConTeXt bibTeX", + suffixes = { "bib", "btx" }, + version = "1.0.0", + + setup = configuration { + pairs = { + { "{", "}" }, + }, + comments = { + inline = "%", + }, + }, + + repository = { + + forget = { + name = style("context.comment", "comment.comment.inline"), + match = forget, + }, + + comment = { + name = style("context.comment", "comment.comment.content"), + ["begin"] = capturedcomment .. capturedopen, + ["end"] = capturedclose, + beginCaptures = { + ["1"] = styled("context.keyword", "comment.name"), + ["2"] = styled("context.grouping", "comment.open"), + }, + endCaptures = { + ["1"] = styled("context.grouping", "comment.close"), + }, + }, + + -- a bit inefficient but good enough + + string_double = { + match = capturedkeyword .. capturedequal .. captureddouble, + captures = { + ["1"] = styled("context.command","doublequoted.key"), + ["2"] = styled("context.operator","doublequoted.equal"), + ["3"] = styled("context.special", "doublequoted.open"), + ["4"] = styled("context.text", "doublequoted.text"), + ["5"] = styled("context.special", "doublequoted.close"), + }, + }, + + string_single = { + match = capturedkeyword .. capturedequal .. capturedsingle, + captures = { + ["1"] = styled("context.command","singlequoted.key"), + ["2"] = styled("context.operator","singlequoted.equal"), + ["3"] = styled("context.special", "singlequoted.open"), + ["4"] = styled("context.text", "singlequoted.text"), + ["5"] = styled("context.special", "singlequoted.close"), + }, + }, + + string_grouped = { + match = capturedkeyword .. capturedequal .. capturedgroup, + captures = { + ["1"] = styled("context.command","grouped.key"), + ["2"] = styled("context.operator","grouped.equal"), + ["3"] = styled("context.operator", "grouped.open"), + ["4"] = styled("context.text", "grouped.text"), + ["5"] = styled("context.operator", "grouped.close"), + }, + }, + + string_value = { + match = capturedkeyword .. capturedequal .. capturedkeyword, + captures = { + ["1"] = styled("context.command", "value.key"), + ["2"] = styled("context.operator", "value.equal"), + ["3"] = styled("context.text", "value.text"), + }, + }, + + string_concat = { + patterns = { + { + match = capturedhash .. captureddouble, + captures = { + ["1"] = styled("context.operator","concat.doublequoted.concatinator"), + ["2"] = styled("context.special", "concat.doublequoted.open"), + ["3"] = styled("context.text", "concat.doublequoted.text"), + ["4"] = styled("context.special", "concat.doublequoted.close"), + } + }, + { + match = capturedhash .. capturedsingle, + captures = { + ["1"] = styled("context.operator","concat.singlequoted.concatinator"), + ["2"] = styled("context.special", "concat.singlequoted.open"), + ["3"] = styled("context.text", "concat.singlequoted.text"), + ["4"] = styled("context.special", "concat.singlequoted.close"), + }, + }, + { + match = capturedhash .. capturedgroup, + captures = { + ["1"] = styled("context.operator","concat.grouped.concatinator"), + ["2"] = styled("context.operator", "concat.grouped.open"), + ["3"] = styled("context.text", "concat.grouped.text"), + ["4"] = styled("context.operator", "concat.grouped.close"), + }, + }, + { + match = capturedhash .. capturedkeyword, + captured = { + ["1"] = styled("context.operator","concat.value.concatinator"), + ["2"] = styled("context.text", "concat.value.text"), + }, + }, + }, + }, + + separator = { + match = capturedcomma, + name = style("context.operator","definition.separator"), + }, + + definition = { + name = style("context.warning","definition.error"), + ["begin"] = capturedkeyword .. capturedopen .. capturedkeyword .. capturedcomma, + ["end"] = capturedclose, + beginCaptures = { + ["1"] = styled("context.keyword", "definition.category"), + ["2"] = styled("context.grouping", "definition.open"), + ["3"] = styled("context.warning", "definition.label.text"), + ["3"] = styled("context.operator", "definition.label.separator"), + }, + endCaptures = { + ["1"] = styled("context.grouping", "definition.close"), + }, + patterns = { + include("#string_double"), + include("#string_single"), + include("#string_grouped"), + include("#string_value"), + include("#string_concat"), + include("#separator"), + }, + }, + + concatinator = { + match = capturedhash, + name = style("context.operator","definition.concatinator"), + }, + + shortcut = { + name = style("context.warning","shortcut.error"), + ["begin"] = capturedshortcut .. capturedopen, + ["end"] = capturedclose, + beginCaptures = { + ["1"] = styled("context.keyword", "shortcut.name"), + ["2"] = styled("context.grouping", "shortcut.open"), + }, + endCaptures = { + ["1"] = styled("context.grouping", "shortcut.close"), + }, + patterns = { + include("#string_double"), + include("#string_single"), + include("#string_grouped"), + include("#string_value"), + include("#string_concat"), + }, + }, + + }, + + patterns = { + include("#forget"), + include("#comment"), + include("#shortcut"), + include("#definition"), + }, + + } + + end + + -- The sql lexer (only needed occasionally in documentation and so). + + do + + -- ANSI SQL 92 | 99 | 2003 + + local function words(list) + table.sort(list,sorter) + local str = table.concat(list,"|") + return "(" .. str .. "|" .. string.upper(str) .. ")" .. "(?=[^a-zA-Z])" + end + + local capturedkeywords = words { + "absolute", "action", "add", "after", "all", "allocate", "alter", "and", "any", + "are", "array", "as", "asc", "asensitive", "assertion", "asymmetric", "at", + "atomic", "authorization", "avg", "before", "begin", "between", "bigint", + "binary", "bit", "bit_length", "blob", "boolean", "both", "breadth", "by", + "call", "called", "cascade", "cascaded", "case", "cast", "catalog", "char", + "char_length", "character", "character_length", "check", "clob", "close", + "coalesce", "collate", "collation", "column", "commit", "condition", "connect", + "connection", "constraint", "constraints", "constructor", "contains", "continue", + "convert", "corresponding", "count", "create", "cross", "cube", "current", + "current_date", "current_default_transform_group", "current_path", + "current_role", "current_time", "current_timestamp", + "current_transform_group_for_type", "current_user", "cursor", "cycle", "data", + "date", "day", "deallocate", "dec", "decimal", "declare", "default", + "deferrable", "deferred", "delete", "depth", "deref", "desc", "describe", + "descriptor", "deterministic", "diagnostics", "disconnect", "distinct", "do", + "domain", "double", "drop", "dynamic", "each", "element", "else", "elseif", + "end", "equals", "escape", "except", "exception", "exec", "execute", "exists", + "exit", "external", "extract", "false", "fetch", "filter", "first", "float", + "for", "foreign", "found", "free", "from", "full", "function", "general", "get", + "global", "go", "goto", "grant", "group", "grouping", "handler", "having", + "hold", "hour", "identity", "if", "immediate", "in", "indicator", "initially", + "inner", "inout", "input", "insensitive", "insert", "int", "integer", + "intersect", "interval", "into", "is", "isolation", "iterate", "join", "key", + "language", "large", "last", "lateral", "leading", "leave", "left", "level", + "like", "local", "localtime", "localtimestamp", "locator", "loop", "lower", + "map", "match", "max", "member", "merge", "method", "min", "minute", "modifies", + "module", "month", "multiset", "names", "national", "natural", "nchar", "nclob", + "new", "next", "no", "none", "not", "null", "nullif", "numeric", "object", + "octet_length", "of", "old", "on", "only", "open", "option", "or", "order", + "ordinality", "out", "outer", "output", "over", "overlaps", "pad", "parameter", + "partial", "partition", "path", "position", "precision", "prepare", "preserve", + "primary", "prior", "privileges", "procedure", "public", "range", "read", + "reads", "real", "recursive", "ref", "references", "referencing", "relative", + "release", "repeat", "resignal", "restrict", "result", "return", "returns", + "revoke", "right", "role", "rollback", "rollup", "routine", "row", "rows", + "savepoint", "schema", "scope", "scroll", "search", "second", "section", + "select", "sensitive", "session", "session_user", "set", "sets", "signal", + "similar", "size", "smallint", "some", "space", "specific", "specifictype", + "sql", "sqlcode", "sqlerror", "sqlexception", "sqlstate", "sqlwarning", "start", + "state", "static", "submultiset", "substring", "sum", "symmetric", "system", + "system_user", "table", "tablesample", "temporary", "then", "time", "timestamp", + "timezone_hour", "timezone_minute", "to", "trailing", "transaction", "translate", + "translation", "treat", "trigger", "trim", "true", "under", "undo", "union", + "unique", "unknown", "unnest", "until", "update", "upper", "usage", "user", + "using", "value", "values", "varchar", "varying", "view", "when", "whenever", + "where", "while", "window", "with", "within", "without", "work", "write", "year", + "zone", + } + + -- The dialects list is taken from drupal.org with standard subtracted. + -- + -- MySQL 3.23.x | 4.x | 5.x + -- PostGreSQL 8.1 + -- MS SQL Server 2000 + -- MS ODBC + -- Oracle 10.2 + + local captureddialects = words { + "a", "abort", "abs", "access", "ada", "admin", "aggregate", "alias", "also", + "always", "analyse", "analyze", "assignment", "attribute", "attributes", "audit", + "auto_increment", "avg_row_length", "backup", "backward", "bernoulli", "bitvar", + "bool", "break", "browse", "bulk", "c", "cache", "cardinality", "catalog_name", + "ceil", "ceiling", "chain", "change", "character_set_catalog", + "character_set_name", "character_set_schema", "characteristics", "characters", + "checked", "checkpoint", "checksum", "class", "class_origin", "cluster", + "clustered", "cobol", "collation_catalog", "collation_name", "collation_schema", + "collect", "column_name", "columns", "command_function", "command_function_code", + "comment", "committed", "completion", "compress", "compute", "condition_number", + "connection_name", "constraint_catalog", "constraint_name", "constraint_schema", + "containstable", "conversion", "copy", "corr", "covar_pop", "covar_samp", + "createdb", "createrole", "createuser", "csv", "cume_dist", "cursor_name", + "database", "databases", "datetime", "datetime_interval_code", + "datetime_interval_precision", "day_hour", "day_microsecond", "day_minute", + "day_second", "dayofmonth", "dayofweek", "dayofyear", "dbcc", "defaults", + "defined", "definer", "degree", "delay_key_write", "delayed", "delimiter", + "delimiters", "dense_rank", "deny", "derived", "destroy", "destructor", + "dictionary", "disable", "disk", "dispatch", "distinctrow", "distributed", "div", + "dual", "dummy", "dump", "dynamic_function", "dynamic_function_code", "enable", + "enclosed", "encoding", "encrypted", "end-exec", "enum", "errlvl", "escaped", + "every", "exclude", "excluding", "exclusive", "existing", "exp", "explain", + "fields", "file", "fillfactor", "final", "float4", "float8", "floor", "flush", + "following", "force", "fortran", "forward", "freetext", "freetexttable", + "freeze", "fulltext", "fusion", "g", "generated", "granted", "grants", + "greatest", "header", "heap", "hierarchy", "high_priority", "holdlock", "host", + "hosts", "hour_microsecond", "hour_minute", "hour_second", "identified", + "identity_insert", "identitycol", "ignore", "ilike", "immutable", + "implementation", "implicit", "include", "including", "increment", "index", + "infile", "infix", "inherit", "inherits", "initial", "initialize", "insert_id", + "instance", "instantiable", "instead", "int1", "int2", "int3", "int4", "int8", + "intersection", "invoker", "isam", "isnull", "k", "key_member", "key_type", + "keys", "kill", "lancompiler", "last_insert_id", "least", "length", "less", + "limit", "lineno", "lines", "listen", "ln", "load", "location", "lock", "login", + "logs", "long", "longblob", "longtext", "low_priority", "m", "matched", + "max_rows", "maxextents", "maxvalue", "mediumblob", "mediumint", "mediumtext", + "message_length", "message_octet_length", "message_text", "middleint", + "min_rows", "minus", "minute_microsecond", "minute_second", "minvalue", + "mlslabel", "mod", "mode", "modify", "monthname", "more", "move", "mumps", + "myisam", "name", "nesting", "no_write_to_binlog", "noaudit", "nocheck", + "nocompress", "nocreatedb", "nocreaterole", "nocreateuser", "noinherit", + "nologin", "nonclustered", "normalize", "normalized", "nosuperuser", "nothing", + "notify", "notnull", "nowait", "nullable", "nulls", "number", "octets", "off", + "offline", "offset", "offsets", "oids", "online", "opendatasource", "openquery", + "openrowset", "openxml", "operation", "operator", "optimize", "optionally", + "options", "ordering", "others", "outfile", "overlay", "overriding", "owner", + "pack_keys", "parameter_mode", "parameter_name", "parameter_ordinal_position", + "parameter_specific_catalog", "parameter_specific_name", + "parameter_specific_schema", "parameters", "pascal", "password", "pctfree", + "percent", "percent_rank", "percentile_cont", "percentile_disc", "placing", + "plan", "pli", "postfix", "power", "preceding", "prefix", "preorder", "prepared", + "print", "proc", "procedural", "process", "processlist", "purge", "quote", + "raid0", "raiserror", "rank", "raw", "readtext", "recheck", "reconfigure", + "regexp", "regr_avgx", "regr_avgy", "regr_count", "regr_intercept", "regr_r2", + "regr_slope", "regr_sxx", "regr_sxy", "regr_syy", "reindex", "reload", "rename", + "repeatable", "replace", "replication", "require", "reset", "resource", + "restart", "restore", "returned_cardinality", "returned_length", + "returned_octet_length", "returned_sqlstate", "rlike", "routine_catalog", + "routine_name", "routine_schema", "row_count", "row_number", "rowcount", + "rowguidcol", "rowid", "rownum", "rule", "save", "scale", "schema_name", + "schemas", "scope_catalog", "scope_name", "scope_schema", "second_microsecond", + "security", "self", "separator", "sequence", "serializable", "server_name", + "setof", "setuser", "share", "show", "shutdown", "simple", "soname", "source", + "spatial", "specific_name", "sql_big_result", "sql_big_selects", + "sql_big_tables", "sql_calc_found_rows", "sql_log_off", "sql_log_update", + "sql_low_priority_updates", "sql_select_limit", "sql_small_result", + "sql_warnings", "sqlca", "sqrt", "ssl", "stable", "starting", "statement", + "statistics", "status", "stddev_pop", "stddev_samp", "stdin", "stdout", + "storage", "straight_join", "strict", "string", "structure", "style", + "subclass_origin", "sublist", "successful", "superuser", "synonym", "sysdate", + "sysid", "table_name", "tables", "tablespace", "temp", "template", "terminate", + "terminated", "text", "textsize", "than", "ties", "tinyblob", "tinyint", + "tinytext", "toast", "top", "top_level_count", "tran", "transaction_active", + "transactions_committed", "transactions_rolled_back", "transform", "transforms", + "trigger_catalog", "trigger_name", "trigger_schema", "truncate", "trusted", + "tsequal", "type", "uescape", "uid", "unbounded", "uncommitted", "unencrypted", + "unlisten", "unlock", "unnamed", "unsigned", "updatetext", "use", + "user_defined_type_catalog", "user_defined_type_code", "user_defined_type_name", + "user_defined_type_schema", "utc_date", "utc_time", "utc_timestamp", "vacuum", + "valid", "validate", "validator", "var_pop", "var_samp", "varbinary", "varchar2", + "varcharacter", "variable", "variables", "verbose", "volatile", "waitfor", + "width_bucket", "writetext", "x509", "xor", "year_month", "zerofill", + } + + local capturedoperators = oneof { + "+", "-", "*", "/", + "%", "^", "!", "&", "|", "?", "~", + "=", "<", ">", + ";", ":", ".", + "{", "}", "[", "]", "(", ")", + } + + local spaces = "\\s*" + local identifier = "[a-zA-Z\\_][a-zA-Z0-9\\_]*" + + local comment = "%.*$\\n?" + local commentopen = "/\\*" + local commentclose = "\\*/" + + local doublequote = "\"" + local singlequote = "\'" + local reversequote = "`" + + local doublecontent = "(?:\\\\\"|[^\"])*" + local singlecontent = "(?:\\\\\'|[^\'])*" + local reversecontent = "(?:\\\\`|[^`])*" + + local decnumber = "[\\-]?[0-9]+(\\.[0-9]+)?([eEpP]\\-?[0-9]+)?" + + local captureddouble = capture(doublequote) .. capture(doublecontent) .. capture(doublequote) + local capturedsingle = capture(singlequote) .. capture(singlecontent) .. capture(singlequote) + local capturedreverse = capture(reversequote) .. capture(reversecontent) .. capture(reversequote) + + local style, styled = styler("sql") + + registerlexer { + + category = "sql", + description = "ConTeXt SQL", + suffixes = { "sql" }, + version = "1.0.0", + + setup = configuration { +-- comments = { +-- inline = "...", +-- display = { "...", "..." }, +-- }, + }, + + repository = { + + comment_short = { + name = style("context.comment", "comment.comment"), + match = comment, + }, + + comment_long = { + name = style("context.comment", "comment.text"), + ["begin"] = commentopen, + ["end"] = commentclose, + beginCaptures = { ["0"] = styled("context.command", "comment.open") }, + endCaptures = { ["0"] = styled("context.command", "comment.close") }, + }, + + keyword_standard = { + name = style("context.keyword", "reserved.standard"), + match = capturedkeywords, + }, + + keyword_dialect = { + name = style("context.keyword", "reserved.dialect"), + match = captureddialects, + }, + + operator = { + name = style("context.special", "operator"), + match = capturedoperators, + + }, + + identifier = { + name = style("context.text", "identifier"), + match = identifier, + }, + + string_double = { + match = captureddouble, + captures = { + ["1"] = styled("context.special", "doublequoted.open"), + ["2"] = styled("context.text", "doublequoted.text"), + ["3"] = styled("context.special", "doublequoted.close"), + }, + }, + + string_single = { + match = capturedsingle, + captures = { + ["1"] = styled("context.special", "singlequoted.open"), + ["2"] = styled("context.text", "singlequoted.text"), + ["3"] = styled("context.special", "singlequoted.close"), + }, + }, + + string_reverse = { + match = capturedreverse, + captures = { + ["1"] = styled("context.special", "reversequoted.open"), + ["2"] = styled("context.text", "reversequoted.text"), + ["3"] = styled("context.special", "reversequoted.close"), + }, + }, + + number = { + name = style("context.number", "number"), + match = decnumber, + }, + + }, + + patterns = { + include("#keyword_standard"), + include("#keyword_dialect"), + include("#identifier"), + include("#string_double"), + include("#string_single"), + include("#string_reverse"), + include("#comment_long"), + include("#comment_short"), + include("#number"), + include("#operator"), + }, + + } + + end + + -- The bnf lexer (only used for documentation, untested). + + do + + local operators = oneof { + "*", "+", "-", "/", + ",", ".", ":", ";", + "(", ")", "<", ">", "{", "}", "[", "]", + "#", "=", "?", "@", "|", " ", "!","$", "%", "&", "\\", "^", "-", "_", "`", "~", + } + + local spaces = "\\s*" + + local text = "[a-zA-Z0-9]|" .. operators + + local doublequote = "\"" + local singlequote = "\'" + + local termopen = "<" + local termclose = ">" + local termcontent = "([a-zA-Z][a-zA-Z0-9\\-]*)" + + local becomes = "::=" + local extra = "|" + + local captureddouble = capture(doublequote) .. capture(text) .. capture(doublequote) + local capturedsingle = capture(singlequote) .. capture(text) .. capture(singlequote) + local capturedterm = capture(termopen) .. capture(termcontent) .. capture(termclose) + + local style, styled = styler("bnf") + + registerlexer { + + category = "bnf", + description = "ConTeXt BNF", + suffixes = { "bnf" }, + version = "1.0.0", + + setup = configuration { + pairs = { + { "<", ">" }, + }, + }, + + repository = { + + term = { + match = capturedterm, + captures = { + ["1"] = styled("context.command", "term.open"), + ["2"] = styled("context.text", "term.text"), + ["3"] = styled("context.command", "term.close"), + }, + }, + + text_single = { + match = capturedsingle, + captures = { + ["1"] = styled("context.special", "singlequoted.open"), + ["2"] = styled("context.text", "singlequoted.text"), + ["3"] = styled("context.special", "singlequoted.close"), + }, + }, + + text_double = { + match = captureddouble, + captures = { + ["1"] = styled("context.special", "doublequoted.open"), + ["2"] = styled("context.text", "doublequoted.text"), + ["3"] = styled("context.special", "doublequoted.close"), + }, + }, + + becomes = { + name = style("context.operator", "symbol.becomes"), + match = becomes, + }, + + extra = { + name = style("context.extra", "symbol.extra"), + match = extra, + }, + + }, + + patterns = { + include("#term"), + include("#text_single"), + include("#text_reverse"), + include("#becomes"), + include("#extra"), + }, + + } + + end + + do + + -- A rather simple one, but consistent with the rest. I don't use an IDE or fancy + -- features. No tricks for me. + + local function words(list) + table.sort(list,sorter) + return "\\b(" .. table.concat(list,"|") .. ")\\b" + end + + local capturedkeywords = words { -- copied from cpp.lua + -- c + "asm", "auto", "break", "case", "const", "continue", "default", "do", "else", + "extern", "false", "for", "goto", "if", "inline", "register", "return", + "sizeof", "static", "switch", "true", "typedef", "volatile", "while", + "restrict", + -- hm + "_Bool", "_Complex", "_Pragma", "_Imaginary", + -- c++. + "catch", "class", "const_cast", "delete", "dynamic_cast", "explicit", + "export", "friend", "mutable", "namespace", "new", "operator", "private", + "protected", "public", "signals", "slots", "reinterpret_cast", + "static_assert", "static_cast", "template", "this", "throw", "try", "typeid", + "typename", "using", "virtual" + } + + local captureddatatypes = words { -- copied from cpp.lua + "bool", "char", "double", "enum", "float", "int", "long", "short", "signed", + "struct", "union", "unsigned", "void" + } + + local capturedluatex = words { -- new + "word", "halfword", "quarterword", "scaled", "pointer", "glueratio", + } + + local capturedmacros = words { -- copied from cpp.lua + "define", "elif", "else", "endif", "error", "if", "ifdef", "ifndef", "import", + "include", "line", "pragma", "undef", "using", "warning" + } + + local operators = oneof { + "*", "+", "-", "/", + "%", "^", "!", "&", "?", "~", "|", + "=", "<", ">", + ";", ":", ".", + "{", "}", "[", "]", "(", ")", + } + + local spaces = "\\s*" + + local identifier = "[A-Za-z_][A-Za-z_0-9]*" + + local comment = "//.*$\\n?" + local commentopen = "/\\*" + local commentclose = "\\*/" + + local doublequote = "\"" + local singlequote = "\'" + local reversequote = "`" + + local doublecontent = "(?:\\\\\"|[^\"])*" + local singlecontent = "(?:\\\\\'|[^\'])*" + + local captureddouble = capture(doublequote) .. capture(doublecontent) .. capture(doublequote) + local capturedsingle = capture(singlequote) .. capture(singlecontent) .. capture(singlequote) + + local texopen = "/\\*tex" + local texclose = "\\*/" + + local hexnumber = "[\\-]?0[xX][A-Fa-f0-9]+(\\.[A-Fa-f0-9]+)?([eEpP]\\-?[A-Fa-f0-9]+)?" + local decnumber = "[\\-]?[0-9]+(\\.[0-9]+)?([eEpP]\\-?[0-9]+)?" + + local capturedmacros = spaces .. capture("#") .. spaces .. capturedmacros + + local style, styled = styler("c") + + registerlexer { + + category = "cpp", + description = "ConTeXt C", + suffixes = { "c", "h", "cpp", "hpp" }, + version = "1.0.0", + + setup = configuration { + pairs = { + { "{", "}" }, + { "[", "]" }, + { "(", ")" }, + }, + }, + + repository = { + + keyword = { + match = capturedkeywords, + name = style("context.keyword","c"), + }, + + datatype = { + match = captureddatatypes, + name = style("context.keyword","datatype"), + }, + + luatex = { + match = capturedluatex, + name = style("context.command","luatex"), + }, + + macro = { + match = capturedmacros, + captures = { + ["1"] = styled("context.data","macro.tag"), + ["2"] = styled("context.data","macro.name"), + } + }, + + texcomment = { + ["begin"] = texopen, + ["end"] = texclose, + patterns = embedded("tex"), + beginCaptures = { ["0"] = styled("context.comment", "tex.open") }, + endCaptures = { ["0"] = styled("context.comment", "tex.close") }, + }, + + longcomment = { + name = style("context.comment","long"), + ["begin"] = commentopen, + ["end"] = commentclose, + }, + + shortcomment = { + name = style("context.comment","short"), + match = comment, + }, + + identifier = { + name = style("context.default","identifier"), + match = identifier, + }, + + operator = { + name = style("context.operator","any"), + match = operators, + }, + + string_double = { + match = captureddouble, + captures = { + ["1"] = styled("context.special", "doublequoted.open"), + ["2"] = styled("context.string", "doublequoted.text"), + ["3"] = styled("context.special", "doublequoted.close"), + }, + }, + + string_single = { + match = capturedsingle, + captures = { + ["1"] = styled("context.special", "singlequoted.open"), + ["2"] = styled("context.string", "singlequoted.text"), + ["3"] = styled("context.special", "singlequoted.close"), + }, + }, + + hexnumber = { + name = style("context.number","hex"), + match = hexnumber, + }, + + decnumber = { + name = style("context.number","dec"), + match = decnumber, + }, + + }, + + patterns = { + include("#keyword"), + include("#datatype"), + include("#luatex"), + include("#identifier"), + include("#macro"), + include("#string_double"), + include("#string_single"), + include("#texcomment"), + include("#longcomment"), + include("#shortcomment"), + include("#hexnumber"), + include("#decnumber"), + include("#operator"), + }, + + } + + end + + -- The pdf lexer. + + do + + -- we can assume no errors in the syntax + + local spaces = "\\s*" + + local reserved = oneof { "true" ,"false" , "null" } + local reference = "R" + + local dictionaryopen = "<<" + local dictionaryclose = ">>" + + local arrayopen = "\\[" + local arrayclose = "\\]" + + local stringopen = "\\(" + local stringcontent = "(?:\\\\[\\(\\)]|[^\\(\\)])*" + local stringclose = "\\)" + + local hexstringopen = "<" + local hexstringcontent = "[^>]*" + local hexstringclose = ">" + + local unicodebomb = "feff" + + local objectopen = "obj" -- maybe also ^ $ + local objectclose = "endobj" -- maybe also ^ $ + + local streamopen = "^stream$" + local streamclose = "^endstream$" + + local name = "/[^\\s<>/\\[\\]\\(\\)]+" -- no need to be more clever than this + local integer = "[\\-]?[0-9]+" -- no need to be more clever than this + local real = "[\\-]?[0-9]*[\\.]?[0-9]+" -- no need to be more clever than this + + local capturedcardinal = "([0-9]+)" + + local captureddictionaryopen = capture(dictionaryopen) + local captureddictionaryclose = capture(dictionaryclose) + + local capturedarrayopen = capture(arrayopen) + local capturedarrayclose = capture(arrayclose) + + local capturedobjectopen = capture(objectopen) + local capturedobjectclose = capture(objectclose) + + local capturedname = capture(name) + local capturedinteger = capture(integer) + local capturedreal = capture(real) + local capturedreserved = capture(reserved) + local capturedreference = capture(reference) + + local capturedunicode = capture(hexstringopen) .. capture(unicodebomb) .. capture(hexstringcontent) .. capture(hexstringclose) + local capturedunicode = capture(hexstringopen) .. capture(unicodebomb) .. capture(hexstringcontent) .. capture(hexstringclose) + local capturedwhatsit = capture(hexstringopen) .. capture(hexstringcontent) .. capture(hexstringclose) + local capturedstring = capture(stringopen) .. capture(stringcontent) .. capture(stringclose) + + local style, styled = styler("pdf") + + -- strings are not ok yet: there can be nested unescaped () but not critical now + + registerlexer { + + category = "pdf", + description = "ConTeXt PDF", + suffixes = { "pdf" }, + version = "1.0.0", + + setup = configuration { + pairs = { + { "<", ">" }, + { "[", "]" }, + { "(", ")" }, + }, + }, + + repository = { + + comment = { + name = style("context.comment","comment"), + match = "%.*$\\n?", + }, + + content = { + patterns = { + { include = "#dictionary" }, + { include = "#stream" }, + { include = "#array" }, + { + name = style("context.constant","object.content.name"), + match = capturedname, + }, + { + match = capturedcardinal .. spaces .. capturedcardinal .. spaces .. capturedreference, + captures = { + ["1"] = styled("context.warning","content.reference.1"), + ["2"] = styled("context.warning","content.reference.2"), + ["3"] = styled("context.command","content.reference.3"), + } + }, + { + name = style("context.number","content.real"), + match = capturedreal, + }, + { + name = style("context.number","content.integer"), + match = capturedinteger, + }, + { + match = capturedstring, + captures = { + ["1"] = styled("context.quote","content.string.open"), + ["2"] = styled("context.string","content.string.text"), + ["3"] = styled("context.quote","content.string.close"), + } + }, + { + name = style("context.number","content.reserved"), + match = capturedreserved, + }, + { + match = capturedunicode, + captures = { + ["1"] = styled("context.quote","content.unicode.open"), + ["2"] = styled("context.plain","content.unicode.bomb"), + ["3"] = styled("context.string","content.unicode.text"), + ["4"] = styled("context.quote","content.unicode.close"), + } + }, + { + match = capturedwhatsit, + captures = { + ["1"] = styled("context.quote","content.whatsit.open"), + ["2"] = styled("context.string","content.whatsit.text"), + ["3"] = styled("context.quote","content.whatsit.close"), + } + }, + }, + }, + + object = { + ["begin"] = capturedcardinal .. spaces .. capturedcardinal .. spaces .. capturedobjectopen, + ["end"] = capturedobjectclose, + patterns = { { include = "#content" } }, + beginCaptures = { + ["1"] = styled("context.warning","object.1"), + ["2"] = styled("context.warning","object.2"), + ["3"] = styled("context.keyword", "object.open") + }, + endCaptures = { + ["1"] = styled("context.keyword", "object.close") + }, + }, + + array = { + ["begin"] = capturedarrayopen, + ["end"] = capturedarrayclose, + patterns = { { include = "#content" } }, + beginCaptures = { ["1"] = styled("context.grouping", "array.open") }, + endCaptures = { ["1"] = styled("context.grouping", "array.close") }, + }, + + dictionary = { + ["begin"] = captureddictionaryopen, + ["end"] = captureddictionaryclose, + beginCaptures = { ["1"] = styled("context.grouping", "dictionary.open") }, + endCaptures = { ["1"] = styled("context.grouping", "dictionary.close") }, + patterns = { + { + ["begin"] = capturedname .. spaces, + ["end"] = "(?=[>])", + beginCaptures = { ["1"] = styled("context.command", "dictionary.name") }, + patterns = { { include = "#content" } }, + }, + }, + }, + + xref = { + ["begin"] = "xref" .. spaces, + ["end"] = "(?=[^0-9])", + captures = { + ["0"] = styled("context.keyword", "xref.1"), + }, + patterns = { + { + ["begin"] = capturedcardinal .. spaces .. capturedcardinal .. spaces, + ["end"] = "(?=[^0-9])", + captures = { + ["1"] = styled("context.number", "xref.2"), + ["2"] = styled("context.number", "xref.3"), + }, + patterns = { + { + ["begin"] = capturedcardinal .. spaces .. capturedcardinal .. spaces .. "([fn])" .. spaces, + ["end"] = "(?=.)", + captures = { + ["1"] = styled("context.number", "xref.4"), + ["2"] = styled("context.number", "xref.5"), + ["3"] = styled("context.keyword", "xref.6"), + }, + }, + }, + }, + }, + }, + + startxref = { + ["begin"] = "startxref" .. spaces, + ["end"] = "(?=[^0-9])", + captures = { + ["0"] = styled("context.keyword", "startxref.1"), + }, + patterns = { + { + ["begin"] = capturedcardinal .. spaces, + ["end"] = "(?=.)", + captures = { + ["1"] = styled("context.number", "startxref.2"), + }, + }, + }, + }, + + trailer = { + name = style("context.keyword", "trailer"), + match = "trailer", + }, + + stream = { + ["begin"] = streamopen, + ["end"] = streamclose, + beginCaptures = { ["0"] = styled("context.keyword", "stream.open") }, + endCaptures = { ["0"] = styled("context.keyword", "stream.close") }, + }, + + }, + + patterns = { + include("#object"), + include("#comment"), + include("#trailer"), + include("#dictionary"), -- cheat: trailer dict + include("#startxref"), + include("#xref"), + }, + + } + + end + + -- The JSON lexer. I don't want to spend time on (and mess up the lexer) with + -- some ugly multistage key/value parser so we just assume that the key is on + -- the same line as the colon and the value. It looks bad otherwise anyway. + + do + + local spaces = "\\s*" + local separator = "\\," + local becomes = "\\:" + + local arrayopen = "\\[" + local arrayclose = "\\]" + + local hashopen = "\\{" + local hashclose = "\\}" + + local stringopen = "\"" + local stringcontent = "(?:\\\\\"|[^\"])*" + local stringclose = stringopen + + local reserved = oneof { "true", "false", "null" } + + local hexnumber = "[\\-]?0[xX][A-Fa-f0-9]+(\\.[A-Fa-f0-9]+)?([eEpP]\\-?[A-Fa-f0-9]+)?" + local decnumber = "[\\-]?[0-9]+(\\.[0-9]+)?([eEpP]\\-?[0-9]+)?" + + local capturedarrayopen = capture(arrayopen) + local capturedarrayclose = capture(arrayclose) + local capturedhashopen = capture(hashopen) + local capturedhashclose = capture(hashclose) + + local capturedreserved = capture(reserved) + local capturedbecomes = capture(becomes) + local capturedseparator = capture(separator) + local capturedstring = capture(stringopen) .. capture(stringcontent) .. capture(stringclose) + local capturedhexnumber = capture(hexnumber) + local captureddecnumber = capture(decnumber) + + local style, styled = styler("json") + + registerlexer { + + category = "json", + description = "ConTeXt JSON", + suffixes = { "json" }, + version = "1.0.0", + + setup = configuration { + pairs = { + { "{", "}" }, + { "[", "]" }, + }, + }, + + repository = { + + separator = { + name = style("context.operator","separator"), + match = spaces .. capturedseparator, + }, + + reserved = { + name = style("context.primitive","reserved"), + match = spaces .. capturedreserved, + }, + + hexnumber = { + name = style("context.number","hex"), + match = spaces .. capturedhexnumber, + }, + + decnumber = { + name = style("context.number","dec"), + match = spaces .. captureddecnumber, + }, + + string = { + match = spaces .. capturedstring, + captures = { + ["1"] = styled("context.quote","string.open"), + ["2"] = styled("context.string","string.text"), + ["3"] = styled("context.quote","string.close"), + }, + }, + + kv_reserved = { + match = capturedstring .. spaces .. capturedbecomes .. spaces .. capturedreserved, + captures = { + ["1"] = styled("context.quote", "reserved.key.open"), + ["2"] = styled("context.text", "reserved.key.text"), + ["3"] = styled("context.quote", "reserved.key.close"), + ["4"] = styled("context.operator", "reserved.becomes"), + ["5"] = styled("context.primitive","reserved.value"), + } + }, + + kv_hexnumber = { + match = capturedstring .. spaces .. capturedbecomes .. spaces .. capturedhexnumber, + captures = { + ["1"] = styled("context.quote", "hex.key.open"), + ["2"] = styled("context.text", "hex.key.text"), + ["3"] = styled("context.quote", "hex.key.close"), + ["4"] = styled("context.operator","hex.becomes"), + ["5"] = styled("context.number", "hex.value"), + } + }, + + kv_decnumber = { + match = capturedstring .. spaces .. capturedbecomes .. spaces .. captureddecnumber, + captures = { + ["1"] = styled("context.quote", "dec.key.open"), + ["2"] = styled("context.text", "dec.key.text"), + ["3"] = styled("context.quote", "dec.key.close"), + ["4"] = styled("context.operator","dec.becomes"), + ["5"] = styled("context.number", "dec.value"), + } + }, + + kv_string = { + match = capturedstring .. spaces .. capturedbecomes .. spaces .. capturedstring, + captures = { + ["1"] = styled("context.quote", "string.key.open"), + ["2"] = styled("context.text", "string.key.text"), + ["3"] = styled("context.quote", "string.key.close"), + ["4"] = styled("context.operator","string.becomes"), + ["5"] = styled("context.quote", "string.value.open"), + ["6"] = styled("context.string", "string.value.text"), + ["7"] = styled("context.quote", "string.value.close"), + }, + }, + + kv_array = { + ["begin"] = capturedstring .. spaces .. capturedbecomes .. spaces .. capturedarrayopen, + ["end"] = arrayclose, + beginCaptures = { + ["1"] = styled("context.quote", "array.key.open"), + ["2"] = styled("context.text", "array.key.text"), + ["3"] = styled("context.quote", "array.key.close"), + ["4"] = styled("context.operator","array.becomes"), + ["5"] = styled("context.grouping","array.value.open") + }, + endCaptures = { + ["0"] = styled("context.grouping","array.value.close") + }, + patterns = { include("#content") }, + }, + + kv_hash = { + ["begin"] = capturedstring .. spaces .. capturedbecomes .. spaces .. capturedhashopen, + ["end"] = hashclose, + beginCaptures = { + ["1"] = styled("context.quote", "hash.key.open"), + ["2"] = styled("context.text", "hash.key.text"), + ["3"] = styled("context.quote", "hash.key.close"), + ["4"] = styled("context.operator","hash.becomes"), + ["5"] = styled("context.grouping","hash.value.open") + }, + endCaptures = { + ["0"] = styled("context.grouping","hash.value.close") + }, + patterns = { include("#kv_content") }, + }, + + content = { + patterns = { + include("#string"), + include("#hexnumber"), + include("#decnumber"), + include("#reserved"), + include("#hash"), + include("#array"), + include("#separator"), + }, + }, + + kv_content = { + patterns = { + include("#kv_string"), + include("#kv_hexnumber"), + include("#kv_decnumber"), + include("#kv_reserved"), + include("#kv_hash"), + include("#kv_array"), + include("#separator"), + }, + }, + + array = { + ["begin"] = arrayopen, + ["end"] = arrayclose, + beginCaptures = { ["0"] = styled("context.grouping","array.open") }, + endCaptures = { ["0"] = styled("context.grouping","array.close") }, + patterns = { include("#content") }, + }, + + hash = { + ["begin"] = hashopen, + ["end"] = hashclose, + beginCaptures = { ["0"] = styled("context.grouping","hash.open") }, + endCaptures = { ["0"] = styled("context.grouping","hash.close") }, + patterns = { include("#kv_content") }, + }, + + }, + + patterns = { + include("#content"), + }, + + } + + end + + savepackage() + +end + +function scripts.vscode.start() + local path = locate() + if path then + local command = 'start "vs code context" code --reuse-window --ignore-gpu-blacklist --extensions-dir "' .. path .. '" --install-extension context' + report("running command: %s",command) + os.execute(command) + end +end + +if environment.arguments.generate then + scripts.vscode.generate() +elseif environment.arguments.start then + scripts.vscode.start() +elseif environment.arguments.exporthelp then + application.export(environment.arguments.exporthelp,environment.files[1]) +else + application.help() +end + +scripts.vscode.generate([[t:/vscode/data/context/extensions]]) diff --git a/Master/texmf-dist/scripts/context/lua/mtx-watch.lua b/Master/texmf-dist/scripts/context/lua/mtx-watch.lua index 8629058e5a2..7815dafdd7a 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-watch.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-watch.lua @@ -208,11 +208,11 @@ report("checking file %s/%s: %s",dirname,basename,ok and "okay" or "skipped") local oldpath = lfs.currentdir() lfs.chdir(newpath) scripts.watch.save_exa_modes(joblog,ctmname) - if pipe then result = os.resultof(command) else result = os.spawn(command) end + if pipe then result = os.resultof(command) else result = os.execute(command) end lfs.chdir(oldpath) else scripts.watch.save_exa_modes(joblog,ctmname) - if pipe then result = os.resultof(command) else result = os.spawn(command) end + if pipe then result = os.resultof(command) else result = os.execute(command) end end report("return value: %s", result) done = true diff --git a/Master/texmf-dist/scripts/context/lua/mtxrun.lua b/Master/texmf-dist/scripts/context/lua/mtxrun.lua index 98fe7f752e3..5907fa33c66 100755 --- a/Master/texmf-dist/scripts/context/lua/mtxrun.lua +++ b/Master/texmf-dist/scripts/context/lua/mtxrun.lua @@ -194,7 +194,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 6330, stripped down to: 2831 +-- original size: 6529, stripped down to: 2933 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -313,6 +313,9 @@ elseif not ffi.number then end if LUAVERSION>5.3 then end +if status and os.setenv then + os.setenv("engine",string.lower(status.luatex_engine or "unknown")) +end end -- of closure @@ -321,7 +324,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-macro"] = package.loaded["l-macro"] or true --- original size: 10131, stripped down to: 5991 +-- original size: 10130, stripped down to: 5990 if not modules then modules={} end modules ['l-macros']={ version=1.001, @@ -546,7 +549,7 @@ end macros.loaded=loaded function required(name,trace) local filename=file.addsuffix(name,"lua") - local fullname=resolvers and resolvers.find_file(filename) or filename + local fullname=resolvers and resolvers.findfile(filename) or filename if not fullname or fullname=="" then return false end @@ -576,7 +579,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-sandbox"] = package.loaded["l-sandbox"] or true --- original size: 9747, stripped down to: 6313 +-- original size: 9604, stripped down to: 6394 if not modules then modules={} end modules ['l-sandbox']={ version=1.001, @@ -808,6 +811,9 @@ local function supported(library) return l end loadfile=register(loadfile,"loadfile") +if supported("lua") then + lua.openfile=register(lua.openfile,"lua.openfile") +end if supported("io") then io.open=register(io.open,"io.open") io.popen=register(io.popen,"io.popen") @@ -1168,7 +1174,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true --- original size: 38434, stripped down to: 19310 +-- original size: 38440, stripped down to: 19316 if not modules then modules={} end modules ['l-lpeg']={ version=1.001, @@ -1337,7 +1343,7 @@ patterns.propername=(uppercase+lowercase+underscore)*(uppercase+lowercase+unders patterns.somecontent=(anything-newline-space)^1 patterns.beginline=#(1-newline) patterns.longtostring=Cs(whitespace^0/""*((patterns.quoted+nonwhitespace^1+whitespace^1/""*(endofstring+Cc(" ")))^0)) -function anywhere(pattern) +local function anywhere(pattern) return (1-P(pattern))^0*P(pattern) end lpeg.anywhere=anywhere @@ -1967,7 +1973,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-string"] = package.loaded["l-string"] or true --- original size: 6461, stripped down to: 3255 +-- original size: 6644, stripped down to: 3410 if not modules then modules={} end modules ['l-string']={ version=1.001, @@ -2031,9 +2037,11 @@ function string.is_empty(str) end end local anything=patterns.anything -local allescapes=Cc("%")*S(".-+%?()[]*") -local someescapes=Cc("%")*S(".-+%()[]") -local matchescapes=Cc(".")*S("*?") +local moreescapes=Cc("%")*S(".-+%?()[]*$^{}") +local allescapes=Cc("%")*S(".-+%?()[]*") +local someescapes=Cc("%")*S(".-+%()[]") +local matchescapes=Cc(".")*S("*?") +local pattern_m=Cs ((moreescapes+anything )^0 ) local pattern_a=Cs ((allescapes+anything )^0 ) local pattern_b=Cs ((someescapes+matchescapes+anything )^0 ) local pattern_c=Cs (Cc("^")*(someescapes+matchescapes+anything )^0*Cc("$") ) @@ -2043,6 +2051,8 @@ end function string.topattern(str,lowercase,strict) if str=="" or type(str)~="string" then return ".*" + elseif strict=="all" then + str=lpegmatch(pattern_m,str) elseif strict then str=lpegmatch(pattern_c,str) else @@ -2092,7 +2102,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-table"] = package.loaded["l-table"] or true --- original size: 41332, stripped down to: 21508 +-- original size: 41758, stripped down to: 22643 if not modules then modules={} end modules ['l-table']={ version=1.001, @@ -2101,7 +2111,7 @@ if not modules then modules={} end modules ['l-table']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local type,next,tostring,tonumber,select=type,next,tostring,tonumber,select +local type,next,tostring,tonumber,select,rawget=type,next,tostring,tonumber,select,rawget local table,string=table,string local concat,sort=table.concat,table.sort local format,lower,dump=string.format,string.lower,string.dump @@ -2435,13 +2445,13 @@ function table.fromhash(t) end return hsh end -local noquotes,hexify,handle,compact,inline,functions,metacheck +local noquotes,hexify,handle,compact,inline,functions,metacheck,accurate local reserved=table.tohash { 'and','break','do','else','elseif','end','false','for','function','if', 'in','local','nil','not','or','repeat','return','then','true','until','while', - 'NaN','goto', + 'NaN','goto','const', } -local function is_simple_table(t,hexify) +local function is_simple_table(t,hexify,accurate) local nt=#t if nt>0 then local n=0 @@ -2460,6 +2470,8 @@ local function is_simple_table(t,hexify) if tv=="number" then if hexify then tt[i]=format("0x%X",v) + elseif accurate then + tt[i]=format("%q",v) else tt[i]=v end @@ -2480,6 +2492,8 @@ local function is_simple_table(t,hexify) if tv=="number" then if hexify then tt[i+1]=format("0x%X",v) + elseif accurate then + tt[i+1]=format("%q",v) else tt[i+1]=v end @@ -2551,6 +2565,8 @@ local function do_serialize(root,name,depth,level,indexed) if tv=="number" then if hexify then handle(format("%s 0x%X,",depth,v)) + elseif accurate then + handle(format("%s %q,",depth,v)) else handle(format("%s %s,",depth,v)) end @@ -2560,7 +2576,7 @@ local function do_serialize(root,name,depth,level,indexed) if next(v)==nil then handle(format("%s {},",depth)) elseif inline then - local st=is_simple_table(v,hexify) + local st=is_simple_table(v,hexify,accurate) if st then handle(format("%s { %s },",depth,concat(st,", "))) else @@ -2588,12 +2604,16 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]=0x%X,",depth,k,v)) + elseif accurate then + handle(format("%s [%s]=%q,",depth,k,v)) else handle(format("%s [%s]=%s,",depth,k,v)) end elseif tk=="boolean" then if hexify then handle(format("%s [%s]=0x%X,",depth,k and "true" or "false",v)) + elseif accurate then + handle(format("%s [%s]=%q,",depth,k and "true" or "false",v)) else handle(format("%s [%s]=%s,",depth,k and "true" or "false",v)) end @@ -2601,12 +2621,16 @@ local function do_serialize(root,name,depth,level,indexed) elseif noquotes and not reserved[k] and lpegmatch(propername,k) then if hexify then handle(format("%s %s=0x%X,",depth,k,v)) + elseif accurate then + handle(format("%s %s=%q,",depth,k,v)) else handle(format("%s %s=%s,",depth,k,v)) end else if hexify then handle(format("%s [%q]=0x%X,",depth,k,v)) + elseif accurate then + handle(format("%s [%q]=%q,",depth,k,v)) else handle(format("%s [%q]=%s,",depth,k,v)) end @@ -2615,6 +2639,8 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]=%q,",depth,k,v)) + elseif accurate then + handle(format("%s [%q]=%q,",depth,k,v)) else handle(format("%s [%s]=%q,",depth,k,v)) end @@ -2631,6 +2657,8 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]={},",depth,k)) + elseif accurate then + handle(format("%s [%q]={},",depth,k)) else handle(format("%s [%s]={},",depth,k)) end @@ -2643,11 +2671,13 @@ local function do_serialize(root,name,depth,level,indexed) handle(format("%s [%q]={},",depth,k)) end elseif inline then - local st=is_simple_table(v,hexify) + local st=is_simple_table(v,hexify,accurate) if st then if tk=="number" then if hexify then handle(format("%s [0x%X]={ %s },",depth,k,concat(st,", "))) + elseif accurate then + handle(format("%s [%q]={ %s },",depth,k,concat(st,", "))) else handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) end @@ -2669,6 +2699,8 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]=%s,",depth,k,v and "true" or "false")) + elseif accurate then + handle(format("%s [%q]=%s,",depth,k,v and "true" or "false")) else handle(format("%s [%s]=%s,",depth,k,v and "true" or "false")) end @@ -2688,6 +2720,8 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]=load(%q),",depth,k,f)) + elseif accurate then + handle(format("%s [%q]=load(%q),",depth,k,f)) else handle(format("%s [%s]=load(%q),",depth,k,f)) end @@ -2705,6 +2739,8 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]=%q,",depth,k,tostring(v))) + elseif accurate then + handle(format("%s [%q]=%q,",depth,k,tostring(v))) else handle(format("%s [%s]=%q,",depth,k,tostring(v))) end @@ -2728,6 +2764,7 @@ local function serialize(_handle,root,name,specification) if type(specification)=="table" then noquotes=specification.noquotes hexify=specification.hexify + accurate=specification.accurate handle=_handle or specification.handle or print functions=specification.functions compact=specification.compact @@ -3043,7 +3080,7 @@ end local function sequenced(t,sep,simple) if not t then return "" - elseif type(t)=="string" then + elseif type(t)~="table" then return t end local n=#t @@ -3082,7 +3119,11 @@ local function sequenced(t,sep,simple) end end end - return concat(s,sep or " | ") + if sep==true then + return "{ "..concat(s,", ").." }" + else + return concat(s,sep or " | ") + end end table.sequenced=sequenced function table.print(t,...) @@ -3213,7 +3254,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-io"] = package.loaded["l-io"] or true --- original size: 11823, stripped down to: 6325 +-- original size: 11829, stripped down to: 6331 if not modules then modules={} end modules ['l-io']={ version=1.001, @@ -3227,7 +3268,7 @@ local open,flush,write,read=io.open,io.flush,io.write,io.read local byte,find,gsub,format=string.byte,string.find,string.gsub,string.format local concat=table.concat local type=type -if string.find(os.getenv("PATH"),";",1,true) then +if string.find(os.getenv("PATH") or "",";",1,true) then io.fileseparator,io.pathseparator="\\",";" else io.fileseparator,io.pathseparator="/",":" @@ -3765,7 +3806,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-os"] = package.loaded["l-os"] or true --- original size: 18916, stripped down to: 10126 +-- original size: 19102, stripped down to: 10192 if not modules then modules={} end modules ['l-os']={ version=1.001, @@ -3779,7 +3820,7 @@ local date,time=os.date,os.time local find,format,gsub,upper,gmatch=string.find,string.format,string.gsub,string.upper,string.gmatch local concat=table.concat local random,ceil,randomseed=math.random,math.ceil,math.randomseed -local rawget,rawset,type,getmetatable,setmetatable,tonumber,tostring=rawget,rawset,type,getmetatable,setmetatable,tonumber,tostring +local type,setmetatable,tonumber,tostring=type,setmetatable,tonumber,tostring do local selfdir=os.selfdir if selfdir=="" then @@ -3930,7 +3971,8 @@ local launchers={ unix="xdg-open %s &> /dev/null &", } function os.launch(str) - execute(format(launchers[os.name] or launchers.unix,str)) + local command=format(launchers[os.name] or launchers.unix,str) + execute(command) end local gettimeofday=os.gettimeofday or os.clock os.gettimeofday=gettimeofday @@ -4185,6 +4227,12 @@ function os.validdate(year,month,day) end return year,month,day end +function os.date(fmt,...) + if not fmt then + fmt="%Y-%m-%d %H:%M" + end + return date(fmt,...) +end local osexit=os.exit local exitcode=nil function os.setexitcode(code) @@ -4207,7 +4255,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-file"] = package.loaded["l-file"] or true --- original size: 21984, stripped down to: 10148 +-- original size: 22175, stripped down to: 10302 if not modules then modules={} end modules ['l-file']={ version=1.001, @@ -4229,15 +4277,24 @@ local checkedsplit=string.checkedsplit local P,R,S,C,Cs,Cp,Cc,Ct=lpeg.P,lpeg.R,lpeg.S,lpeg.C,lpeg.Cs,lpeg.Cp,lpeg.Cc,lpeg.Ct local attributes=lfs.attributes function lfs.isdir(name) - return attributes(name,"mode")=="directory" + if name then + return attributes(name,"mode")=="directory" + end end function lfs.isfile(name) - local a=attributes(name,"mode") - return a=="file" or a=="link" or nil + if name then + local a=attributes(name,"mode") + return a=="file" or a=="link" or nil + end end function lfs.isfound(name) - local a=attributes(name,"mode") - return (a=="file" or a=="link") and name or nil + if name then + local a=attributes(name,"mode") + return (a=="file" or a=="link") and name or nil + end +end +function lfs.modification(name) + return name and attributes(name,"modification") or nil end if sandbox then sandbox.redefine(lfs.isfile,"lfs.isfile") @@ -4616,7 +4673,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-gzip"] = package.loaded["l-gzip"] or true --- original size: 1211, stripped down to: 951 +-- original size: 5115, stripped down to: 1699 if not modules then modules={} end modules ['l-gzip']={ version=1.001, @@ -4624,43 +4681,75 @@ if not modules then modules={} end modules ['l-gzip']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -if not gzip then - return -end -local suffix,suffixes=file.suffix,file.suffixes -function gzip.load(filename) - local f=io.open(filename,"rb") - if not f then - elseif suffix(filename)=="gz" then - f:close() - local g=gzip.open(filename,"rb") - if g then - local str=g:read("*all") - g:close() - return str +gzip=gzip or {} +if not zlib then + zlib=xzip +elseif not xzip then + xzip=zlib +end +if zlib then + local suffix=file.suffix + local suffixes=file.suffixes + local find=string.find + local openfile=io.open + local gzipwindow=15+16 + local gziplevel=3 + local identifier="^\x1F\x8B\x08" + local compress=zlib.compress + local decompress=zlib.decompress + function gzip.load(filename) + local f=openfile(filename,"rb") + if not f then + else + local data=f:read("*all") + f:close() + if data and data~="" then + if suffix(filename)=="gz" then + data=decompress(data,gzipwindow) + end + return data + end end - else - local str=f:read("*all") - f:close() - return str end -end -function gzip.save(filename,data) - if suffix(filename)~="gz" then - filename=filename..".gz" + function gzip.save(filename,data,level) + if suffix(filename)~="gz" then + filename=filename..".gz" + end + local f=openfile(filename,"wb") + if f then + data=compress(data or "",level or gziplevel,nil,gzipwindow) + f:write(data) + f:close() + return #data + end end - local f=io.open(filename,"wb") - if f then - local s=zlib.compress(data or "",9,nil,15+16) - f:write(s) - f:close() - return #s + function gzip.suffix(filename) + local suffix,extra=suffixes(filename) + local gzipped=extra=="gz" + return suffix,gzipped + end + function gzip.compressed(s) + return s and find(s,identifier) + end + function gzip.compress(s,level) + if s and not find(s,identifier) then + if not level then + level=gziplevel + elseif level<=0 then + return s + elseif level>9 then + level=9 + end + return compress(s,level or gziplevel,nil,gzipwindow) or s + end + end + function gzip.decompress(s) + if s and find(s,identifier) then + return decompress(s,gzipwindow) + else + return s + end end -end -function gzip.suffix(filename) - local suffix,extra=suffixes(filename) - local gzipped=extra=="gz" - return suffix,gzipped end @@ -4670,7 +4759,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-md5"] = package.loaded["l-md5"] or true --- original size: 3309, stripped down to: 2218 +-- original size: 3414, stripped down to: 2307 if not modules then modules={} end modules ['l-md5']={ version=1.001, @@ -4689,6 +4778,8 @@ if not md5 then end local md5,file=md5,file local gsub=string.gsub +local modification,isfile,touch=lfs.modification,lfs.isfile,lfs.touch +local loaddata,savedata=io.loaddata,io.savedata do local patterns=lpeg and lpeg.patterns if patterns then @@ -4704,10 +4795,11 @@ do md5.sumHEXA=md5.HEX end end +local md5HEX=md5.HEX function file.needsupdating(oldname,newname,threshold) - local oldtime=lfs.attributes(oldname,"modification") + local oldtime=modification(oldname) if oldtime then - local newtime=lfs.attributes(newname,"modification") + local newtime=modification(newname) if not newtime then return true elseif newtime>=oldtime then @@ -4723,31 +4815,32 @@ function file.needsupdating(oldname,newname,threshold) end file.needs_updating=file.needsupdating function file.syncmtimes(oldname,newname) - local oldtime=lfs.attributes(oldname,"modification") - if oldtime and lfs.isfile(newname) then - lfs.touch(newname,oldtime,oldtime) + local oldtime=modification(oldname) + if oldtime and isfile(newname) then + touch(newname,oldtime,oldtime) end end -function file.checksum(name) +local function checksum(name) if md5 then - local data=io.loaddata(name) + local data=loaddata(name) if data then - return md5.HEX(data) + return md5HEX(data) end end return nil end +file.checksum=checksum function file.loadchecksum(name) if md5 then - local data=io.loaddata(name..".md5") + local data=loaddata(name..".md5") return data and (gsub(data,"%s","")) end return nil end function file.savechecksum(name,checksum) - if not checksum then checksum=file.checksum(name) end + if not checksum then checksum=checksum(name) end if checksum then - io.savedata(name..".md5",checksum) + savedata(name..".md5",checksum) return checksum end return nil @@ -5636,7 +5729,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-unicode"] = package.loaded["l-unicode"] or true --- original size: 41047, stripped down to: 17171 +-- original size: 41281, stripped down to: 17261 if not modules then modules={} end modules ['l-unicode']={ version=1.001, @@ -6151,49 +6244,52 @@ end function utf.utf32_to_utf8_t(t,endian) return endian and utf32_to_utf8_be_t(t) or utf32_to_utf8_le_t(t) or t end -local function little(b) - if b<0x10000 then - return char(b%256,rshift(b,8)) - else - b=b-0x10000 - local b1=rshift(b,10)+0xD800 - local b2=b%1024+0xDC00 - return char(b1%256,rshift(b1,8),b2%256,rshift(b2,8)) +if bit32 then + local rshift=bit32.rshift + local function little(b) + if b<0x10000 then + return char(b%256,rshift(b,8)) + else + b=b-0x10000 + local b1=rshift(b,10)+0xD800 + local b2=b%1024+0xDC00 + return char(b1%256,rshift(b1,8),b2%256,rshift(b2,8)) + end end -end -local function big(b) - if b<0x10000 then - return char(rshift(b,8),b%256) - else - b=b-0x10000 - local b1=rshift(b,10)+0xD800 - local b2=b%1024+0xDC00 - return char(rshift(b1,8),b1%256,rshift(b2,8),b2%256) + local function big(b) + if b<0x10000 then + return char(rshift(b,8),b%256) + else + b=b-0x10000 + local b1=rshift(b,10)+0xD800 + local b2=b%1024+0xDC00 + return char(rshift(b1,8),b1%256,rshift(b2,8),b2%256) + end end -end -local l_remap=Cs((p_utf8byte/little+P(1)/"")^0) -local b_remap=Cs((p_utf8byte/big+P(1)/"")^0) -local function utf8_to_utf16_be(str,nobom) - if nobom then - return lpegmatch(b_remap,str) - else - return char(254,255)..lpegmatch(b_remap,str) + local l_remap=Cs((p_utf8byte/little+P(1)/"")^0) + local b_remap=Cs((p_utf8byte/big+P(1)/"")^0) + local function utf8_to_utf16_be(str,nobom) + if nobom then + return lpegmatch(b_remap,str) + else + return char(254,255)..lpegmatch(b_remap,str) + end end -end -local function utf8_to_utf16_le(str,nobom) - if nobom then - return lpegmatch(l_remap,str) - else - return char(255,254)..lpegmatch(l_remap,str) + local function utf8_to_utf16_le(str,nobom) + if nobom then + return lpegmatch(l_remap,str) + else + return char(255,254)..lpegmatch(l_remap,str) + end end -end -utf.utf8_to_utf16_be=utf8_to_utf16_be -utf.utf8_to_utf16_le=utf8_to_utf16_le -function utf.utf8_to_utf16(str,littleendian,nobom) - if littleendian then - return utf8_to_utf16_le(str,nobom) - else - return utf8_to_utf16_be(str,nobom) + utf.utf8_to_utf16_be=utf8_to_utf16_be + utf.utf8_to_utf16_le=utf8_to_utf16_le + function utf.utf8_to_utf16(str,littleendian,nobom) + if littleendian then + return utf8_to_utf16_le(str,nobom) + else + return utf8_to_utf16_be(str,nobom) + end end end local pattern=Cs ( @@ -6467,7 +6563,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-str"] = package.loaded["util-str"] or true --- original size: 43488, stripped down to: 21595 +-- original size: 45188, stripped down to: 22734 if not modules then modules={} end modules ['util-str']={ version=1.001, @@ -6797,6 +6893,13 @@ local template=[[ %s return function(%s) return %s end ]] +local pattern=Cs(Cc('"')*( + (1-S('"\\\n\r'))^1+P('"')/'\\"'+P('\\')/'\\\\'+P('\n')/'\\n'+P('\r')/'\\r' +)^0*Cc('"')) +patterns.escapedquotes=pattern +function string.escapedquotes(s) + return lpegmatch(pattern,s) +end local preamble="" local environment={ global=global or _G, @@ -6821,9 +6924,10 @@ local environment={ formattednumber=number.formatted, sparseexponent=number.sparseexponent, formattedfloat=number.formattedfloat, - stripzero=lpeg.patterns.stripzero, - stripzeros=lpeg.patterns.stripzeros, - FORMAT=string.f9, + stripzero=patterns.stripzero, + stripzeros=patterns.stripzeros, + escapedquotes=string.escapedquotes, + FORMAT=string.f6, } local arguments={ "a1" } setmetatable(arguments,{ __index=function(t,k) @@ -6874,13 +6978,16 @@ local format_left=function(f) return format("a%s..utfpadding(a%s,%i)",n,n,-f) end end -local format_q=function() +local format_q=JITSUPPORTED and function() n=n+1 return format("(a%s ~= nil and format('%%q',tostring(a%s)) or '')",n,n) +end or function() + n=n+1 + return format("(a%s ~= nil and format('%%q',a%s) or '')",n,n) end local format_Q=function() n=n+1 - return format("format('%%q',tostring(a%s))",n) + return format("escapedquotes(tostring(a%s))",n) end local format_i=function(f) n=n+1 @@ -7031,12 +7138,25 @@ local format_n=function() n=n+1 return format("((a%s %% 1 == 0) and format('%%i',a%s) or tostring(a%s))",n,n,n) end -local format_N=function(f) - n=n+1 - if not f or f=="" then - f=".9" - end - return format("(((a%s %% 1 == 0) and format('%%i',a%s)) or lpegmatch(stripzero,format('%%%sf',a%s)))",n,n,f,n) +local format_N if environment.FORMAT then + format_N=function(f) + n=n+1 + if not f or f=="" then + return format("FORMAT(a%s,'%%.9f')",n) + elseif f==".6" or f=="0.6" then + return format("FORMAT(a%s)",n) + else + return format("FORMAT(a%s,'%%%sf')",n,f) + end + end +else + format_N=function(f) + n=n+1 + if not f or f=="" then + f=".9" + end + return format("(((a%s %% 1 == 0) and format('%%i',a%s)) or lpegmatch(stripzero,format('%%%sf',a%s)))",n,n,f,n) + end end local format_a=function(f) n=n+1 @@ -7265,9 +7385,9 @@ patterns.xmlescape=Cs((P("<")/"<"+P(">")/">"+P("&")/"&"+P('"')/"" patterns.texescape=Cs((C(S("#$%\\{}"))/"\\%1"+anything)^0) patterns.luaescape=Cs(((1-S('"\n'))^1+P('"')/'\\"'+P('\n')/'\\n"')^0) patterns.luaquoted=Cs(Cc('"')*((1-S('"\n'))^1+P('"')/'\\"'+P('\n')/'\\n"')^0*Cc('"')) -add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],{ xmlescape=lpeg.patterns.xmlescape }) -add(formatters,"tex",[[lpegmatch(texescape,%s)]],{ texescape=lpeg.patterns.texescape }) -add(formatters,"lua",[[lpegmatch(luaescape,%s)]],{ luaescape=lpeg.patterns.luaescape }) +add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],{ xmlescape=patterns.xmlescape }) +add(formatters,"tex",[[lpegmatch(texescape,%s)]],{ texescape=patterns.texescape }) +add(formatters,"lua",[[lpegmatch(luaescape,%s)]],{ luaescape=patterns.luaescape }) local dquote=patterns.dquote local equote=patterns.escaped+dquote/'\\"'+1 local cquote=Cc('"') @@ -7299,6 +7419,27 @@ local f_16_16=formatters["%0.5N"] function number.to16dot16(n) return f_16_16(n/65536.0) end +if not string.explode then + local tsplitat=lpeg.tsplitat + local p_utf=patterns.utf8character + local p_check=C(p_utf)*(P("+")*Cc(true))^0 + local p_split=Ct(C(p_utf)^0) + local p_space=Ct((C(1-P(" ")^1)+P(" ")^1)^0) + function string.explode(str,symbol) + if symbol=="" then + return lpegmatch(p_split,str) + elseif symbol then + local a,b=lpegmatch(p_check,symbol) + if b then + return lpegmatch(tsplitat(P(a)^1),str) + else + return lpegmatch(tsplitat(a),str) + end + else + return lpegmatch(p_space,str) + end + end +end end -- of closure @@ -7307,7 +7448,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tab"] = package.loaded["util-tab"] or true --- original size: 28866, stripped down to: 16134 +-- original size: 32649, stripped down to: 18257 if not modules then modules={} end modules ['util-tab']={ version=1.001, @@ -7544,78 +7685,160 @@ function tables.encapsulate(core,capsule,protect) } ) end end -local f_hashed_string=formatters["[%q]=%q,"] -local f_hashed_number=formatters["[%q]=%s,"] -local f_hashed_boolean=formatters["[%q]=%l,"] -local f_hashed_table=formatters["[%q]="] -local f_indexed_string=formatters["[%s]=%q,"] -local f_indexed_number=formatters["[%s]=%s,"] -local f_indexed_boolean=formatters["[%s]=%l,"] -local f_indexed_table=formatters["[%s]="] -local f_ordered_string=formatters["%q,"] -local f_ordered_number=formatters["%s,"] -local f_ordered_boolean=formatters["%l,"] -function table.fastserialize(t,prefix) - local r={ type(prefix)=="string" and prefix or "return" } - local m=1 - local function fastserialize(t,outer) - local n=#t - m=m+1 - r[m]="{" - if n>0 then - for i=0,n do - local v=t[i] - local tv=type(v) - if tv=="string" then - m=m+1 r[m]=f_ordered_string(v) - elseif tv=="number" then - m=m+1 r[m]=f_ordered_number(v) - elseif tv=="table" then - fastserialize(v) - elseif tv=="boolean" then - m=m+1 r[m]=f_ordered_boolean(v) +if JITSUPPORTED then + local f_hashed_string=formatters["[%Q]=%Q,"] + local f_hashed_number=formatters["[%Q]=%s,"] + local f_hashed_boolean=formatters["[%Q]=%l,"] + local f_hashed_table=formatters["[%Q]="] + local f_indexed_string=formatters["[%s]=%Q,"] + local f_indexed_number=formatters["[%s]=%s,"] + local f_indexed_boolean=formatters["[%s]=%l,"] + local f_indexed_table=formatters["[%s]="] + local f_ordered_string=formatters["%Q,"] + local f_ordered_number=formatters["%s,"] + local f_ordered_boolean=formatters["%l,"] + function table.fastserialize(t,prefix) + local r={ type(prefix)=="string" and prefix or "return" } + local m=1 + local function fastserialize(t,outer) + local n=#t + m=m+1 + r[m]="{" + if n>0 then + local v=t[0] + if v then + local tv=type(v) + if tv=="string" then + m=m+1 r[m]=f_indexed_string(0,v) + elseif tv=="number" then + m=m+1 r[m]=f_indexed_number(0,v) + elseif tv=="table" then + m=m+1 r[m]=f_indexed_table(0) + fastserialize(v) + m=m+1 r[m]=f_indexed_table(0) + elseif tv=="boolean" then + m=m+1 r[m]=f_indexed_boolean(0,v) + end + end + for i=1,n do + local v=t[i] + local tv=type(v) + if tv=="string" then + m=m+1 r[m]=f_ordered_string(v) + elseif tv=="number" then + m=m+1 r[m]=f_ordered_number(v) + elseif tv=="table" then + fastserialize(v) + elseif tv=="boolean" then + m=m+1 r[m]=f_ordered_boolean(v) + end end end - end - for k,v in next,t do - local tk=type(k) - if tk=="number" then - if k>n or k<0 then + for k,v in next,t do + local tk=type(k) + if tk=="number" then + if k>n or k<0 then + local tv=type(v) + if tv=="string" then + m=m+1 r[m]=f_indexed_string(k,v) + elseif tv=="number" then + m=m+1 r[m]=f_indexed_number(k,v) + elseif tv=="table" then + m=m+1 r[m]=f_indexed_table(k) + fastserialize(v) + elseif tv=="boolean" then + m=m+1 r[m]=f_indexed_boolean(k,v) + end + end + else local tv=type(v) if tv=="string" then - m=m+1 r[m]=f_indexed_string(k,v) + m=m+1 r[m]=f_hashed_string(k,v) elseif tv=="number" then - m=m+1 r[m]=f_indexed_number(k,v) + m=m+1 r[m]=f_hashed_number(k,v) elseif tv=="table" then - m=m+1 r[m]=f_indexed_table(k) + m=m+1 r[m]=f_hashed_table(k) fastserialize(v) elseif tv=="boolean" then - m=m+1 r[m]=f_indexed_boolean(k,v) + m=m+1 r[m]=f_hashed_boolean(k,v) end end + end + m=m+1 + if outer then + r[m]="}" else - local tv=type(v) - if tv=="string" then - m=m+1 r[m]=f_hashed_string(k,v) - elseif tv=="number" then - m=m+1 r[m]=f_hashed_number(k,v) - elseif tv=="table" then - m=m+1 r[m]=f_hashed_table(k) - fastserialize(v) - elseif tv=="boolean" then - m=m+1 r[m]=f_hashed_boolean(k,v) - end + r[m]="}," end + return r end - m=m+1 - if outer then - r[m]="}" - else - r[m]="}," + return concat(fastserialize(t,true)) + end +else + local f_v=formatters["[%q]=%q,"] + local f_t=formatters["[%q]="] + local f_q=formatters["%q,"] + function table.fastserialize(t,prefix) + local r={ type(prefix)=="string" and prefix or "return" } + local m=1 + local function fastserialize(t,outer) + local n=#t + m=m+1 + r[m]="{" + if n>0 then + local v=t[0] + if v then + m=m+1 + r[m]="[0]='" + if type(v)=="table" then + fastserialize(v) + else + r[m]=format("%q,",v) + end + end + for i=1,n do + local v=t[i] + m=m+1 + if type(v)=="table" then + r[m]=format("[%i]=",i) + fastserialize(v) + else + r[m]=format("[%i]=%q,",i,v) + end + end + end + for k,v in next,t do + local tk=type(k) + if tk=="number" then + if k>n or k<0 then + m=m+1 + if type(v)=="table" then + r[m]=format("[%i]=",k) + fastserialize(v) + else + r[m]=format("[%i]=%q,",k,v) + end + end + else + m=m+1 + if type(v)=="table" then + r[m]=format("[%q]=",k) + fastserialize(v) + else + r[m]=format("[%q]=%q,",k,v) + end + end + end + m=m+1 + if outer then + r[m]="}" + else + r[m]="}," + end + return r end - return r + return concat(fastserialize(t,true)) end - return concat(fastserialize(t,true)) end function table.deserialize(str) if not str or str=="" then @@ -7709,28 +7932,28 @@ function table.twowaymapper(t) return t end local f_start_key_idx=formatters["%w{"] -local f_start_key_num=formatters["%w[%s]={"] +local f_start_key_num=JITSUPPORTED and formatters["%w[%s]={"] or formatters["%w[%q]={"] local f_start_key_str=formatters["%w[%q]={"] local f_start_key_boo=formatters["%w[%l]={"] local f_start_key_nop=formatters["%w{"] local f_stop=formatters["%w},"] -local f_key_num_value_num=formatters["%w[%s]=%s,"] -local f_key_str_value_num=formatters["%w[%q]=%s,"] -local f_key_boo_value_num=formatters["%w[%l]=%s,"] -local f_key_num_value_str=formatters["%w[%s]=%q,"] -local f_key_str_value_str=formatters["%w[%q]=%q,"] -local f_key_boo_value_str=formatters["%w[%l]=%q,"] -local f_key_num_value_boo=formatters["%w[%s]=%l,"] -local f_key_str_value_boo=formatters["%w[%q]=%l,"] +local f_key_num_value_num=JITSUPPORTED and formatters["%w[%s]=%s,"] or formatters["%w[%s]=%q,"] +local f_key_str_value_num=JITSUPPORTED and formatters["%w[%Q]=%s,"] or formatters["%w[%Q]=%q,"] +local f_key_boo_value_num=JITSUPPORTED and formatters["%w[%l]=%s,"] or formatters["%w[%l]=%q,"] +local f_key_num_value_str=JITSUPPORTED and formatters["%w[%s]=%Q,"] or formatters["%w[%q]=%Q,"] +local f_key_str_value_str=formatters["%w[%Q]=%Q,"] +local f_key_boo_value_str=formatters["%w[%l]=%Q,"] +local f_key_num_value_boo=JITSUPPORTED and formatters["%w[%s]=%l,"] or formatters["%w[%q]=%l,"] +local f_key_str_value_boo=formatters["%w[%Q]=%l,"] local f_key_boo_value_boo=formatters["%w[%l]=%l,"] -local f_key_num_value_not=formatters["%w[%s]={},"] -local f_key_str_value_not=formatters["%w[%q]={},"] +local f_key_num_value_not=JITSUPPORTED and formatters["%w[%s]={},"] or formatters["%w[%q]={},"] +local f_key_str_value_not=formatters["%w[%Q]={},"] local f_key_boo_value_not=formatters["%w[%l]={},"] -local f_key_num_value_seq=formatters["%w[%s]={ %, t },"] -local f_key_str_value_seq=formatters["%w[%q]={ %, t },"] +local f_key_num_value_seq=JITSUPPORTED and formatters["%w[%s]={ %, t },"] or formatters["%w[%q]={ %, t },"] +local f_key_str_value_seq=formatters["%w[%Q]={ %, t },"] local f_key_boo_value_seq=formatters["%w[%l]={ %, t },"] -local f_val_num=formatters["%w%s,"] -local f_val_str=formatters["%w%q,"] +local f_val_num=JITSUPPORTED and formatters["%w%s,"] or formatters["%w%q,"] +local f_val_str=formatters["%w%Q,"] local f_val_boo=formatters["%w%l,"] local f_val_not=formatters["%w{},"] local f_val_seq=formatters["%w{ %, t },"] @@ -7738,7 +7961,7 @@ local f_fin_seq=formatters[" %, t }"] local f_table_return=formatters["return {"] local f_table_name=formatters["%s={"] local f_table_direct=formatters["{"] -local f_table_entry=formatters["[%q]={"] +local f_table_entry=formatters["[%Q]={"] local f_table_finish=formatters["}"] local spaces=utilities.strings.newrepeater(" ") local original_serialize=table.serialize @@ -8339,7 +8562,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sac"] = package.loaded["util-sac"] or true --- original size: 11332, stripped down to: 8420 +-- original size: 12946, stripped down to: 9507 if not modules then modules={} end modules ['util-sac']={ version=1.001, @@ -8374,6 +8597,7 @@ end function streams.size(f) return f and f[3] or 0 end +streams.getsize=streams.size function streams.setposition(f,i) if f[4] then if i<=0 then @@ -8566,9 +8790,9 @@ function streams.readfixed2(f) f[2]=j+1 local a,b=byte(f[1],i,j) if a>=0x80 then - tonumber((a-0x100).."."..b) + return tonumber((a-0x100).."."..b) or 0 else - tonumber((a ).."."..b) + return tonumber((a ).."."..b) or 0 end end function streams.readfixed4(f) @@ -8577,9 +8801,9 @@ function streams.readfixed4(f) f[2]=j+1 local a,b,c,d=byte(f[1],i,j) if a>=0x80 then - tonumber((0x100*a+b-0x10000).."."..(0x100*c+d)) + return tonumber((0x100*a+b-0x10000).."."..(0x100*c+d)) or 0 else - tonumber((0x100*a+b ).."."..(0x100*c+d)) + return tonumber((0x100*a+b ).."."..(0x100*c+d)) or 0 end end if bit32 then @@ -8659,6 +8883,16 @@ if sio and sio.readcardinal2 then f[2]=i+4 return readinteger4(f[1],i) end + function streams.readfixed2(f) + local i=f[2] + f[2]=i+2 + return readfixed2(f[1],i) + end + function streams.readfixed4(f) + local i=f[2] + f[2]=i+4 + return readfixed4(f[1],i) + end function streams.read2dot4(f) local i=f[2] f[2]=i+2 @@ -8758,6 +8992,50 @@ else return t end end +do + local files=utilities.files + if files then + local openfile=files.open + local openstream=streams.open + local openstring=streams.openstring + local setmetatable=setmetatable + function io.newreader(str,method) + local f,m + if method=="string" then + f=openstring(str) + m=streams + elseif method=="stream" then + f=openstream(str) + m=streams + else + f=openfile(str,"rb") + m=files + end + if f then + local t={} + setmetatable(t,{ + __index=function(t,k) + local r=m[k] + if k=="close" then + if f then + m.close(f) + f=nil + end + return function() end + elseif r then + local v=function(_,a,b) return r(f,a,b) end + t[k]=v + return v + else + print("unknown key",k) + end + end + } ) + return t + end + end + end +end end -- of closure @@ -9801,7 +10079,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-soc-imp-copas"] = package.loaded["util-soc-imp-copas"] or true --- original size: 25844, stripped down to: 14821 +-- original size: 25959, stripped down to: 14893 local socket=socket or require("socket") @@ -9838,6 +10116,7 @@ local copas={ autoclose=true, running=false, report=report, + trace=false, } local function statushandler(status,...) if status then @@ -9847,7 +10126,9 @@ local function statushandler(status,...) if type(err)=="table" then err=err[1] end - report("error: %s",tostring(err)) + if copas.trace then + report("error: %s",tostring(err)) + end return nil,err end function socket.protect(func) @@ -9861,7 +10142,9 @@ function socket.newtry(finalizer) if not status then local detail=select(2,...) pcall(finalizer,detail) - report("error: %s",tostring(detail)) + if copas.trace then + report("error: %s",tostring(detail)) + end return end return... @@ -12429,7 +12712,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 13340, stripped down to: 8826 +-- original size: 13394, stripped down to: 8882 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -12439,8 +12722,9 @@ if not modules then modules={} end modules ['trac-set']={ license="see context related readme files" } local type,next,tostring,tonumber=type,next,tostring,tonumber +local print=print local concat,sortedhash=table.concat,table.sortedhash -local format,find,lower,gsub,topattern=string.format,string.find,string.lower,string.gsub,string.topattern +local formatters,find,lower,gsub,topattern=string.formatters,string.find,string.lower,string.gsub,string.topattern local is_boolean=string.is_boolean local settings_to_hash=utilities.parsers.settings_to_hash local allocate=utilities.storage.allocate @@ -12450,10 +12734,10 @@ local setters=utilities.setters or {} utilities.setters=setters local data={} local trace_initialize=false +local frozen=true function setters.initialize(filename,name,values) local setter=data[name] if setter then - frozen=true local data=setter.data if data then for key,newvalue in sortedhash(values) do @@ -12647,8 +12931,8 @@ function setters.show(t) end end local enable,disable,register,list,show=setters.enable,setters.disable,setters.register,setters.list,setters.show -function setters.report(setter,...) - print(format("%-15s : %s\n",setter.name,format(...))) +function setters.report(setter,fmt,...) + print(formatters["%-15s : %s\n"](setter.name,formatters[fmt](...))) end local function default(setter,name) local d=setter.data[name] @@ -12668,7 +12952,7 @@ function setters.new(name) disable=function(...) disable (setter,...) end, reset=function(...) reset (setter,...) end, register=function(...) register(setter,...) end, - list=function(...) list (setter,...) end, + list=function(...) return list (setter,...) end, show=function(...) show (setter,...) end, default=function(...) return default (setter,...) end, value=function(...) return value (setter,...) end, @@ -12771,7 +13055,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 32618, stripped down to: 20935 +-- original size: 33003, stripped down to: 21667 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -12816,14 +13100,30 @@ local function ignore() end setmetatableindex(logs,function(t,k) t[k]=ignore;return ignore end) local report,subreport,status,settarget,setformats,settranslations local direct,subdirect,writer,pushtarget,poptarget,setlogfile,settimedlog,setprocessor,setformatters,newline +local function ansisupported(specification) + if specification~="ansi" and specification~="ansilog" then + return false + elseif os and os.enableansi then + return os.enableansi() + else + return false + end +end if runningtex and texio then if texio.setescape then texio.setescape(0) end - if arg then + if arg and ansisupported then for k,v in next,arg do if v=="--ansi" or v=="--c:ansi" then - variant="ansi" + if ansisupported("ansi") then + variant="ansi" + end + break + elseif v=="--ansilog" or v=="--c:ansilog" then + if ansisupported("ansilog") then + variant="ansilog" + end break end end @@ -12928,6 +13228,10 @@ if runningtex and texio then }, } } + variants.ansilog={ + formats=variants.ansi.formats, + targets=variants.default.targets, + } logs.flush=io.flush writer=function(...) write_nl(target,...) @@ -13034,6 +13338,9 @@ if runningtex and texio then t=specification.targets f=specification.formats or specification else + if not ansisupported(specification) then + specification="default" + end local v=variants[specification] if v then t=v.targets @@ -13060,7 +13367,7 @@ if runningtex and texio then subdirect_nop=f.subdirect_nop status_yes=f.status_yes status_nop=f.status_nop - if variant=="ansi" then + if variant=="ansi" or variant=="ansilog" then useluawrites() end settarget(whereto) @@ -13153,6 +13460,9 @@ else if type(specification)=="table" then f=specification.formats or specification else + if not ansisupported(specification) then + specification="default" + end local v=variants[specification] if v then f=v.formats @@ -13408,12 +13718,6 @@ end local nesting=0 local verbose=false local hasscheme=url.hasscheme -function logs.show_open(name) -end -function logs.show_close(name) -end -function logs.show_load(name) -end local simple=logs.reporter("comment") logs.simple=simple logs.simpleline=simple @@ -13480,6 +13784,13 @@ local exporters={ logs.reporters=reporters logs.exporters=exporters function logs.application(t) + local arguments=environment and environment.arguments + if arguments then + local ansi=arguments.ansi or arguments.ansilog + if ansi then + logs.setformatters(arguments.ansi and "ansi" or "ansilog") + end + end t.name=t.name or "unknown" t.banner=t.banner t.moreinfo=moreinfo @@ -13553,8 +13864,6 @@ else print(format(...)) end end -io.stdout:setvbuf('no') -io.stderr:setvbuf('no') if package.helpers.report then package.helpers.report=logs.reporter("package loader") end @@ -13652,7 +13961,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-inf"] = package.loaded["trac-inf"] or true --- original size: 8966, stripped down to: 5972 +-- original size: 9973, stripped down to: 7492 if not modules then modules={} end modules ['trac-inf']={ version=1.001, @@ -13674,7 +13983,7 @@ statistics.enable=true statistics.threshold=0.01 local statusinfo,n,registered,timers={},0,{},{} setmetatableindex(timers,function(t,k) - local v={ timing=0,loadtime=0 } + local v={ timing=0,loadtime=0,offset=0 } t[k]=v return v end) @@ -13682,10 +13991,40 @@ local function hastiming(instance) return instance and timers[instance] end local function resettiming(instance) - timers[instance or "notimer"]={ timing=0,loadtime=0 } + timers[instance or "notimer"]={ timing=0,loadtime=0,offset=0 } end local ticks=clock local seconds=function(n) return n or 0 end +if lua.getpreciseticks then + ticks=lua.getpreciseticks + seconds=lua.getpreciseseconds +elseif FFISUPPORTED and ffi and os.type=="windows" then + local okay,kernel=pcall(ffi.load,"kernel32") + if kernel then + local tonumber=ffi.number or tonumber + ffi.cdef[[ + int QueryPerformanceFrequency(int64_t *lpFrequency); + int QueryPerformanceCounter(int64_t *lpPerformanceCount); + ]] + local target=ffi.new("__int64[1]") + ticks=function() + if kernel.QueryPerformanceCounter(target)==1 then + return tonumber(target[0]) + else + return 0 + end + end + local target=ffi.new("__int64[1]") + seconds=function(ticks) + if kernel.QueryPerformanceFrequency(target)==1 then + return ticks/tonumber(target[0]) + else + return 0 + end + end + end +else +end local function starttiming(instance,reset) local timer=timers[instance or "notimer"] local it=timer.timing @@ -13720,12 +14059,26 @@ local function stoptiming(instance) end return 0 end +local function benchmarktimer(instance) + local timer=timers[instance or "notimer"] + local it=timer.timing + if it>1 then + timer.timing=it-1 + else + local starttime=timer.starttime + if starttime and starttime>0 then + timer.offset=ticks()-starttime + else + timer.offset=0 + end + end +end local function elapsed(instance) if type(instance)=="number" then return instance else local timer=timers[instance or "notimer"] - return timer and seconds(timer.loadtime) or 0 + return timer and seconds(timer.loadtime-2*(timer.offset or 0)) or 0 end end local function currenttime(instance) @@ -13738,7 +14091,7 @@ local function currenttime(instance) else local starttime=timer.starttime if starttime and starttime>0 then - return seconds(timer.loadtime+ticks()-starttime) + return seconds(timer.loadtime+ticks()-starttime-2*(timer.offset or 0)) end end return 0 @@ -13764,6 +14117,7 @@ statistics.elapsed=elapsed statistics.elapsedtime=elapsedtime statistics.elapsedindeed=elapsedindeed statistics.elapsedseconds=elapsedseconds +statistics.benchmarktimer=benchmarktimer function statistics.register(tag,fnc) if statistics.enable and type(fnc)=="function" then local rt=registered[tag] or (#statusinfo+1) @@ -13780,10 +14134,17 @@ function statistics.show() return format("%s, type: %s, binary subtree: %s", os.platform or "unknown",os.type or "unknown",environment.texos or "unknown") end) - register("used engine",function() - return format("%s version %s with functionality level %s, banner: %s", - LUATEXENGINE,LUATEXVERSION,LUATEXFUNCTIONALITY,lower(status.banner)) - end) + if LUATEXENGINE=="luametatex" then + register("used engine",function() + return format("%s version %s, functionality level %s, format id %s", + LUATEXENGINE,LUATEXVERSION,LUATEXFUNCTIONALITY,LUATEXFORMATID) + end) + else + register("used engine",function() + return format("%s version %s with functionality level %s, banner: %s", + LUATEXENGINE,LUATEXVERSION,LUATEXFUNCTIONALITY,lower(status.banner)) + end) + end register("control sequences",function() return format("%s of %s + %s",status.cs_count,status.hash_size,status.hash_extra) end) @@ -13822,7 +14183,11 @@ function statistics.show() end function statistics.memused() local round=math.round or math.floor - return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000),round(status.luastate_bytes/1000000)) + return format("%s MB, ctx: %s MB, max: %s MB)", + round(collectgarbage("count")/1000), + round(status.luastate_bytes/1000000), + status.luastate_bytes_max and round(status.luastate_bytes_max/1000000) or "unknown" + ) end starttiming(statistics) function statistics.formatruntime(runtime) @@ -14013,7 +14378,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lua"] = package.loaded["util-lua"] or true --- original size: 6664, stripped down to: 4589 +-- original size: 7149, stripped down to: 4997 if not modules then modules={} end modules ['util-lua']={ version=1.001, @@ -14038,16 +14403,21 @@ luautilities.nofstrippedchunks=0 luautilities.nofstrippedbytes=0 local strippedchunks={} luautilities.strippedchunks=strippedchunks +if not LUATEXENGINE then + LUATEXENGINE=status.luatex_engine and string.lower(status.luatex_engine) + JITSUPPORTED=LUATEXENGINE=="luajittex" or jit + CONTEXTLMTXMODE=CONTEXTLMTXMODE or (LUATEXENGINE=="luametatex" and 1) or 0 +end luautilities.suffixes={ tma="tma", - tmc=jit and "tmb" or "tmc", + tmc=(CONTEXTLMTXMODE and CONTEXTLMTXMODE>0 and "tmd") or (jit and "tmb") or "tmc", lua="lua", - luc=jit and "lub" or "luc", + luc=(CONTEXTLMTXMODE and CONTEXTLMTXMODE>0 and "lud") or (jit and "lub") or "luc", lui="lui", luv="luv", luj="luj", tua="tua", - tuc="tuc", + tuc=(CONTEXTLMTXMODE and CONTEXTLMTXMODE>0 and "tud") or (jit and "tub") or "tuc", } local function register(name) if tracestripping then @@ -14186,7 +14556,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-deb"] = package.loaded["util-deb"] or true --- original size: 9955, stripped down to: 6693 +-- original size: 10136, stripped down to: 6832 if not modules then modules={} end modules ['util-deb']={ version=1.001, @@ -14210,7 +14580,13 @@ local dummycalls=10*1000 local nesting=0 local names={} local initialize=false -if not (FFISUPPORTED and ffi) then +if lua.getpreciseticks then + initialize=function() + ticks=lua.getpreciseticks + seconds=lua.getpreciseseconds + initialize=false + end +elseif not (FFISUPPORTED and ffi) then elseif os.type=="windows" then initialize=function() local kernel=ffilib("kernel32","system") @@ -14482,7 +14858,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tpl"] = package.loaded["util-tpl"] or true --- original size: 7112, stripped down to: 3887 +-- original size: 7722, stripped down to: 4212 if not modules then modules={} end modules ['util-tpl']={ version=1.001, @@ -14498,6 +14874,7 @@ local report_template=logs.reporter("template") local tostring,next=tostring,next local format,sub,byte=string.format,string.sub,string.byte local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns +local formatters=string.formatters local replacer local function replacekey(k,t,how,recursive) local v=t[k] @@ -14566,6 +14943,10 @@ local function replaceoptional(l,m,r,t,how,recurse) local v=t[l] return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" end +local function replaceformatted(l,m,r,t,how,recurse) + local v=t[r] + return v and formatters[l](v) +end local single=P("%") local double=P("%%") local lquoted=P("%[") @@ -14579,16 +14960,19 @@ local nolquoted=lquoted/'' local norquoted=rquoted/'' local nolquotedq=lquotedq/'' local norquotedq=rquotedq/'' +local nolformatted=P(":")/"%%" +local norformatted=P(":")/"" local noloptional=P("%?")/'' local noroptional=P("?%")/'' local nomoptional=P(":")/'' local args=Carg(1)*Carg(2)*Carg(3) -local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle -local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq -local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted +local key=nosingle*((C((1-nosingle)^1)*args)/replacekey)*nosingle +local quoted=nolquotedq*((C((1-norquotedq)^1)*args)/replacekeyquoted)*norquotedq +local unquoted=nolquoted*((C((1-norquoted)^1)*args)/replacekeyunquoted)*norquoted local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional +local formatted=nosingle*((Cs(nolformatted*(1-norformatted )^1)*norformatted*C((1-nosingle)^1)*args)/replaceformatted)*nosingle local any=P(1) - replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) + replacer=Cs((unquoted+quoted+formatted+escape+optional+key+any)^0) local function replace(str,mapping,how,recurse) if mapping and str then return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str @@ -14627,7 +15011,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sbx"] = package.loaded["util-sbx"] or true --- original size: 20393, stripped down to: 13121 +-- original size: 21084, stripped down to: 13214 if not modules then modules={} end modules ['util-sbx']={ version=1.001, @@ -14873,37 +15257,50 @@ local iopopen=sandbox.original(io.popen) local reported={} local function validcommand(name,program,template,checkers,defaults,variables,reporter,strict) if validbinaries~=false and (validbinaries==true or validbinaries[program]) then + local binpath=nil if variables then for variable,value in next,variables do - local checker=validators[checkers[variable]] - if checker then - value=checker(unquoted(value),strict) - if value then - variables[variable]=optionalquoted(value) + local chktype=checkers[variable] + if chktype=="verbose" then + else + local checker=validators[chktype] + if checker then + value=checker(unquoted(value),strict) + if value then + variables[variable]=optionalquoted(value) + else + report("variable %a with value %a fails the check",variable,value) + return + end else - report("variable %a with value %a fails the check",variable,value) + report("variable %a has no checker",variable) return end - else - report("variable %a has no checker",variable) - return end end for variable,default in next,defaults do local value=variables[variable] if not value or value=="" then - local checker=validators[checkers[variable]] - if checker then - default=checker(unquoted(default),strict) - if default then - variables[variable]=optionalquoted(default) - else - report("variable %a with default %a fails the check",variable,default) - return + local chktype=checkers[variable] + if chktype=="verbose" then + else + local checker=validators[chktype] + if checker then + default=checker(unquoted(default),strict) + if default then + variables[variable]=optionalquoted(default) + else + report("variable %a with default %a fails the check",variable,default) + return + end end end end end + binpath=variables.binarypath + end + if type(binpath)=="string" and binpath~="" then + program=binpath.."/"..program end local command=program.." "..replace(template,variables) if reporter then @@ -14938,7 +15335,8 @@ local runners={ if trace then report("execute: %s",command) end - return osexecute(command) + local okay=osexecute(command) + return okay end end, pipeto=function(...) @@ -14972,7 +15370,7 @@ function sandbox.registerrunner(specification) return end if validrunners[name] then - report("invalid name, runner %a already defined") + report("invalid name, runner %a already defined",name) return end local program=specification.program @@ -15090,8 +15488,8 @@ if io then end if os then overload(os.execute,binaryrunner,"os.execute") - overload(os.spawn,dummyrunner,"os.spawn") - overload(os.exec,dummyrunner,"os.exec") + overload(os.spawn,dummyrunner,"os.spawn") + overload(os.exec,dummyrunner,"os.exec") overload(os.resultof,binaryrunner,"os.resultof") overload(os.pipeto,binaryrunner,"os.pipeto") overload(os.rename,filehandlertwo,"os.rename") @@ -15116,13 +15514,6 @@ end if zip then zip.open=register(zip.open,filehandlerone,"zip.open") end -if fontloader then - fontloader.open=register(fontloader.open,filehandlerone,"fontloader.open") - fontloader.info=register(fontloader.info,filehandlerone,"fontloader.info") -end -if epdf then - epdf.open=register(epdf.open,filehandlerone,"epdf.open") -end sandbox.registerroot=registerroot sandbox.registerbinary=registerbinary sandbox.registerlibrary=registerlibrary @@ -15528,7 +15919,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-env"] = package.loaded["luat-env"] or true --- original size: 6134, stripped down to: 4118 +-- original size: 6551, stripped down to: 4315 if not modules then modules={} end modules ['luat-env']={ version=1.001, @@ -15537,7 +15928,7 @@ package.loaded["luat-env"] = package.loaded["luat-env"] or true copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local rawset,rawget,loadfile=rawset,rawget,loadfile +local rawset,loadfile=rawset,loadfile local gsub=string.gsub local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_lua=logs.reporter("resolvers","lua") @@ -15584,6 +15975,12 @@ function environment.texfile(filename) return resolvers.findfile(filename,'tex') end function environment.luafile(filename) + if CONTEXTLMTXMODE and CONTEXTLMTXMODE>0 and file.suffix(filename)=="lua" then + local resolved=resolvers.findfile(file.replacesuffix(filename,"lmt")) or "" + if resolved~="" then + return resolved + end + end local resolved=resolvers.findfile(filename,'tex') or "" if resolved~="" then return resolved @@ -15695,7 +16092,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-zip"] = package.loaded["util-zip"] or true --- original size: 18645, stripped down to: 11291 +-- original size: 19496, stripped down to: 10858 if not modules then modules={} end modules ['util-zip']={ version=1.001, @@ -15706,7 +16103,7 @@ if not modules then modules={} end modules ['util-zip']={ local type,tostring,tonumber=type,tostring,tonumber local sort=table.sort local find,format,sub,gsub=string.find,string.format,string.sub,string.gsub -local osdate,ostime=os.date,os.time +local osdate,ostime,osclock=os.date,os.time,os.clock local ioopen=io.open local loaddata,savedata=io.loaddata,io.savedata local filejoin,isdir,dirname,mkdirs=file.join,lfs.isdir,file.dirname,dir.mkdirs @@ -15721,25 +16118,19 @@ local getposition=files.getposition local band=bit32.band local rshift=bit32.rshift local lshift=bit32.lshift -local decompress,calculatecrc -if flate then - decompress=flate.flate_decompress - calculatecrc=flate.update_crc32 -else +local decompress,expandsize,calculatecrc local zlibdecompress=zlib.decompress + local zlibexpandsize=zlib.expandsize local zlibchecksum=zlib.crc32 - decompress=function(source,targetsize) - local target=zlibdecompress(source,-15) - if target then - return target - else - return false,1 - end + decompress=function(source) + return zlibdecompress(source,-15) end + expandsize=zlibexpandsize and function(source,targetsize) + return zlibexpandsize(source,targetsize,-15) + end or decompress calculatecrc=function(buffer,initial) return zlibchecksum(initial or 0,buffer) end -end local zipfiles={} utilities.zipfiles=zipfiles local openzipfile,closezipfile,unzipfile,foundzipfile,getziphash,getziplist do @@ -15864,7 +16255,11 @@ local openzipfile,closezipfile,unzipfile,foundzipfile,getziphash,getziplist do setposition(handle,position) local result=readstring(handle,compressed) if data.method==8 then - result=decompress(result,data.uncompressed) + if expandsize then + result=expandsize(result,data.uncompressed) + else + result=decompress(result) + end end if check and data.crc32~=calculatecrc(result) then print("checksum mismatch") @@ -15883,14 +16278,14 @@ local openzipfile,closezipfile,unzipfile,foundzipfile,getziphash,getziplist do zipfiles.list=getziplist zipfiles.found=foundzipfile end -if flate then do +if xzip then local writecardinal1=files.writebyte local writecardinal2=files.writecardinal2le local writecardinal4=files.writecardinal4le local logwriter=logs.writer local globpattern=dir.globpattern - local compress=flate.flate_compress - local checksum=flate.update_crc32 + local compress=xzip.compress + local checksum=xzip.crc32 local function fromdostime(dostime,dosdate) return ostime { year=rshift(dosdate,9)+1980, @@ -16087,27 +16482,33 @@ if flate then do local count=#list local step=number.idiv(count,10) local done=0 + local steps=verbose=="steps" + local time=steps and osclock() for i=1,count do local l=list[i] local n=l.filename local d=unzipfile(z,n) - local p=filejoin(path,n) - if mkdirs(dirname(p)) then - if verbose=="steps" then - total=total+#d - done=done+1 - if done>=step then - done=0 - logwriter(format("%4i files of %4i done, %10i bytes",i,count,total)) + if d then + local p=filejoin(path,n) + if mkdirs(dirname(p)) then + if steps then + total=total+#d + done=done+1 + if done>=step then + done=0 + logwriter(format("%4i files of %4i done, %10i bytes, %0.3f seconds",i,count,total,osclock()-time)) + end + elseif verbose then + logwriter(n) end - elseif verbose then - logwriter(n) + savedata(p,d) end - savedata(p,d) + else + logwriter(format("problem with file %s",n)) end end - if verbose=="steps" then - logwriter(format("%4i files of %4i done, %10i bytes",count,count,total)) + if steps then + logwriter(format("%4i files of %4i done, %10i bytes, %0.3f seconds",count,count,total,osclock()-time)) end closezipfile(z) return true @@ -16118,37 +16519,8 @@ if flate then do end zipfiles.zipdir=zipdir zipfiles.unzipdir=unzipdir -end end -if flate then - local streams=utilities.streams - local openfile=streams.open - local closestream=streams.close - local setposition=streams.setposition - local getsize=streams.size - local readcardinal4=streams.readcardinal4le - local getstring=streams.getstring - local decompress=flate.gz_decompress - function zipfiles.gunzipfile(filename) - local strm=openfile(filename) - if strm then - setposition(strm,getsize(strm)-4+1) - local size=readcardinal4(strm) - local data=decompress(getstring(strm),size) - closestream(strm) - return data - end - end -elseif gzip then - local openfile=gzip.open - function zipfiles.gunzipfile(filename) - local g=openfile(filename,"rb") - if g then - local d=g:read("*a") - d:close() - return d - end - end end +zipfiles.gunzipfile=gzip.load end -- of closure @@ -16157,7 +16529,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-tab"] = package.loaded["lxml-tab"] or true --- original size: 60383, stripped down to: 35698 +-- original size: 61191, stripped down to: 35864 if not modules then modules={} end modules ['lxml-tab']={ version=1.001, @@ -16881,7 +17253,10 @@ local slash=P('/') local colon=P(':') local semicolon=P(';') local ampersand=P('&') -local valid=R('az','AZ','09')+S('_-.') +local valid_0=R("\128\255") +local valid_1=R('az','AZ')+S('_')+valid_0 +local valid_2=valid_1+R('09')+S('-.') +local valid=valid_1*valid_2^0 local name_yes=C(valid^1)*colon*C(valid^1) local name_nop=C(P(true))*C(valid^1) local name=name_yes+name_nop @@ -16917,8 +17292,9 @@ end local function entityfile(pattern,k,v,n) if n then local okay,data - if resolvers then - okay,data=resolvers.loadbinfile(n) + local loadbinfile=resolvers and resolvers.loadbinfile + if loadbinfile then + okay,data=loadbinfile(n) else data=io.loaddata(n) okay=data and data~="" @@ -17027,12 +17403,14 @@ publicentityfile+publicdoctype+systemdoctype+definitiondoctype+simpledoctype)*op } return grammar_parsed_text_one,grammar_parsed_text_two,grammar_unparsed_text end -grammar_parsed_text_one_nop, -grammar_parsed_text_two_nop, -grammar_unparsed_text_nop=install(space,spacing,anything) -grammar_parsed_text_one_yes, -grammar_parsed_text_two_yes, -grammar_unparsed_text_yes=install(space_nl,spacing_nl,anything_nl) +local + grammar_parsed_text_one_nop, + grammar_parsed_text_two_nop, + grammar_unparsed_text_nop=install(space,spacing,anything) +local + grammar_parsed_text_one_yes, + grammar_parsed_text_two_yes, + grammar_unparsed_text_yes=install(space_nl,spacing_nl,anything_nl) local function _xmlconvert_(data,settings,detail) settings=settings or {} preparexmlstate(settings) @@ -17613,7 +17991,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-lpt"] = package.loaded["lxml-lpt"] or true --- original size: 55145, stripped down to: 30992 +-- original size: 54626, stripped down to: 31255 if not modules then modules={} end modules ['lxml-lpt']={ version=1.001, @@ -18095,6 +18473,8 @@ local builtin={ lastindex="(#ll.__p__.dt or 1)", lastelement="(ll.__p__.en or 1)", last="#list", + list="list", + self="ll", rootposition="order", order="order", element="(ll.ei or 1)", @@ -18203,7 +18583,8 @@ local function register_selector(specification) end local function register_expression(expression) local converted=lpegmatch(converter,expression) - local runner=load(format(template_e,converted)) + local wrapped=format(template_e,converted) + local runner=load(wrapped) runner=(runner and runner()) or function() errorrunner_e(expression,converted) end return { kind="expression",expression=expression,converted=converted,evaluator=runner } end @@ -18575,6 +18956,20 @@ expressions.count=function(e,pattern) local collected=applylpath(e,pattern) return pattern and (collected and #collected) or 0 end +expressions.attribute=function(e,name,value) + if type(e)=="table" and name then + local a=e.at + if a then + local v=a[name] + if value then + return v==value + else + return v + end + end + end + return nil +end expressions.oneof=function(s,...) for i=1,select("#",...) do if s==select(i,...) then @@ -18621,7 +19016,7 @@ function expressions.contains(str,pattern) end return false end -function xml.expressions.idstring(str) +function expressions.idstring(str) return type(str)=="string" and gsub(str,"^#","") or "" end local function traverse(root,pattern,handle) @@ -20394,7 +20789,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 11099, stripped down to: 7152 +-- original size: 11019, stripped down to: 7086 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -20407,9 +20802,9 @@ local next,type,getmetatable,rawset=next,type,getmetatable,rawset local gsub,find,gmatch,char=string.gsub,string.find,string.gmatch,string.char local filedirname,filebasename,filejoin=file.dirname,file.basename,file.join local ostype,osname,osuname,ossetenv,osgetenv=os.type,os.name,os.uname,os.setenv,os.getenv +local sortedpairs=table.sortedpairs local P,S,R,C,Cs,Cc,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) -local trace_detail=false trackers.register("resolvers.details",function(v) trace_detail=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) local report_initialization=logs.reporter("resolvers","initialization") resolvers=resolvers or {} @@ -20618,7 +21013,7 @@ if ostype=="unix" then rawset(t,k,v) end local colon=P(":") - for k,v in table.sortedpairs(prefixes) do + for k,v in sortedpairs(prefixes) do if p then p=P(k)+p else @@ -20645,7 +21040,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 18154, stripped down to: 10416 +-- original size: 18179, stripped down to: 10432 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -20929,7 +21324,7 @@ local function scan(files,remap,spec,path,n,m,r,onlyone,tolerant) scancache[sub(full,1,-2)]=files return files,remap,n,m,r end -function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) +local function scanfiles(path,branch,usecache,onlyonce,tolerant) local realpath=resolveprefix(path) if usecache then local content=fullcache[realpath] @@ -20984,8 +21379,9 @@ function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) statistics.stoptiming(timer) return content end +resolvers.scanfiles=scanfiles function resolvers.simplescanfiles(path,branch,usecache) - return resolvers.scanfiles(path,branch,usecache,true,true) + return scanfiles(path,branch,usecache,true,true) end function resolvers.scandata() table.sort(scanned) @@ -21052,7 +21448,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 9360, stripped down to: 6312 +-- original size: 9400, stripped down to: 6347 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -21062,7 +21458,7 @@ if not modules then modules={} end modules ['data-env']={ license="see context related readme files", } local lower,gsub=string.lower,string.gsub -local next=next +local next,rawget=next,rawget local resolvers=resolvers local allocate=utilities.storage.allocate local setmetatableindex=table.setmetatableindex @@ -21143,13 +21539,13 @@ local relations=allocate { mp={ names={ "mp" }, variable='MPINPUTS', - suffixes={ 'mp','mpvi','mpiv','mpii' }, + suffixes={ 'mp','mpvi','mpiv','mpxl','mpii' }, usertype=true, }, tex={ names={ "tex" }, variable='TEXINPUTS', - suffixes={ "tex","mkvi","mkiv","mkii","cld","lfg","xml" }, + suffixes={ "tex","mkiv","mkvi","mkxl","mklx","mkii","cld","lfg","xml" }, usertype=true, }, icc={ @@ -21337,7 +21733,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 16284, stripped down to: 10938 +-- original size: 16099, stripped down to: 11379 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -21346,48 +21742,83 @@ if not modules then modules={} end modules ['data-tmp']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local format,lower,gsub,concat=string.format,string.lower,string.gsub,table.concat -local concat=table.concat -local mkdirs,isdir,isfile=dir.mkdirs,lfs.isdir,lfs.isfile -local addsuffix,is_writable,is_readable=file.addsuffix,file.is_writable,file.is_readable -local formatters=string.formatters local next,type=next,type +local pcall,loadfile,collectgarbage=pcall,loadfile,collectgarbage +local format,lower,gsub=string.format,string.lower,string.gsub +local concat,serialize,fastserialize,serializetofile=table.concat,table.serialize,table.fastserialize,table.tofile +local mkdirs,expanddirname,isdir,isfile=dir.mkdirs,dir.expandname,lfs.isdir,lfs.isfile +local is_writable,is_readable=file.is_writable,file.is_readable +local collapsepath,joinfile,addsuffix,dirname=file.collapsepath,file.join,file.addsuffix,file.dirname +local savedata=file.savedata +local formatters=string.formatters +local osexit,osdate,osuuid=os.exit,os.date,os.uuid +local removefile=os.remove +local md5hex=md5.hex local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) local report_caches=logs.reporter("resolvers","caches") local report_resolvers=logs.reporter("resolvers","caching") local resolvers=resolvers local cleanpath=resolvers.cleanpath -local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) -local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) -local compile=utilities.lua.compile -function utilities.lua.compile(luafile,lucfile,cleanup,strip) - if cleanup==nil then cleanup=directive_cleanup end - if strip==nil then strip=directive_strip end - return compile(luafile,lucfile,cleanup,strip) +local resolvepath=resolvers.resolve +local luautilities=utilities.lua +do + local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) + local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) + local compilelua=luautilities.compile + function luautilities.compile(luafile,lucfile,cleanup,strip) + if cleanup==nil then cleanup=directive_cleanup end + if strip==nil then strip=directive_strip end + return compilelua(luafile,lucfile,cleanup,strip) + end end caches=caches or {} local caches=caches -local luasuffixes=utilities.lua.suffixes -caches.base=caches.base or "luatex-cache" -caches.more=caches.more or "context" -caches.direct=false -caches.tree=false -caches.force=true -caches.ask=false -caches.relocate=false +local writable=nil +local readables={} +local usedreadables={} +local compilelua=luautilities.compile +local luasuffixes=luautilities.suffixes +caches.base=caches.base or "luatex-cache" +caches.more=caches.more or "context" caches.defaults={ "TMPDIR","TEMPDIR","TMP","TEMP","HOME","HOMEPATH" } -directives.register("system.caches.fast",function(v) caches.fast=true end) -local writable,readables,usedreadables=nil,{},{} +local direct_cache=false +local fast_cache=false +local cache_tree=false +directives.register("system.caches.direct",function(v) direct_cache=true end) +directives.register("system.caches.fast",function(v) fast_cache=true end) +local function configfiles() + return concat(resolvers.configurationfiles(),";") +end +local function hashed(tree) + tree=gsub(tree,"[\\/]+$","") + tree=lower(tree) + local hash=md5hex(tree) + if trace_cache or trace_locating then + report_caches("hashing tree %a, hash %a",tree,hash) + end + return hash +end +local function treehash() + local tree=configfiles() + if not tree or tree=="" then + return false + else + return hashed(tree) + end +end +caches.hashed=hashed +caches.treehash=treehash +caches.configfiles=configfiles local function identify() local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") if texmfcaches then for k=1,#texmfcaches do local cachepath=texmfcaches[k] if cachepath~="" then - cachepath=resolvers.resolve(cachepath) - cachepath=resolvers.cleanpath(cachepath) - cachepath=file.collapsepath(cachepath) + cachepath=resolvepath(cachepath) + cachepath=cleanpath(cachepath) + cachepath=collapsepath(cachepath) local valid=isdir(cachepath) if valid then if is_readable(cachepath) then @@ -21396,16 +21827,14 @@ local function identify() writable=cachepath end end - elseif not writable and caches.force then - local cacheparent=file.dirname(cachepath) - if is_writable(cacheparent) and true then - if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath),"no",{ "yes","no" })=="yes" then - mkdirs(cachepath) - if isdir(cachepath) and is_writable(cachepath) then - report_caches("path %a created",cachepath) - writable=cachepath - readables[#readables+1]=cachepath - end + elseif not writable then + local cacheparent=dirname(cachepath) + if is_writable(cacheparent) then + mkdirs(cachepath) + if isdir(cachepath) and is_writable(cachepath) then + report_caches("path %a created",cachepath) + writable=cachepath + readables[#readables+1]=cachepath end end end @@ -21418,8 +21847,8 @@ local function identify() local cachepath=texmfcaches[k] cachepath=resolvers.expansion(cachepath) if cachepath~="" then - cachepath=resolvers.resolve(cachepath) - cachepath=resolvers.cleanpath(cachepath) + cachepath=resolvepath(cachepath) + cachepath=cleanpath(cachepath) local valid=isdir(cachepath) if valid and is_readable(cachepath) then if not writable and is_writable(cachepath) then @@ -21433,23 +21862,25 @@ local function identify() end if not writable then report_caches("fatal error: there is no valid writable cache path defined") - os.exit() + osexit() elseif #readables==0 then report_caches("fatal error: there is no valid readable cache path defined") - os.exit() + osexit() end - writable=dir.expandname(resolvers.cleanpath(writable)) - local base,more,tree=caches.base,caches.more,caches.tree or caches.treehash() + writable=expanddirname(cleanpath(writable)) + local base=caches.base + local more=caches.more + local tree=cache_tree or treehash() if tree then - caches.tree=tree + cache_tree=tree writable=mkdirs(writable,base,more,tree) for i=1,#readables do - readables[i]=file.join(readables[i],base,more,tree) + readables[i]=joinfile(readables[i],base,more,tree) end else writable=mkdirs(writable,base,more) for i=1,#readables do - readables[i]=file.join(readables[i],base,more) + readables[i]=joinfile(readables[i],base,more) end end if trace_cache then @@ -21486,27 +21917,8 @@ function caches.usedpaths(separator) return writable or "?" end end -function caches.configfiles() - return concat(resolvers.configurationfiles(),";") -end -function caches.hashed(tree) - tree=gsub(tree,"[\\/]+$","") - tree=lower(tree) - local hash=md5.hex(tree) - if trace_cache or trace_locating then - report_caches("hashing tree %a, hash %a",tree,hash) - end - return hash -end -function caches.treehash() - local tree=caches.configfiles() - if not tree or tree=="" then - return false - else - return caches.hashed(tree) - end -end -local r_cache,w_cache={},{} +local r_cache={} +local w_cache={} local function getreadablepaths(...) local tags={... } local hash=concat(tags,"/") @@ -21516,7 +21928,7 @@ local function getreadablepaths(...) if #tags>0 then done={} for i=1,#readables do - done[i]=file.join(readables[i],...) + done[i]=joinfile(readables[i],...) end else done=readables @@ -21540,17 +21952,25 @@ local function getwritablepath(...) end return done end -caches.getreadablepaths=getreadablepaths -caches.getwritablepath=getwritablepath -function caches.getfirstreadablefile(filename,...) - local fullname,path=caches.setfirstwritablefile(filename,...) +local function setfirstwritablefile(filename,...) + local wr=getwritablepath(...) + local fullname=joinfile(wr,filename) + return fullname,wr +end +local function setluanames(path,name) + return + format("%s/%s.%s",path,name,luasuffixes.tma), + format("%s/%s.%s",path,name,luasuffixes.tmc) +end +local function getfirstreadablefile(filename,...) + local fullname,path=setfirstwritablefile(filename,...) if is_readable(fullname) then return fullname,path end local rd=getreadablepaths(...) for i=1,#rd do local path=rd[i] - local fullname=file.join(path,filename) + local fullname=joinfile(path,filename) if is_readable(fullname) then usedreadables[i]=true return fullname,path @@ -21558,19 +21978,11 @@ function caches.getfirstreadablefile(filename,...) end return fullname,path end -function caches.setfirstwritablefile(filename,...) - local wr=getwritablepath(...) - local fullname=file.join(wr,filename) - return fullname,wr -end -function caches.define(category,subcategory) - return function() - return getwritablepath(category,subcategory) - end -end -function caches.setluanames(path,name) - return format("%s/%s.%s",path,name,luasuffixes.tma),format("%s/%s.%s",path,name,luasuffixes.tmc) -end +caches.getreadablepaths=getreadablepaths +caches.getwritablepath=getwritablepath +caches.setfirstwritablefile=setfirstwritablefile +caches.getfirstreadablefile=getfirstreadablefile +caches.setluanames=setluanames function caches.loaddata(readables,name,writable) if type(readables)=="string" then readables={ readables } @@ -21578,21 +21990,22 @@ function caches.loaddata(readables,name,writable) for i=1,#readables do local path=readables[i] local loader=false - local tmaname,tmcname=caches.setluanames(path,name) + local state=false + local tmaname,tmcname=setluanames(path,name) if isfile(tmcname) then - loader=loadfile(tmcname) + state,loader=pcall(loadfile,tmcname) end if not loader and isfile(tmaname) then - local tmacrap,tmcname=caches.setluanames(writable,name) + local tmacrap,tmcname=setluanames(writable,name) if isfile(tmcname) then - loader=loadfile(tmcname) + state,loader=pcall(loadfile,tmcname) end - utilities.lua.compile(tmaname,tmcname) + compilelua(tmaname,tmcname) if isfile(tmcname) then - loader=loadfile(tmcname) + state,loader=pcall(loadfile,tmcname) end if not loader then - loader=loadfile(tmaname) + state,loader=pcall(loadfile,tmaname) end end if loader then @@ -21604,21 +22017,21 @@ function caches.loaddata(readables,name,writable) return false end function caches.is_writable(filepath,filename) - local tmaname,tmcname=caches.setluanames(filepath,filename) + local tmaname,tmcname=setluanames(filepath,filename) return is_writable(tmaname) end -local saveoptions={ compact=true } -function caches.savedata(filepath,filename,data,raw) - local tmaname,tmcname=caches.setluanames(filepath,filename) - data.cache_uuid=os.uuid() - if caches.fast then - file.savedata(tmaname,table.fastserialize(data,true)) - elseif caches.direct then - file.savedata(tmaname,table.serialize(data,true,saveoptions)) +local saveoptions={ compact=true,accurate=not JITSUPPORTED } +function caches.savedata(filepath,filename,data,fast) + local tmaname,tmcname=setluanames(filepath,filename) + data.cache_uuid=osuuid() + if fast or fast_cache then + savedata(tmaname,fastserialize(data,true)) + elseif direct_cache then + savedata(tmaname,serialize(data,true,saveoptions)) else - table.tofile(tmaname,data,true,saveoptions) + serializetofile(tmaname,data,true,saveoptions) end - utilities.lua.compile(tmaname,tmcname) + compilelua(tmaname,tmcname) end local content_state={} function caches.contentstate() @@ -21626,11 +22039,14 @@ function caches.contentstate() end function caches.loadcontent(cachename,dataname,filename) if not filename then - local name=caches.hashed(cachename) - local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") - filename=file.join(path,name) + local name=hashed(cachename) + local full,path=getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=joinfile(path,name) + end + local state,blob=pcall(loadfile,addsuffix(filename,luasuffixes.luc)) + if not blob then + state,blob=pcall(loadfile,addsuffix(filename,luasuffixes.lua)) end - local blob=loadfile(addsuffix(filename,luasuffixes.luc)) or loadfile(addsuffix(filename,luasuffixes.lua)) if blob then local data=blob() if data and data.content then @@ -21663,9 +22079,9 @@ function caches.collapsecontent(content) end function caches.savecontent(cachename,dataname,content,filename) if not filename then - local name=caches.hashed(cachename) - local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") - filename=file.join(path,name) + local name=hashed(cachename) + local full,path=setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=joinfile(path,name) end local luaname=addsuffix(filename,luasuffixes.lua) local lucname=addsuffix(filename,luasuffixes.luc) @@ -21676,17 +22092,17 @@ function caches.savecontent(cachename,dataname,content,filename) type=dataname, root=cachename, version=resolvers.cacheversion, - date=os.date("%Y-%m-%d"), - time=os.date("%H:%M:%S"), + date=osdate("%Y-%m-%d"), + time=osdate("%H:%M:%S"), content=content, - uuid=os.uuid(), + uuid=osuuid(), } - local ok=io.savedata(luaname,table.serialize(data,true)) + local ok=savedata(luaname,serialize(data,true)) if ok then if trace_locating then report_resolvers("category %a, cachename %a saved in %a",dataname,cachename,luaname) end - if utilities.lua.compile(luaname,lucname) then + if compilelua(luaname,lucname) then if trace_locating then report_resolvers("%a compiled to %a",dataname,lucname) end @@ -21695,7 +22111,7 @@ function caches.savecontent(cachename,dataname,content,filename) if trace_locating then report_resolvers("compiling failed for %a, deleting file %a",dataname,lucname) end - os.remove(lucname) + removefile(lucname) end elseif trace_locating then report_resolvers("unable to save %a in %a (access error)",dataname,luaname) @@ -21709,7 +22125,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5310, stripped down to: 3784 +-- original size: 5518, stripped down to: 3854 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -21718,31 +22134,45 @@ if not modules then modules={} end modules ['data-met']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local find,format=string.find,string.format -local sequenced=table.sequenced +local type=type +local find=string.find local addurlscheme,urlhashed=url.addscheme,url.hashed +local collapsepath,joinfile=file.collapsepath,file.join +local report_methods=logs.reporter("resolvers","methods") local trace_locating=false local trace_methods=false trackers.register("resolvers.locating",function(v) trace_methods=v end) trackers.register("resolvers.methods",function(v) trace_methods=v end) -local report_methods=logs.reporter("resolvers","methods") local allocate=utilities.storage.allocate local resolvers=resolvers local registered={} local function splitmethod(filename) if not filename then - return { scheme="unknown",original=filename } + return { + scheme="unknown", + original=filename, + } end if type(filename)=="table" then return filename end - filename=file.collapsepath(filename,".") + filename=collapsepath(filename,".") if not find(filename,"://",1,true) then - return { scheme="file",path=filename,original=filename,filename=filename } + return { + scheme="file", + path=filename, + original=filename, + filename=filename, + } end - local specification=url.hashed(filename) + local specification=urlhashed(filename) if not specification.scheme or specification.scheme=="" then - return { scheme="file",path=filename,original=filename,filename=filename } + return { + scheme="file", + path=filename, + original=filename, + filename=filename, + } else return specification end @@ -21751,7 +22181,8 @@ resolvers.splitmethod=splitmethod local function methodhandler(what,first,...) local method=registered[what] if method then - local how,namespace=method.how,method.namespace + local how=method.how + local namespace=method.namespace if how=="uri" or how=="url" then local specification=splitmethod(first) local scheme=specification.scheme @@ -21797,7 +22228,10 @@ local function methodhandler(what,first,...) end resolvers.methodhandler=methodhandler function resolvers.registermethod(name,namespace,how) - registered[name]={ how=how or "tag",namespace=namespace } + registered[name]={ + how=how or "tag", + namespace=namespace + } namespace["byscheme"]=function(scheme,filename,...) if scheme=="file" then return methodhandler(name,filename,...) @@ -21806,7 +22240,7 @@ function resolvers.registermethod(name,namespace,how) end end end -local concatinators=allocate { notfound=file.join } +local concatinators=allocate { notfound=joinfile } local locators=allocate { notfound=function() end } local hashers=allocate { notfound=function() end } local generators=allocate { notfound=function() end } @@ -21827,7 +22261,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 68195, stripped down to: 43680 +-- original size: 69576, stripped down to: 44470 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -21838,7 +22272,8 @@ if not modules then modules={} end modules ['data-res']={ } local gsub,find,lower,upper,match,gmatch=string.gsub,string.find,string.lower,string.upper,string.match,string.gmatch local concat,insert,remove=table.concat,table.insert,table.remove -local next,type,rawget=next,type,rawget +local next,type,rawget,loadfile=next,type,rawget,loadfile +local mergedtable=table.merged local os=os local P,S,R,C,Cc,Cs,Ct,Carg=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Carg local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns @@ -21854,13 +22289,14 @@ local joinpath=file.joinpath local is_qualified_path=file.is_qualified_path local allocate=utilities.storage.allocate local settings_to_array=utilities.parsers.settings_to_array +local urlhasscheme=url.hasscheme local getcurrentdir=lfs.currentdir local isfile=lfs.isfile local isdir=lfs.isdir local setmetatableindex=table.setmetatableindex local luasuffixes=utilities.lua.suffixes local trace_locating=false trackers .register("resolvers.locating",function(v) trace_locating=v end) -local trace_detail=false trackers .register("resolvers.details",function(v) trace_detail=v end) +local trace_details=false trackers .register("resolvers.details",function(v) trace_details=v end) local trace_expansions=false trackers .register("resolvers.expansions",function(v) trace_expansions=v end) local trace_paths=false trackers .register("resolvers.paths",function(v) trace_paths=v end) local resolve_otherwise=true directives.register("resolvers.otherwise",function(v) resolve_otherwise=v end) @@ -21879,10 +22315,17 @@ local ostype,osname,osenv,ossetenv,osgetenv=os.type,os.name,os.env,os.setenv,os. resolvers.cacheversion="1.100" resolvers.configbanner="" resolvers.homedir=environment.homedir -resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } resolvers.luacnfname="texmfcnf.lua" resolvers.luacnffallback="contextcnf.lua" resolvers.luacnfstate="unknown" +local criticalvars={ + "SELFAUTOLOC", + "SELFAUTODIR", + "SELFAUTOPARENT", + "TEXMFCNF", + "TEXMF", + "TEXOS", +} if environment.default_texmfcnf then resolvers.luacnfspec="home:texmf/web2c;"..environment.default_texmfcnf else @@ -21902,13 +22345,20 @@ local dangerous=resolvers.dangerous local suffixmap=resolvers.suffixmap resolvers.defaultsuffixes={ "tex" } local instance=nil -function resolvers.setenv(key,value,raw) +local variable +local expansion +local setenv +local getenv +local formatofsuffix=resolvers.formatofsuffix +local splitpath=resolvers.splitpath +local splitmethod=resolvers.splitmethod +setenv=function(key,value,raw) if instance then instance.environment[key]=value ossetenv(key,raw and value or resolveprefix(value)) end end -local function getenv(key) +getenv=function(key) local value=rawget(instance.environment,key) if value and value~="" then return value @@ -21918,94 +22368,97 @@ local function getenv(key) end end resolvers.getenv=getenv -resolvers.env=getenv -local function resolvevariable(k) - return instance.expansions[k] -end +resolvers.setenv=setenv local dollarstripper=lpeg.stripper("$") local inhibitstripper=P("!")^0*Cs(P(1)^0) -local somevariable=P("$")/"" -local somekey=C(R("az","AZ","09","__","--")^1) -local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) -local variableexpander=Cs((somevariable*(somekey/resolvevariable)+somethingelse)^1 ) -local cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" -local variablecleaner=Cs((cleaner+P(1))^0) -local somevariable=R("az","AZ","09","__","--")^1/resolvevariable -local variable=(P("$")/"")*(somevariable+(P("{")/"")*somevariable*(P("}")/"")) -local variableresolver=Cs((variable+P(1))^0) -local function expandedvariable(var) - return lpegmatch(variableexpander,var) or var -end -function resolvers.reset() - if trace_locating then - report_resolving("creating instance") - end - local environment={} - local variables={} - local expansions={} - local order={} - instance={ - environment=environment, - variables=variables, - expansions=expansions, - order=order, - files={}, - setups={}, - found={}, - foundintrees={}, - hashes={}, - hashed={}, - pathlists=false, - specification={}, - lists={}, - data={}, - fakepaths={}, - remember=true, - diskcache=true, - renewcache=false, - renewtree=false, - loaderror=false, - savelists=true, - pattern=nil, - force_suffixes=true, - pathstack={}, - } - setmetatableindex(variables,function(t,k) - local v - for i=1,#order do - v=order[i][k] +local expandedvariable,resolvedvariable do + local function resolveinstancevariable(k) + return instance.expansions[k] + end + local p_variable=P("$")/"" + local p_key=C(R("az","AZ","09","__","--")^1) + local p_whatever=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) + local variableexpander=Cs((p_variable*(p_key/resolveinstancevariable)+p_whatever)^1 ) + local p_cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" + local variablecleaner=Cs((p_cleaner+P(1))^0) + local p_variable=R("az","AZ","09","__","--")^1/resolveinstancevariable + local p_variable=(P("$")/"")*(p_variable+(P("{")/"")*p_variable*(P("}")/"")) + local variableresolver=Cs((p_variable+P(1))^0) + expandedvariable=function(var) + return lpegmatch(variableexpander,var) or var + end + function resolvers.reset() + if trace_locating then + report_resolving("creating instance") + end + local environment={} + local variables={} + local expansions={} + local order={} + instance={ + environment=environment, + variables=variables, + expansions=expansions, + order=order, + files={}, + setups={}, + found={}, + foundintrees={}, + hashes={}, + hashed={}, + pathlists=false, + specification={}, + lists={}, + data={}, + fakepaths={}, + remember=true, + diskcache=true, + renewcache=false, + renewtree=false, + loaderror=false, + savelists=true, + pattern=nil, + force_suffixes=true, + pathstack={}, + } + setmetatableindex(variables,function(t,k) + local v + for i=1,#order do + v=order[i][k] + if v~=nil then + t[k]=v + return v + end + end + if v==nil then + v="" + end + t[k]=v + return v + end) + local repath=resolvers.repath + setmetatableindex(environment,function(t,k) + local v=osgetenv(k) + if v==nil then + v=variables[k] + end if v~=nil then - t[k]=v - return v + v=checkedvariable(v) or "" end - end - if v==nil then - v="" - end - t[k]=v - return v - end) - setmetatableindex(environment,function(t,k) - local v=osgetenv(k) - if v==nil then - v=variables[k] - end - if v~=nil then - v=checkedvariable(v) or "" - end - v=resolvers.repath(v) - t[k]=v - return v - end) - setmetatableindex(expansions,function(t,k) - local v=environment[k] - if type(v)=="string" then - v=lpegmatch(variableresolver,v) - v=lpegmatch(variablecleaner,v) - end - t[k]=v - return v - end) + v=repath(v) + t[k]=v + return v + end) + setmetatableindex(expansions,function(t,k) + local v=environment[k] + if type(v)=="string" then + v=lpegmatch(variableresolver,v) + v=lpegmatch(variablecleaner,v) + end + t[k]=v + return v + end) + end end function resolvers.initialized() return instance~=nil @@ -22019,31 +22472,33 @@ local function reset_caches() instance.lists={} instance.pathlists=false end -local slash=P("/") -local pathexpressionpattern=Cs ( - Cc("^")*( - Cc("%")*S(".-")+slash^2*P(-1)/"/.*" +local makepathexpression do + local slash=P("/") + local pathexpressionpattern=Cs ( + Cc("^")*( + Cc("%")*S(".-")+slash^2*P(-1)/"/.*" +slash^2/"/"+(1-slash)*P(-1)*Cc("/")+P(1) - )^1*Cc("$") -) -local cache={} -local function makepathexpression(str) - if str=="." then - return "^%./$" - else - local c=cache[str] - if not c then - c=lpegmatch(pathexpressionpattern,str) - cache[str]=c + )^1*Cc("$") + ) + local cache={} + makepathexpression=function(str) + if str=="." then + return "^%./$" + else + local c=cache[str] + if not c then + c=lpegmatch(pathexpressionpattern,str) + cache[str]=c + end + return c end - return c end end local function reportcriticalvariables(cnfspec) if trace_locating then - for i=1,#resolvers.criticalvars do - local k=resolvers.criticalvars[i] - local v=resolvers.getenv(k) or "unknown" + for i=1,#criticalvars do + local k=criticalvars[i] + local v=getenv(k) or "unknown" report_resolving("variable %a set to %a",k,v) end report_resolving() @@ -22065,7 +22520,7 @@ local function identify_configuration_files() resolvers.luacnfstate="environment" end reportcriticalvariables(cnfspec) - local cnfpaths=expandedpathfromlist(resolvers.splitpath(cnfspec)) + local cnfpaths=expandedpathfromlist(splitpath(cnfspec)) local function locatecnf(luacnfname,kind) for i=1,#cnfpaths do local filepath=cnfpaths[i] @@ -22098,6 +22553,8 @@ local function identify_configuration_files() end local function load_configuration_files() local specification=instance.specification + local setups=instance.setups + local order=instance.order if #specification>0 then local luacnfname=resolvers.luacnfname for i=1,#specification do @@ -22107,7 +22564,6 @@ local function load_configuration_files() local realname=resolveprefix(filename) local blob=loadfile(realname) if blob then - local setups=instance.setups local data=blob() local parent=data and data.parent if parent then @@ -22118,7 +22574,7 @@ local function load_configuration_files() local parentdata=blob() if parentdata then report_resolving("loading configuration file %a",filename) - data=table.merged(parentdata,data) + data=mergedtable(parentdata,data) end end end @@ -22150,7 +22606,7 @@ local function load_configuration_files() if trace_locating then report_resolving("reloading configuration due to TEXMF redefinition") end - resolvers.setenv("TEXMFCNF",cnfspec) + setenv("TEXMFCNF",cnfspec) instance.specification={} identify_configuration_files() load_configuration_files() @@ -22168,7 +22624,7 @@ local function load_configuration_files() elseif trace_locating then report_resolving("skipping configuration file %a (no valid format)",filename) end - instance.order[#instance.order+1]=instance.setups[pathname] + order[#order+1]=setups[pathname] if instance.loaderror then break end @@ -22177,6 +22633,8 @@ local function load_configuration_files() report_resolving("warning: no lua configuration files found") end end +local expandedpathlist +local unexpandedpathlist function resolvers.configurationfiles() return instance.specification or {} end @@ -22193,7 +22651,7 @@ local function load_file_databases() end end local function locate_file_databases() - local texmfpaths=resolvers.expandedpathlist("TEXMF") + local texmfpaths=expandedpathlist("TEXMF") if #texmfpaths>0 then for i=1,#texmfpaths do local path=collapsepath(texmfpaths[i]) @@ -22202,7 +22660,7 @@ local function locate_file_databases() if stripped~="" then local runtime=stripped==path path=cleanpath(path) - local spec=resolvers.splitmethod(stripped) + local spec=splitmethod(stripped) if runtime and (spec.noscheme or spec.scheme=="file") then stripped="tree:///"..stripped elseif spec.scheme=="cache" or spec.scheme=="file" then @@ -22236,11 +22694,13 @@ local function generate_file_databases() end end local function save_file_databases() - for i=1,#instance.hashes do - local hash=instance.hashes[i] + local hashes=instance.hashes + local files=instance.files + for i=1,#hashes do + local hash=hashes[i] local cachename=hash.name if hash.cache then - local content=instance.files[cachename] + local content=files[cachename] caches.collapsecontent(content) if trace_locating then report_resolving("saving tree %a",cachename) @@ -22252,8 +22712,9 @@ local function save_file_databases() end end function resolvers.renew(hashname) + local files=instance.files if hashname and hashname~="" then - local expanded=resolvers.expansion(hashname) or "" + local expanded=expansion(hashname) or "" if expanded~="" then if trace_locating then report_resolving("identifying tree %a from %a",expanded,hashname) @@ -22270,7 +22731,7 @@ function resolvers.renew(hashname) report_resolving("using path %a",realpath) end methodhandler('generators',hashname) - local content=instance.files[hashname] + local content=files[hashname] caches.collapsecontent(content) if trace_locating then report_resolving("saving tree %a",hashname) @@ -22297,38 +22758,46 @@ local function load_databases() end end function resolvers.appendhash(type,name,cache) - if not instance.hashed[name] then + local hashed=instance.hashed + local hashes=instance.hashes + if hashed[name] then + else if trace_locating then report_resolving("hash %a appended",name) end - insert(instance.hashes,{ type=type,name=name,cache=cache } ) - instance.hashed[name]=cache + insert(hashes,{ type=type,name=name,cache=cache } ) + hashed[name]=cache end end function resolvers.prependhash(type,name,cache) - if not instance.hashed[name] then + local hashed=instance.hashed + local hashes=instance.hashes + if hashed[name] then + else if trace_locating then report_resolving("hash %a prepended",name) end - insert(instance.hashes,1,{ type=type,name=name,cache=cache } ) - instance.hashed[name]=cache + insert(hashes,1,{ type=type,name=name,cache=cache } ) + hashed[name]=cache end end function resolvers.extendtexmfvariable(specification) - local t=resolvers.splitpath(getenv("TEXMF")) - insert(t,1,specification) - local newspec=concat(t,",") - if instance.environment["TEXMF"] then - instance.environment["TEXMF"]=newspec - elseif instance.variables["TEXMF"] then - instance.variables["TEXMF"]=newspec + local environment=instance.environment + local variables=instance.variables + local texmftrees=splitpath(getenv("TEXMF")) + insert(texmftrees,1,specification) + texmftrees=concat(texmftrees,",") + if environment["TEXMF"] then + environment["TEXMF"]=texmftrees + elseif variables["TEXMF"] then + variables["TEXMF"]=texmftrees else end reset_hashes() end function resolvers.splitexpansions() - local ie=instance.expansions - for k,v in next,ie do + local expansions=instance.expansions + for k,v in next,expansions do local t,tn,h,p={},0,{},splitconfigurationpath(v) for kk=1,#p do local vv=p[kk] @@ -22338,33 +22807,37 @@ function resolvers.splitexpansions() h[vv]=true end end - if #t>1 then - ie[k]=t + if tn>1 then + expansions[k]=t else - ie[k]=t[1] + expansions[k]=t[1] end end end function resolvers.datastate() return caches.contentstate() end -function resolvers.variable(name) +variable=function(name) + local variables=instance.variables local name=name and lpegmatch(dollarstripper,name) - local result=name and instance.variables[name] + local result=name and variables[name] return result~=nil and result or "" end -function resolvers.expansion(name) +expansion=function(name) + local expansions=instance.expansions local name=name and lpegmatch(dollarstripper,name) - local result=name and instance.expansions[name] + local result=name and expansions[name] return result~=nil and result or "" end -function resolvers.unexpandedpathlist(str) - local pth=resolvers.variable(str) - local lst=resolvers.splitpath(pth) +resolvers.variable=variable +resolvers.expansion=expansion +unexpandedpathlist=function(str) + local pth=variable(str) + local lst=splitpath(pth) return expandedpathfromlist(lst) end function resolvers.unexpandedpath(str) - return joinpath(resolvers.unexpandedpathlist(str)) + return joinpath(unexpandedpathlist(str)) end function resolvers.pushpath(name) local pathstack=instance.pathstack @@ -22394,8 +22867,8 @@ function resolvers.stackpath() end local done={} function resolvers.resetextrapaths() - local ep=instance.extra_paths - if not ep then + local extra_paths=instance.extra_paths + if not extra_paths then done={} instance.extra_paths={} elseif #ep>0 then @@ -22416,8 +22889,8 @@ function resolvers.registerextrapath(paths,subpaths) end local paths=settings_to_array(paths) local subpaths=settings_to_array(subpaths) - local ep=instance.extra_paths or {} - local oldn=#ep + local extra_paths=instance.extra_paths or {} + local oldn=#extra_paths local newn=oldn local nofpaths=#paths local nofsubpaths=#subpaths @@ -22430,7 +22903,7 @@ function resolvers.registerextrapath(paths,subpaths) local ps=p.."/"..s if not done[ps] then newn=newn+1 - ep[newn]=cleanpath(ps) + extra_paths[newn]=cleanpath(ps) done[ps]=true end end @@ -22440,7 +22913,7 @@ function resolvers.registerextrapath(paths,subpaths) local p=paths[i] if not done[p] then newn=newn+1 - ep[newn]=cleanpath(p) + extra_paths[newn]=cleanpath(p) done[p]=true end end @@ -22449,17 +22922,17 @@ function resolvers.registerextrapath(paths,subpaths) for i=1,oldn do for j=1,nofsubpaths do local s=subpaths[j] - local ps=ep[i].."/"..s + local ps=extra_paths[i].."/"..s if not done[ps] then newn=newn+1 - ep[newn]=cleanpath(ps) + extra_paths[newn]=cleanpath(ps) done[ps]=true end end end end if newn>0 then - instance.extra_paths=ep + instance.extra_paths=extra_paths end if newn~=oldn then reset_caches() @@ -22467,17 +22940,19 @@ function resolvers.registerextrapath(paths,subpaths) end function resolvers.pushextrapath(path) local paths=settings_to_array(path) - if instance.extra_stack then - insert(instance.extra_stack,1,paths) + local extra_stack=instance.extra_stack + if extra_stack then + insert(extra_stack,1,paths) else instance.extra_stack={ paths } end reset_caches() end function resolvers.popextrapath() - if instance.extra_stack then + local extra_stack=instance.extra_stack + if extra_stack then reset_caches() - return remove(instance.extra_stack,1) + return remove(extra_stack,1) end end local function made_list(instance,list,extra_too) @@ -22506,33 +22981,21 @@ local function made_list(instance,list,extra_too) end end if extra_too then - local es=instance.extra_stack - if es and #es>0 then - for k=1,#es do - add(es[k]) + local extra_stack=instance.extra_stack + local extra_paths=instance.extra_paths + if extra_stack and #extra_stack>0 then + for k=1,#extra_stack do + add(extra_stack[k]) end end - local ep=instance.extra_paths - if ep and #ep>0 then - add(ep) + if extra_paths and #extra_paths>0 then + add(extra_paths) end end add(list) return new end -function resolvers.cleanpathlist(str) - local t=resolvers.expandedpathlist(str) - if t then - for i=1,#t do - t[i]=collapsepath(cleanpath(t[i])) - end - end - return t -end -function resolvers.expandpath(str) - return joinpath(resolvers.expandedpathlist(str)) -end -function resolvers.expandedpathlist(str,extra_too) +expandedpathlist=function(str,extra_too) if not str then return {} elseif instance.savelists then @@ -22540,40 +23003,56 @@ function resolvers.expandedpathlist(str,extra_too) local lists=instance.lists local lst=lists[str] if not lst then - local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str)),extra_too) + local l=made_list(instance,splitpath(expansion(str)),extra_too) lst=expandedpathfromlist(l) lists[str]=lst end return lst else - local lst=resolvers.splitpath(resolvers.expansion(str)) + local lst=splitpath(expansion(str)) return made_list(instance,expandedpathfromlist(lst),extra_too) end end -function resolvers.expandedpathlistfromvariable(str) +resolvers.expandedpathlist=expandedpathlist +resolvers.unexpandedpathlist=unexpandedpathlist +function resolvers.cleanpathlist(str) + local t=expandedpathlist(str) + if t then + for i=1,#t do + t[i]=collapsepath(cleanpath(t[i])) + end + end + return t +end +function resolvers.expandpath(str) + return joinpath(expandedpathlist(str)) +end +local function expandedpathlistfromvariable(str) str=lpegmatch(dollarstripper,str) local tmp=resolvers.variableofformatorsuffix(str) - return resolvers.expandedpathlist(tmp~="" and tmp or str) + return expandedpathlist(tmp~="" and tmp or str) end function resolvers.expandpathfromvariable(str) - return joinpath(resolvers.expandedpathlistfromvariable(str)) + return joinpath(expandedpathlistfromvariable(str)) end +resolvers.expandedpathlistfromvariable=expandedpathlistfromvariable function resolvers.cleanedpathlist(v) - local t=resolvers.expandedpathlist(v) + local t=expandedpathlist(v) for i=1,#t do - t[i]=resolvers.resolve(resolvers.cleanpath(t[i])) + t[i]=resolveprefix(cleanpath(t[i])) end return t end function resolvers.expandbraces(str) - local pth=expandedpathfromlist(resolvers.splitpath(str)) + local pth=expandedpathfromlist(splitpath(str)) return joinpath(pth) end function resolvers.registerfilehash(name,content,someerror) + local files=instance.files if content then - instance.files[name]=content + files[name]=content else - instance.files[name]={} + files[name]={} if somerror==true then instance.loaderror=someerror end @@ -22592,7 +23071,7 @@ function resolvers.renewcache() end local function isreadable(name) local readable=isfile(name) - if trace_detail then + if trace_details then if readable then report_resolving("file %a is readable",name) else @@ -22609,7 +23088,7 @@ local function collect_files(names) local variant=hash.type local search=filejoin(root,path,name) local result=methodhandler('concatinators',variant,root,path,name) - if trace_detail then + if trace_details then report_resolving("match: variant %a, search %a, result %a",variant,search,result) end noffiles=noffiles+1 @@ -22618,7 +23097,7 @@ local function collect_files(names) end for k=1,#names do local filename=names[k] - if trace_detail then + if trace_details then report_resolving("checking name %a",filename) end local basename=filebasename(filename) @@ -22630,12 +23109,13 @@ local function collect_files(names) pathname="/"..pathname.."$" end local hashes=instance.hashes + local files=instance.files for h=1,#hashes do local hash=hashes[h] local hashname=hash.name - local content=hashname and instance.files[hashname] + local content=hashname and files[hashname] if content then - if trace_detail then + if trace_details then report_resolving("deep checking %a, base %a, pattern %a",hashname,basename,pathname) end local path,name=lookup(content,basename) @@ -22703,7 +23183,6 @@ local function find_analyze(filename,askedformat,allresults) if askedformat=="" then if filesuffix=="" or not suffixmap[filesuffix] then local defaultsuffixes=resolvers.defaultsuffixes - local formatofsuffix=resolvers.formatofsuffix for i=1,#defaultsuffixes do local forcedname=filename..'.'..defaultsuffixes[i] wantedfiles[#wantedfiles+1]=forcedname @@ -22713,7 +23192,7 @@ local function find_analyze(filename,askedformat,allresults) end end else - filetype=resolvers.formatofsuffix(filename) + filetype=formatofsuffix(filename) if trace_locating then report_resolving("using suffix based filetype %a",filetype) end @@ -22736,7 +23215,7 @@ local function find_analyze(filename,askedformat,allresults) end local function find_direct(filename,allresults) if not dangerous[askedformat] and isreadable(filename) then - if trace_detail then + if trace_details then report_resolving("file %a found directly",filename) end return "direct",{ filename } @@ -22761,12 +23240,12 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) report_resolving("checking qualified name %a",filename) end if isreadable(filename) then - if trace_detail then + if trace_details then report_resolving("qualified file %a found",filename) end return "qualified",{ filename } end - if trace_detail then + if trace_details then report_resolving("locating qualified file %a",filename) end local forcedname,suffix="",suffixonly(filename) @@ -22774,11 +23253,11 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) local format_suffixes=askedformat=="" and resolvers.defaultsuffixes or suffixes[askedformat] if format_suffixes then for i=1,#format_suffixes do - local s=format_suffixes[i] - forcedname=filename.."."..s + local suffix=format_suffixes[i] + forcedname=filename.."."..suffix if isreadable(forcedname) then if trace_locating then - report_resolving("no suffix, forcing format filetype %a",s) + report_resolving("no suffix, forcing format filetype %a",suffix) end return "qualified",{ forcedname } end @@ -22791,7 +23270,7 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) local savedformat=askedformat local format=savedformat or "" if format=="" then - askedformat=resolvers.formatofsuffix(suffix) + askedformat=formatofsuffix(suffix) end if not format then askedformat="othertextfiles" @@ -22822,7 +23301,7 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) end local function check_subpath(fname) if isreadable(fname) then - if trace_detail then + if trace_details then report_resolving("found %a by deep scanning",fname) end return fname @@ -22830,7 +23309,7 @@ local function check_subpath(fname) end local function makepathlist(list,filetype) local typespec=resolvers.variableofformat(filetype) - local pathlist=resolvers.expandedpathlist(typespec,filetype and usertypes[filetype]) + local pathlist=expandedpathlist(typespec,filetype and usertypes[filetype]) local entry={} if pathlist and #pathlist>0 then for k=1,#pathlist do @@ -22841,7 +23320,7 @@ local function makepathlist(list,filetype) local expression=makepathexpression(pathname) local barename=gsub(pathname,"/+$","") barename=resolveprefix(barename) - local scheme=url.hasscheme(barename) + local scheme=urlhasscheme(barename) local schemename=gsub(barename,"%.%*$",'') entry[k]={ path=path, @@ -22878,7 +23357,7 @@ local function find_intree(filename,filetype,wantedfiles,allresults) dirlist[i]=filedirname(filelist[i][3]).."/" end end - if trace_detail then + if trace_details then report_resolving("checking filename %a in tree",filename) end for k=1,#pathlist do @@ -22888,7 +23367,7 @@ local function find_intree(filename,filetype,wantedfiles,allresults) local done=false if filelist then local expression=entry.expression - if trace_detail then + if trace_details then report_resolving("using pattern %a for path %a",expression,pathname) end for k=1,#filelist do @@ -22899,16 +23378,16 @@ local function find_intree(filename,filetype,wantedfiles,allresults) result[#result+1]=resolveprefix(fl[3]) done=true if allresults then - if trace_detail then + if trace_details then report_resolving("match to %a in hash for file %a and path %a, continue scanning",expression,f,d) end else - if trace_detail then + if trace_details then report_resolving("match to %a in hash for file %a and path %a, quit scanning",expression,f,d) end break end - elseif trace_detail then + elseif trace_details then report_resolving("no match to %a in hash for file %a and path %a",expression,f,d) end end @@ -22923,7 +23402,7 @@ local function find_intree(filename,filetype,wantedfiles,allresults) if not find(pname,"*",1,true) then if can_be_dir(pname) then if not done and not entry.prescanned then - if trace_detail then + if trace_details then report_resolving("quick root scan for %a",pname) end for k=1,#wantedfiles do @@ -22938,7 +23417,7 @@ local function find_intree(filename,filetype,wantedfiles,allresults) end end if not done and entry.recursive then - if trace_detail then + if trace_details then report_resolving("scanning filesystem for %a",pname) end local files=resolvers.simplescanfiles(pname,false,true) @@ -23004,7 +23483,7 @@ local function find_intree(filename,filetype,wantedfiles,allresults) end end local function find_onpath(filename,filetype,wantedfiles,allresults) - if trace_detail then + if trace_details then report_resolving("checking filename %a, filetype %a, wanted files %a",filename,filetype,concat(wantedfiles," | ")) end local result={} @@ -23046,7 +23525,9 @@ collect_instance_files=function(filename,askedformat,allresults) { find_onpath (filename,filetype,wantedfiles,true) }, { find_otherwise(filename,filetype,wantedfiles,true) }, } - local result,status,done={},{},{} + local result={} + local status={} + local done={} for k,r in next,results do local method,list=r[1],r[2] if method and list then @@ -23060,7 +23541,7 @@ collect_instance_files=function(filename,askedformat,allresults) end end end - if trace_detail then + if trace_details then report_resolving("lookup status: %s",table.serialize(status,filename)) end return result,status @@ -23117,6 +23598,9 @@ local function findfiles(filename,filetype,allresults) if not filename or filename=="" then return {} end + if allresults==nil then + allresults=true + end local result,status=collect_instance_files(filename,filetype or "",allresults) if not result or #result==0 then local lowered=lower(filename) @@ -23126,27 +23610,25 @@ local function findfiles(filename,filetype,allresults) end return result or {},status end -function resolvers.findfiles(filename,filetype) - if not filename or filename=="" then - return "" - else - return findfiles(filename,filetype,true) - end -end -function resolvers.findfile(filename,filetype) +local function findfile(filename,filetype) if not filename or filename=="" then return "" else return findfiles(filename,filetype,false)[1] or "" end end +resolvers.findfiles=findfiles +resolvers.findfile=findfile +resolvers.find_file=findfile +resolvers.find_files=findfiles function resolvers.findpath(filename,filetype) return filedirname(findfiles(filename,filetype,false)[1] or "") end local function findgivenfiles(filename,allresults) + local hashes=instance.hashes + local files=instance.files local base=filebasename(filename) local result={} - local hashes=instance.hashes local function okay(hash,path,name) local found=methodhandler('concatinators',hash.type,hash.name,path,name) if found and found~="" then @@ -23156,7 +23638,7 @@ local function findgivenfiles(filename,allresults) end for k=1,#hashes do local hash=hashes[k] - local content=instance.files[hash.name] + local content=files[hash.name] if content then local path,name=lookup(content,base) if not path then @@ -23188,14 +23670,14 @@ function resolvers.wildcardpattern(pattern) return lpegmatch(makewildcard,pattern) or pattern end local function findwildcardfiles(filename,allresults,result) + local files=instance.files + local hashes=instance.hashes local result=result or {} local base=filebasename(filename) local dirn=filedirname(filename) local path=lower(lpegmatch(makewildcard,dirn) or dirn) local name=lower(lpegmatch(makewildcard,base) or base) - local files=instance.files if find(name,"*",1,true) then - local hashes=instance.hashes local function okay(found,path,base,hashname,hashtype) if find(found,path) then local full=methodhandler('concatinators',hashtype,hashname,found,base) @@ -23235,7 +23717,6 @@ local function findwildcardfiles(filename,allresults,result) end end end - local hashes=instance.hashes for k=1,#hashes do local hash=hashes[k] local hashname=hash.name @@ -23265,13 +23746,21 @@ end function resolvers.findwildcardfile(filename) return findwildcardfiles(filename,false)[1] or "" end -function resolvers.automount() -end -function resolvers.starttiming() - statistics.starttiming(instance) +do + local starttiming=statistics.starttiming + local stoptiming=statistics.stoptiming + local elapsedtime=statistics.elapsedtime + function resolvers.starttiming() + starttiming(instance) + end + function resolvers.stoptiming() + stoptiming(instance) + end + function resolvers.loadtime() + return elapsedtime(instance) + end end -function resolvers.stoptiming() - statistics.stoptiming(instance) +function resolvers.automount() end function resolvers.load(option) resolvers.starttiming() @@ -23285,9 +23774,6 @@ function resolvers.load(option) local files=instance.files return files and next(files) and true end -function resolvers.loadtime() - return statistics.elapsedtime(instance) -end local function report(str) if trace_locating then report_resolving(str) @@ -23317,7 +23803,7 @@ function resolvers.dowithfilesandreport(command,files,...) end end function resolvers.showpath(str) - return joinpath(resolvers.expandedpathlist(resolvers.formatofvariable(str))) + return joinpath(expandedpathlist(resolvers.formatofvariable(str))) end function resolvers.registerfile(files,name,path) if files[name] then @@ -23331,7 +23817,7 @@ function resolvers.registerfile(files,name,path) end end function resolvers.dowithpath(name,func) - local pathlist=resolvers.expandedpathlist(name) + local pathlist=expandedpathlist(name) for i=1,#pathlist do func("^"..cleanpath(pathlist[i])) end @@ -23341,11 +23827,11 @@ function resolvers.dowithvariable(name,func) end function resolvers.locateformat(name) local engine=environment.ownmain or "luatex" - local barename=removesuffix(name) + local barename=removesuffix(file.basename(name)) local fullname=addsuffix(barename,"fmt") local fmtname=caches.getfirstreadablefile(fullname,"formats",engine) or "" if fmtname=="" then - fmtname=resolvers.findfile(fullname) + fmtname=findfile(fullname) fmtname=cleanpath(fmtname) end if fmtname~="" then @@ -23354,17 +23840,17 @@ function resolvers.locateformat(name) local lucname=addsuffix(barename,luasuffixes.luc) local luiname=addsuffix(barename,luasuffixes.lui) if isfile(luiname) then - return barename,luiname + return fmtname,luiname elseif isfile(lucname) then - return barename,lucname + return fmtname,lucname elseif isfile(luaname) then - return barename,luaname + return fmtname,luaname end end return nil,nil end function resolvers.booleanvariable(str,default) - local b=resolvers.expansion(str) + local b=expansion(str) if b=="" then return default else @@ -23374,6 +23860,7 @@ function resolvers.booleanvariable(str,default) end function resolvers.dowithfilesintree(pattern,handle,before,after) local hashes=instance.hashes + local files=instance.files for i=1,#hashes do local hash=hashes[i] local blobtype=hash.type @@ -23385,7 +23872,7 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) if before then before(blobtype,blobpath,pattern) end - for path,name in filtered(instance.files[blobpath],pattern) do + for path,name in filtered(files[blobpath],pattern) do if type(path)=="string" then checked=checked+1 if handle(blobtype,blobpath,path,name) then @@ -23406,10 +23893,6 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) end end end -local obsolete=resolvers.obsolete or {} -resolvers.obsolete=obsolete -resolvers.find_file=resolvers.findfile obsolete.find_file=resolvers.findfile -resolvers.find_files=resolvers.findfiles obsolete.find_files=resolvers.findfiles function resolvers.knownvariables(pattern) if instance then local environment=instance.environment @@ -23443,7 +23926,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 4854, stripped down to: 2889 +-- original size: 5088, stripped down to: 3144 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -23452,6 +23935,7 @@ if not modules then modules={} end modules ['data-pre']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local insert,remove=table.insert,table.remove local resolvers=resolvers local prefixes=resolvers.prefixes local cleanpath=resolvers.cleanpath @@ -23529,8 +24013,9 @@ prefixes.kpse=prefixes.locate prefixes.full=prefixes.locate prefixes.file=prefixes.filename prefixes.path=prefixes.pathname +local inputstack={} +local stackpath=resolvers.stackpath local function toppath() - local inputstack=resolvers.inputstack if not inputstack then return "." end @@ -23542,15 +24027,23 @@ local function toppath() end end local function jobpath() - local path=resolvers.stackpath() + local path=stackpath() if not path or path=="" then return "." else return path end end +local function pushinputname(name) + insert(inputstack,name) +end +local function popinputname(name) + return remove(inputstack) +end resolvers.toppath=toppath resolvers.jobpath=jobpath +resolvers.pushinputname=pushinputname +resolvers.popinputname=popinputname prefixes.toppath=function(str) return cleanpath(joinpath(toppath(),str)) end prefixes.jobpath=function(str) return cleanpath(joinpath(jobpath(),str)) end resolvers.setdynamic("toppath") @@ -23593,7 +24086,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-out"] = package.loaded["data-out"] or true --- original size: 530, stripped down to: 470 +-- original size: 551, stripped down to: 470 if not modules then modules={} end modules ['data-out']={ version=1.001, @@ -23616,7 +24109,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3863, stripped down to: 3170 +-- original size: 4365, stripped down to: 3588 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -23625,39 +24118,46 @@ if not modules then modules={} end modules ['data-fil']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local ioopen=io.open +local isdir=lfs.isdir local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_files=logs.reporter("resolvers","files") local resolvers=resolvers local resolveprefix=resolvers.resolve -local finders,openers,loaders,savers=resolvers.finders,resolvers.openers,resolvers.loaders,resolvers.savers -local locators,hashers,generators,concatinators=resolvers.locators,resolvers.hashers,resolvers.generators,resolvers.concatinators +local findfile=resolvers.findfile +local scanfiles=resolvers.scanfiles +local registerfilehash=resolvers.registerfilehash +local appendhash=resolvers.appendhash +local loadcachecontent=caches.loadcontent local checkgarbage=utilities.garbagecollector and utilities.garbagecollector.check -function locators.file(specification) +function resolvers.locators.file(specification) local filename=specification.filename local realname=resolveprefix(filename) - if realname and realname~='' and lfs.isdir(realname) then + if realname and realname~='' and isdir(realname) then if trace_locating then report_files("file locator %a found as %a",filename,realname) end - resolvers.appendhash('file',filename,true) + appendhash('file',filename,true) elseif trace_locating then report_files("file locator %a not found",filename) end end -function hashers.file(specification) +function resolvers.hashers.file(specification) local pathname=specification.filename - local content=caches.loadcontent(pathname,'files') - resolvers.registerfilehash(pathname,content,content==nil) + local content=loadcachecontent(pathname,'files') + registerfilehash(pathname,content,content==nil) end -function generators.file(specification) +function resolvers.generators.file(specification) local pathname=specification.filename - local content=resolvers.scanfiles(pathname,false,true) - resolvers.registerfilehash(pathname,content,true) + local content=scanfiles(pathname,false,true) + registerfilehash(pathname,content,true) end -concatinators.file=file.join +resolvers.concatinators.file=file.join +local finders=resolvers.finders +local notfound=finders.notfound function finders.file(specification,filetype) local filename=specification.filename - local foundname=resolvers.findfile(filename,filetype) + local foundname=findfile(filename,filetype) if foundname and foundname~="" then if trace_locating then report_files("file finder: %a found",filename) @@ -23667,37 +24167,55 @@ function finders.file(specification,filetype) if trace_locating then report_files("file finder: %a not found",filename) end - return finders.notfound() + return notfound() end end -function openers.helpers.textopener(tag,filename,f) +local openers=resolvers.openers +local notfound=openers.notfound +local overloaded=false +local function textopener(tag,filename,f) return { - reader=function() return f:read () end, - close=function() logs.show_close(filename) return f:close() end, + reader=function() return f:read () end, + close=function() return f:close() end, } end +function openers.helpers.textopener(...) + return textopener(...) +end +function openers.helpers.settextopener(opener) + if overloaded then + report_files("file opener: %s overloaded","already") + else + if trace_locating then + report_files("file opener: %s overloaded","once") + end + overloaded=true + textopener=opener + end +end function openers.file(specification,filetype) local filename=specification.filename if filename and filename~="" then - local f=io.open(filename,"r") + local f=ioopen(filename,"r") if f then if trace_locating then report_files("file opener: %a opened",filename) end - return openers.helpers.textopener("file",filename,f) + return textopener("file",filename,f) end end if trace_locating then report_files("file opener: %a not found",filename) end - return openers.notfound() + return notfound() end +local loaders=resolvers.loaders +local notfound=loaders.notfound function loaders.file(specification,filetype) local filename=specification.filename if filename and filename~="" then - local f=io.open(filename,"rb") + local f=ioopen(filename,"rb") if f then - logs.show_load(filename) if trace_locating then report_files("file loader: %a loaded",filename) end @@ -23714,7 +24232,7 @@ function loaders.file(specification,filetype) if trace_locating then report_files("file loader: %a not found",filename) end - return loaders.notfound() + return notfound() end @@ -23724,7 +24242,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-con"] = package.loaded["data-con"] or true --- original size: 5029, stripped down to: 3432 +-- original size: 5388, stripped down to: 3685 if not modules then modules={} end modules ['data-con']={ version=1.100, @@ -23733,6 +24251,7 @@ if not modules then modules={} end modules ['data-con']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local setmetatable=setmetatable local format,lower,gsub=string.format,string.lower,string.gsub local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) local trace_containers=false trackers.register("resolvers.containers",function(v) trace_containers=v end) @@ -23740,16 +24259,21 @@ local trace_storage=false trackers.register("resolvers.storage",function(v) tra containers=containers or {} local containers=containers containers.usecache=true +local getwritablepath=caches.getwritablepath +local getreadablepaths=caches.getreadablepaths +local cacheiswritable=caches.is_writable +local loaddatafromcache=caches.loaddata +local savedataincache=caches.savedata local report_containers=logs.reporter("resolvers","containers") local allocated={} local mt={ __index=function(t,k) if k=="writable" then - local writable=caches.getwritablepath(t.category,t.subcategory) or { "." } + local writable=getwritablepath(t.category,t.subcategory) or { "." } t.writable=writable return writable elseif k=="readables" then - local readables=caches.getreadablepaths(t.category,t.subcategory) or { "." } + local readables=getreadablepaths(t.category,t.subcategory) or { "." } t.readables=readables return readables end @@ -23780,7 +24304,7 @@ function containers.define(category,subcategory,version,enabled) end end function containers.is_usable(container,name) - return container.enabled and caches and caches.is_writable(container.writable,name) + return container.enabled and caches and cacheiswritable(container.writable,name) end function containers.is_valid(container,name) if name and name~="" then @@ -23794,7 +24318,7 @@ function containers.read(container,name) local storage=container.storage local stored=storage[name] if not stored and container.enabled and caches and containers.usecache then - stored=caches.loaddata(container.readables,name,container.writable) + stored=loaddatafromcache(container.readables,name,container.writable) if stored and stored.cache_version==container.version then if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","load",container.subcategory,name) @@ -23810,17 +24334,20 @@ function containers.read(container,name) end return stored end -function containers.write(container,name,data) +function containers.write(container,name,data,fast) if data then data.cache_version=container.version if container.enabled and caches then - local unique,shared=data.unique,data.shared - data.unique,data.shared=nil,nil - caches.savedata(container.writable,name,data) + local unique=data.unique + local shared=data.shared + data.unique=nil + data.shared=nil + savedataincache(container.writable,name,data,fast) if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","save",container.subcategory,name) end - data.unique,data.shared=unique,shared + data.unique=unique + data.shared=shared end if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","store",container.subcategory,name) @@ -23843,7 +24370,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-use"] = package.loaded["data-use"] or true --- original size: 4434, stripped down to: 3180 +-- original size: 5790, stripped down to: 2910 if not modules then modules={} end modules ['data-use']={ version=1.001, @@ -23852,40 +24379,11 @@ if not modules then modules={} end modules ['data-use']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local format,lower,gsub,find=string.format,string.lower,string.gsub,string.find +local format=string.format local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_mounts=logs.reporter("resolvers","mounts") local resolvers=resolvers -resolvers.automounted=resolvers.automounted or {} -function resolvers.automount(usecache) - local mountpaths=resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) - if (not mountpaths or #mountpaths==0) and usecache then - mountpaths=caches.getreadablepaths("mount") - end - if mountpaths and #mountpaths>0 then - resolvers.starttiming() - for k=1,#mountpaths do - local root=mountpaths[k] - local f=io.open(root.."/url.tmi") - if f then - for line in f:lines() do - if line then - if find(line,"^[%%#%-]") then - elseif find(line,"^zip://") then - if trace_locating then - report_mounts("mounting %a",line) - end - table.insert(resolvers.automounted,line) - resolvers.usezipfile(line) - end - end - end - f:close() - end - end - resolvers.stoptiming() - end -end +local findfile=resolvers.findfile statistics.register("used config file",function() return caches.configfiles() end) statistics.register("used cache path",function() return caches.usedpaths() end) function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) @@ -23895,9 +24393,11 @@ function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) local luvdata={ enginebanner=enginebanner, formatbanner=formatbanner, - sourcehash=md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), + sourcehash=md5.hex(io.loaddata(findfile(sourcefile)) or "unknown"), sourcefile=sourcefile, luaversion=LUAVERSION, + formatid=LUATEXFORMATID, + functionality=LUATEXFUNCTIONALITY, } io.savedata(luvname,table.serialize(luvdata,true)) lua.registerfinalizer(function() @@ -23917,7 +24417,7 @@ function statistics.checkfmtstatus(texname) if lfs.isfile(luvname) then local luv=dofile(luvname) if luv and luv.sourcefile then - local sourcehash=md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") + local sourcehash=md5.hex(io.loaddata(findfile(luv.sourcefile)) or "unknown") local luvbanner=luv.enginebanner or "?" if luvbanner~=enginebanner then return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) @@ -23927,8 +24427,19 @@ function statistics.checkfmtstatus(texname) return format("source mismatch (luv: %s <> bin: %s)",luvhash,sourcehash) end local luvluaversion=luv.luaversion or 0 - if luvluaversion~=LUAVERSION then - return format("lua mismatch (luv: %s <> bin: %s)",luvluaversion,LUAVERSION) + local engluaversion=LUAVERSION or 0 + if luvluaversion~=engluaversion then + return format("lua mismatch (luv: %s <> bin: %s)",luvluaversion,engluaversion) + end + local luvfunctionality=luv.functionality or 0 + local engfunctionality=status.development_id or 0 + if luvfunctionality~=engfunctionality then + return format("functionality mismatch (luv: %s <> bin: %s)",luvfunctionality,engfunctionality) + end + local luvformatid=luv.formatid or 0 + local engformatid=status.format_id or 0 + if luvformatid~=engformatid then + return format("formatid mismatch (luv: %s <> bin: %s)",luvformatid,engformatid) end else return "invalid status file" @@ -23947,7 +24458,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 10263, stripped down to: 7556 +-- original size: 10725, stripped down to: 7949 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -23960,6 +24471,14 @@ local format,find,match=string.format,string.find,string.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_zip=logs.reporter("resolvers","zip") local resolvers=resolvers +local findfile=resolvers.findfile +local registerfile=resolvers.registerfile +local splitmethod=resolvers.splitmethod +local prependhash=resolvers.prependhash +local starttiming=resolvers.starttiming +local extendtexmf=resolvers.extendtexmfvariable +local stoptiming=resolvers.stoptiming +local urlquery=url.query zip=zip or {} local zip=zip local archives=zip.archives or {} @@ -23974,8 +24493,9 @@ if zipfiles then closezip=zipfiles.close validfile=zipfiles.found wholefile=zipfiles.unzip + local listzip=zipfiles.list traversezip=function(zfile) - return ipairs(zipfiles.list(zfile)) + return ipairs(listzip(zfile)) end local streams=utilities.streams local openstream=streams.open @@ -24032,28 +24552,30 @@ local function validzip(str) return str end end -function zip.openarchive(name) +local function openarchive(name) if not name or name=="" then return nil else local arch=archives[name] if not arch then - local full=resolvers.findfile(name) or "" + local full=findfile(name) or "" arch=full~="" and openzip(full) or false archives[name]=arch end return arch end end -function zip.closearchive(name) +local function closearchive(name) if not name or (name=="" and archives[name]) then closezip(archives[name]) archives[name]=nil end end +zip.openarchive=openarchive +zip.closearchive=closearchive function resolvers.locators.zip(specification) local archive=specification.filename - local zipfile=archive and archive~="" and zip.openarchive(archive) + local zipfile=archive and archive~="" and openarchive(archive) if trace_locating then if zipfile then report_zip("locator: archive %a found",archive) @@ -24062,13 +24584,6 @@ function resolvers.locators.zip(specification) end end end -function resolvers.hashers.zip(specification) - local archive=specification.filename - if trace_locating then - report_zip("loading file %a",archive) - end - resolvers.usezipfile(specification.original) -end function resolvers.concatinators.zip(zipfile,path,name) if not path or path=="" then return format('%s?name=%s',zipfile,name) @@ -24076,14 +24591,16 @@ function resolvers.concatinators.zip(zipfile,path,name) return format('%s?name=%s/%s',zipfile,path,name) end end -function resolvers.finders.zip(specification) +local finders=resolvers.finders +local notfound=finders.notfound +function finders.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("finder: archive %a found",archive) @@ -24104,16 +24621,19 @@ function resolvers.finders.zip(specification) if trace_locating then report_zip("finder: %a not found",original) end - return resolvers.finders.notfound() + return notfound() end -function resolvers.openers.zip(specification) +local openers=resolvers.openers +local notfound=openers.notfound +local textopener=openers.helpers.textopener +function openers.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("opener; archive %a opened",archive) @@ -24123,7 +24643,7 @@ function resolvers.openers.zip(specification) if trace_locating then report_zip("opener: file %a found",queryname) end - return resolvers.openers.helpers.textopener('zip',original,handle) + return textopener('zip',original,handle) elseif trace_locating then report_zip("opener: file %a not found",queryname) end @@ -24135,23 +24655,24 @@ function resolvers.openers.zip(specification) if trace_locating then report_zip("opener: %a not found",original) end - return resolvers.openers.notfound() + return notfound() end -function resolvers.loaders.zip(specification) +local loaders=resolvers.loaders +local notfound=loaders.notfound +function loaders.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("loader: archive %a opened",archive) end local data=wholefile(zfile,queryname) if data then - logs.show_load(original) if trace_locating then report_zip("loader; file %a loaded",original) end @@ -24167,47 +24688,24 @@ function resolvers.loaders.zip(specification) if trace_locating then report_zip("loader: %a not found",original) end - return resolvers.openers.notfound() -end -function resolvers.usezipfile(archive) - local specification=resolvers.splitmethod(archive) - local archive=specification.filename - if archive and not registeredfiles[archive] then - local z=zip.openarchive(archive) - if z then - local tree=url.query(specification.query).tree or "" - if trace_locating then - report_zip("registering: archive %a",archive) - end - resolvers.starttiming() - resolvers.prependhash('zip',archive) - resolvers.extendtexmfvariable(archive) - registeredfiles[archive]=z - resolvers.registerfilehash(archive,resolvers.registerzipfile(z,tree)) - resolvers.stoptiming() - elseif trace_locating then - report_zip("registering: unknown archive %a",archive) - end - elseif trace_locating then - report_zip("registering: archive %a not found",archive) - end + return notfound() end -function resolvers.registerzipfile(z,tree) +local function registerzipfile(z,tree) local names={} local files={} local remap={} local n=0 local filter=tree=="" and "^(.+)/(.-)$" or format("^%s/(.+)/(.-)$",tree) - local register=resolvers.registerfile if trace_locating then report_zip("registering: using filter %a",filter) end + starttiming() for i in traversezip(z) do local filename=i.filename local path,name=match(filename,filter) if not path then n=n+1 - register(names,filename,"") + registerfile(names,filename,"") local usedname=lower(filename) files[usedname]="" if usedname~=filename then @@ -24224,12 +24722,43 @@ function resolvers.registerzipfile(z,tree) else end end + stoptiming() report_zip("registering: %s files registered",n) return { files=files, remap=remap, } end +local function usezipfile(archive) + local specification=splitmethod(archive) + local archive=specification.filename + if archive and not registeredfiles[archive] then + local z=openarchive(archive) + if z then + local tree=urlquery(specification.query).tree or "" + if trace_locating then + report_zip("registering: archive %a",archive) + end + prependhash('zip',archive) + extendtexmf(archive) + registeredfiles[archive]=z + registerfilehash(archive,registerzipfile(z,tree)) + elseif trace_locating then + report_zip("registering: unknown archive %a",archive) + end + elseif trace_locating then + report_zip("registering: archive %a not found",archive) + end +end +resolvers.usezipfile=usezipfile +resolvers.registerzipfile=registerzipfile +function resolvers.hashers.zip(specification) + local archive=specification.filename + if trace_locating then + report_zip("loading file %a",archive) + end + usezipfile(specification.original) +end end -- of closure @@ -24238,7 +24767,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 8478, stripped down to: 5223 +-- original size: 10802, stripped down to: 6619 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -24247,180 +24776,255 @@ if not modules then modules={} end modules ['data-tre']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local type=type local find,gsub,lower=string.find,string.gsub,string.lower -local basename,dirname,joinname=file.basename,file.dirname,file .join +local basename,dirname,joinname=file.basename,file.dirname,file.join local globdir,isdir,isfile=dir.glob,lfs.isdir,lfs.isfile local P,lpegmatch=lpeg.P,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_trees=logs.reporter("resolvers","trees") local resolvers=resolvers -local resolveprefix=resolvers.resolve -local notfound=resolvers.finders.notfound -local lookup=resolvers.get_from_content -local collectors={} -local found={} -function resolvers.finders.tree(specification) - local spec=specification.filename - local okay=found[spec] - if okay==nil then - if spec~="" then - local path=dirname(spec) - local name=basename(spec) - if path=="" then - path="." - end - local names=collectors[path] - if not names then - local pattern=find(path,"/%*+$") and path or (path.."/*") - names=globdir(pattern) - collectors[path]=names - end - local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" - for i=1,#names do - local fullname=names[i] - if find(fullname,pattern) then - found[spec]=fullname - return fullname - end - end - local pattern=lower(pattern) - for i=1,#names do - local fullname=lower(names[i]) - if find(fullname,pattern) then - if isfile(fullname) then +local finders=resolvers.finders +local openers=resolvers.openers +local loaders=resolvers.loaders +local locators=resolvers.locators +local hashers=resolvers.hashers +local generators=resolvers.generators +do + local collectors={} + local found={} + local notfound=finders.notfound + function finders.tree(specification) + local spec=specification.filename + local okay=found[spec] + if okay==nil then + if spec~="" then + local path=dirname(spec) + local name=basename(spec) + if path=="" then + path="." + end + local names=collectors[path] + if not names then + local pattern=find(path,"/%*+$") and path or (path.."/*") + names=globdir(pattern) + collectors[path]=names + end + local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" + for i=1,#names do + local fullname=names[i] + if find(fullname,pattern) then found[spec]=fullname return fullname - else - break + end + end + local pattern=lower(pattern) + for i=1,#names do + local fullname=lower(names[i]) + if find(fullname,pattern) then + if isfile(fullname) then + found[spec]=fullname + return fullname + else + break + end end end end + okay=notfound() + found[spec]=okay end - okay=notfound() - found[spec]=okay + return okay end - return okay end -function resolvers.locators.tree(specification) - local name=specification.filename - local realname=resolveprefix(name) - if realname and realname~='' and isdir(realname) then - if trace_locating then - report_trees("locator %a found",realname) +do + local resolveprefix=resolvers.resolve + local appendhash=resolvers.appendhash + local function dolocate(specification) + local name=specification.filename + local realname=resolveprefix(name) + if realname and realname~='' and isdir(realname) then + if trace_locating then + report_trees("locator %a found",realname) + end + appendhash('tree',name,false) + elseif trace_locating then + report_trees("locator %a not found",name) end - resolvers.appendhash('tree',name,false) - elseif trace_locating then - report_trees("locator %a not found",name) end + locators.tree=dolocate + locators.dirlist=dolocate + locators.dirfile=dolocate end -function resolvers.hashers.tree(specification) - local name=specification.filename - if trace_locating then - report_trees("analyzing %a",name) - end - resolvers.methodhandler("hashers",name) - resolvers.generators.file(specification) -end -local collectors={} -local splitter=lpeg.splitat("/**/") -local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } -table.setmetatableindex(collectors,function(t,k) - local rootname=lpegmatch(stripper,k) - local dataname=joinname(rootname,"dirlist") - local content=caches.loadcontent(dataname,"files",dataname) - if not content then - content=resolvers.scanfiles(rootname,nil,nil,false,true) - caches.savecontent(dataname,"files",content,dataname) - end - t[k]=content - return content -end) -local function checked(root,p,n) - if p then - if type(p)=="table" then - for i=1,#p do - local fullname=joinname(root,p[i],n) +do + local filegenerator=generators.file + generators.dirlist=filegenerator + generators.dirfile=filegenerator +end +do + local filegenerator=generators.file + local methodhandler=resolvers.methodhandler + local function dohash(specification) + local name=specification.filename + if trace_locating then + report_trees("analyzing %a",name) + end + methodhandler("hashers",name) + filegenerator(specification) + end + hashers.tree=dohash + hashers.dirlist=dohash + hashers.dirfile=dohash +end +local resolve do + local collectors={} + local splitter=lpeg.splitat("/**/") + local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } + local loadcontent=caches.loadcontent + local savecontent=caches.savecontent + local notfound=finders.notfound + local scanfiles=resolvers.scanfiles + local lookup=resolvers.get_from_content + table.setmetatableindex(collectors,function(t,k) + local rootname=lpegmatch(stripper,k) + local dataname=joinname(rootname,"dirlist") + local content=loadcontent(dataname,"files",dataname) + if not content then + content=scanfiles(rootname,nil,nil,false,true) + savecontent(dataname,"files",content,dataname) + end + t[k]=content + return content + end) + local function checked(root,p,n) + if p then + if type(p)=="table" then + for i=1,#p do + local fullname=joinname(root,p[i],n) + if isfile(fullname) then + return fullname + end + end + else + local fullname=joinname(root,p,n) if isfile(fullname) then return fullname end end - else - local fullname=joinname(root,p,n) - if isfile(fullname) then - return fullname - end end + return notfound() end - return notfound() -end -local function resolve(specification) - local filename=specification.filename - if filename~="" then - local root,rest=lpegmatch(splitter,filename) - if root and rest then - local path,name=dirname(rest),basename(rest) - if name~=rest then - local content=collectors[root] - local p,n=lookup(content,name) - if not p then - return notfound() - end - local pattern=".*/"..path.."$" - local istable=type(p)=="table" - if istable then - for i=1,#p do - local pi=p[i] - if pi==path or find(pi,pattern) then - local fullname=joinname(root,pi,n) - if isfile(fullname) then - return fullname + resolve=function(specification) + local filename=specification.filename + if filename~="" then + local root,rest=lpegmatch(splitter,filename) + if root and rest then + local path,name=dirname(rest),basename(rest) + if name~=rest then + local content=collectors[root] + local p,n=lookup(content,name) + if not p then + return notfound() + end + local pattern=".*/"..path.."$" + local istable=type(p)=="table" + if istable then + for i=1,#p do + local pi=p[i] + if pi==path or find(pi,pattern) then + local fullname=joinname(root,pi,n) + if isfile(fullname) then + return fullname + end end end + elseif p==path or find(p,pattern) then + local fullname=joinname(root,p,n) + if isfile(fullname) then + return fullname + end end - elseif p==path or find(p,pattern) then - local fullname=joinname(root,p,n) - if isfile(fullname) then - return fullname + local queries=specification.queries + if queries and queries.option=="fileonly" then + return checked(root,p,n) + else + return notfound() end end - local queries=specification.queries - if queries and queries.option=="fileonly" then - return checked(root,p,n) - else - return notfound() - end + end + local path=dirname(filename) + local name=basename(filename) + local root=lpegmatch(stripper,path) + local content=collectors[path] + local p,n=lookup(content,name) + if p then + return checked(root,p,n) end end - local path,name=dirname(filename),basename(filename) - local root=lpegmatch(stripper,path) - local content=collectors[path] - local p,n=lookup(content,name) - if p then - return checked(root,p,n) + return notfound() + end + finders.dirlist=resolve + function finders.dirfile(specification) + local queries=specification.queries + if queries then + queries.option="fileonly" + else + specification.queries={ option="fileonly" } end + return resolve(specification) end - return notfound() end -resolvers.finders .dirlist=resolve -resolvers.locators .dirlist=resolvers.locators .tree -resolvers.hashers .dirlist=resolvers.hashers .tree -resolvers.generators.dirlist=resolvers.generators.file -resolvers.openers .dirlist=resolvers.openers .file -resolvers.loaders .dirlist=resolvers.loaders .file -function resolvers.finders.dirfile(specification) - local queries=specification.queries - if queries then - queries.option="fileonly" - else - specification.queries={ option="fileonly" } +do + local fileopener=openers.file + local fileloader=loaders.file + openers.dirlist=fileopener + loaders.dirlist=fileloader + openers.dirfile=fileopener + loaders.dirfile=fileloader +end +do + local hashfile="dirhash.lua" + local kind="HASH256" + local version=1.0 + local loadtable=table.load + local savetable=table.save + local loaddata=io.loaddata + function resolvers.dirstatus(patterns) + local t=type(patterns) + if t=="string" then + patterns={ patterns } + elseif t~="table" then + return false + end + local status=loadtable(hashfile) + if not status or status.version~=version or status.kind~=kind then + status={ + version=1.0, + kind=kind, + hashes={}, + } + end + local hashes=status.hashes + local changed={} + local action=sha2[kind] + local update={} + for i=1,#patterns do + local pattern=patterns[i] + local files=globdir(pattern) + for i=1,#files do + local name=files[i] + local hash=action(loaddata(name)) + if hashes[name]~=hash then + changed[#changed+1]=name + end + update[name]=hash + end + end + status.hashes=update + savetable(hashfile,status) + return #changed>0 and changed or false end - return resolve(specification) end -resolvers.locators .dirfile=resolvers.locators .dirlist -resolvers.hashers .dirfile=resolvers.hashers .dirlist -resolvers.generators.dirfile=resolvers.generators.dirlist -resolvers.openers .dirfile=resolvers.openers .dirlist -resolvers.loaders .dirfile=resolvers.loaders .dirlist end -- of closure @@ -24429,7 +25033,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6753, stripped down to: 5268 +-- original size: 6945, stripped down to: 5408 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -24439,8 +25043,11 @@ if not modules then modules={} end modules ['data-sch']={ license="see context related readme files" } local load,tonumber=load,tonumber -local gsub,concat,format=string.gsub,table.concat,string.format +local gsub,format=string.gsub,string.format +local sortedhash,concat=table.sortedhash,table.concat local finders,openers,loaders=resolvers.finders,resolvers.openers,resolvers.loaders +local addsuffix,suffix,splitbase=file.addsuffix,file.suffix,file.splitbase +local md5hex=md5.hex local trace_schemes=false trackers.register("resolvers.schemes",function(v) trace_schemes=v end) local report_schemes=logs.reporter("resolvers","schemes") local http=require("socket.http") @@ -24457,7 +25064,7 @@ function cleaners.none(specification) return specification.original end function cleaners.strip(specification) - local path,name=file.splitbase(specification.original) + local path,name=splitbase(specification.original) if path=="" then return (gsub(name,"[^%a%d%.]+","-")) else @@ -24465,7 +25072,7 @@ function cleaners.strip(specification) end end function cleaners.md5(specification) - return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) + return addsuffix(md5hex(specification.original),suffix(specification.path)) end local cleaner=cleaners.strip directives.register("schemes.cleanmethod",function(v) cleaner=cleaners[v] or cleaners.strip end) @@ -24485,7 +25092,7 @@ local runner=sandbox.registerrunner { name="curl resolver", method="execute", program="curl", - template="--silent --insecure --create-dirs --output %cachename% %original%", + template='--silent --insecure --create-dirs --output "%cachename%" "%original%"', checkers={ cachename="cache", original="url", @@ -24556,10 +25163,10 @@ end schemes.install=install local function http_handler(specification,cachename) local tempname=cachename..".tmp" - local f=io.open(tempname,"wb") + local handle=io.open(tempname,"wb") local status,message=http.request { url=specification.original, - sink=ltn12.sink.file(f) + sink=ltn12.sink.file(handle) } if not status then os.remove(tempname) @@ -24574,13 +25181,13 @@ install('https') install('ftp') statistics.register("scheme handling time",function() local l,r,nl,nr={},{},0,0 - for k,v in table.sortedhash(loaded) do + for k,v in sortedhash(loaded) do if v>0 then nl=nl+1 l[nl]=k..":"..v end end - for k,v in table.sortedhash(reused) do + for k,v in sortedhash(reused) do if v>0 then nr=nr+1 r[nr]=k..":"..v @@ -24588,10 +25195,10 @@ statistics.register("scheme handling time",function() end local n=nl+nr if n>0 then - l=nl>0 and concat(l) or "none" - r=nr>0 and concat(r) or "none" + if nl==0 then l={ "none" } end + if nr==0 then r={ "none" } end return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s", - statistics.elapsedtime(schemes),n,threshold,l,r) + statistics.elapsedtime(schemes),n,threshold,concat(l," "),concat(l," ")) else return nil end @@ -24624,7 +25231,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4207, stripped down to: 3041 +-- original size: 4227, stripped down to: 3049 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -24634,8 +25241,7 @@ if not modules then modules={} end modules ['data-lua']={ license="see context related readme files" } local package,lpeg=package,lpeg -local gsub=string.gsub -local concat=table.concat +local loadfile=loadfile local addsuffix=file.addsuffix local P,S,Cs,lpegmatch=lpeg.P,lpeg.S,lpeg.Cs,lpeg.match local luasuffixes={ 'tex','lua' } @@ -24646,6 +25252,8 @@ local helpers=package.helpers or {} local methods=helpers.methods or {} local resolvers=resolvers local resolveprefix=resolvers.resolve +local expandedpaths=resolvers.expandedpathlistfromvariable +local findfile=resolvers.findfile helpers.report=logs.reporter("resolvers","libraries") trackers.register("resolvers.libraries",function(v) helpers.trace=v end) trackers.register("resolvers.locating",function(v) helpers.trace=v end) @@ -24674,7 +25282,7 @@ local function getluaformatpaths() if not luaformatpaths then luaformatpaths={} for i=1,#luaformats do - registerpath("lua format","lua",luaformatpaths,resolvers.expandedpathlistfromvariable(luaformats[i])) + registerpath("lua format","lua",luaformatpaths,expandedpaths(luaformats[i])) end end return luaformatpaths @@ -24683,7 +25291,7 @@ local function getlibformatpaths() if not libformatpaths then libformatpaths={} for i=1,#libformats do - registerpath("lib format","lib",libformatpaths,resolvers.expandedpathlistfromvariable(libformats[i])) + registerpath("lib format","lib",libformatpaths,expandedpaths(libformats[i])) end end return libformatpaths @@ -24693,7 +25301,7 @@ local function loadedbyformat(name,rawname,suffixes,islib,what) local report=helpers.report for i=1,#suffixes do local format=suffixes[i] - local resolved=resolvers.findfile(name,format) or "" + local resolved=findfile(name,format) or "" if trace then report("%s format, identifying %a using format %a",what,name,format) end @@ -24731,7 +25339,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-aux"] = package.loaded["data-aux"] or true --- original size: 2452, stripped down to: 1877 +-- original size: 2610, stripped down to: 2019 if not modules then modules={} end modules ['data-aux']={ version=1.001, @@ -24742,24 +25350,28 @@ if not modules then modules={} end modules ['data-aux']={ } local find=string.find local type,next=type,next +local addsuffix,removesuffix=file.addsuffix,file.removesuffix +local loaddata,savedata=io.loaddata,io.savedata local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local resolvers=resolvers +local cleanpath=resolvers.cleanpath +local findfiles=resolvers.findfiles local report_scripts=logs.reporter("resolvers","scripts") function resolvers.updatescript(oldname,newname) local scriptpath="context/lua" - newname=file.addsuffix(newname,"lua") - local oldscript=resolvers.cleanpath(oldname) + local oldscript=cleanpath(oldname) + local newname=addsuffix(newname,"lua") + local newscripts=findfiles(newname) or {} if trace_locating then report_scripts("to be replaced old script %a",oldscript) end - local newscripts=resolvers.findfiles(newname) or {} if #newscripts==0 then if trace_locating then report_scripts("unable to locate new script") end else for i=1,#newscripts do - local newscript=resolvers.cleanpath(newscripts[i]) + local newscript=cleanpath(newscripts[i]) if trace_locating then report_scripts("checking new script %a",newscript) end @@ -24771,17 +25383,17 @@ function resolvers.updatescript(oldname,newname) if trace_locating then report_scripts("new script should come from %a",scriptpath) end - elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then + elseif not (find(oldscript,removesuffix(newname).."$") or find(oldscript,newname.."$")) then if trace_locating then report_scripts("invalid new script name") end else - local newdata=io.loaddata(newscript) + local newdata=loaddata(newscript) if newdata then if trace_locating then report_scripts("old script content replaced by new content: %s",oldscript) end - io.savedata(oldscript,newdata) + savedata(oldscript,newdata) break elseif trace_locating then report_scripts("unable to load new script") @@ -24854,7 +25466,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 1823, stripped down to: 1542 +-- original size: 2038, stripped down to: 1696 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -24864,15 +25476,22 @@ if not modules then modules={} end modules ['data-lst']={ license="see context related readme files" } local type=type -local concat,sortedhash=table.concat,table.sortedhash +local sortedhash=table.sortedhash +local isdir=lfs.isdir local resolvers=resolvers local listers=resolvers.listers or {} resolvers.listers=listers local resolveprefix=resolvers.resolve +local configurationfiles=resolvers.configurationfiles +local expandedpathfromlist=resolvers.expandedpathfromlist +local splitpath=resolvers.splitpath +local knownvariables=resolvers.knownvariables local report_lists=logs.reporter("resolvers","lists") local report_resolved=logs.reporter("system","resolved") local function tabstr(str) - if type(str)=='table' then + if not str then + return "unset" + elseif type(str)=='table' then return concat(str," | ") else return str @@ -24882,22 +25501,22 @@ function listers.variables(pattern) local result=resolvers.knownvariables(pattern) for key,value in sortedhash(result) do report_lists(key) - report_lists(" env: %s",tabstr(value.environment or "unset")) - report_lists(" var: %s",tabstr(value.variable or "unset")) - report_lists(" exp: %s",tabstr(value.expansion or "unset")) - report_lists(" res: %s",tabstr(value.resolved or "unset")) + report_lists(" env: %s",tabstr(value.environment)) + report_lists(" var: %s",tabstr(value.variable)) + report_lists(" exp: %s",tabstr(value.expansion)) + report_lists(" res: %s",tabstr(value.resolved)) end end function listers.configurations() - local configurations=resolvers.configurationfiles() + local configurations=configurationfiles() for i=1,#configurations do report_resolved("file : %s",resolveprefix(configurations[i])) end report_resolved("") - local list=resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) + local list=expandedpathfromlist(splitpath(resolvers.luacnfspec)) for i=1,#list do local li=resolveprefix(list[i]) - if lfs.isdir(li) then + if isdir(li) then report_resolved("path - %s",li) else report_resolved("path + %s",li) @@ -24910,318 +25529,147 @@ end -- of closure do -- create closure to overcome 200 locals limit -package.loaded["util-lib"] = package.loaded["util-lib"] or true +package.loaded["libs-ini"] = package.loaded["libs-ini"] or true --- original size: 16094, stripped down to: 8443 +-- original size: 5822, stripped down to: 3629 -if not modules then modules={} end modules ['util-lib']={ +if not modules then modules={} end modules ['libs-ini']={ version=1.001, comment="companion to luat-lib.mkiv", author="Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright="PRAGMA ADE / ConTeXt Development Team", - license="see context related readme files", + license="see context related readme files" } +local type,unpack=type,unpack local type=type -local next=next -local pcall=pcall -local gsub=string.gsub -local find=string.find -local sort=table.sort -local pathpart=file.pathpart local nameonly=file.nameonly local joinfile=file.join -local removesuffix=file.removesuffix local addsuffix=file.addsuffix -local findfile=resolvers.findfile -local findfiles=resolvers.findfiles -local expandpaths=resolvers.expandedpathlistfromvariable local qualifiedpath=file.is_qualified_path local isfile=lfs.isfile -local done=false -local function locate(required,version,trace,report,action) - if type(required)~="string" then - report("provide a proper library name") - return - end - if trace then - report("requiring library %a with version %a",required,version or "any") - end - local found_library=nil - local required_full=gsub(required,"%.","/") - local required_path=pathpart(required_full) - local required_base=nameonly(required_full) - if qualifiedpath(required) then - if isfile(addsuffix(required,os.libsuffix)) then - if trace then - report("qualified name %a found",required) +local findfile=resolvers.findfile +local expandpaths=resolvers.expandedpathlistfromvariable +local report=logs.reporter("resolvers","libraries") +local trace=false +trackers.register("resolvers.lib",function(v) trace=v end) +local function findlib(required) + local suffix=os.libsuffix or "so" + if not qualifiedpath(required) then + local list=directives.value("system.librarynames" ) + local only=nameonly(required) + if type(list)=="table" then + list=list[only] + if type(list)~="table" then + list={ only } end - found_library=required else - if trace then - report("qualified name %a not found",required) - end + list={ only } end - else - local required_name=required_base.."."..os.libsuffix - local version=type(version)=="string" and version~="" and version or false - local engine="luatex" - if trace and not done then - local list=expandpaths("lib") - for i=1,#list do - report("tds path %i: %s",i,list[i]) - end - end - local function found(locate,asked_library,how,...) - if trace then - report("checking %s: %a",how,asked_library) - end - return locate(asked_library,...) + if trace then + report("using lookup list for library %a: % | t",only,list) end - local function check(locate,...) - local found=nil - if version then - local asked_library=joinfile(required_path,version,required_name) - if trace then - report("checking %s: %a","with version",asked_library) - end - found=locate(asked_library,...) + for i=1,#list do + local name=list[i] + local found=findfile(name,"lib") + if not found then + found=findfile(addsuffix(name,suffix),"lib") end - if not found or found=="" then - local asked_library=joinfile(required_path,required_name) + if found then if trace then - report("checking %s: %a","with version",asked_library) + report("library %a resolved via %a path to %a",name,"tds lib",found) end - found=locate(asked_library,...) - end - return found and found~="" and found or false - end - local function attempt(checkpattern) - if trace then - report("checking tds lib paths strictly") - end - local found=findfile and check(findfile,"lib") - if found and (not checkpattern or find(found,checkpattern)) then return found end - if trace then - report("checking tds lib paths with wildcard") - end - local asked_library=joinfile(required_path,".*",required_name) - if trace then - report("checking %s: %a","latest version",asked_library) - end - local list=findfiles(asked_library,"lib",true) - if list and #list>0 then - sort(list) - local found=list[#list] - if found and (not checkpattern or find(found,checkpattern)) then - return found - end - end - if trace then - report("checking lib paths") - end - package.extralibpath(environment.ownpath) - local paths=package.libpaths() - local pattern="/[^/]+%."..os.libsuffix.."$" - for i=1,#paths do - required_path=gsub(paths[i],pattern,"") - local found=check(lfs.isfound) - if type(found)=="string" and (not checkpattern or find(found,checkpattern)) then - return found - end - end - return false end - if engine then - if trace then - report("attemp 1, engine %a",engine) - end - found_library=attempt("/"..engine.."/") - if not found_library then - if trace then - report("attemp 2, no engine",asked_library) + if expandpaths then + local list=expandpaths("PATH") + local base=addsuffix(only,suffix) + for i=1,#list do + local full=joinfile(list[i],base) + local found=isfile(full) and full + if found then + if trace then + report("library %a resolved via %a path to %a",name,"system",found) + end + return found end - found_library=attempt() end - else - found_library=attempt() end - end - if not found_library then + elseif isfile(addsuffix(required,suffix)) then if trace then - report("not found: %a",required) + report("library with qualified name %a %sfound",required,"") end - library=false + return required else if trace then - report("found: %a",found_library) - end - local result,message=action(found_library,required_base) - if result then - library=result - else - library=false - report("load error: message %a, library %a",tostring(message or "unknown"),found_library or "no library") + report("library with qualified name %a %sfound",required,"not ") end end - if trace then - if not library then - report("unknown library: %a",required) - else - report("stored library: %a",required) - end - end - return library or nil + return false end -do - local report_swiglib=logs.reporter("swiglib") - local trace_swiglib=false - local savedrequire=require - local loadedlibs={} - local loadlib=package.loadlib - local pushdir=dir.push - local popdir=dir.pop - trackers.register("resolvers.swiglib",function(v) trace_swiglib=v end) - function requireswiglib(required,version) - local library=loadedlibs[library] - if library==nil then - local trace_swiglib=trace_swiglib or package.helpers.trace - library=locate(required,version,trace_swiglib,report_swiglib,function(name,base) - pushdir(pathpart(name)) - local opener="luaopen_"..base - if trace_swiglib then - report_swiglib("opening: %a with %a",name,opener) - end - local library,message=loadlib(name,opener) - local libtype=type(library) - if libtype=="function" then - library=library() +local foundlibraries=table.setmetatableindex(function(t,k) + local v=findlib(k) + t[k]=v + return v +end) +function resolvers.findlib(required) + return foundlibraries[required] +end +local libraries={} +resolvers.libraries=libraries +local report=logs.reporter("optional") +if optional then optional.loaded={} end +function libraries.validoptional(name) + local thelib=optional and optional[name] + if not thelib then + elseif thelib.initialize then + return thelib + else + report("invalid optional library %a",libname) + end +end +function libraries.optionalloaded(name,libnames) + local thelib=optional and optional[name] + if not thelib then + report("no optional %a library found",name) + else + local thelib_initialize=thelib.initialize + if not thelib_initialize then + report("invalid optional library %a",name) + else + if type(libnames)=="string" then + libnames={ libnames } + end + if type(libnames)=="table" then + for i=1,#libnames do + local libname=libnames[i] + local filename=foundlibraries[libname] + if filename then + libnames[i]=filename + else + report("unable to locate library %a",libname) + return + end + end + local initialized=thelib_initialize(unpack(libnames)) + if initialized then + report("using library '% + t'",libnames) else - report_swiglib("load error: %a returns %a, message %a, library %a",opener,libtype,(string.gsub(message or "no message","[%s]+$","")),found_library or "no library") - library=false + report("unable to initialize library '% + t'",libnames) end - popdir() - return library - end) - loadedlibs[required]=library or false - end - return library - end - function require(name,version) - if find(name,"^swiglib%.") then - return requireswiglib(name,version) - else - return savedrequire(name) - end - end - local swiglibs={} - local initializer="core" - function swiglib(name,version) - local library=swiglibs[name] - if not library then - statistics.starttiming(swiglibs) - if trace_swiglib then - report_swiglib("loading %a",name) - end - if not find(name,"%."..initializer.."$") then - fullname="swiglib."..name.."."..initializer - else - fullname="swiglib."..name + return initialized end - library=requireswiglib(fullname,version) - swiglibs[name]=library - statistics.stoptiming(swiglibs) end - return library end - statistics.register("used swiglibs",function() - if next(swiglibs) then - return string.format("%s, initial load time %s seconds",table.concat(table.sortedkeys(swiglibs)," "),statistics.elapsedtime(swiglibs)) - end - end) end if FFISUPPORTED and ffi and ffi.load then - local report_ffilib=logs.reporter("ffilib") - local trace_ffilib=false - local savedffiload=ffi.load - trackers.register("resolvers.ffilib",function(v) trace_ffilib=v end) - local loaded={} - local function locateindeed(name) - name=removesuffix(name) - local l=loaded[name] - if l==nil then - local state,library=pcall(savedffiload,name) - if type(library)=="userdata" then - l=library - elseif type(state)=="userdata" then - l=state - else - l=false - end - loaded[name]=l - elseif trace_ffilib then - report_ffilib("reusing already loaded %a",name) - end - return l - end - local function getlist(required) - local list=directives.value("system.librarynames" ) - if type(list)=="table" then - list=list[required] - if type(list)=="table" then - if trace then - report("using lookup list for library %a: % | t",required,list) - end - return list - end - end - return { required } - end - function ffilib(name,version) - name=removesuffix(name) - local l=loaded[name] - if l~=nil then - if trace_ffilib then - report_ffilib("reusing already loaded %a",name) - end - return l - end - local list=getlist(name) - if version=="system" then - for i=1,#list do - local library=locateindeed(list[i]) - if type(library)=="userdata" then - return library - end - end - else - for i=1,#list do - local library=locate(list[i],version,trace_ffilib,report_ffilib,locateindeed) - if type(library)=="userdata" then - return library - end - end - end - end + local ffiload=ffi.load function ffi.load(name) - local list=getlist(name) - for i=1,#list do - local library=ffilib(list[i]) - if type(library)=="userdata" then - return library - end - end - if trace_ffilib then - report_ffilib("trying to load %a using normal loader",name) - end - for i=1,#list do - local state,library=pcall(savedffiload,list[i]) - if type(library)=="userdata" then - return library - elseif type(state)=="userdata" then - return library - end + local full=name and foundlibraries[name] + if full then + return ffiload(full) + else + return ffiload(name) end end end @@ -25336,7 +25784,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 9637, stripped down to: 7253 +-- original size: 13964, stripped down to: 10026 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -25350,16 +25798,14 @@ local concat=table.concat local quoted=string.quoted local luasuffixes=utilities.lua.suffixes local report_format=logs.reporter("resolvers","formats") -local function primaryflags() - local arguments=environment.arguments +local function primaryflags(arguments) local flags={} if arguments.silent then flags[#flags+1]="--interaction=batchmode" end return concat(flags," ") end -local function secondaryflags() - local arguments=environment.arguments +local function secondaryflags(arguments) local trackers=arguments.trackers local directives=arguments.directives local flags={} @@ -25381,6 +25827,9 @@ local function secondaryflags() if arguments.ansi then flags[#flags+1]="--c:ansi" end + if arguments.ansilog then + flags[#flags+1]="--c:ansilog" + end if arguments.strip then flags[#flags+1]="--c:strip" end @@ -25391,12 +25840,13 @@ local function secondaryflags() end local template=[[--ini %primaryflags% --lua=%luafile% %texfile% %secondaryflags% %dump% %redirect%]] local checkers={ - primaryflags="string", - secondaryflags="string", + primaryflags="verbose", + secondaryflags="verbose", luafile="readable", texfile="readable", redirect="string", dump="string", + binarypath="string", } local runners={ luatex=sandbox.registerrunner { @@ -25421,46 +25871,88 @@ local runners={ reporter=report_format, }, } -function environment.make_format(name,arguments) +local function validbinarypath() + if not environment.arguments.nobinarypath then + local path=environment.ownpath or file.dirname(environment.ownname) + if path and path~="" then + path=dir.expandname(path) + if path~="" and lfs.isdir(path) then + return path + end + end + end +end +function environment.make_format(formatname) + local arguments=environment.arguments local engine=environment.ownmain or "luatex" - local silent=environment.arguments.silent - local errors=environment.arguments.errors - local olddir=dir.current() - local path=caches.getwritablepath("formats",engine) or "" - if path~="" then - lfs.chdir(path) - end - report_format("using format path %a",dir.current()) - local texsourcename=file.addsuffix(name,"mkiv") - local fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + local silent=arguments.silent + local errors=arguments.errors + local texsourcename="" + local texsourcepath="" + local fulltexsourcename="" + if engine=="luametatex" then + texsourcename=file.addsuffix(formatname,"mkxl") + fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + end + if fulltexsourcename=="" then + texsourcename=file.addsuffix(formatname,"mkiv") + fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + end if fulltexsourcename=="" then - texsourcename=file.addsuffix(name,"tex") + texsourcename=file.addsuffix(formatname,"tex") fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" end if fulltexsourcename=="" then - report_format("no tex source file with name %a (mkiv or tex)",name) - lfs.chdir(olddir) + report_format("no tex source file with name %a (mkiv or tex)",formatname) return - else - report_format("using tex source file %a",fulltexsourcename) end - local texsourcepath=dir.expandname(file.dirname(fulltexsourcename)) - local specificationname=file.replacesuffix(fulltexsourcename,"lus") - local fullspecificationname=resolvers.findfile(specificationname,"tex") or "" - if fullspecificationname=="" then - specificationname=file.join(texsourcepath,"context.lus") - fullspecificationname=resolvers.findfile(specificationname,"tex") or "" + report_format("using tex source file %a",fulltexsourcename) + fulltexsourcename=dir.expandname(fulltexsourcename) + texsourcepath=file.dirname(fulltexsourcename) + if not lfs.isfile(fulltexsourcename) then + report_format("no accessible tex source file with name %a",fulltexsourcename) + return end + local specificationname="context.lus" + local specificationpath="" + local fullspecificationname=resolvers.findfile(specificationname) or "" if fullspecificationname=="" then - report_format("unknown stub specification %a",specificationname) - lfs.chdir(olddir) + report_format("unable to locate specification file %a",specificationname) + return + end + report_format("using specification file %a",fullspecificationname) + fullspecificationname=dir.expandname(fullspecificationname) + specificationpath=file.dirname(fullspecificationname) + if texsourcepath~=specificationpath then + report_format("tex source file and specification file are on different paths") + return + end + if not lfs.isfile(fulltexsourcename) then + report_format("no accessible tex source file with name %a",fulltexsourcename) + return + end + if not lfs.isfile(fullspecificationname) then + report_format("no accessible specification file with name %a",fulltexsourcename) + return + end + report_format("using tex source path %a",texsourcepath) + local validformatpath=caches.getwritablepath("formats",engine) or "" + local startupdir=dir.current() + if validformatpath=="" then + report_format("invalid format path, insufficient write access") + return + end + local binarypath=validbinarypath() + report_format("changing to format path %a",validformatpath) + lfs.chdir(validformatpath) + if dir.current()~=validformatpath then + report_format("unable to change to format path %a",validformatpath) return end - local specificationpath=file.dirname(fullspecificationname) local usedluastub=nil local usedlualibs=dofile(fullspecificationname) if type(usedlualibs)=="string" then - usedluastub=file.join(file.dirname(fullspecificationname),usedlualibs) + usedluastub=file.join(specificationpath,usedlualibs) elseif type(usedlualibs)=="table" then report_format("using stub specification %a",fullspecificationname) local texbasename=file.basename(name) @@ -25477,48 +25969,57 @@ function environment.make_format(name,arguments) end else report_format("invalid stub specification %a",fullspecificationname) - lfs.chdir(olddir) + lfs.chdir(startupdir) return end + local runner=runners[engine] + if not runner then + report_format("the format %a cannot be generated, no runner available for engine %a",name,engine) + lfs.chdir(startupdir) + return + end + local primaryflags=primaryflags(arguments) + local secondaryflags=secondaryflags(arguments) local specification={ - primaryflags=primaryflags(), - secondaryflags=secondaryflags(), + binarypath=binarypath, + primaryflags=primaryflags, + secondaryflags=secondaryflags, luafile=quoted(usedluastub), texfile=quoted(fulltexsourcename), dump=os.platform=="unix" and "\\\\dump" or "\\dump", } - local runner=runners[engine] - if not runner then - report_format("format %a cannot be generated, no runner available for engine %a",name,engine) - elseif silent then - statistics.starttiming() + if silent then specification.redirect="> temp.log" - local result=runner(specification) - local runtime=statistics.stoptiming() - if result~=0 then - print(format("%s silent make > fatal error when making format %q",engine,name)) - else - print(format("%s silent make > format %q made in %.3f seconds",engine,name,runtime)) - end + end + statistics.starttiming() + local result=runner(specification) + local runtime=statistics.stoptiming() + if silent then os.remove("temp.log") - else - runner(specification) end - local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" - local mp=dir.glob(pattern) - if mp then - for i=1,#mp do - local name=mp[i] - report_format("removing related mplib format %a",file.basename(name)) - os.remove(name) + report_format() + if binarypath and binarypath~="" then + report_format("binary path : %s",binarypath or "?") end - end - lfs.chdir(olddir) + report_format("format path : %s",validformatpath) + report_format("luatex engine : %s",engine) + report_format("lua startup file : %s",usedluastub) + if primaryflags~="" then + report_format("primary flags : %s",primaryflags) + end + if secondaryflags~="" then + report_format("secondary flags : %s",secondaryflags) + end + report_format("context file : %s",fulltexsourcename) + report_format("run time : %.3f seconds",runtime) + report_format("return value : %s",result==0 and "okay" or "error") + report_format() + lfs.chdir(startupdir) end -local template=[[%flags% --fmt=%fmtfile% --lua=%luafile% %texfile% %more%]] +local template=[[%primaryflags% --fmt=%fmtfile% --lua=%luafile% %texfile% %secondaryflags%]] local checkers={ - flags="string", - more="string", + primaryflags="verbose", + secondaryflags="verbose", fmtfile="readable", luafile="readable", texfile="readable", @@ -25531,6 +26032,13 @@ local runners={ checkers=checkers, reporter=report_format, }, + luametatex=sandbox.registerrunner { + name="run luametatex format", + program="luametatex", + template=template, + checkers=checkers, + reporter=report_format, + }, luajittex=sandbox.registerrunner { name="run luajittex format", program="luajittex", @@ -25539,51 +26047,78 @@ local runners={ reporter=report_format, }, } -function environment.run_format(name,data,more) - if name and name~="" then - local engine=environment.ownmain or "luatex" - local barename=file.removesuffix(name) - local fmtname=caches.getfirstreadablefile(file.addsuffix(barename,"fmt"),"formats",engine) - if fmtname=="" then - fmtname=resolvers.findfile(file.addsuffix(barename,"fmt")) or "" - end - fmtname=resolvers.cleanpath(fmtname) - if fmtname=="" then - report_format("no format with name %a",name) - else - local barename=file.removesuffix(name) - local luaname=file.addsuffix(barename,"luc") - if not lfs.isfile(luaname) then - luaname=file.addsuffix(barename,"lua") - end - if not lfs.isfile(luaname) then - report_format("using format name %a",fmtname) - report_format("no luc/lua file with name %a",barename) - else - local runner=runners[engine] - if not runner then - report_format("format %a cannot be run, no runner available for engine %a",name,engine) - else - runner { - flags=primaryflags(), - fmtfile=quoted(barename), - luafile=quoted(luaname), - texfile=quoted(data), - more=more, - } - end - end - end +function environment.run_format(formatname,scriptname,filename,primaryflags,secondaryflags,verbose) + local engine=environment.ownmain or "luatex" + if not formatname or formatname=="" then + report_format("missing format name") + return + end + if not scriptname or scriptname=="" then + report_format("missing script name") + return + end + if not lfs.isfile(formatname) or not lfs.isfile(scriptname) then + formatname,scriptname=resolvers.locateformat(formatname) end + if not formatname or formatname=="" then + report_format("invalid format name") + return + end + if not scriptname or scriptname=="" then + report_format("invalid script name") + return + end + local runner=runners[engine] + if not runner then + report_format("format %a cannot be run, no runner available for engine %a",file.nameonly(name),engine) + return + end + if not filename then + filename "" + end + local binarypath=validbinarypath() + local specification={ + binarypath=binarypath, + primaryflags=primaryflags or "", + secondaryflags=secondaryflags or "", + fmtfile=quoted(formatname), + luafile=quoted(scriptname), + texfile=filename~="" and quoted(filename) or "", + } + statistics.starttiming() + local result=runner(specification) + local runtime=statistics.stoptiming() + if verbose then + report_format() + if binarypath and binarypath~="" then + report_format("binary path : %s",binarypath) + end + report_format("luatex engine : %s",engine) + report_format("lua startup file : %s",scriptname) + report_format("tex format file : %s",formatname) + if filename~="" then + report_format("tex input file : %s",filename) + end + if primaryflags~="" then + report_format("primary flags : %s",primaryflags) + end + if secondaryflags~="" then + report_format("secondary flags : %s",secondaryflags) + end + report_format("run time : %.3f seconds",runtime) + report_format("return value : %s",result==0 and "okay" or "error") + report_format() + end + return result end end -- of closure --- used libraries : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua +-- used libraries : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua libs-ini.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 1019480 --- stripped bytes : 403728 +-- original bytes : 1038245 +-- stripped bytes : 409841 -- end library merge @@ -25697,7 +26232,7 @@ local ownlibs = { -- order can be made better 'data-tmf.lua', 'data-lst.lua', - 'util-lib.lua', -- swiglib + 'libs-ini.lua', 'luat-sta.lua', 'luat-fmt.lua', @@ -25846,7 +26381,7 @@ local helpinfo = [[ <category name="basic"> <subcategory> <flag name="script"><short>run an mtx script (lua prefered method) (<ref name="noquotes"/>), no script gives list</short></flag> - <flag name="evaluate"><short>run code passed on the commandline (between quotes)</short></flag> + <flag name="evaluate"><short>run code passed on the commandline (between quotes) (=loop) (exit|quit aborts)</short></flag> <flag name="execute"><short>run a script or program (texmfstart method) (<ref name="noquotes"/>)</short></flag> <flag name="resolve"><short>resolve prefixed arguments</short></flag> <flag name="ctxlua"><short>run internally (using preloaded libs)</short></flag> @@ -25865,6 +26400,7 @@ local helpinfo = [[ <flag name="stubpath" value="binpath"><short>paths where stubs wil be written</short></flag> <flag name="windows"><short>create windows (mswin) stubs</short></flag> <flag name="unix"><short>create unix (linux) stubs</short></flag> + <flag name="addbinarypath"><short>prepend the (found) binarypath to runners</short></flag> </subcategory> <subcategory> <flag name="verbose"><short>give a bit more info</short></flag> @@ -26086,8 +26622,7 @@ end report() io.flush() end - -- no os.exec because otherwise we get the wrong return value - local code = os.execute(command) -- maybe spawn + local code = os.execute(command) if code == 0 then return true else @@ -26130,7 +26665,7 @@ function runners.execute_program(fullname) report() report() io.flush() - local code = os.exec(command) -- (fullname,unpack(after)) does not work / maybe spawn + local code = os.execute(command) return code == 0 end end @@ -26517,16 +27052,22 @@ function runners.associate(filename) end function runners.evaluate(code,filename) -- for Luigi + local environment = table.setmetatableindex(_G) if code == "loop" then while true do - io.write("> ") + io.write("lua > ") local code = io.read() - if code ~= "" then + if code == "quit" or code == "exit" then + break + elseif code ~= "" then local temp = string.match(code,"^= (.*)$") if temp then - code = "print("..temp..")" + code = "inspect("..temp..")" + end + local compiled, message = load(code,"console","t",environment) + if type(compiled) ~= "function" then + compiled = load("inspect("..code..")","console","t",environment) end - local compiled, message = loadstring(code) if type(compiled) ~= "function" then io.write("! " .. (message or code).."\n") else @@ -26539,7 +27080,10 @@ function runners.evaluate(code,filename) -- for Luigi code = filename end if code ~= "" then - local compiled, message = loadstring(code) + local compiled, message = load(code,"console","t",environment) + if type(compiled) ~= "function" then + compiled = load("inspect("..code..")","console","t",environment) + end if type(compiled) ~= "function" then io.write("invalid lua code: " .. (message or code)) return @@ -26716,17 +27260,17 @@ do end -if e_argument("ansi") then +-- if e_argument("ansi") or e_argument("ansilog") then - logs.setformatters("ansi") +-- logs.setformatters(e_argument("ansi") and "ansi" or "ansilog") - local script = e_argument("script") or e_argument("scripts") +-- -- local script = e_argument("script") or e_argument("scripts") +-- -- +-- -- if type(script) == "string" then +-- -- logs.writer("]0;"..script.."") -- for Alan to test +-- -- end - if type(script) == "string" then - logs.writer("]0;"..script.."") -- for Alan to test - end - -end +-- end if e_argument("script") or e_argument("scripts") then @@ -26975,11 +27519,11 @@ elseif e_argument("format-path") then resolvers.load() report(caches.getwritablepath("format")) -elseif e_argument("pattern") then - - -- luatools - - runners.execute_ctx_script("mtx-base","--pattern='" .. e_argument("pattern") .. "'",filename) +-- elseif e_argument("pattern") then +-- +-- -- luatools +-- +-- runners.execute_ctx_script("mtx-base","--pattern='" .. e_argument("pattern") .. "'",filename) elseif e_argument("generate") then diff --git a/Master/texmf-dist/scripts/context/stubs/install/first-setup.sh b/Master/texmf-dist/scripts/context/stubs/install/first-setup.sh index 71fe32c1629..83af6a97e67 100755 --- a/Master/texmf-dist/scripts/context/stubs/install/first-setup.sh +++ b/Master/texmf-dist/scripts/context/stubs/install/first-setup.sh @@ -48,7 +48,7 @@ case "$system" in # a) binutils, this should work almost everywhere if $(which readelf >/dev/null 2>&1); then readelf -A /proc/self/exe | grep -q '^ \+Tag_ABI_VFP_args' - if [ ! $? ]; then + if [ $? != 0 ]; then platform="linux-armel" fi # b) debian-specific fallback @@ -86,16 +86,18 @@ case "$system" in esac ;; # OpenBSD OpenBSD) + version=`uname -r` case "$cpu" in - i*86) platform="openbsd" ;; - amd64) platform="openbsd-amd64" ;; + i*86) platform="openbsd${version}" ;; + amd64) platform="openbsd${version}-amd64" ;; *) platform="unknown" ;; esac ;; # cygwin CYGWIN*) case "$cpu" in - i*86) platform="cygwin" ;; - x86_64|ia64) platform="cygwin" ;; + i*86) platform="mswin" ;; # cygwin + # Pavneet Arora. 20190924. For Cygwin 64-bit platform should be win64. + x86_64|ia64) platform="win64" ;; # cygwin-64 *) platform="unknown" ;; esac ;; # UWIN @@ -145,15 +147,18 @@ fi rsync -rlptv rsync://contextgarden.net/minimals/setup/$platform/bin . # use native windows binaries on cygwin -if test "$platform" = "cygwin" ; then - platform=mswin -fi +# Pavneet Arora. 20190924. +# ..Commented out the following section entirely. +#if test "$platform" = "cygwin" ; then +# platform=mswin +#fi # download or update the distribution # you may remove the --context=beta switch if you want to use "current" # you can use --engine=luatex if you want just mkiv env PATH="$PWD/bin:$CONTEXTROOT/texmf-$platform/bin:$PATH" MTX_PLATFORM="$platform" \ -./bin/mtxrun --script ./bin/mtx-update.lua --force --update --make --context=beta --platform="$platform" --texroot="$CONTEXTROOT" $@ +./bin/mtxrun --script ./bin/mtx-update.lua --force --update --make --context=beta --engine=luatex --platform="$platform" --texroot="$CONTEXTROOT" $@ +echo "./bin/mtxrun --script ./bin/mtx-update.lua --force --update --make --context=beta --engine=luatex --platform=\"$platform\" --texroot=\"$CONTEXTROOT\" $@" echo echo "When you want to use context, you need to initialize the tree by typing:" diff --git a/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.lua b/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.lua index 98fe7f752e3..5907fa33c66 100644 --- a/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.lua +++ b/Master/texmf-dist/scripts/context/stubs/mswin/mtxrun.lua @@ -194,7 +194,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 6330, stripped down to: 2831 +-- original size: 6529, stripped down to: 2933 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -313,6 +313,9 @@ elseif not ffi.number then end if LUAVERSION>5.3 then end +if status and os.setenv then + os.setenv("engine",string.lower(status.luatex_engine or "unknown")) +end end -- of closure @@ -321,7 +324,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-macro"] = package.loaded["l-macro"] or true --- original size: 10131, stripped down to: 5991 +-- original size: 10130, stripped down to: 5990 if not modules then modules={} end modules ['l-macros']={ version=1.001, @@ -546,7 +549,7 @@ end macros.loaded=loaded function required(name,trace) local filename=file.addsuffix(name,"lua") - local fullname=resolvers and resolvers.find_file(filename) or filename + local fullname=resolvers and resolvers.findfile(filename) or filename if not fullname or fullname=="" then return false end @@ -576,7 +579,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-sandbox"] = package.loaded["l-sandbox"] or true --- original size: 9747, stripped down to: 6313 +-- original size: 9604, stripped down to: 6394 if not modules then modules={} end modules ['l-sandbox']={ version=1.001, @@ -808,6 +811,9 @@ local function supported(library) return l end loadfile=register(loadfile,"loadfile") +if supported("lua") then + lua.openfile=register(lua.openfile,"lua.openfile") +end if supported("io") then io.open=register(io.open,"io.open") io.popen=register(io.popen,"io.popen") @@ -1168,7 +1174,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true --- original size: 38434, stripped down to: 19310 +-- original size: 38440, stripped down to: 19316 if not modules then modules={} end modules ['l-lpeg']={ version=1.001, @@ -1337,7 +1343,7 @@ patterns.propername=(uppercase+lowercase+underscore)*(uppercase+lowercase+unders patterns.somecontent=(anything-newline-space)^1 patterns.beginline=#(1-newline) patterns.longtostring=Cs(whitespace^0/""*((patterns.quoted+nonwhitespace^1+whitespace^1/""*(endofstring+Cc(" ")))^0)) -function anywhere(pattern) +local function anywhere(pattern) return (1-P(pattern))^0*P(pattern) end lpeg.anywhere=anywhere @@ -1967,7 +1973,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-string"] = package.loaded["l-string"] or true --- original size: 6461, stripped down to: 3255 +-- original size: 6644, stripped down to: 3410 if not modules then modules={} end modules ['l-string']={ version=1.001, @@ -2031,9 +2037,11 @@ function string.is_empty(str) end end local anything=patterns.anything -local allescapes=Cc("%")*S(".-+%?()[]*") -local someescapes=Cc("%")*S(".-+%()[]") -local matchescapes=Cc(".")*S("*?") +local moreescapes=Cc("%")*S(".-+%?()[]*$^{}") +local allescapes=Cc("%")*S(".-+%?()[]*") +local someescapes=Cc("%")*S(".-+%()[]") +local matchescapes=Cc(".")*S("*?") +local pattern_m=Cs ((moreescapes+anything )^0 ) local pattern_a=Cs ((allescapes+anything )^0 ) local pattern_b=Cs ((someescapes+matchescapes+anything )^0 ) local pattern_c=Cs (Cc("^")*(someescapes+matchescapes+anything )^0*Cc("$") ) @@ -2043,6 +2051,8 @@ end function string.topattern(str,lowercase,strict) if str=="" or type(str)~="string" then return ".*" + elseif strict=="all" then + str=lpegmatch(pattern_m,str) elseif strict then str=lpegmatch(pattern_c,str) else @@ -2092,7 +2102,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-table"] = package.loaded["l-table"] or true --- original size: 41332, stripped down to: 21508 +-- original size: 41758, stripped down to: 22643 if not modules then modules={} end modules ['l-table']={ version=1.001, @@ -2101,7 +2111,7 @@ if not modules then modules={} end modules ['l-table']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local type,next,tostring,tonumber,select=type,next,tostring,tonumber,select +local type,next,tostring,tonumber,select,rawget=type,next,tostring,tonumber,select,rawget local table,string=table,string local concat,sort=table.concat,table.sort local format,lower,dump=string.format,string.lower,string.dump @@ -2435,13 +2445,13 @@ function table.fromhash(t) end return hsh end -local noquotes,hexify,handle,compact,inline,functions,metacheck +local noquotes,hexify,handle,compact,inline,functions,metacheck,accurate local reserved=table.tohash { 'and','break','do','else','elseif','end','false','for','function','if', 'in','local','nil','not','or','repeat','return','then','true','until','while', - 'NaN','goto', + 'NaN','goto','const', } -local function is_simple_table(t,hexify) +local function is_simple_table(t,hexify,accurate) local nt=#t if nt>0 then local n=0 @@ -2460,6 +2470,8 @@ local function is_simple_table(t,hexify) if tv=="number" then if hexify then tt[i]=format("0x%X",v) + elseif accurate then + tt[i]=format("%q",v) else tt[i]=v end @@ -2480,6 +2492,8 @@ local function is_simple_table(t,hexify) if tv=="number" then if hexify then tt[i+1]=format("0x%X",v) + elseif accurate then + tt[i+1]=format("%q",v) else tt[i+1]=v end @@ -2551,6 +2565,8 @@ local function do_serialize(root,name,depth,level,indexed) if tv=="number" then if hexify then handle(format("%s 0x%X,",depth,v)) + elseif accurate then + handle(format("%s %q,",depth,v)) else handle(format("%s %s,",depth,v)) end @@ -2560,7 +2576,7 @@ local function do_serialize(root,name,depth,level,indexed) if next(v)==nil then handle(format("%s {},",depth)) elseif inline then - local st=is_simple_table(v,hexify) + local st=is_simple_table(v,hexify,accurate) if st then handle(format("%s { %s },",depth,concat(st,", "))) else @@ -2588,12 +2604,16 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]=0x%X,",depth,k,v)) + elseif accurate then + handle(format("%s [%s]=%q,",depth,k,v)) else handle(format("%s [%s]=%s,",depth,k,v)) end elseif tk=="boolean" then if hexify then handle(format("%s [%s]=0x%X,",depth,k and "true" or "false",v)) + elseif accurate then + handle(format("%s [%s]=%q,",depth,k and "true" or "false",v)) else handle(format("%s [%s]=%s,",depth,k and "true" or "false",v)) end @@ -2601,12 +2621,16 @@ local function do_serialize(root,name,depth,level,indexed) elseif noquotes and not reserved[k] and lpegmatch(propername,k) then if hexify then handle(format("%s %s=0x%X,",depth,k,v)) + elseif accurate then + handle(format("%s %s=%q,",depth,k,v)) else handle(format("%s %s=%s,",depth,k,v)) end else if hexify then handle(format("%s [%q]=0x%X,",depth,k,v)) + elseif accurate then + handle(format("%s [%q]=%q,",depth,k,v)) else handle(format("%s [%q]=%s,",depth,k,v)) end @@ -2615,6 +2639,8 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]=%q,",depth,k,v)) + elseif accurate then + handle(format("%s [%q]=%q,",depth,k,v)) else handle(format("%s [%s]=%q,",depth,k,v)) end @@ -2631,6 +2657,8 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]={},",depth,k)) + elseif accurate then + handle(format("%s [%q]={},",depth,k)) else handle(format("%s [%s]={},",depth,k)) end @@ -2643,11 +2671,13 @@ local function do_serialize(root,name,depth,level,indexed) handle(format("%s [%q]={},",depth,k)) end elseif inline then - local st=is_simple_table(v,hexify) + local st=is_simple_table(v,hexify,accurate) if st then if tk=="number" then if hexify then handle(format("%s [0x%X]={ %s },",depth,k,concat(st,", "))) + elseif accurate then + handle(format("%s [%q]={ %s },",depth,k,concat(st,", "))) else handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) end @@ -2669,6 +2699,8 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]=%s,",depth,k,v and "true" or "false")) + elseif accurate then + handle(format("%s [%q]=%s,",depth,k,v and "true" or "false")) else handle(format("%s [%s]=%s,",depth,k,v and "true" or "false")) end @@ -2688,6 +2720,8 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]=load(%q),",depth,k,f)) + elseif accurate then + handle(format("%s [%q]=load(%q),",depth,k,f)) else handle(format("%s [%s]=load(%q),",depth,k,f)) end @@ -2705,6 +2739,8 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]=%q,",depth,k,tostring(v))) + elseif accurate then + handle(format("%s [%q]=%q,",depth,k,tostring(v))) else handle(format("%s [%s]=%q,",depth,k,tostring(v))) end @@ -2728,6 +2764,7 @@ local function serialize(_handle,root,name,specification) if type(specification)=="table" then noquotes=specification.noquotes hexify=specification.hexify + accurate=specification.accurate handle=_handle or specification.handle or print functions=specification.functions compact=specification.compact @@ -3043,7 +3080,7 @@ end local function sequenced(t,sep,simple) if not t then return "" - elseif type(t)=="string" then + elseif type(t)~="table" then return t end local n=#t @@ -3082,7 +3119,11 @@ local function sequenced(t,sep,simple) end end end - return concat(s,sep or " | ") + if sep==true then + return "{ "..concat(s,", ").." }" + else + return concat(s,sep or " | ") + end end table.sequenced=sequenced function table.print(t,...) @@ -3213,7 +3254,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-io"] = package.loaded["l-io"] or true --- original size: 11823, stripped down to: 6325 +-- original size: 11829, stripped down to: 6331 if not modules then modules={} end modules ['l-io']={ version=1.001, @@ -3227,7 +3268,7 @@ local open,flush,write,read=io.open,io.flush,io.write,io.read local byte,find,gsub,format=string.byte,string.find,string.gsub,string.format local concat=table.concat local type=type -if string.find(os.getenv("PATH"),";",1,true) then +if string.find(os.getenv("PATH") or "",";",1,true) then io.fileseparator,io.pathseparator="\\",";" else io.fileseparator,io.pathseparator="/",":" @@ -3765,7 +3806,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-os"] = package.loaded["l-os"] or true --- original size: 18916, stripped down to: 10126 +-- original size: 19102, stripped down to: 10192 if not modules then modules={} end modules ['l-os']={ version=1.001, @@ -3779,7 +3820,7 @@ local date,time=os.date,os.time local find,format,gsub,upper,gmatch=string.find,string.format,string.gsub,string.upper,string.gmatch local concat=table.concat local random,ceil,randomseed=math.random,math.ceil,math.randomseed -local rawget,rawset,type,getmetatable,setmetatable,tonumber,tostring=rawget,rawset,type,getmetatable,setmetatable,tonumber,tostring +local type,setmetatable,tonumber,tostring=type,setmetatable,tonumber,tostring do local selfdir=os.selfdir if selfdir=="" then @@ -3930,7 +3971,8 @@ local launchers={ unix="xdg-open %s &> /dev/null &", } function os.launch(str) - execute(format(launchers[os.name] or launchers.unix,str)) + local command=format(launchers[os.name] or launchers.unix,str) + execute(command) end local gettimeofday=os.gettimeofday or os.clock os.gettimeofday=gettimeofday @@ -4185,6 +4227,12 @@ function os.validdate(year,month,day) end return year,month,day end +function os.date(fmt,...) + if not fmt then + fmt="%Y-%m-%d %H:%M" + end + return date(fmt,...) +end local osexit=os.exit local exitcode=nil function os.setexitcode(code) @@ -4207,7 +4255,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-file"] = package.loaded["l-file"] or true --- original size: 21984, stripped down to: 10148 +-- original size: 22175, stripped down to: 10302 if not modules then modules={} end modules ['l-file']={ version=1.001, @@ -4229,15 +4277,24 @@ local checkedsplit=string.checkedsplit local P,R,S,C,Cs,Cp,Cc,Ct=lpeg.P,lpeg.R,lpeg.S,lpeg.C,lpeg.Cs,lpeg.Cp,lpeg.Cc,lpeg.Ct local attributes=lfs.attributes function lfs.isdir(name) - return attributes(name,"mode")=="directory" + if name then + return attributes(name,"mode")=="directory" + end end function lfs.isfile(name) - local a=attributes(name,"mode") - return a=="file" or a=="link" or nil + if name then + local a=attributes(name,"mode") + return a=="file" or a=="link" or nil + end end function lfs.isfound(name) - local a=attributes(name,"mode") - return (a=="file" or a=="link") and name or nil + if name then + local a=attributes(name,"mode") + return (a=="file" or a=="link") and name or nil + end +end +function lfs.modification(name) + return name and attributes(name,"modification") or nil end if sandbox then sandbox.redefine(lfs.isfile,"lfs.isfile") @@ -4616,7 +4673,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-gzip"] = package.loaded["l-gzip"] or true --- original size: 1211, stripped down to: 951 +-- original size: 5115, stripped down to: 1699 if not modules then modules={} end modules ['l-gzip']={ version=1.001, @@ -4624,43 +4681,75 @@ if not modules then modules={} end modules ['l-gzip']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -if not gzip then - return -end -local suffix,suffixes=file.suffix,file.suffixes -function gzip.load(filename) - local f=io.open(filename,"rb") - if not f then - elseif suffix(filename)=="gz" then - f:close() - local g=gzip.open(filename,"rb") - if g then - local str=g:read("*all") - g:close() - return str +gzip=gzip or {} +if not zlib then + zlib=xzip +elseif not xzip then + xzip=zlib +end +if zlib then + local suffix=file.suffix + local suffixes=file.suffixes + local find=string.find + local openfile=io.open + local gzipwindow=15+16 + local gziplevel=3 + local identifier="^\x1F\x8B\x08" + local compress=zlib.compress + local decompress=zlib.decompress + function gzip.load(filename) + local f=openfile(filename,"rb") + if not f then + else + local data=f:read("*all") + f:close() + if data and data~="" then + if suffix(filename)=="gz" then + data=decompress(data,gzipwindow) + end + return data + end end - else - local str=f:read("*all") - f:close() - return str end -end -function gzip.save(filename,data) - if suffix(filename)~="gz" then - filename=filename..".gz" + function gzip.save(filename,data,level) + if suffix(filename)~="gz" then + filename=filename..".gz" + end + local f=openfile(filename,"wb") + if f then + data=compress(data or "",level or gziplevel,nil,gzipwindow) + f:write(data) + f:close() + return #data + end end - local f=io.open(filename,"wb") - if f then - local s=zlib.compress(data or "",9,nil,15+16) - f:write(s) - f:close() - return #s + function gzip.suffix(filename) + local suffix,extra=suffixes(filename) + local gzipped=extra=="gz" + return suffix,gzipped + end + function gzip.compressed(s) + return s and find(s,identifier) + end + function gzip.compress(s,level) + if s and not find(s,identifier) then + if not level then + level=gziplevel + elseif level<=0 then + return s + elseif level>9 then + level=9 + end + return compress(s,level or gziplevel,nil,gzipwindow) or s + end + end + function gzip.decompress(s) + if s and find(s,identifier) then + return decompress(s,gzipwindow) + else + return s + end end -end -function gzip.suffix(filename) - local suffix,extra=suffixes(filename) - local gzipped=extra=="gz" - return suffix,gzipped end @@ -4670,7 +4759,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-md5"] = package.loaded["l-md5"] or true --- original size: 3309, stripped down to: 2218 +-- original size: 3414, stripped down to: 2307 if not modules then modules={} end modules ['l-md5']={ version=1.001, @@ -4689,6 +4778,8 @@ if not md5 then end local md5,file=md5,file local gsub=string.gsub +local modification,isfile,touch=lfs.modification,lfs.isfile,lfs.touch +local loaddata,savedata=io.loaddata,io.savedata do local patterns=lpeg and lpeg.patterns if patterns then @@ -4704,10 +4795,11 @@ do md5.sumHEXA=md5.HEX end end +local md5HEX=md5.HEX function file.needsupdating(oldname,newname,threshold) - local oldtime=lfs.attributes(oldname,"modification") + local oldtime=modification(oldname) if oldtime then - local newtime=lfs.attributes(newname,"modification") + local newtime=modification(newname) if not newtime then return true elseif newtime>=oldtime then @@ -4723,31 +4815,32 @@ function file.needsupdating(oldname,newname,threshold) end file.needs_updating=file.needsupdating function file.syncmtimes(oldname,newname) - local oldtime=lfs.attributes(oldname,"modification") - if oldtime and lfs.isfile(newname) then - lfs.touch(newname,oldtime,oldtime) + local oldtime=modification(oldname) + if oldtime and isfile(newname) then + touch(newname,oldtime,oldtime) end end -function file.checksum(name) +local function checksum(name) if md5 then - local data=io.loaddata(name) + local data=loaddata(name) if data then - return md5.HEX(data) + return md5HEX(data) end end return nil end +file.checksum=checksum function file.loadchecksum(name) if md5 then - local data=io.loaddata(name..".md5") + local data=loaddata(name..".md5") return data and (gsub(data,"%s","")) end return nil end function file.savechecksum(name,checksum) - if not checksum then checksum=file.checksum(name) end + if not checksum then checksum=checksum(name) end if checksum then - io.savedata(name..".md5",checksum) + savedata(name..".md5",checksum) return checksum end return nil @@ -5636,7 +5729,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-unicode"] = package.loaded["l-unicode"] or true --- original size: 41047, stripped down to: 17171 +-- original size: 41281, stripped down to: 17261 if not modules then modules={} end modules ['l-unicode']={ version=1.001, @@ -6151,49 +6244,52 @@ end function utf.utf32_to_utf8_t(t,endian) return endian and utf32_to_utf8_be_t(t) or utf32_to_utf8_le_t(t) or t end -local function little(b) - if b<0x10000 then - return char(b%256,rshift(b,8)) - else - b=b-0x10000 - local b1=rshift(b,10)+0xD800 - local b2=b%1024+0xDC00 - return char(b1%256,rshift(b1,8),b2%256,rshift(b2,8)) +if bit32 then + local rshift=bit32.rshift + local function little(b) + if b<0x10000 then + return char(b%256,rshift(b,8)) + else + b=b-0x10000 + local b1=rshift(b,10)+0xD800 + local b2=b%1024+0xDC00 + return char(b1%256,rshift(b1,8),b2%256,rshift(b2,8)) + end end -end -local function big(b) - if b<0x10000 then - return char(rshift(b,8),b%256) - else - b=b-0x10000 - local b1=rshift(b,10)+0xD800 - local b2=b%1024+0xDC00 - return char(rshift(b1,8),b1%256,rshift(b2,8),b2%256) + local function big(b) + if b<0x10000 then + return char(rshift(b,8),b%256) + else + b=b-0x10000 + local b1=rshift(b,10)+0xD800 + local b2=b%1024+0xDC00 + return char(rshift(b1,8),b1%256,rshift(b2,8),b2%256) + end end -end -local l_remap=Cs((p_utf8byte/little+P(1)/"")^0) -local b_remap=Cs((p_utf8byte/big+P(1)/"")^0) -local function utf8_to_utf16_be(str,nobom) - if nobom then - return lpegmatch(b_remap,str) - else - return char(254,255)..lpegmatch(b_remap,str) + local l_remap=Cs((p_utf8byte/little+P(1)/"")^0) + local b_remap=Cs((p_utf8byte/big+P(1)/"")^0) + local function utf8_to_utf16_be(str,nobom) + if nobom then + return lpegmatch(b_remap,str) + else + return char(254,255)..lpegmatch(b_remap,str) + end end -end -local function utf8_to_utf16_le(str,nobom) - if nobom then - return lpegmatch(l_remap,str) - else - return char(255,254)..lpegmatch(l_remap,str) + local function utf8_to_utf16_le(str,nobom) + if nobom then + return lpegmatch(l_remap,str) + else + return char(255,254)..lpegmatch(l_remap,str) + end end -end -utf.utf8_to_utf16_be=utf8_to_utf16_be -utf.utf8_to_utf16_le=utf8_to_utf16_le -function utf.utf8_to_utf16(str,littleendian,nobom) - if littleendian then - return utf8_to_utf16_le(str,nobom) - else - return utf8_to_utf16_be(str,nobom) + utf.utf8_to_utf16_be=utf8_to_utf16_be + utf.utf8_to_utf16_le=utf8_to_utf16_le + function utf.utf8_to_utf16(str,littleendian,nobom) + if littleendian then + return utf8_to_utf16_le(str,nobom) + else + return utf8_to_utf16_be(str,nobom) + end end end local pattern=Cs ( @@ -6467,7 +6563,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-str"] = package.loaded["util-str"] or true --- original size: 43488, stripped down to: 21595 +-- original size: 45188, stripped down to: 22734 if not modules then modules={} end modules ['util-str']={ version=1.001, @@ -6797,6 +6893,13 @@ local template=[[ %s return function(%s) return %s end ]] +local pattern=Cs(Cc('"')*( + (1-S('"\\\n\r'))^1+P('"')/'\\"'+P('\\')/'\\\\'+P('\n')/'\\n'+P('\r')/'\\r' +)^0*Cc('"')) +patterns.escapedquotes=pattern +function string.escapedquotes(s) + return lpegmatch(pattern,s) +end local preamble="" local environment={ global=global or _G, @@ -6821,9 +6924,10 @@ local environment={ formattednumber=number.formatted, sparseexponent=number.sparseexponent, formattedfloat=number.formattedfloat, - stripzero=lpeg.patterns.stripzero, - stripzeros=lpeg.patterns.stripzeros, - FORMAT=string.f9, + stripzero=patterns.stripzero, + stripzeros=patterns.stripzeros, + escapedquotes=string.escapedquotes, + FORMAT=string.f6, } local arguments={ "a1" } setmetatable(arguments,{ __index=function(t,k) @@ -6874,13 +6978,16 @@ local format_left=function(f) return format("a%s..utfpadding(a%s,%i)",n,n,-f) end end -local format_q=function() +local format_q=JITSUPPORTED and function() n=n+1 return format("(a%s ~= nil and format('%%q',tostring(a%s)) or '')",n,n) +end or function() + n=n+1 + return format("(a%s ~= nil and format('%%q',a%s) or '')",n,n) end local format_Q=function() n=n+1 - return format("format('%%q',tostring(a%s))",n) + return format("escapedquotes(tostring(a%s))",n) end local format_i=function(f) n=n+1 @@ -7031,12 +7138,25 @@ local format_n=function() n=n+1 return format("((a%s %% 1 == 0) and format('%%i',a%s) or tostring(a%s))",n,n,n) end -local format_N=function(f) - n=n+1 - if not f or f=="" then - f=".9" - end - return format("(((a%s %% 1 == 0) and format('%%i',a%s)) or lpegmatch(stripzero,format('%%%sf',a%s)))",n,n,f,n) +local format_N if environment.FORMAT then + format_N=function(f) + n=n+1 + if not f or f=="" then + return format("FORMAT(a%s,'%%.9f')",n) + elseif f==".6" or f=="0.6" then + return format("FORMAT(a%s)",n) + else + return format("FORMAT(a%s,'%%%sf')",n,f) + end + end +else + format_N=function(f) + n=n+1 + if not f or f=="" then + f=".9" + end + return format("(((a%s %% 1 == 0) and format('%%i',a%s)) or lpegmatch(stripzero,format('%%%sf',a%s)))",n,n,f,n) + end end local format_a=function(f) n=n+1 @@ -7265,9 +7385,9 @@ patterns.xmlescape=Cs((P("<")/"<"+P(">")/">"+P("&")/"&"+P('"')/"" patterns.texescape=Cs((C(S("#$%\\{}"))/"\\%1"+anything)^0) patterns.luaescape=Cs(((1-S('"\n'))^1+P('"')/'\\"'+P('\n')/'\\n"')^0) patterns.luaquoted=Cs(Cc('"')*((1-S('"\n'))^1+P('"')/'\\"'+P('\n')/'\\n"')^0*Cc('"')) -add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],{ xmlescape=lpeg.patterns.xmlescape }) -add(formatters,"tex",[[lpegmatch(texescape,%s)]],{ texescape=lpeg.patterns.texescape }) -add(formatters,"lua",[[lpegmatch(luaescape,%s)]],{ luaescape=lpeg.patterns.luaescape }) +add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],{ xmlescape=patterns.xmlescape }) +add(formatters,"tex",[[lpegmatch(texescape,%s)]],{ texescape=patterns.texescape }) +add(formatters,"lua",[[lpegmatch(luaescape,%s)]],{ luaescape=patterns.luaescape }) local dquote=patterns.dquote local equote=patterns.escaped+dquote/'\\"'+1 local cquote=Cc('"') @@ -7299,6 +7419,27 @@ local f_16_16=formatters["%0.5N"] function number.to16dot16(n) return f_16_16(n/65536.0) end +if not string.explode then + local tsplitat=lpeg.tsplitat + local p_utf=patterns.utf8character + local p_check=C(p_utf)*(P("+")*Cc(true))^0 + local p_split=Ct(C(p_utf)^0) + local p_space=Ct((C(1-P(" ")^1)+P(" ")^1)^0) + function string.explode(str,symbol) + if symbol=="" then + return lpegmatch(p_split,str) + elseif symbol then + local a,b=lpegmatch(p_check,symbol) + if b then + return lpegmatch(tsplitat(P(a)^1),str) + else + return lpegmatch(tsplitat(a),str) + end + else + return lpegmatch(p_space,str) + end + end +end end -- of closure @@ -7307,7 +7448,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tab"] = package.loaded["util-tab"] or true --- original size: 28866, stripped down to: 16134 +-- original size: 32649, stripped down to: 18257 if not modules then modules={} end modules ['util-tab']={ version=1.001, @@ -7544,78 +7685,160 @@ function tables.encapsulate(core,capsule,protect) } ) end end -local f_hashed_string=formatters["[%q]=%q,"] -local f_hashed_number=formatters["[%q]=%s,"] -local f_hashed_boolean=formatters["[%q]=%l,"] -local f_hashed_table=formatters["[%q]="] -local f_indexed_string=formatters["[%s]=%q,"] -local f_indexed_number=formatters["[%s]=%s,"] -local f_indexed_boolean=formatters["[%s]=%l,"] -local f_indexed_table=formatters["[%s]="] -local f_ordered_string=formatters["%q,"] -local f_ordered_number=formatters["%s,"] -local f_ordered_boolean=formatters["%l,"] -function table.fastserialize(t,prefix) - local r={ type(prefix)=="string" and prefix or "return" } - local m=1 - local function fastserialize(t,outer) - local n=#t - m=m+1 - r[m]="{" - if n>0 then - for i=0,n do - local v=t[i] - local tv=type(v) - if tv=="string" then - m=m+1 r[m]=f_ordered_string(v) - elseif tv=="number" then - m=m+1 r[m]=f_ordered_number(v) - elseif tv=="table" then - fastserialize(v) - elseif tv=="boolean" then - m=m+1 r[m]=f_ordered_boolean(v) +if JITSUPPORTED then + local f_hashed_string=formatters["[%Q]=%Q,"] + local f_hashed_number=formatters["[%Q]=%s,"] + local f_hashed_boolean=formatters["[%Q]=%l,"] + local f_hashed_table=formatters["[%Q]="] + local f_indexed_string=formatters["[%s]=%Q,"] + local f_indexed_number=formatters["[%s]=%s,"] + local f_indexed_boolean=formatters["[%s]=%l,"] + local f_indexed_table=formatters["[%s]="] + local f_ordered_string=formatters["%Q,"] + local f_ordered_number=formatters["%s,"] + local f_ordered_boolean=formatters["%l,"] + function table.fastserialize(t,prefix) + local r={ type(prefix)=="string" and prefix or "return" } + local m=1 + local function fastserialize(t,outer) + local n=#t + m=m+1 + r[m]="{" + if n>0 then + local v=t[0] + if v then + local tv=type(v) + if tv=="string" then + m=m+1 r[m]=f_indexed_string(0,v) + elseif tv=="number" then + m=m+1 r[m]=f_indexed_number(0,v) + elseif tv=="table" then + m=m+1 r[m]=f_indexed_table(0) + fastserialize(v) + m=m+1 r[m]=f_indexed_table(0) + elseif tv=="boolean" then + m=m+1 r[m]=f_indexed_boolean(0,v) + end + end + for i=1,n do + local v=t[i] + local tv=type(v) + if tv=="string" then + m=m+1 r[m]=f_ordered_string(v) + elseif tv=="number" then + m=m+1 r[m]=f_ordered_number(v) + elseif tv=="table" then + fastserialize(v) + elseif tv=="boolean" then + m=m+1 r[m]=f_ordered_boolean(v) + end end end - end - for k,v in next,t do - local tk=type(k) - if tk=="number" then - if k>n or k<0 then + for k,v in next,t do + local tk=type(k) + if tk=="number" then + if k>n or k<0 then + local tv=type(v) + if tv=="string" then + m=m+1 r[m]=f_indexed_string(k,v) + elseif tv=="number" then + m=m+1 r[m]=f_indexed_number(k,v) + elseif tv=="table" then + m=m+1 r[m]=f_indexed_table(k) + fastserialize(v) + elseif tv=="boolean" then + m=m+1 r[m]=f_indexed_boolean(k,v) + end + end + else local tv=type(v) if tv=="string" then - m=m+1 r[m]=f_indexed_string(k,v) + m=m+1 r[m]=f_hashed_string(k,v) elseif tv=="number" then - m=m+1 r[m]=f_indexed_number(k,v) + m=m+1 r[m]=f_hashed_number(k,v) elseif tv=="table" then - m=m+1 r[m]=f_indexed_table(k) + m=m+1 r[m]=f_hashed_table(k) fastserialize(v) elseif tv=="boolean" then - m=m+1 r[m]=f_indexed_boolean(k,v) + m=m+1 r[m]=f_hashed_boolean(k,v) end end + end + m=m+1 + if outer then + r[m]="}" else - local tv=type(v) - if tv=="string" then - m=m+1 r[m]=f_hashed_string(k,v) - elseif tv=="number" then - m=m+1 r[m]=f_hashed_number(k,v) - elseif tv=="table" then - m=m+1 r[m]=f_hashed_table(k) - fastserialize(v) - elseif tv=="boolean" then - m=m+1 r[m]=f_hashed_boolean(k,v) - end + r[m]="}," end + return r end - m=m+1 - if outer then - r[m]="}" - else - r[m]="}," + return concat(fastserialize(t,true)) + end +else + local f_v=formatters["[%q]=%q,"] + local f_t=formatters["[%q]="] + local f_q=formatters["%q,"] + function table.fastserialize(t,prefix) + local r={ type(prefix)=="string" and prefix or "return" } + local m=1 + local function fastserialize(t,outer) + local n=#t + m=m+1 + r[m]="{" + if n>0 then + local v=t[0] + if v then + m=m+1 + r[m]="[0]='" + if type(v)=="table" then + fastserialize(v) + else + r[m]=format("%q,",v) + end + end + for i=1,n do + local v=t[i] + m=m+1 + if type(v)=="table" then + r[m]=format("[%i]=",i) + fastserialize(v) + else + r[m]=format("[%i]=%q,",i,v) + end + end + end + for k,v in next,t do + local tk=type(k) + if tk=="number" then + if k>n or k<0 then + m=m+1 + if type(v)=="table" then + r[m]=format("[%i]=",k) + fastserialize(v) + else + r[m]=format("[%i]=%q,",k,v) + end + end + else + m=m+1 + if type(v)=="table" then + r[m]=format("[%q]=",k) + fastserialize(v) + else + r[m]=format("[%q]=%q,",k,v) + end + end + end + m=m+1 + if outer then + r[m]="}" + else + r[m]="}," + end + return r end - return r + return concat(fastserialize(t,true)) end - return concat(fastserialize(t,true)) end function table.deserialize(str) if not str or str=="" then @@ -7709,28 +7932,28 @@ function table.twowaymapper(t) return t end local f_start_key_idx=formatters["%w{"] -local f_start_key_num=formatters["%w[%s]={"] +local f_start_key_num=JITSUPPORTED and formatters["%w[%s]={"] or formatters["%w[%q]={"] local f_start_key_str=formatters["%w[%q]={"] local f_start_key_boo=formatters["%w[%l]={"] local f_start_key_nop=formatters["%w{"] local f_stop=formatters["%w},"] -local f_key_num_value_num=formatters["%w[%s]=%s,"] -local f_key_str_value_num=formatters["%w[%q]=%s,"] -local f_key_boo_value_num=formatters["%w[%l]=%s,"] -local f_key_num_value_str=formatters["%w[%s]=%q,"] -local f_key_str_value_str=formatters["%w[%q]=%q,"] -local f_key_boo_value_str=formatters["%w[%l]=%q,"] -local f_key_num_value_boo=formatters["%w[%s]=%l,"] -local f_key_str_value_boo=formatters["%w[%q]=%l,"] +local f_key_num_value_num=JITSUPPORTED and formatters["%w[%s]=%s,"] or formatters["%w[%s]=%q,"] +local f_key_str_value_num=JITSUPPORTED and formatters["%w[%Q]=%s,"] or formatters["%w[%Q]=%q,"] +local f_key_boo_value_num=JITSUPPORTED and formatters["%w[%l]=%s,"] or formatters["%w[%l]=%q,"] +local f_key_num_value_str=JITSUPPORTED and formatters["%w[%s]=%Q,"] or formatters["%w[%q]=%Q,"] +local f_key_str_value_str=formatters["%w[%Q]=%Q,"] +local f_key_boo_value_str=formatters["%w[%l]=%Q,"] +local f_key_num_value_boo=JITSUPPORTED and formatters["%w[%s]=%l,"] or formatters["%w[%q]=%l,"] +local f_key_str_value_boo=formatters["%w[%Q]=%l,"] local f_key_boo_value_boo=formatters["%w[%l]=%l,"] -local f_key_num_value_not=formatters["%w[%s]={},"] -local f_key_str_value_not=formatters["%w[%q]={},"] +local f_key_num_value_not=JITSUPPORTED and formatters["%w[%s]={},"] or formatters["%w[%q]={},"] +local f_key_str_value_not=formatters["%w[%Q]={},"] local f_key_boo_value_not=formatters["%w[%l]={},"] -local f_key_num_value_seq=formatters["%w[%s]={ %, t },"] -local f_key_str_value_seq=formatters["%w[%q]={ %, t },"] +local f_key_num_value_seq=JITSUPPORTED and formatters["%w[%s]={ %, t },"] or formatters["%w[%q]={ %, t },"] +local f_key_str_value_seq=formatters["%w[%Q]={ %, t },"] local f_key_boo_value_seq=formatters["%w[%l]={ %, t },"] -local f_val_num=formatters["%w%s,"] -local f_val_str=formatters["%w%q,"] +local f_val_num=JITSUPPORTED and formatters["%w%s,"] or formatters["%w%q,"] +local f_val_str=formatters["%w%Q,"] local f_val_boo=formatters["%w%l,"] local f_val_not=formatters["%w{},"] local f_val_seq=formatters["%w{ %, t },"] @@ -7738,7 +7961,7 @@ local f_fin_seq=formatters[" %, t }"] local f_table_return=formatters["return {"] local f_table_name=formatters["%s={"] local f_table_direct=formatters["{"] -local f_table_entry=formatters["[%q]={"] +local f_table_entry=formatters["[%Q]={"] local f_table_finish=formatters["}"] local spaces=utilities.strings.newrepeater(" ") local original_serialize=table.serialize @@ -8339,7 +8562,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sac"] = package.loaded["util-sac"] or true --- original size: 11332, stripped down to: 8420 +-- original size: 12946, stripped down to: 9507 if not modules then modules={} end modules ['util-sac']={ version=1.001, @@ -8374,6 +8597,7 @@ end function streams.size(f) return f and f[3] or 0 end +streams.getsize=streams.size function streams.setposition(f,i) if f[4] then if i<=0 then @@ -8566,9 +8790,9 @@ function streams.readfixed2(f) f[2]=j+1 local a,b=byte(f[1],i,j) if a>=0x80 then - tonumber((a-0x100).."."..b) + return tonumber((a-0x100).."."..b) or 0 else - tonumber((a ).."."..b) + return tonumber((a ).."."..b) or 0 end end function streams.readfixed4(f) @@ -8577,9 +8801,9 @@ function streams.readfixed4(f) f[2]=j+1 local a,b,c,d=byte(f[1],i,j) if a>=0x80 then - tonumber((0x100*a+b-0x10000).."."..(0x100*c+d)) + return tonumber((0x100*a+b-0x10000).."."..(0x100*c+d)) or 0 else - tonumber((0x100*a+b ).."."..(0x100*c+d)) + return tonumber((0x100*a+b ).."."..(0x100*c+d)) or 0 end end if bit32 then @@ -8659,6 +8883,16 @@ if sio and sio.readcardinal2 then f[2]=i+4 return readinteger4(f[1],i) end + function streams.readfixed2(f) + local i=f[2] + f[2]=i+2 + return readfixed2(f[1],i) + end + function streams.readfixed4(f) + local i=f[2] + f[2]=i+4 + return readfixed4(f[1],i) + end function streams.read2dot4(f) local i=f[2] f[2]=i+2 @@ -8758,6 +8992,50 @@ else return t end end +do + local files=utilities.files + if files then + local openfile=files.open + local openstream=streams.open + local openstring=streams.openstring + local setmetatable=setmetatable + function io.newreader(str,method) + local f,m + if method=="string" then + f=openstring(str) + m=streams + elseif method=="stream" then + f=openstream(str) + m=streams + else + f=openfile(str,"rb") + m=files + end + if f then + local t={} + setmetatable(t,{ + __index=function(t,k) + local r=m[k] + if k=="close" then + if f then + m.close(f) + f=nil + end + return function() end + elseif r then + local v=function(_,a,b) return r(f,a,b) end + t[k]=v + return v + else + print("unknown key",k) + end + end + } ) + return t + end + end + end +end end -- of closure @@ -9801,7 +10079,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-soc-imp-copas"] = package.loaded["util-soc-imp-copas"] or true --- original size: 25844, stripped down to: 14821 +-- original size: 25959, stripped down to: 14893 local socket=socket or require("socket") @@ -9838,6 +10116,7 @@ local copas={ autoclose=true, running=false, report=report, + trace=false, } local function statushandler(status,...) if status then @@ -9847,7 +10126,9 @@ local function statushandler(status,...) if type(err)=="table" then err=err[1] end - report("error: %s",tostring(err)) + if copas.trace then + report("error: %s",tostring(err)) + end return nil,err end function socket.protect(func) @@ -9861,7 +10142,9 @@ function socket.newtry(finalizer) if not status then local detail=select(2,...) pcall(finalizer,detail) - report("error: %s",tostring(detail)) + if copas.trace then + report("error: %s",tostring(detail)) + end return end return... @@ -12429,7 +12712,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 13340, stripped down to: 8826 +-- original size: 13394, stripped down to: 8882 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -12439,8 +12722,9 @@ if not modules then modules={} end modules ['trac-set']={ license="see context related readme files" } local type,next,tostring,tonumber=type,next,tostring,tonumber +local print=print local concat,sortedhash=table.concat,table.sortedhash -local format,find,lower,gsub,topattern=string.format,string.find,string.lower,string.gsub,string.topattern +local formatters,find,lower,gsub,topattern=string.formatters,string.find,string.lower,string.gsub,string.topattern local is_boolean=string.is_boolean local settings_to_hash=utilities.parsers.settings_to_hash local allocate=utilities.storage.allocate @@ -12450,10 +12734,10 @@ local setters=utilities.setters or {} utilities.setters=setters local data={} local trace_initialize=false +local frozen=true function setters.initialize(filename,name,values) local setter=data[name] if setter then - frozen=true local data=setter.data if data then for key,newvalue in sortedhash(values) do @@ -12647,8 +12931,8 @@ function setters.show(t) end end local enable,disable,register,list,show=setters.enable,setters.disable,setters.register,setters.list,setters.show -function setters.report(setter,...) - print(format("%-15s : %s\n",setter.name,format(...))) +function setters.report(setter,fmt,...) + print(formatters["%-15s : %s\n"](setter.name,formatters[fmt](...))) end local function default(setter,name) local d=setter.data[name] @@ -12668,7 +12952,7 @@ function setters.new(name) disable=function(...) disable (setter,...) end, reset=function(...) reset (setter,...) end, register=function(...) register(setter,...) end, - list=function(...) list (setter,...) end, + list=function(...) return list (setter,...) end, show=function(...) show (setter,...) end, default=function(...) return default (setter,...) end, value=function(...) return value (setter,...) end, @@ -12771,7 +13055,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 32618, stripped down to: 20935 +-- original size: 33003, stripped down to: 21667 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -12816,14 +13100,30 @@ local function ignore() end setmetatableindex(logs,function(t,k) t[k]=ignore;return ignore end) local report,subreport,status,settarget,setformats,settranslations local direct,subdirect,writer,pushtarget,poptarget,setlogfile,settimedlog,setprocessor,setformatters,newline +local function ansisupported(specification) + if specification~="ansi" and specification~="ansilog" then + return false + elseif os and os.enableansi then + return os.enableansi() + else + return false + end +end if runningtex and texio then if texio.setescape then texio.setescape(0) end - if arg then + if arg and ansisupported then for k,v in next,arg do if v=="--ansi" or v=="--c:ansi" then - variant="ansi" + if ansisupported("ansi") then + variant="ansi" + end + break + elseif v=="--ansilog" or v=="--c:ansilog" then + if ansisupported("ansilog") then + variant="ansilog" + end break end end @@ -12928,6 +13228,10 @@ if runningtex and texio then }, } } + variants.ansilog={ + formats=variants.ansi.formats, + targets=variants.default.targets, + } logs.flush=io.flush writer=function(...) write_nl(target,...) @@ -13034,6 +13338,9 @@ if runningtex and texio then t=specification.targets f=specification.formats or specification else + if not ansisupported(specification) then + specification="default" + end local v=variants[specification] if v then t=v.targets @@ -13060,7 +13367,7 @@ if runningtex and texio then subdirect_nop=f.subdirect_nop status_yes=f.status_yes status_nop=f.status_nop - if variant=="ansi" then + if variant=="ansi" or variant=="ansilog" then useluawrites() end settarget(whereto) @@ -13153,6 +13460,9 @@ else if type(specification)=="table" then f=specification.formats or specification else + if not ansisupported(specification) then + specification="default" + end local v=variants[specification] if v then f=v.formats @@ -13408,12 +13718,6 @@ end local nesting=0 local verbose=false local hasscheme=url.hasscheme -function logs.show_open(name) -end -function logs.show_close(name) -end -function logs.show_load(name) -end local simple=logs.reporter("comment") logs.simple=simple logs.simpleline=simple @@ -13480,6 +13784,13 @@ local exporters={ logs.reporters=reporters logs.exporters=exporters function logs.application(t) + local arguments=environment and environment.arguments + if arguments then + local ansi=arguments.ansi or arguments.ansilog + if ansi then + logs.setformatters(arguments.ansi and "ansi" or "ansilog") + end + end t.name=t.name or "unknown" t.banner=t.banner t.moreinfo=moreinfo @@ -13553,8 +13864,6 @@ else print(format(...)) end end -io.stdout:setvbuf('no') -io.stderr:setvbuf('no') if package.helpers.report then package.helpers.report=logs.reporter("package loader") end @@ -13652,7 +13961,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-inf"] = package.loaded["trac-inf"] or true --- original size: 8966, stripped down to: 5972 +-- original size: 9973, stripped down to: 7492 if not modules then modules={} end modules ['trac-inf']={ version=1.001, @@ -13674,7 +13983,7 @@ statistics.enable=true statistics.threshold=0.01 local statusinfo,n,registered,timers={},0,{},{} setmetatableindex(timers,function(t,k) - local v={ timing=0,loadtime=0 } + local v={ timing=0,loadtime=0,offset=0 } t[k]=v return v end) @@ -13682,10 +13991,40 @@ local function hastiming(instance) return instance and timers[instance] end local function resettiming(instance) - timers[instance or "notimer"]={ timing=0,loadtime=0 } + timers[instance or "notimer"]={ timing=0,loadtime=0,offset=0 } end local ticks=clock local seconds=function(n) return n or 0 end +if lua.getpreciseticks then + ticks=lua.getpreciseticks + seconds=lua.getpreciseseconds +elseif FFISUPPORTED and ffi and os.type=="windows" then + local okay,kernel=pcall(ffi.load,"kernel32") + if kernel then + local tonumber=ffi.number or tonumber + ffi.cdef[[ + int QueryPerformanceFrequency(int64_t *lpFrequency); + int QueryPerformanceCounter(int64_t *lpPerformanceCount); + ]] + local target=ffi.new("__int64[1]") + ticks=function() + if kernel.QueryPerformanceCounter(target)==1 then + return tonumber(target[0]) + else + return 0 + end + end + local target=ffi.new("__int64[1]") + seconds=function(ticks) + if kernel.QueryPerformanceFrequency(target)==1 then + return ticks/tonumber(target[0]) + else + return 0 + end + end + end +else +end local function starttiming(instance,reset) local timer=timers[instance or "notimer"] local it=timer.timing @@ -13720,12 +14059,26 @@ local function stoptiming(instance) end return 0 end +local function benchmarktimer(instance) + local timer=timers[instance or "notimer"] + local it=timer.timing + if it>1 then + timer.timing=it-1 + else + local starttime=timer.starttime + if starttime and starttime>0 then + timer.offset=ticks()-starttime + else + timer.offset=0 + end + end +end local function elapsed(instance) if type(instance)=="number" then return instance else local timer=timers[instance or "notimer"] - return timer and seconds(timer.loadtime) or 0 + return timer and seconds(timer.loadtime-2*(timer.offset or 0)) or 0 end end local function currenttime(instance) @@ -13738,7 +14091,7 @@ local function currenttime(instance) else local starttime=timer.starttime if starttime and starttime>0 then - return seconds(timer.loadtime+ticks()-starttime) + return seconds(timer.loadtime+ticks()-starttime-2*(timer.offset or 0)) end end return 0 @@ -13764,6 +14117,7 @@ statistics.elapsed=elapsed statistics.elapsedtime=elapsedtime statistics.elapsedindeed=elapsedindeed statistics.elapsedseconds=elapsedseconds +statistics.benchmarktimer=benchmarktimer function statistics.register(tag,fnc) if statistics.enable and type(fnc)=="function" then local rt=registered[tag] or (#statusinfo+1) @@ -13780,10 +14134,17 @@ function statistics.show() return format("%s, type: %s, binary subtree: %s", os.platform or "unknown",os.type or "unknown",environment.texos or "unknown") end) - register("used engine",function() - return format("%s version %s with functionality level %s, banner: %s", - LUATEXENGINE,LUATEXVERSION,LUATEXFUNCTIONALITY,lower(status.banner)) - end) + if LUATEXENGINE=="luametatex" then + register("used engine",function() + return format("%s version %s, functionality level %s, format id %s", + LUATEXENGINE,LUATEXVERSION,LUATEXFUNCTIONALITY,LUATEXFORMATID) + end) + else + register("used engine",function() + return format("%s version %s with functionality level %s, banner: %s", + LUATEXENGINE,LUATEXVERSION,LUATEXFUNCTIONALITY,lower(status.banner)) + end) + end register("control sequences",function() return format("%s of %s + %s",status.cs_count,status.hash_size,status.hash_extra) end) @@ -13822,7 +14183,11 @@ function statistics.show() end function statistics.memused() local round=math.round or math.floor - return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000),round(status.luastate_bytes/1000000)) + return format("%s MB, ctx: %s MB, max: %s MB)", + round(collectgarbage("count")/1000), + round(status.luastate_bytes/1000000), + status.luastate_bytes_max and round(status.luastate_bytes_max/1000000) or "unknown" + ) end starttiming(statistics) function statistics.formatruntime(runtime) @@ -14013,7 +14378,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lua"] = package.loaded["util-lua"] or true --- original size: 6664, stripped down to: 4589 +-- original size: 7149, stripped down to: 4997 if not modules then modules={} end modules ['util-lua']={ version=1.001, @@ -14038,16 +14403,21 @@ luautilities.nofstrippedchunks=0 luautilities.nofstrippedbytes=0 local strippedchunks={} luautilities.strippedchunks=strippedchunks +if not LUATEXENGINE then + LUATEXENGINE=status.luatex_engine and string.lower(status.luatex_engine) + JITSUPPORTED=LUATEXENGINE=="luajittex" or jit + CONTEXTLMTXMODE=CONTEXTLMTXMODE or (LUATEXENGINE=="luametatex" and 1) or 0 +end luautilities.suffixes={ tma="tma", - tmc=jit and "tmb" or "tmc", + tmc=(CONTEXTLMTXMODE and CONTEXTLMTXMODE>0 and "tmd") or (jit and "tmb") or "tmc", lua="lua", - luc=jit and "lub" or "luc", + luc=(CONTEXTLMTXMODE and CONTEXTLMTXMODE>0 and "lud") or (jit and "lub") or "luc", lui="lui", luv="luv", luj="luj", tua="tua", - tuc="tuc", + tuc=(CONTEXTLMTXMODE and CONTEXTLMTXMODE>0 and "tud") or (jit and "tub") or "tuc", } local function register(name) if tracestripping then @@ -14186,7 +14556,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-deb"] = package.loaded["util-deb"] or true --- original size: 9955, stripped down to: 6693 +-- original size: 10136, stripped down to: 6832 if not modules then modules={} end modules ['util-deb']={ version=1.001, @@ -14210,7 +14580,13 @@ local dummycalls=10*1000 local nesting=0 local names={} local initialize=false -if not (FFISUPPORTED and ffi) then +if lua.getpreciseticks then + initialize=function() + ticks=lua.getpreciseticks + seconds=lua.getpreciseseconds + initialize=false + end +elseif not (FFISUPPORTED and ffi) then elseif os.type=="windows" then initialize=function() local kernel=ffilib("kernel32","system") @@ -14482,7 +14858,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tpl"] = package.loaded["util-tpl"] or true --- original size: 7112, stripped down to: 3887 +-- original size: 7722, stripped down to: 4212 if not modules then modules={} end modules ['util-tpl']={ version=1.001, @@ -14498,6 +14874,7 @@ local report_template=logs.reporter("template") local tostring,next=tostring,next local format,sub,byte=string.format,string.sub,string.byte local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns +local formatters=string.formatters local replacer local function replacekey(k,t,how,recursive) local v=t[k] @@ -14566,6 +14943,10 @@ local function replaceoptional(l,m,r,t,how,recurse) local v=t[l] return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" end +local function replaceformatted(l,m,r,t,how,recurse) + local v=t[r] + return v and formatters[l](v) +end local single=P("%") local double=P("%%") local lquoted=P("%[") @@ -14579,16 +14960,19 @@ local nolquoted=lquoted/'' local norquoted=rquoted/'' local nolquotedq=lquotedq/'' local norquotedq=rquotedq/'' +local nolformatted=P(":")/"%%" +local norformatted=P(":")/"" local noloptional=P("%?")/'' local noroptional=P("?%")/'' local nomoptional=P(":")/'' local args=Carg(1)*Carg(2)*Carg(3) -local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle -local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq -local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted +local key=nosingle*((C((1-nosingle)^1)*args)/replacekey)*nosingle +local quoted=nolquotedq*((C((1-norquotedq)^1)*args)/replacekeyquoted)*norquotedq +local unquoted=nolquoted*((C((1-norquoted)^1)*args)/replacekeyunquoted)*norquoted local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional +local formatted=nosingle*((Cs(nolformatted*(1-norformatted )^1)*norformatted*C((1-nosingle)^1)*args)/replaceformatted)*nosingle local any=P(1) - replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) + replacer=Cs((unquoted+quoted+formatted+escape+optional+key+any)^0) local function replace(str,mapping,how,recurse) if mapping and str then return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str @@ -14627,7 +15011,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sbx"] = package.loaded["util-sbx"] or true --- original size: 20393, stripped down to: 13121 +-- original size: 21084, stripped down to: 13214 if not modules then modules={} end modules ['util-sbx']={ version=1.001, @@ -14873,37 +15257,50 @@ local iopopen=sandbox.original(io.popen) local reported={} local function validcommand(name,program,template,checkers,defaults,variables,reporter,strict) if validbinaries~=false and (validbinaries==true or validbinaries[program]) then + local binpath=nil if variables then for variable,value in next,variables do - local checker=validators[checkers[variable]] - if checker then - value=checker(unquoted(value),strict) - if value then - variables[variable]=optionalquoted(value) + local chktype=checkers[variable] + if chktype=="verbose" then + else + local checker=validators[chktype] + if checker then + value=checker(unquoted(value),strict) + if value then + variables[variable]=optionalquoted(value) + else + report("variable %a with value %a fails the check",variable,value) + return + end else - report("variable %a with value %a fails the check",variable,value) + report("variable %a has no checker",variable) return end - else - report("variable %a has no checker",variable) - return end end for variable,default in next,defaults do local value=variables[variable] if not value or value=="" then - local checker=validators[checkers[variable]] - if checker then - default=checker(unquoted(default),strict) - if default then - variables[variable]=optionalquoted(default) - else - report("variable %a with default %a fails the check",variable,default) - return + local chktype=checkers[variable] + if chktype=="verbose" then + else + local checker=validators[chktype] + if checker then + default=checker(unquoted(default),strict) + if default then + variables[variable]=optionalquoted(default) + else + report("variable %a with default %a fails the check",variable,default) + return + end end end end end + binpath=variables.binarypath + end + if type(binpath)=="string" and binpath~="" then + program=binpath.."/"..program end local command=program.." "..replace(template,variables) if reporter then @@ -14938,7 +15335,8 @@ local runners={ if trace then report("execute: %s",command) end - return osexecute(command) + local okay=osexecute(command) + return okay end end, pipeto=function(...) @@ -14972,7 +15370,7 @@ function sandbox.registerrunner(specification) return end if validrunners[name] then - report("invalid name, runner %a already defined") + report("invalid name, runner %a already defined",name) return end local program=specification.program @@ -15090,8 +15488,8 @@ if io then end if os then overload(os.execute,binaryrunner,"os.execute") - overload(os.spawn,dummyrunner,"os.spawn") - overload(os.exec,dummyrunner,"os.exec") + overload(os.spawn,dummyrunner,"os.spawn") + overload(os.exec,dummyrunner,"os.exec") overload(os.resultof,binaryrunner,"os.resultof") overload(os.pipeto,binaryrunner,"os.pipeto") overload(os.rename,filehandlertwo,"os.rename") @@ -15116,13 +15514,6 @@ end if zip then zip.open=register(zip.open,filehandlerone,"zip.open") end -if fontloader then - fontloader.open=register(fontloader.open,filehandlerone,"fontloader.open") - fontloader.info=register(fontloader.info,filehandlerone,"fontloader.info") -end -if epdf then - epdf.open=register(epdf.open,filehandlerone,"epdf.open") -end sandbox.registerroot=registerroot sandbox.registerbinary=registerbinary sandbox.registerlibrary=registerlibrary @@ -15528,7 +15919,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-env"] = package.loaded["luat-env"] or true --- original size: 6134, stripped down to: 4118 +-- original size: 6551, stripped down to: 4315 if not modules then modules={} end modules ['luat-env']={ version=1.001, @@ -15537,7 +15928,7 @@ package.loaded["luat-env"] = package.loaded["luat-env"] or true copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local rawset,rawget,loadfile=rawset,rawget,loadfile +local rawset,loadfile=rawset,loadfile local gsub=string.gsub local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_lua=logs.reporter("resolvers","lua") @@ -15584,6 +15975,12 @@ function environment.texfile(filename) return resolvers.findfile(filename,'tex') end function environment.luafile(filename) + if CONTEXTLMTXMODE and CONTEXTLMTXMODE>0 and file.suffix(filename)=="lua" then + local resolved=resolvers.findfile(file.replacesuffix(filename,"lmt")) or "" + if resolved~="" then + return resolved + end + end local resolved=resolvers.findfile(filename,'tex') or "" if resolved~="" then return resolved @@ -15695,7 +16092,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-zip"] = package.loaded["util-zip"] or true --- original size: 18645, stripped down to: 11291 +-- original size: 19496, stripped down to: 10858 if not modules then modules={} end modules ['util-zip']={ version=1.001, @@ -15706,7 +16103,7 @@ if not modules then modules={} end modules ['util-zip']={ local type,tostring,tonumber=type,tostring,tonumber local sort=table.sort local find,format,sub,gsub=string.find,string.format,string.sub,string.gsub -local osdate,ostime=os.date,os.time +local osdate,ostime,osclock=os.date,os.time,os.clock local ioopen=io.open local loaddata,savedata=io.loaddata,io.savedata local filejoin,isdir,dirname,mkdirs=file.join,lfs.isdir,file.dirname,dir.mkdirs @@ -15721,25 +16118,19 @@ local getposition=files.getposition local band=bit32.band local rshift=bit32.rshift local lshift=bit32.lshift -local decompress,calculatecrc -if flate then - decompress=flate.flate_decompress - calculatecrc=flate.update_crc32 -else +local decompress,expandsize,calculatecrc local zlibdecompress=zlib.decompress + local zlibexpandsize=zlib.expandsize local zlibchecksum=zlib.crc32 - decompress=function(source,targetsize) - local target=zlibdecompress(source,-15) - if target then - return target - else - return false,1 - end + decompress=function(source) + return zlibdecompress(source,-15) end + expandsize=zlibexpandsize and function(source,targetsize) + return zlibexpandsize(source,targetsize,-15) + end or decompress calculatecrc=function(buffer,initial) return zlibchecksum(initial or 0,buffer) end -end local zipfiles={} utilities.zipfiles=zipfiles local openzipfile,closezipfile,unzipfile,foundzipfile,getziphash,getziplist do @@ -15864,7 +16255,11 @@ local openzipfile,closezipfile,unzipfile,foundzipfile,getziphash,getziplist do setposition(handle,position) local result=readstring(handle,compressed) if data.method==8 then - result=decompress(result,data.uncompressed) + if expandsize then + result=expandsize(result,data.uncompressed) + else + result=decompress(result) + end end if check and data.crc32~=calculatecrc(result) then print("checksum mismatch") @@ -15883,14 +16278,14 @@ local openzipfile,closezipfile,unzipfile,foundzipfile,getziphash,getziplist do zipfiles.list=getziplist zipfiles.found=foundzipfile end -if flate then do +if xzip then local writecardinal1=files.writebyte local writecardinal2=files.writecardinal2le local writecardinal4=files.writecardinal4le local logwriter=logs.writer local globpattern=dir.globpattern - local compress=flate.flate_compress - local checksum=flate.update_crc32 + local compress=xzip.compress + local checksum=xzip.crc32 local function fromdostime(dostime,dosdate) return ostime { year=rshift(dosdate,9)+1980, @@ -16087,27 +16482,33 @@ if flate then do local count=#list local step=number.idiv(count,10) local done=0 + local steps=verbose=="steps" + local time=steps and osclock() for i=1,count do local l=list[i] local n=l.filename local d=unzipfile(z,n) - local p=filejoin(path,n) - if mkdirs(dirname(p)) then - if verbose=="steps" then - total=total+#d - done=done+1 - if done>=step then - done=0 - logwriter(format("%4i files of %4i done, %10i bytes",i,count,total)) + if d then + local p=filejoin(path,n) + if mkdirs(dirname(p)) then + if steps then + total=total+#d + done=done+1 + if done>=step then + done=0 + logwriter(format("%4i files of %4i done, %10i bytes, %0.3f seconds",i,count,total,osclock()-time)) + end + elseif verbose then + logwriter(n) end - elseif verbose then - logwriter(n) + savedata(p,d) end - savedata(p,d) + else + logwriter(format("problem with file %s",n)) end end - if verbose=="steps" then - logwriter(format("%4i files of %4i done, %10i bytes",count,count,total)) + if steps then + logwriter(format("%4i files of %4i done, %10i bytes, %0.3f seconds",count,count,total,osclock()-time)) end closezipfile(z) return true @@ -16118,37 +16519,8 @@ if flate then do end zipfiles.zipdir=zipdir zipfiles.unzipdir=unzipdir -end end -if flate then - local streams=utilities.streams - local openfile=streams.open - local closestream=streams.close - local setposition=streams.setposition - local getsize=streams.size - local readcardinal4=streams.readcardinal4le - local getstring=streams.getstring - local decompress=flate.gz_decompress - function zipfiles.gunzipfile(filename) - local strm=openfile(filename) - if strm then - setposition(strm,getsize(strm)-4+1) - local size=readcardinal4(strm) - local data=decompress(getstring(strm),size) - closestream(strm) - return data - end - end -elseif gzip then - local openfile=gzip.open - function zipfiles.gunzipfile(filename) - local g=openfile(filename,"rb") - if g then - local d=g:read("*a") - d:close() - return d - end - end end +zipfiles.gunzipfile=gzip.load end -- of closure @@ -16157,7 +16529,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-tab"] = package.loaded["lxml-tab"] or true --- original size: 60383, stripped down to: 35698 +-- original size: 61191, stripped down to: 35864 if not modules then modules={} end modules ['lxml-tab']={ version=1.001, @@ -16881,7 +17253,10 @@ local slash=P('/') local colon=P(':') local semicolon=P(';') local ampersand=P('&') -local valid=R('az','AZ','09')+S('_-.') +local valid_0=R("\128\255") +local valid_1=R('az','AZ')+S('_')+valid_0 +local valid_2=valid_1+R('09')+S('-.') +local valid=valid_1*valid_2^0 local name_yes=C(valid^1)*colon*C(valid^1) local name_nop=C(P(true))*C(valid^1) local name=name_yes+name_nop @@ -16917,8 +17292,9 @@ end local function entityfile(pattern,k,v,n) if n then local okay,data - if resolvers then - okay,data=resolvers.loadbinfile(n) + local loadbinfile=resolvers and resolvers.loadbinfile + if loadbinfile then + okay,data=loadbinfile(n) else data=io.loaddata(n) okay=data and data~="" @@ -17027,12 +17403,14 @@ publicentityfile+publicdoctype+systemdoctype+definitiondoctype+simpledoctype)*op } return grammar_parsed_text_one,grammar_parsed_text_two,grammar_unparsed_text end -grammar_parsed_text_one_nop, -grammar_parsed_text_two_nop, -grammar_unparsed_text_nop=install(space,spacing,anything) -grammar_parsed_text_one_yes, -grammar_parsed_text_two_yes, -grammar_unparsed_text_yes=install(space_nl,spacing_nl,anything_nl) +local + grammar_parsed_text_one_nop, + grammar_parsed_text_two_nop, + grammar_unparsed_text_nop=install(space,spacing,anything) +local + grammar_parsed_text_one_yes, + grammar_parsed_text_two_yes, + grammar_unparsed_text_yes=install(space_nl,spacing_nl,anything_nl) local function _xmlconvert_(data,settings,detail) settings=settings or {} preparexmlstate(settings) @@ -17613,7 +17991,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-lpt"] = package.loaded["lxml-lpt"] or true --- original size: 55145, stripped down to: 30992 +-- original size: 54626, stripped down to: 31255 if not modules then modules={} end modules ['lxml-lpt']={ version=1.001, @@ -18095,6 +18473,8 @@ local builtin={ lastindex="(#ll.__p__.dt or 1)", lastelement="(ll.__p__.en or 1)", last="#list", + list="list", + self="ll", rootposition="order", order="order", element="(ll.ei or 1)", @@ -18203,7 +18583,8 @@ local function register_selector(specification) end local function register_expression(expression) local converted=lpegmatch(converter,expression) - local runner=load(format(template_e,converted)) + local wrapped=format(template_e,converted) + local runner=load(wrapped) runner=(runner and runner()) or function() errorrunner_e(expression,converted) end return { kind="expression",expression=expression,converted=converted,evaluator=runner } end @@ -18575,6 +18956,20 @@ expressions.count=function(e,pattern) local collected=applylpath(e,pattern) return pattern and (collected and #collected) or 0 end +expressions.attribute=function(e,name,value) + if type(e)=="table" and name then + local a=e.at + if a then + local v=a[name] + if value then + return v==value + else + return v + end + end + end + return nil +end expressions.oneof=function(s,...) for i=1,select("#",...) do if s==select(i,...) then @@ -18621,7 +19016,7 @@ function expressions.contains(str,pattern) end return false end -function xml.expressions.idstring(str) +function expressions.idstring(str) return type(str)=="string" and gsub(str,"^#","") or "" end local function traverse(root,pattern,handle) @@ -20394,7 +20789,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 11099, stripped down to: 7152 +-- original size: 11019, stripped down to: 7086 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -20407,9 +20802,9 @@ local next,type,getmetatable,rawset=next,type,getmetatable,rawset local gsub,find,gmatch,char=string.gsub,string.find,string.gmatch,string.char local filedirname,filebasename,filejoin=file.dirname,file.basename,file.join local ostype,osname,osuname,ossetenv,osgetenv=os.type,os.name,os.uname,os.setenv,os.getenv +local sortedpairs=table.sortedpairs local P,S,R,C,Cs,Cc,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) -local trace_detail=false trackers.register("resolvers.details",function(v) trace_detail=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) local report_initialization=logs.reporter("resolvers","initialization") resolvers=resolvers or {} @@ -20618,7 +21013,7 @@ if ostype=="unix" then rawset(t,k,v) end local colon=P(":") - for k,v in table.sortedpairs(prefixes) do + for k,v in sortedpairs(prefixes) do if p then p=P(k)+p else @@ -20645,7 +21040,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 18154, stripped down to: 10416 +-- original size: 18179, stripped down to: 10432 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -20929,7 +21324,7 @@ local function scan(files,remap,spec,path,n,m,r,onlyone,tolerant) scancache[sub(full,1,-2)]=files return files,remap,n,m,r end -function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) +local function scanfiles(path,branch,usecache,onlyonce,tolerant) local realpath=resolveprefix(path) if usecache then local content=fullcache[realpath] @@ -20984,8 +21379,9 @@ function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) statistics.stoptiming(timer) return content end +resolvers.scanfiles=scanfiles function resolvers.simplescanfiles(path,branch,usecache) - return resolvers.scanfiles(path,branch,usecache,true,true) + return scanfiles(path,branch,usecache,true,true) end function resolvers.scandata() table.sort(scanned) @@ -21052,7 +21448,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 9360, stripped down to: 6312 +-- original size: 9400, stripped down to: 6347 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -21062,7 +21458,7 @@ if not modules then modules={} end modules ['data-env']={ license="see context related readme files", } local lower,gsub=string.lower,string.gsub -local next=next +local next,rawget=next,rawget local resolvers=resolvers local allocate=utilities.storage.allocate local setmetatableindex=table.setmetatableindex @@ -21143,13 +21539,13 @@ local relations=allocate { mp={ names={ "mp" }, variable='MPINPUTS', - suffixes={ 'mp','mpvi','mpiv','mpii' }, + suffixes={ 'mp','mpvi','mpiv','mpxl','mpii' }, usertype=true, }, tex={ names={ "tex" }, variable='TEXINPUTS', - suffixes={ "tex","mkvi","mkiv","mkii","cld","lfg","xml" }, + suffixes={ "tex","mkiv","mkvi","mkxl","mklx","mkii","cld","lfg","xml" }, usertype=true, }, icc={ @@ -21337,7 +21733,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 16284, stripped down to: 10938 +-- original size: 16099, stripped down to: 11379 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -21346,48 +21742,83 @@ if not modules then modules={} end modules ['data-tmp']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local format,lower,gsub,concat=string.format,string.lower,string.gsub,table.concat -local concat=table.concat -local mkdirs,isdir,isfile=dir.mkdirs,lfs.isdir,lfs.isfile -local addsuffix,is_writable,is_readable=file.addsuffix,file.is_writable,file.is_readable -local formatters=string.formatters local next,type=next,type +local pcall,loadfile,collectgarbage=pcall,loadfile,collectgarbage +local format,lower,gsub=string.format,string.lower,string.gsub +local concat,serialize,fastserialize,serializetofile=table.concat,table.serialize,table.fastserialize,table.tofile +local mkdirs,expanddirname,isdir,isfile=dir.mkdirs,dir.expandname,lfs.isdir,lfs.isfile +local is_writable,is_readable=file.is_writable,file.is_readable +local collapsepath,joinfile,addsuffix,dirname=file.collapsepath,file.join,file.addsuffix,file.dirname +local savedata=file.savedata +local formatters=string.formatters +local osexit,osdate,osuuid=os.exit,os.date,os.uuid +local removefile=os.remove +local md5hex=md5.hex local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) local report_caches=logs.reporter("resolvers","caches") local report_resolvers=logs.reporter("resolvers","caching") local resolvers=resolvers local cleanpath=resolvers.cleanpath -local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) -local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) -local compile=utilities.lua.compile -function utilities.lua.compile(luafile,lucfile,cleanup,strip) - if cleanup==nil then cleanup=directive_cleanup end - if strip==nil then strip=directive_strip end - return compile(luafile,lucfile,cleanup,strip) +local resolvepath=resolvers.resolve +local luautilities=utilities.lua +do + local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) + local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) + local compilelua=luautilities.compile + function luautilities.compile(luafile,lucfile,cleanup,strip) + if cleanup==nil then cleanup=directive_cleanup end + if strip==nil then strip=directive_strip end + return compilelua(luafile,lucfile,cleanup,strip) + end end caches=caches or {} local caches=caches -local luasuffixes=utilities.lua.suffixes -caches.base=caches.base or "luatex-cache" -caches.more=caches.more or "context" -caches.direct=false -caches.tree=false -caches.force=true -caches.ask=false -caches.relocate=false +local writable=nil +local readables={} +local usedreadables={} +local compilelua=luautilities.compile +local luasuffixes=luautilities.suffixes +caches.base=caches.base or "luatex-cache" +caches.more=caches.more or "context" caches.defaults={ "TMPDIR","TEMPDIR","TMP","TEMP","HOME","HOMEPATH" } -directives.register("system.caches.fast",function(v) caches.fast=true end) -local writable,readables,usedreadables=nil,{},{} +local direct_cache=false +local fast_cache=false +local cache_tree=false +directives.register("system.caches.direct",function(v) direct_cache=true end) +directives.register("system.caches.fast",function(v) fast_cache=true end) +local function configfiles() + return concat(resolvers.configurationfiles(),";") +end +local function hashed(tree) + tree=gsub(tree,"[\\/]+$","") + tree=lower(tree) + local hash=md5hex(tree) + if trace_cache or trace_locating then + report_caches("hashing tree %a, hash %a",tree,hash) + end + return hash +end +local function treehash() + local tree=configfiles() + if not tree or tree=="" then + return false + else + return hashed(tree) + end +end +caches.hashed=hashed +caches.treehash=treehash +caches.configfiles=configfiles local function identify() local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") if texmfcaches then for k=1,#texmfcaches do local cachepath=texmfcaches[k] if cachepath~="" then - cachepath=resolvers.resolve(cachepath) - cachepath=resolvers.cleanpath(cachepath) - cachepath=file.collapsepath(cachepath) + cachepath=resolvepath(cachepath) + cachepath=cleanpath(cachepath) + cachepath=collapsepath(cachepath) local valid=isdir(cachepath) if valid then if is_readable(cachepath) then @@ -21396,16 +21827,14 @@ local function identify() writable=cachepath end end - elseif not writable and caches.force then - local cacheparent=file.dirname(cachepath) - if is_writable(cacheparent) and true then - if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath),"no",{ "yes","no" })=="yes" then - mkdirs(cachepath) - if isdir(cachepath) and is_writable(cachepath) then - report_caches("path %a created",cachepath) - writable=cachepath - readables[#readables+1]=cachepath - end + elseif not writable then + local cacheparent=dirname(cachepath) + if is_writable(cacheparent) then + mkdirs(cachepath) + if isdir(cachepath) and is_writable(cachepath) then + report_caches("path %a created",cachepath) + writable=cachepath + readables[#readables+1]=cachepath end end end @@ -21418,8 +21847,8 @@ local function identify() local cachepath=texmfcaches[k] cachepath=resolvers.expansion(cachepath) if cachepath~="" then - cachepath=resolvers.resolve(cachepath) - cachepath=resolvers.cleanpath(cachepath) + cachepath=resolvepath(cachepath) + cachepath=cleanpath(cachepath) local valid=isdir(cachepath) if valid and is_readable(cachepath) then if not writable and is_writable(cachepath) then @@ -21433,23 +21862,25 @@ local function identify() end if not writable then report_caches("fatal error: there is no valid writable cache path defined") - os.exit() + osexit() elseif #readables==0 then report_caches("fatal error: there is no valid readable cache path defined") - os.exit() + osexit() end - writable=dir.expandname(resolvers.cleanpath(writable)) - local base,more,tree=caches.base,caches.more,caches.tree or caches.treehash() + writable=expanddirname(cleanpath(writable)) + local base=caches.base + local more=caches.more + local tree=cache_tree or treehash() if tree then - caches.tree=tree + cache_tree=tree writable=mkdirs(writable,base,more,tree) for i=1,#readables do - readables[i]=file.join(readables[i],base,more,tree) + readables[i]=joinfile(readables[i],base,more,tree) end else writable=mkdirs(writable,base,more) for i=1,#readables do - readables[i]=file.join(readables[i],base,more) + readables[i]=joinfile(readables[i],base,more) end end if trace_cache then @@ -21486,27 +21917,8 @@ function caches.usedpaths(separator) return writable or "?" end end -function caches.configfiles() - return concat(resolvers.configurationfiles(),";") -end -function caches.hashed(tree) - tree=gsub(tree,"[\\/]+$","") - tree=lower(tree) - local hash=md5.hex(tree) - if trace_cache or trace_locating then - report_caches("hashing tree %a, hash %a",tree,hash) - end - return hash -end -function caches.treehash() - local tree=caches.configfiles() - if not tree or tree=="" then - return false - else - return caches.hashed(tree) - end -end -local r_cache,w_cache={},{} +local r_cache={} +local w_cache={} local function getreadablepaths(...) local tags={... } local hash=concat(tags,"/") @@ -21516,7 +21928,7 @@ local function getreadablepaths(...) if #tags>0 then done={} for i=1,#readables do - done[i]=file.join(readables[i],...) + done[i]=joinfile(readables[i],...) end else done=readables @@ -21540,17 +21952,25 @@ local function getwritablepath(...) end return done end -caches.getreadablepaths=getreadablepaths -caches.getwritablepath=getwritablepath -function caches.getfirstreadablefile(filename,...) - local fullname,path=caches.setfirstwritablefile(filename,...) +local function setfirstwritablefile(filename,...) + local wr=getwritablepath(...) + local fullname=joinfile(wr,filename) + return fullname,wr +end +local function setluanames(path,name) + return + format("%s/%s.%s",path,name,luasuffixes.tma), + format("%s/%s.%s",path,name,luasuffixes.tmc) +end +local function getfirstreadablefile(filename,...) + local fullname,path=setfirstwritablefile(filename,...) if is_readable(fullname) then return fullname,path end local rd=getreadablepaths(...) for i=1,#rd do local path=rd[i] - local fullname=file.join(path,filename) + local fullname=joinfile(path,filename) if is_readable(fullname) then usedreadables[i]=true return fullname,path @@ -21558,19 +21978,11 @@ function caches.getfirstreadablefile(filename,...) end return fullname,path end -function caches.setfirstwritablefile(filename,...) - local wr=getwritablepath(...) - local fullname=file.join(wr,filename) - return fullname,wr -end -function caches.define(category,subcategory) - return function() - return getwritablepath(category,subcategory) - end -end -function caches.setluanames(path,name) - return format("%s/%s.%s",path,name,luasuffixes.tma),format("%s/%s.%s",path,name,luasuffixes.tmc) -end +caches.getreadablepaths=getreadablepaths +caches.getwritablepath=getwritablepath +caches.setfirstwritablefile=setfirstwritablefile +caches.getfirstreadablefile=getfirstreadablefile +caches.setluanames=setluanames function caches.loaddata(readables,name,writable) if type(readables)=="string" then readables={ readables } @@ -21578,21 +21990,22 @@ function caches.loaddata(readables,name,writable) for i=1,#readables do local path=readables[i] local loader=false - local tmaname,tmcname=caches.setluanames(path,name) + local state=false + local tmaname,tmcname=setluanames(path,name) if isfile(tmcname) then - loader=loadfile(tmcname) + state,loader=pcall(loadfile,tmcname) end if not loader and isfile(tmaname) then - local tmacrap,tmcname=caches.setluanames(writable,name) + local tmacrap,tmcname=setluanames(writable,name) if isfile(tmcname) then - loader=loadfile(tmcname) + state,loader=pcall(loadfile,tmcname) end - utilities.lua.compile(tmaname,tmcname) + compilelua(tmaname,tmcname) if isfile(tmcname) then - loader=loadfile(tmcname) + state,loader=pcall(loadfile,tmcname) end if not loader then - loader=loadfile(tmaname) + state,loader=pcall(loadfile,tmaname) end end if loader then @@ -21604,21 +22017,21 @@ function caches.loaddata(readables,name,writable) return false end function caches.is_writable(filepath,filename) - local tmaname,tmcname=caches.setluanames(filepath,filename) + local tmaname,tmcname=setluanames(filepath,filename) return is_writable(tmaname) end -local saveoptions={ compact=true } -function caches.savedata(filepath,filename,data,raw) - local tmaname,tmcname=caches.setluanames(filepath,filename) - data.cache_uuid=os.uuid() - if caches.fast then - file.savedata(tmaname,table.fastserialize(data,true)) - elseif caches.direct then - file.savedata(tmaname,table.serialize(data,true,saveoptions)) +local saveoptions={ compact=true,accurate=not JITSUPPORTED } +function caches.savedata(filepath,filename,data,fast) + local tmaname,tmcname=setluanames(filepath,filename) + data.cache_uuid=osuuid() + if fast or fast_cache then + savedata(tmaname,fastserialize(data,true)) + elseif direct_cache then + savedata(tmaname,serialize(data,true,saveoptions)) else - table.tofile(tmaname,data,true,saveoptions) + serializetofile(tmaname,data,true,saveoptions) end - utilities.lua.compile(tmaname,tmcname) + compilelua(tmaname,tmcname) end local content_state={} function caches.contentstate() @@ -21626,11 +22039,14 @@ function caches.contentstate() end function caches.loadcontent(cachename,dataname,filename) if not filename then - local name=caches.hashed(cachename) - local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") - filename=file.join(path,name) + local name=hashed(cachename) + local full,path=getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=joinfile(path,name) + end + local state,blob=pcall(loadfile,addsuffix(filename,luasuffixes.luc)) + if not blob then + state,blob=pcall(loadfile,addsuffix(filename,luasuffixes.lua)) end - local blob=loadfile(addsuffix(filename,luasuffixes.luc)) or loadfile(addsuffix(filename,luasuffixes.lua)) if blob then local data=blob() if data and data.content then @@ -21663,9 +22079,9 @@ function caches.collapsecontent(content) end function caches.savecontent(cachename,dataname,content,filename) if not filename then - local name=caches.hashed(cachename) - local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") - filename=file.join(path,name) + local name=hashed(cachename) + local full,path=setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=joinfile(path,name) end local luaname=addsuffix(filename,luasuffixes.lua) local lucname=addsuffix(filename,luasuffixes.luc) @@ -21676,17 +22092,17 @@ function caches.savecontent(cachename,dataname,content,filename) type=dataname, root=cachename, version=resolvers.cacheversion, - date=os.date("%Y-%m-%d"), - time=os.date("%H:%M:%S"), + date=osdate("%Y-%m-%d"), + time=osdate("%H:%M:%S"), content=content, - uuid=os.uuid(), + uuid=osuuid(), } - local ok=io.savedata(luaname,table.serialize(data,true)) + local ok=savedata(luaname,serialize(data,true)) if ok then if trace_locating then report_resolvers("category %a, cachename %a saved in %a",dataname,cachename,luaname) end - if utilities.lua.compile(luaname,lucname) then + if compilelua(luaname,lucname) then if trace_locating then report_resolvers("%a compiled to %a",dataname,lucname) end @@ -21695,7 +22111,7 @@ function caches.savecontent(cachename,dataname,content,filename) if trace_locating then report_resolvers("compiling failed for %a, deleting file %a",dataname,lucname) end - os.remove(lucname) + removefile(lucname) end elseif trace_locating then report_resolvers("unable to save %a in %a (access error)",dataname,luaname) @@ -21709,7 +22125,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5310, stripped down to: 3784 +-- original size: 5518, stripped down to: 3854 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -21718,31 +22134,45 @@ if not modules then modules={} end modules ['data-met']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local find,format=string.find,string.format -local sequenced=table.sequenced +local type=type +local find=string.find local addurlscheme,urlhashed=url.addscheme,url.hashed +local collapsepath,joinfile=file.collapsepath,file.join +local report_methods=logs.reporter("resolvers","methods") local trace_locating=false local trace_methods=false trackers.register("resolvers.locating",function(v) trace_methods=v end) trackers.register("resolvers.methods",function(v) trace_methods=v end) -local report_methods=logs.reporter("resolvers","methods") local allocate=utilities.storage.allocate local resolvers=resolvers local registered={} local function splitmethod(filename) if not filename then - return { scheme="unknown",original=filename } + return { + scheme="unknown", + original=filename, + } end if type(filename)=="table" then return filename end - filename=file.collapsepath(filename,".") + filename=collapsepath(filename,".") if not find(filename,"://",1,true) then - return { scheme="file",path=filename,original=filename,filename=filename } + return { + scheme="file", + path=filename, + original=filename, + filename=filename, + } end - local specification=url.hashed(filename) + local specification=urlhashed(filename) if not specification.scheme or specification.scheme=="" then - return { scheme="file",path=filename,original=filename,filename=filename } + return { + scheme="file", + path=filename, + original=filename, + filename=filename, + } else return specification end @@ -21751,7 +22181,8 @@ resolvers.splitmethod=splitmethod local function methodhandler(what,first,...) local method=registered[what] if method then - local how,namespace=method.how,method.namespace + local how=method.how + local namespace=method.namespace if how=="uri" or how=="url" then local specification=splitmethod(first) local scheme=specification.scheme @@ -21797,7 +22228,10 @@ local function methodhandler(what,first,...) end resolvers.methodhandler=methodhandler function resolvers.registermethod(name,namespace,how) - registered[name]={ how=how or "tag",namespace=namespace } + registered[name]={ + how=how or "tag", + namespace=namespace + } namespace["byscheme"]=function(scheme,filename,...) if scheme=="file" then return methodhandler(name,filename,...) @@ -21806,7 +22240,7 @@ function resolvers.registermethod(name,namespace,how) end end end -local concatinators=allocate { notfound=file.join } +local concatinators=allocate { notfound=joinfile } local locators=allocate { notfound=function() end } local hashers=allocate { notfound=function() end } local generators=allocate { notfound=function() end } @@ -21827,7 +22261,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 68195, stripped down to: 43680 +-- original size: 69576, stripped down to: 44470 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -21838,7 +22272,8 @@ if not modules then modules={} end modules ['data-res']={ } local gsub,find,lower,upper,match,gmatch=string.gsub,string.find,string.lower,string.upper,string.match,string.gmatch local concat,insert,remove=table.concat,table.insert,table.remove -local next,type,rawget=next,type,rawget +local next,type,rawget,loadfile=next,type,rawget,loadfile +local mergedtable=table.merged local os=os local P,S,R,C,Cc,Cs,Ct,Carg=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Carg local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns @@ -21854,13 +22289,14 @@ local joinpath=file.joinpath local is_qualified_path=file.is_qualified_path local allocate=utilities.storage.allocate local settings_to_array=utilities.parsers.settings_to_array +local urlhasscheme=url.hasscheme local getcurrentdir=lfs.currentdir local isfile=lfs.isfile local isdir=lfs.isdir local setmetatableindex=table.setmetatableindex local luasuffixes=utilities.lua.suffixes local trace_locating=false trackers .register("resolvers.locating",function(v) trace_locating=v end) -local trace_detail=false trackers .register("resolvers.details",function(v) trace_detail=v end) +local trace_details=false trackers .register("resolvers.details",function(v) trace_details=v end) local trace_expansions=false trackers .register("resolvers.expansions",function(v) trace_expansions=v end) local trace_paths=false trackers .register("resolvers.paths",function(v) trace_paths=v end) local resolve_otherwise=true directives.register("resolvers.otherwise",function(v) resolve_otherwise=v end) @@ -21879,10 +22315,17 @@ local ostype,osname,osenv,ossetenv,osgetenv=os.type,os.name,os.env,os.setenv,os. resolvers.cacheversion="1.100" resolvers.configbanner="" resolvers.homedir=environment.homedir -resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } resolvers.luacnfname="texmfcnf.lua" resolvers.luacnffallback="contextcnf.lua" resolvers.luacnfstate="unknown" +local criticalvars={ + "SELFAUTOLOC", + "SELFAUTODIR", + "SELFAUTOPARENT", + "TEXMFCNF", + "TEXMF", + "TEXOS", +} if environment.default_texmfcnf then resolvers.luacnfspec="home:texmf/web2c;"..environment.default_texmfcnf else @@ -21902,13 +22345,20 @@ local dangerous=resolvers.dangerous local suffixmap=resolvers.suffixmap resolvers.defaultsuffixes={ "tex" } local instance=nil -function resolvers.setenv(key,value,raw) +local variable +local expansion +local setenv +local getenv +local formatofsuffix=resolvers.formatofsuffix +local splitpath=resolvers.splitpath +local splitmethod=resolvers.splitmethod +setenv=function(key,value,raw) if instance then instance.environment[key]=value ossetenv(key,raw and value or resolveprefix(value)) end end -local function getenv(key) +getenv=function(key) local value=rawget(instance.environment,key) if value and value~="" then return value @@ -21918,94 +22368,97 @@ local function getenv(key) end end resolvers.getenv=getenv -resolvers.env=getenv -local function resolvevariable(k) - return instance.expansions[k] -end +resolvers.setenv=setenv local dollarstripper=lpeg.stripper("$") local inhibitstripper=P("!")^0*Cs(P(1)^0) -local somevariable=P("$")/"" -local somekey=C(R("az","AZ","09","__","--")^1) -local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) -local variableexpander=Cs((somevariable*(somekey/resolvevariable)+somethingelse)^1 ) -local cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" -local variablecleaner=Cs((cleaner+P(1))^0) -local somevariable=R("az","AZ","09","__","--")^1/resolvevariable -local variable=(P("$")/"")*(somevariable+(P("{")/"")*somevariable*(P("}")/"")) -local variableresolver=Cs((variable+P(1))^0) -local function expandedvariable(var) - return lpegmatch(variableexpander,var) or var -end -function resolvers.reset() - if trace_locating then - report_resolving("creating instance") - end - local environment={} - local variables={} - local expansions={} - local order={} - instance={ - environment=environment, - variables=variables, - expansions=expansions, - order=order, - files={}, - setups={}, - found={}, - foundintrees={}, - hashes={}, - hashed={}, - pathlists=false, - specification={}, - lists={}, - data={}, - fakepaths={}, - remember=true, - diskcache=true, - renewcache=false, - renewtree=false, - loaderror=false, - savelists=true, - pattern=nil, - force_suffixes=true, - pathstack={}, - } - setmetatableindex(variables,function(t,k) - local v - for i=1,#order do - v=order[i][k] +local expandedvariable,resolvedvariable do + local function resolveinstancevariable(k) + return instance.expansions[k] + end + local p_variable=P("$")/"" + local p_key=C(R("az","AZ","09","__","--")^1) + local p_whatever=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) + local variableexpander=Cs((p_variable*(p_key/resolveinstancevariable)+p_whatever)^1 ) + local p_cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" + local variablecleaner=Cs((p_cleaner+P(1))^0) + local p_variable=R("az","AZ","09","__","--")^1/resolveinstancevariable + local p_variable=(P("$")/"")*(p_variable+(P("{")/"")*p_variable*(P("}")/"")) + local variableresolver=Cs((p_variable+P(1))^0) + expandedvariable=function(var) + return lpegmatch(variableexpander,var) or var + end + function resolvers.reset() + if trace_locating then + report_resolving("creating instance") + end + local environment={} + local variables={} + local expansions={} + local order={} + instance={ + environment=environment, + variables=variables, + expansions=expansions, + order=order, + files={}, + setups={}, + found={}, + foundintrees={}, + hashes={}, + hashed={}, + pathlists=false, + specification={}, + lists={}, + data={}, + fakepaths={}, + remember=true, + diskcache=true, + renewcache=false, + renewtree=false, + loaderror=false, + savelists=true, + pattern=nil, + force_suffixes=true, + pathstack={}, + } + setmetatableindex(variables,function(t,k) + local v + for i=1,#order do + v=order[i][k] + if v~=nil then + t[k]=v + return v + end + end + if v==nil then + v="" + end + t[k]=v + return v + end) + local repath=resolvers.repath + setmetatableindex(environment,function(t,k) + local v=osgetenv(k) + if v==nil then + v=variables[k] + end if v~=nil then - t[k]=v - return v + v=checkedvariable(v) or "" end - end - if v==nil then - v="" - end - t[k]=v - return v - end) - setmetatableindex(environment,function(t,k) - local v=osgetenv(k) - if v==nil then - v=variables[k] - end - if v~=nil then - v=checkedvariable(v) or "" - end - v=resolvers.repath(v) - t[k]=v - return v - end) - setmetatableindex(expansions,function(t,k) - local v=environment[k] - if type(v)=="string" then - v=lpegmatch(variableresolver,v) - v=lpegmatch(variablecleaner,v) - end - t[k]=v - return v - end) + v=repath(v) + t[k]=v + return v + end) + setmetatableindex(expansions,function(t,k) + local v=environment[k] + if type(v)=="string" then + v=lpegmatch(variableresolver,v) + v=lpegmatch(variablecleaner,v) + end + t[k]=v + return v + end) + end end function resolvers.initialized() return instance~=nil @@ -22019,31 +22472,33 @@ local function reset_caches() instance.lists={} instance.pathlists=false end -local slash=P("/") -local pathexpressionpattern=Cs ( - Cc("^")*( - Cc("%")*S(".-")+slash^2*P(-1)/"/.*" +local makepathexpression do + local slash=P("/") + local pathexpressionpattern=Cs ( + Cc("^")*( + Cc("%")*S(".-")+slash^2*P(-1)/"/.*" +slash^2/"/"+(1-slash)*P(-1)*Cc("/")+P(1) - )^1*Cc("$") -) -local cache={} -local function makepathexpression(str) - if str=="." then - return "^%./$" - else - local c=cache[str] - if not c then - c=lpegmatch(pathexpressionpattern,str) - cache[str]=c + )^1*Cc("$") + ) + local cache={} + makepathexpression=function(str) + if str=="." then + return "^%./$" + else + local c=cache[str] + if not c then + c=lpegmatch(pathexpressionpattern,str) + cache[str]=c + end + return c end - return c end end local function reportcriticalvariables(cnfspec) if trace_locating then - for i=1,#resolvers.criticalvars do - local k=resolvers.criticalvars[i] - local v=resolvers.getenv(k) or "unknown" + for i=1,#criticalvars do + local k=criticalvars[i] + local v=getenv(k) or "unknown" report_resolving("variable %a set to %a",k,v) end report_resolving() @@ -22065,7 +22520,7 @@ local function identify_configuration_files() resolvers.luacnfstate="environment" end reportcriticalvariables(cnfspec) - local cnfpaths=expandedpathfromlist(resolvers.splitpath(cnfspec)) + local cnfpaths=expandedpathfromlist(splitpath(cnfspec)) local function locatecnf(luacnfname,kind) for i=1,#cnfpaths do local filepath=cnfpaths[i] @@ -22098,6 +22553,8 @@ local function identify_configuration_files() end local function load_configuration_files() local specification=instance.specification + local setups=instance.setups + local order=instance.order if #specification>0 then local luacnfname=resolvers.luacnfname for i=1,#specification do @@ -22107,7 +22564,6 @@ local function load_configuration_files() local realname=resolveprefix(filename) local blob=loadfile(realname) if blob then - local setups=instance.setups local data=blob() local parent=data and data.parent if parent then @@ -22118,7 +22574,7 @@ local function load_configuration_files() local parentdata=blob() if parentdata then report_resolving("loading configuration file %a",filename) - data=table.merged(parentdata,data) + data=mergedtable(parentdata,data) end end end @@ -22150,7 +22606,7 @@ local function load_configuration_files() if trace_locating then report_resolving("reloading configuration due to TEXMF redefinition") end - resolvers.setenv("TEXMFCNF",cnfspec) + setenv("TEXMFCNF",cnfspec) instance.specification={} identify_configuration_files() load_configuration_files() @@ -22168,7 +22624,7 @@ local function load_configuration_files() elseif trace_locating then report_resolving("skipping configuration file %a (no valid format)",filename) end - instance.order[#instance.order+1]=instance.setups[pathname] + order[#order+1]=setups[pathname] if instance.loaderror then break end @@ -22177,6 +22633,8 @@ local function load_configuration_files() report_resolving("warning: no lua configuration files found") end end +local expandedpathlist +local unexpandedpathlist function resolvers.configurationfiles() return instance.specification or {} end @@ -22193,7 +22651,7 @@ local function load_file_databases() end end local function locate_file_databases() - local texmfpaths=resolvers.expandedpathlist("TEXMF") + local texmfpaths=expandedpathlist("TEXMF") if #texmfpaths>0 then for i=1,#texmfpaths do local path=collapsepath(texmfpaths[i]) @@ -22202,7 +22660,7 @@ local function locate_file_databases() if stripped~="" then local runtime=stripped==path path=cleanpath(path) - local spec=resolvers.splitmethod(stripped) + local spec=splitmethod(stripped) if runtime and (spec.noscheme or spec.scheme=="file") then stripped="tree:///"..stripped elseif spec.scheme=="cache" or spec.scheme=="file" then @@ -22236,11 +22694,13 @@ local function generate_file_databases() end end local function save_file_databases() - for i=1,#instance.hashes do - local hash=instance.hashes[i] + local hashes=instance.hashes + local files=instance.files + for i=1,#hashes do + local hash=hashes[i] local cachename=hash.name if hash.cache then - local content=instance.files[cachename] + local content=files[cachename] caches.collapsecontent(content) if trace_locating then report_resolving("saving tree %a",cachename) @@ -22252,8 +22712,9 @@ local function save_file_databases() end end function resolvers.renew(hashname) + local files=instance.files if hashname and hashname~="" then - local expanded=resolvers.expansion(hashname) or "" + local expanded=expansion(hashname) or "" if expanded~="" then if trace_locating then report_resolving("identifying tree %a from %a",expanded,hashname) @@ -22270,7 +22731,7 @@ function resolvers.renew(hashname) report_resolving("using path %a",realpath) end methodhandler('generators',hashname) - local content=instance.files[hashname] + local content=files[hashname] caches.collapsecontent(content) if trace_locating then report_resolving("saving tree %a",hashname) @@ -22297,38 +22758,46 @@ local function load_databases() end end function resolvers.appendhash(type,name,cache) - if not instance.hashed[name] then + local hashed=instance.hashed + local hashes=instance.hashes + if hashed[name] then + else if trace_locating then report_resolving("hash %a appended",name) end - insert(instance.hashes,{ type=type,name=name,cache=cache } ) - instance.hashed[name]=cache + insert(hashes,{ type=type,name=name,cache=cache } ) + hashed[name]=cache end end function resolvers.prependhash(type,name,cache) - if not instance.hashed[name] then + local hashed=instance.hashed + local hashes=instance.hashes + if hashed[name] then + else if trace_locating then report_resolving("hash %a prepended",name) end - insert(instance.hashes,1,{ type=type,name=name,cache=cache } ) - instance.hashed[name]=cache + insert(hashes,1,{ type=type,name=name,cache=cache } ) + hashed[name]=cache end end function resolvers.extendtexmfvariable(specification) - local t=resolvers.splitpath(getenv("TEXMF")) - insert(t,1,specification) - local newspec=concat(t,",") - if instance.environment["TEXMF"] then - instance.environment["TEXMF"]=newspec - elseif instance.variables["TEXMF"] then - instance.variables["TEXMF"]=newspec + local environment=instance.environment + local variables=instance.variables + local texmftrees=splitpath(getenv("TEXMF")) + insert(texmftrees,1,specification) + texmftrees=concat(texmftrees,",") + if environment["TEXMF"] then + environment["TEXMF"]=texmftrees + elseif variables["TEXMF"] then + variables["TEXMF"]=texmftrees else end reset_hashes() end function resolvers.splitexpansions() - local ie=instance.expansions - for k,v in next,ie do + local expansions=instance.expansions + for k,v in next,expansions do local t,tn,h,p={},0,{},splitconfigurationpath(v) for kk=1,#p do local vv=p[kk] @@ -22338,33 +22807,37 @@ function resolvers.splitexpansions() h[vv]=true end end - if #t>1 then - ie[k]=t + if tn>1 then + expansions[k]=t else - ie[k]=t[1] + expansions[k]=t[1] end end end function resolvers.datastate() return caches.contentstate() end -function resolvers.variable(name) +variable=function(name) + local variables=instance.variables local name=name and lpegmatch(dollarstripper,name) - local result=name and instance.variables[name] + local result=name and variables[name] return result~=nil and result or "" end -function resolvers.expansion(name) +expansion=function(name) + local expansions=instance.expansions local name=name and lpegmatch(dollarstripper,name) - local result=name and instance.expansions[name] + local result=name and expansions[name] return result~=nil and result or "" end -function resolvers.unexpandedpathlist(str) - local pth=resolvers.variable(str) - local lst=resolvers.splitpath(pth) +resolvers.variable=variable +resolvers.expansion=expansion +unexpandedpathlist=function(str) + local pth=variable(str) + local lst=splitpath(pth) return expandedpathfromlist(lst) end function resolvers.unexpandedpath(str) - return joinpath(resolvers.unexpandedpathlist(str)) + return joinpath(unexpandedpathlist(str)) end function resolvers.pushpath(name) local pathstack=instance.pathstack @@ -22394,8 +22867,8 @@ function resolvers.stackpath() end local done={} function resolvers.resetextrapaths() - local ep=instance.extra_paths - if not ep then + local extra_paths=instance.extra_paths + if not extra_paths then done={} instance.extra_paths={} elseif #ep>0 then @@ -22416,8 +22889,8 @@ function resolvers.registerextrapath(paths,subpaths) end local paths=settings_to_array(paths) local subpaths=settings_to_array(subpaths) - local ep=instance.extra_paths or {} - local oldn=#ep + local extra_paths=instance.extra_paths or {} + local oldn=#extra_paths local newn=oldn local nofpaths=#paths local nofsubpaths=#subpaths @@ -22430,7 +22903,7 @@ function resolvers.registerextrapath(paths,subpaths) local ps=p.."/"..s if not done[ps] then newn=newn+1 - ep[newn]=cleanpath(ps) + extra_paths[newn]=cleanpath(ps) done[ps]=true end end @@ -22440,7 +22913,7 @@ function resolvers.registerextrapath(paths,subpaths) local p=paths[i] if not done[p] then newn=newn+1 - ep[newn]=cleanpath(p) + extra_paths[newn]=cleanpath(p) done[p]=true end end @@ -22449,17 +22922,17 @@ function resolvers.registerextrapath(paths,subpaths) for i=1,oldn do for j=1,nofsubpaths do local s=subpaths[j] - local ps=ep[i].."/"..s + local ps=extra_paths[i].."/"..s if not done[ps] then newn=newn+1 - ep[newn]=cleanpath(ps) + extra_paths[newn]=cleanpath(ps) done[ps]=true end end end end if newn>0 then - instance.extra_paths=ep + instance.extra_paths=extra_paths end if newn~=oldn then reset_caches() @@ -22467,17 +22940,19 @@ function resolvers.registerextrapath(paths,subpaths) end function resolvers.pushextrapath(path) local paths=settings_to_array(path) - if instance.extra_stack then - insert(instance.extra_stack,1,paths) + local extra_stack=instance.extra_stack + if extra_stack then + insert(extra_stack,1,paths) else instance.extra_stack={ paths } end reset_caches() end function resolvers.popextrapath() - if instance.extra_stack then + local extra_stack=instance.extra_stack + if extra_stack then reset_caches() - return remove(instance.extra_stack,1) + return remove(extra_stack,1) end end local function made_list(instance,list,extra_too) @@ -22506,33 +22981,21 @@ local function made_list(instance,list,extra_too) end end if extra_too then - local es=instance.extra_stack - if es and #es>0 then - for k=1,#es do - add(es[k]) + local extra_stack=instance.extra_stack + local extra_paths=instance.extra_paths + if extra_stack and #extra_stack>0 then + for k=1,#extra_stack do + add(extra_stack[k]) end end - local ep=instance.extra_paths - if ep and #ep>0 then - add(ep) + if extra_paths and #extra_paths>0 then + add(extra_paths) end end add(list) return new end -function resolvers.cleanpathlist(str) - local t=resolvers.expandedpathlist(str) - if t then - for i=1,#t do - t[i]=collapsepath(cleanpath(t[i])) - end - end - return t -end -function resolvers.expandpath(str) - return joinpath(resolvers.expandedpathlist(str)) -end -function resolvers.expandedpathlist(str,extra_too) +expandedpathlist=function(str,extra_too) if not str then return {} elseif instance.savelists then @@ -22540,40 +23003,56 @@ function resolvers.expandedpathlist(str,extra_too) local lists=instance.lists local lst=lists[str] if not lst then - local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str)),extra_too) + local l=made_list(instance,splitpath(expansion(str)),extra_too) lst=expandedpathfromlist(l) lists[str]=lst end return lst else - local lst=resolvers.splitpath(resolvers.expansion(str)) + local lst=splitpath(expansion(str)) return made_list(instance,expandedpathfromlist(lst),extra_too) end end -function resolvers.expandedpathlistfromvariable(str) +resolvers.expandedpathlist=expandedpathlist +resolvers.unexpandedpathlist=unexpandedpathlist +function resolvers.cleanpathlist(str) + local t=expandedpathlist(str) + if t then + for i=1,#t do + t[i]=collapsepath(cleanpath(t[i])) + end + end + return t +end +function resolvers.expandpath(str) + return joinpath(expandedpathlist(str)) +end +local function expandedpathlistfromvariable(str) str=lpegmatch(dollarstripper,str) local tmp=resolvers.variableofformatorsuffix(str) - return resolvers.expandedpathlist(tmp~="" and tmp or str) + return expandedpathlist(tmp~="" and tmp or str) end function resolvers.expandpathfromvariable(str) - return joinpath(resolvers.expandedpathlistfromvariable(str)) + return joinpath(expandedpathlistfromvariable(str)) end +resolvers.expandedpathlistfromvariable=expandedpathlistfromvariable function resolvers.cleanedpathlist(v) - local t=resolvers.expandedpathlist(v) + local t=expandedpathlist(v) for i=1,#t do - t[i]=resolvers.resolve(resolvers.cleanpath(t[i])) + t[i]=resolveprefix(cleanpath(t[i])) end return t end function resolvers.expandbraces(str) - local pth=expandedpathfromlist(resolvers.splitpath(str)) + local pth=expandedpathfromlist(splitpath(str)) return joinpath(pth) end function resolvers.registerfilehash(name,content,someerror) + local files=instance.files if content then - instance.files[name]=content + files[name]=content else - instance.files[name]={} + files[name]={} if somerror==true then instance.loaderror=someerror end @@ -22592,7 +23071,7 @@ function resolvers.renewcache() end local function isreadable(name) local readable=isfile(name) - if trace_detail then + if trace_details then if readable then report_resolving("file %a is readable",name) else @@ -22609,7 +23088,7 @@ local function collect_files(names) local variant=hash.type local search=filejoin(root,path,name) local result=methodhandler('concatinators',variant,root,path,name) - if trace_detail then + if trace_details then report_resolving("match: variant %a, search %a, result %a",variant,search,result) end noffiles=noffiles+1 @@ -22618,7 +23097,7 @@ local function collect_files(names) end for k=1,#names do local filename=names[k] - if trace_detail then + if trace_details then report_resolving("checking name %a",filename) end local basename=filebasename(filename) @@ -22630,12 +23109,13 @@ local function collect_files(names) pathname="/"..pathname.."$" end local hashes=instance.hashes + local files=instance.files for h=1,#hashes do local hash=hashes[h] local hashname=hash.name - local content=hashname and instance.files[hashname] + local content=hashname and files[hashname] if content then - if trace_detail then + if trace_details then report_resolving("deep checking %a, base %a, pattern %a",hashname,basename,pathname) end local path,name=lookup(content,basename) @@ -22703,7 +23183,6 @@ local function find_analyze(filename,askedformat,allresults) if askedformat=="" then if filesuffix=="" or not suffixmap[filesuffix] then local defaultsuffixes=resolvers.defaultsuffixes - local formatofsuffix=resolvers.formatofsuffix for i=1,#defaultsuffixes do local forcedname=filename..'.'..defaultsuffixes[i] wantedfiles[#wantedfiles+1]=forcedname @@ -22713,7 +23192,7 @@ local function find_analyze(filename,askedformat,allresults) end end else - filetype=resolvers.formatofsuffix(filename) + filetype=formatofsuffix(filename) if trace_locating then report_resolving("using suffix based filetype %a",filetype) end @@ -22736,7 +23215,7 @@ local function find_analyze(filename,askedformat,allresults) end local function find_direct(filename,allresults) if not dangerous[askedformat] and isreadable(filename) then - if trace_detail then + if trace_details then report_resolving("file %a found directly",filename) end return "direct",{ filename } @@ -22761,12 +23240,12 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) report_resolving("checking qualified name %a",filename) end if isreadable(filename) then - if trace_detail then + if trace_details then report_resolving("qualified file %a found",filename) end return "qualified",{ filename } end - if trace_detail then + if trace_details then report_resolving("locating qualified file %a",filename) end local forcedname,suffix="",suffixonly(filename) @@ -22774,11 +23253,11 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) local format_suffixes=askedformat=="" and resolvers.defaultsuffixes or suffixes[askedformat] if format_suffixes then for i=1,#format_suffixes do - local s=format_suffixes[i] - forcedname=filename.."."..s + local suffix=format_suffixes[i] + forcedname=filename.."."..suffix if isreadable(forcedname) then if trace_locating then - report_resolving("no suffix, forcing format filetype %a",s) + report_resolving("no suffix, forcing format filetype %a",suffix) end return "qualified",{ forcedname } end @@ -22791,7 +23270,7 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) local savedformat=askedformat local format=savedformat or "" if format=="" then - askedformat=resolvers.formatofsuffix(suffix) + askedformat=formatofsuffix(suffix) end if not format then askedformat="othertextfiles" @@ -22822,7 +23301,7 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) end local function check_subpath(fname) if isreadable(fname) then - if trace_detail then + if trace_details then report_resolving("found %a by deep scanning",fname) end return fname @@ -22830,7 +23309,7 @@ local function check_subpath(fname) end local function makepathlist(list,filetype) local typespec=resolvers.variableofformat(filetype) - local pathlist=resolvers.expandedpathlist(typespec,filetype and usertypes[filetype]) + local pathlist=expandedpathlist(typespec,filetype and usertypes[filetype]) local entry={} if pathlist and #pathlist>0 then for k=1,#pathlist do @@ -22841,7 +23320,7 @@ local function makepathlist(list,filetype) local expression=makepathexpression(pathname) local barename=gsub(pathname,"/+$","") barename=resolveprefix(barename) - local scheme=url.hasscheme(barename) + local scheme=urlhasscheme(barename) local schemename=gsub(barename,"%.%*$",'') entry[k]={ path=path, @@ -22878,7 +23357,7 @@ local function find_intree(filename,filetype,wantedfiles,allresults) dirlist[i]=filedirname(filelist[i][3]).."/" end end - if trace_detail then + if trace_details then report_resolving("checking filename %a in tree",filename) end for k=1,#pathlist do @@ -22888,7 +23367,7 @@ local function find_intree(filename,filetype,wantedfiles,allresults) local done=false if filelist then local expression=entry.expression - if trace_detail then + if trace_details then report_resolving("using pattern %a for path %a",expression,pathname) end for k=1,#filelist do @@ -22899,16 +23378,16 @@ local function find_intree(filename,filetype,wantedfiles,allresults) result[#result+1]=resolveprefix(fl[3]) done=true if allresults then - if trace_detail then + if trace_details then report_resolving("match to %a in hash for file %a and path %a, continue scanning",expression,f,d) end else - if trace_detail then + if trace_details then report_resolving("match to %a in hash for file %a and path %a, quit scanning",expression,f,d) end break end - elseif trace_detail then + elseif trace_details then report_resolving("no match to %a in hash for file %a and path %a",expression,f,d) end end @@ -22923,7 +23402,7 @@ local function find_intree(filename,filetype,wantedfiles,allresults) if not find(pname,"*",1,true) then if can_be_dir(pname) then if not done and not entry.prescanned then - if trace_detail then + if trace_details then report_resolving("quick root scan for %a",pname) end for k=1,#wantedfiles do @@ -22938,7 +23417,7 @@ local function find_intree(filename,filetype,wantedfiles,allresults) end end if not done and entry.recursive then - if trace_detail then + if trace_details then report_resolving("scanning filesystem for %a",pname) end local files=resolvers.simplescanfiles(pname,false,true) @@ -23004,7 +23483,7 @@ local function find_intree(filename,filetype,wantedfiles,allresults) end end local function find_onpath(filename,filetype,wantedfiles,allresults) - if trace_detail then + if trace_details then report_resolving("checking filename %a, filetype %a, wanted files %a",filename,filetype,concat(wantedfiles," | ")) end local result={} @@ -23046,7 +23525,9 @@ collect_instance_files=function(filename,askedformat,allresults) { find_onpath (filename,filetype,wantedfiles,true) }, { find_otherwise(filename,filetype,wantedfiles,true) }, } - local result,status,done={},{},{} + local result={} + local status={} + local done={} for k,r in next,results do local method,list=r[1],r[2] if method and list then @@ -23060,7 +23541,7 @@ collect_instance_files=function(filename,askedformat,allresults) end end end - if trace_detail then + if trace_details then report_resolving("lookup status: %s",table.serialize(status,filename)) end return result,status @@ -23117,6 +23598,9 @@ local function findfiles(filename,filetype,allresults) if not filename or filename=="" then return {} end + if allresults==nil then + allresults=true + end local result,status=collect_instance_files(filename,filetype or "",allresults) if not result or #result==0 then local lowered=lower(filename) @@ -23126,27 +23610,25 @@ local function findfiles(filename,filetype,allresults) end return result or {},status end -function resolvers.findfiles(filename,filetype) - if not filename or filename=="" then - return "" - else - return findfiles(filename,filetype,true) - end -end -function resolvers.findfile(filename,filetype) +local function findfile(filename,filetype) if not filename or filename=="" then return "" else return findfiles(filename,filetype,false)[1] or "" end end +resolvers.findfiles=findfiles +resolvers.findfile=findfile +resolvers.find_file=findfile +resolvers.find_files=findfiles function resolvers.findpath(filename,filetype) return filedirname(findfiles(filename,filetype,false)[1] or "") end local function findgivenfiles(filename,allresults) + local hashes=instance.hashes + local files=instance.files local base=filebasename(filename) local result={} - local hashes=instance.hashes local function okay(hash,path,name) local found=methodhandler('concatinators',hash.type,hash.name,path,name) if found and found~="" then @@ -23156,7 +23638,7 @@ local function findgivenfiles(filename,allresults) end for k=1,#hashes do local hash=hashes[k] - local content=instance.files[hash.name] + local content=files[hash.name] if content then local path,name=lookup(content,base) if not path then @@ -23188,14 +23670,14 @@ function resolvers.wildcardpattern(pattern) return lpegmatch(makewildcard,pattern) or pattern end local function findwildcardfiles(filename,allresults,result) + local files=instance.files + local hashes=instance.hashes local result=result or {} local base=filebasename(filename) local dirn=filedirname(filename) local path=lower(lpegmatch(makewildcard,dirn) or dirn) local name=lower(lpegmatch(makewildcard,base) or base) - local files=instance.files if find(name,"*",1,true) then - local hashes=instance.hashes local function okay(found,path,base,hashname,hashtype) if find(found,path) then local full=methodhandler('concatinators',hashtype,hashname,found,base) @@ -23235,7 +23717,6 @@ local function findwildcardfiles(filename,allresults,result) end end end - local hashes=instance.hashes for k=1,#hashes do local hash=hashes[k] local hashname=hash.name @@ -23265,13 +23746,21 @@ end function resolvers.findwildcardfile(filename) return findwildcardfiles(filename,false)[1] or "" end -function resolvers.automount() -end -function resolvers.starttiming() - statistics.starttiming(instance) +do + local starttiming=statistics.starttiming + local stoptiming=statistics.stoptiming + local elapsedtime=statistics.elapsedtime + function resolvers.starttiming() + starttiming(instance) + end + function resolvers.stoptiming() + stoptiming(instance) + end + function resolvers.loadtime() + return elapsedtime(instance) + end end -function resolvers.stoptiming() - statistics.stoptiming(instance) +function resolvers.automount() end function resolvers.load(option) resolvers.starttiming() @@ -23285,9 +23774,6 @@ function resolvers.load(option) local files=instance.files return files and next(files) and true end -function resolvers.loadtime() - return statistics.elapsedtime(instance) -end local function report(str) if trace_locating then report_resolving(str) @@ -23317,7 +23803,7 @@ function resolvers.dowithfilesandreport(command,files,...) end end function resolvers.showpath(str) - return joinpath(resolvers.expandedpathlist(resolvers.formatofvariable(str))) + return joinpath(expandedpathlist(resolvers.formatofvariable(str))) end function resolvers.registerfile(files,name,path) if files[name] then @@ -23331,7 +23817,7 @@ function resolvers.registerfile(files,name,path) end end function resolvers.dowithpath(name,func) - local pathlist=resolvers.expandedpathlist(name) + local pathlist=expandedpathlist(name) for i=1,#pathlist do func("^"..cleanpath(pathlist[i])) end @@ -23341,11 +23827,11 @@ function resolvers.dowithvariable(name,func) end function resolvers.locateformat(name) local engine=environment.ownmain or "luatex" - local barename=removesuffix(name) + local barename=removesuffix(file.basename(name)) local fullname=addsuffix(barename,"fmt") local fmtname=caches.getfirstreadablefile(fullname,"formats",engine) or "" if fmtname=="" then - fmtname=resolvers.findfile(fullname) + fmtname=findfile(fullname) fmtname=cleanpath(fmtname) end if fmtname~="" then @@ -23354,17 +23840,17 @@ function resolvers.locateformat(name) local lucname=addsuffix(barename,luasuffixes.luc) local luiname=addsuffix(barename,luasuffixes.lui) if isfile(luiname) then - return barename,luiname + return fmtname,luiname elseif isfile(lucname) then - return barename,lucname + return fmtname,lucname elseif isfile(luaname) then - return barename,luaname + return fmtname,luaname end end return nil,nil end function resolvers.booleanvariable(str,default) - local b=resolvers.expansion(str) + local b=expansion(str) if b=="" then return default else @@ -23374,6 +23860,7 @@ function resolvers.booleanvariable(str,default) end function resolvers.dowithfilesintree(pattern,handle,before,after) local hashes=instance.hashes + local files=instance.files for i=1,#hashes do local hash=hashes[i] local blobtype=hash.type @@ -23385,7 +23872,7 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) if before then before(blobtype,blobpath,pattern) end - for path,name in filtered(instance.files[blobpath],pattern) do + for path,name in filtered(files[blobpath],pattern) do if type(path)=="string" then checked=checked+1 if handle(blobtype,blobpath,path,name) then @@ -23406,10 +23893,6 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) end end end -local obsolete=resolvers.obsolete or {} -resolvers.obsolete=obsolete -resolvers.find_file=resolvers.findfile obsolete.find_file=resolvers.findfile -resolvers.find_files=resolvers.findfiles obsolete.find_files=resolvers.findfiles function resolvers.knownvariables(pattern) if instance then local environment=instance.environment @@ -23443,7 +23926,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 4854, stripped down to: 2889 +-- original size: 5088, stripped down to: 3144 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -23452,6 +23935,7 @@ if not modules then modules={} end modules ['data-pre']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local insert,remove=table.insert,table.remove local resolvers=resolvers local prefixes=resolvers.prefixes local cleanpath=resolvers.cleanpath @@ -23529,8 +24013,9 @@ prefixes.kpse=prefixes.locate prefixes.full=prefixes.locate prefixes.file=prefixes.filename prefixes.path=prefixes.pathname +local inputstack={} +local stackpath=resolvers.stackpath local function toppath() - local inputstack=resolvers.inputstack if not inputstack then return "." end @@ -23542,15 +24027,23 @@ local function toppath() end end local function jobpath() - local path=resolvers.stackpath() + local path=stackpath() if not path or path=="" then return "." else return path end end +local function pushinputname(name) + insert(inputstack,name) +end +local function popinputname(name) + return remove(inputstack) +end resolvers.toppath=toppath resolvers.jobpath=jobpath +resolvers.pushinputname=pushinputname +resolvers.popinputname=popinputname prefixes.toppath=function(str) return cleanpath(joinpath(toppath(),str)) end prefixes.jobpath=function(str) return cleanpath(joinpath(jobpath(),str)) end resolvers.setdynamic("toppath") @@ -23593,7 +24086,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-out"] = package.loaded["data-out"] or true --- original size: 530, stripped down to: 470 +-- original size: 551, stripped down to: 470 if not modules then modules={} end modules ['data-out']={ version=1.001, @@ -23616,7 +24109,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3863, stripped down to: 3170 +-- original size: 4365, stripped down to: 3588 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -23625,39 +24118,46 @@ if not modules then modules={} end modules ['data-fil']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local ioopen=io.open +local isdir=lfs.isdir local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_files=logs.reporter("resolvers","files") local resolvers=resolvers local resolveprefix=resolvers.resolve -local finders,openers,loaders,savers=resolvers.finders,resolvers.openers,resolvers.loaders,resolvers.savers -local locators,hashers,generators,concatinators=resolvers.locators,resolvers.hashers,resolvers.generators,resolvers.concatinators +local findfile=resolvers.findfile +local scanfiles=resolvers.scanfiles +local registerfilehash=resolvers.registerfilehash +local appendhash=resolvers.appendhash +local loadcachecontent=caches.loadcontent local checkgarbage=utilities.garbagecollector and utilities.garbagecollector.check -function locators.file(specification) +function resolvers.locators.file(specification) local filename=specification.filename local realname=resolveprefix(filename) - if realname and realname~='' and lfs.isdir(realname) then + if realname and realname~='' and isdir(realname) then if trace_locating then report_files("file locator %a found as %a",filename,realname) end - resolvers.appendhash('file',filename,true) + appendhash('file',filename,true) elseif trace_locating then report_files("file locator %a not found",filename) end end -function hashers.file(specification) +function resolvers.hashers.file(specification) local pathname=specification.filename - local content=caches.loadcontent(pathname,'files') - resolvers.registerfilehash(pathname,content,content==nil) + local content=loadcachecontent(pathname,'files') + registerfilehash(pathname,content,content==nil) end -function generators.file(specification) +function resolvers.generators.file(specification) local pathname=specification.filename - local content=resolvers.scanfiles(pathname,false,true) - resolvers.registerfilehash(pathname,content,true) + local content=scanfiles(pathname,false,true) + registerfilehash(pathname,content,true) end -concatinators.file=file.join +resolvers.concatinators.file=file.join +local finders=resolvers.finders +local notfound=finders.notfound function finders.file(specification,filetype) local filename=specification.filename - local foundname=resolvers.findfile(filename,filetype) + local foundname=findfile(filename,filetype) if foundname and foundname~="" then if trace_locating then report_files("file finder: %a found",filename) @@ -23667,37 +24167,55 @@ function finders.file(specification,filetype) if trace_locating then report_files("file finder: %a not found",filename) end - return finders.notfound() + return notfound() end end -function openers.helpers.textopener(tag,filename,f) +local openers=resolvers.openers +local notfound=openers.notfound +local overloaded=false +local function textopener(tag,filename,f) return { - reader=function() return f:read () end, - close=function() logs.show_close(filename) return f:close() end, + reader=function() return f:read () end, + close=function() return f:close() end, } end +function openers.helpers.textopener(...) + return textopener(...) +end +function openers.helpers.settextopener(opener) + if overloaded then + report_files("file opener: %s overloaded","already") + else + if trace_locating then + report_files("file opener: %s overloaded","once") + end + overloaded=true + textopener=opener + end +end function openers.file(specification,filetype) local filename=specification.filename if filename and filename~="" then - local f=io.open(filename,"r") + local f=ioopen(filename,"r") if f then if trace_locating then report_files("file opener: %a opened",filename) end - return openers.helpers.textopener("file",filename,f) + return textopener("file",filename,f) end end if trace_locating then report_files("file opener: %a not found",filename) end - return openers.notfound() + return notfound() end +local loaders=resolvers.loaders +local notfound=loaders.notfound function loaders.file(specification,filetype) local filename=specification.filename if filename and filename~="" then - local f=io.open(filename,"rb") + local f=ioopen(filename,"rb") if f then - logs.show_load(filename) if trace_locating then report_files("file loader: %a loaded",filename) end @@ -23714,7 +24232,7 @@ function loaders.file(specification,filetype) if trace_locating then report_files("file loader: %a not found",filename) end - return loaders.notfound() + return notfound() end @@ -23724,7 +24242,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-con"] = package.loaded["data-con"] or true --- original size: 5029, stripped down to: 3432 +-- original size: 5388, stripped down to: 3685 if not modules then modules={} end modules ['data-con']={ version=1.100, @@ -23733,6 +24251,7 @@ if not modules then modules={} end modules ['data-con']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local setmetatable=setmetatable local format,lower,gsub=string.format,string.lower,string.gsub local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) local trace_containers=false trackers.register("resolvers.containers",function(v) trace_containers=v end) @@ -23740,16 +24259,21 @@ local trace_storage=false trackers.register("resolvers.storage",function(v) tra containers=containers or {} local containers=containers containers.usecache=true +local getwritablepath=caches.getwritablepath +local getreadablepaths=caches.getreadablepaths +local cacheiswritable=caches.is_writable +local loaddatafromcache=caches.loaddata +local savedataincache=caches.savedata local report_containers=logs.reporter("resolvers","containers") local allocated={} local mt={ __index=function(t,k) if k=="writable" then - local writable=caches.getwritablepath(t.category,t.subcategory) or { "." } + local writable=getwritablepath(t.category,t.subcategory) or { "." } t.writable=writable return writable elseif k=="readables" then - local readables=caches.getreadablepaths(t.category,t.subcategory) or { "." } + local readables=getreadablepaths(t.category,t.subcategory) or { "." } t.readables=readables return readables end @@ -23780,7 +24304,7 @@ function containers.define(category,subcategory,version,enabled) end end function containers.is_usable(container,name) - return container.enabled and caches and caches.is_writable(container.writable,name) + return container.enabled and caches and cacheiswritable(container.writable,name) end function containers.is_valid(container,name) if name and name~="" then @@ -23794,7 +24318,7 @@ function containers.read(container,name) local storage=container.storage local stored=storage[name] if not stored and container.enabled and caches and containers.usecache then - stored=caches.loaddata(container.readables,name,container.writable) + stored=loaddatafromcache(container.readables,name,container.writable) if stored and stored.cache_version==container.version then if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","load",container.subcategory,name) @@ -23810,17 +24334,20 @@ function containers.read(container,name) end return stored end -function containers.write(container,name,data) +function containers.write(container,name,data,fast) if data then data.cache_version=container.version if container.enabled and caches then - local unique,shared=data.unique,data.shared - data.unique,data.shared=nil,nil - caches.savedata(container.writable,name,data) + local unique=data.unique + local shared=data.shared + data.unique=nil + data.shared=nil + savedataincache(container.writable,name,data,fast) if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","save",container.subcategory,name) end - data.unique,data.shared=unique,shared + data.unique=unique + data.shared=shared end if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","store",container.subcategory,name) @@ -23843,7 +24370,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-use"] = package.loaded["data-use"] or true --- original size: 4434, stripped down to: 3180 +-- original size: 5790, stripped down to: 2910 if not modules then modules={} end modules ['data-use']={ version=1.001, @@ -23852,40 +24379,11 @@ if not modules then modules={} end modules ['data-use']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local format,lower,gsub,find=string.format,string.lower,string.gsub,string.find +local format=string.format local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_mounts=logs.reporter("resolvers","mounts") local resolvers=resolvers -resolvers.automounted=resolvers.automounted or {} -function resolvers.automount(usecache) - local mountpaths=resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) - if (not mountpaths or #mountpaths==0) and usecache then - mountpaths=caches.getreadablepaths("mount") - end - if mountpaths and #mountpaths>0 then - resolvers.starttiming() - for k=1,#mountpaths do - local root=mountpaths[k] - local f=io.open(root.."/url.tmi") - if f then - for line in f:lines() do - if line then - if find(line,"^[%%#%-]") then - elseif find(line,"^zip://") then - if trace_locating then - report_mounts("mounting %a",line) - end - table.insert(resolvers.automounted,line) - resolvers.usezipfile(line) - end - end - end - f:close() - end - end - resolvers.stoptiming() - end -end +local findfile=resolvers.findfile statistics.register("used config file",function() return caches.configfiles() end) statistics.register("used cache path",function() return caches.usedpaths() end) function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) @@ -23895,9 +24393,11 @@ function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) local luvdata={ enginebanner=enginebanner, formatbanner=formatbanner, - sourcehash=md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), + sourcehash=md5.hex(io.loaddata(findfile(sourcefile)) or "unknown"), sourcefile=sourcefile, luaversion=LUAVERSION, + formatid=LUATEXFORMATID, + functionality=LUATEXFUNCTIONALITY, } io.savedata(luvname,table.serialize(luvdata,true)) lua.registerfinalizer(function() @@ -23917,7 +24417,7 @@ function statistics.checkfmtstatus(texname) if lfs.isfile(luvname) then local luv=dofile(luvname) if luv and luv.sourcefile then - local sourcehash=md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") + local sourcehash=md5.hex(io.loaddata(findfile(luv.sourcefile)) or "unknown") local luvbanner=luv.enginebanner or "?" if luvbanner~=enginebanner then return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) @@ -23927,8 +24427,19 @@ function statistics.checkfmtstatus(texname) return format("source mismatch (luv: %s <> bin: %s)",luvhash,sourcehash) end local luvluaversion=luv.luaversion or 0 - if luvluaversion~=LUAVERSION then - return format("lua mismatch (luv: %s <> bin: %s)",luvluaversion,LUAVERSION) + local engluaversion=LUAVERSION or 0 + if luvluaversion~=engluaversion then + return format("lua mismatch (luv: %s <> bin: %s)",luvluaversion,engluaversion) + end + local luvfunctionality=luv.functionality or 0 + local engfunctionality=status.development_id or 0 + if luvfunctionality~=engfunctionality then + return format("functionality mismatch (luv: %s <> bin: %s)",luvfunctionality,engfunctionality) + end + local luvformatid=luv.formatid or 0 + local engformatid=status.format_id or 0 + if luvformatid~=engformatid then + return format("formatid mismatch (luv: %s <> bin: %s)",luvformatid,engformatid) end else return "invalid status file" @@ -23947,7 +24458,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 10263, stripped down to: 7556 +-- original size: 10725, stripped down to: 7949 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -23960,6 +24471,14 @@ local format,find,match=string.format,string.find,string.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_zip=logs.reporter("resolvers","zip") local resolvers=resolvers +local findfile=resolvers.findfile +local registerfile=resolvers.registerfile +local splitmethod=resolvers.splitmethod +local prependhash=resolvers.prependhash +local starttiming=resolvers.starttiming +local extendtexmf=resolvers.extendtexmfvariable +local stoptiming=resolvers.stoptiming +local urlquery=url.query zip=zip or {} local zip=zip local archives=zip.archives or {} @@ -23974,8 +24493,9 @@ if zipfiles then closezip=zipfiles.close validfile=zipfiles.found wholefile=zipfiles.unzip + local listzip=zipfiles.list traversezip=function(zfile) - return ipairs(zipfiles.list(zfile)) + return ipairs(listzip(zfile)) end local streams=utilities.streams local openstream=streams.open @@ -24032,28 +24552,30 @@ local function validzip(str) return str end end -function zip.openarchive(name) +local function openarchive(name) if not name or name=="" then return nil else local arch=archives[name] if not arch then - local full=resolvers.findfile(name) or "" + local full=findfile(name) or "" arch=full~="" and openzip(full) or false archives[name]=arch end return arch end end -function zip.closearchive(name) +local function closearchive(name) if not name or (name=="" and archives[name]) then closezip(archives[name]) archives[name]=nil end end +zip.openarchive=openarchive +zip.closearchive=closearchive function resolvers.locators.zip(specification) local archive=specification.filename - local zipfile=archive and archive~="" and zip.openarchive(archive) + local zipfile=archive and archive~="" and openarchive(archive) if trace_locating then if zipfile then report_zip("locator: archive %a found",archive) @@ -24062,13 +24584,6 @@ function resolvers.locators.zip(specification) end end end -function resolvers.hashers.zip(specification) - local archive=specification.filename - if trace_locating then - report_zip("loading file %a",archive) - end - resolvers.usezipfile(specification.original) -end function resolvers.concatinators.zip(zipfile,path,name) if not path or path=="" then return format('%s?name=%s',zipfile,name) @@ -24076,14 +24591,16 @@ function resolvers.concatinators.zip(zipfile,path,name) return format('%s?name=%s/%s',zipfile,path,name) end end -function resolvers.finders.zip(specification) +local finders=resolvers.finders +local notfound=finders.notfound +function finders.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("finder: archive %a found",archive) @@ -24104,16 +24621,19 @@ function resolvers.finders.zip(specification) if trace_locating then report_zip("finder: %a not found",original) end - return resolvers.finders.notfound() + return notfound() end -function resolvers.openers.zip(specification) +local openers=resolvers.openers +local notfound=openers.notfound +local textopener=openers.helpers.textopener +function openers.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("opener; archive %a opened",archive) @@ -24123,7 +24643,7 @@ function resolvers.openers.zip(specification) if trace_locating then report_zip("opener: file %a found",queryname) end - return resolvers.openers.helpers.textopener('zip',original,handle) + return textopener('zip',original,handle) elseif trace_locating then report_zip("opener: file %a not found",queryname) end @@ -24135,23 +24655,24 @@ function resolvers.openers.zip(specification) if trace_locating then report_zip("opener: %a not found",original) end - return resolvers.openers.notfound() + return notfound() end -function resolvers.loaders.zip(specification) +local loaders=resolvers.loaders +local notfound=loaders.notfound +function loaders.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("loader: archive %a opened",archive) end local data=wholefile(zfile,queryname) if data then - logs.show_load(original) if trace_locating then report_zip("loader; file %a loaded",original) end @@ -24167,47 +24688,24 @@ function resolvers.loaders.zip(specification) if trace_locating then report_zip("loader: %a not found",original) end - return resolvers.openers.notfound() -end -function resolvers.usezipfile(archive) - local specification=resolvers.splitmethod(archive) - local archive=specification.filename - if archive and not registeredfiles[archive] then - local z=zip.openarchive(archive) - if z then - local tree=url.query(specification.query).tree or "" - if trace_locating then - report_zip("registering: archive %a",archive) - end - resolvers.starttiming() - resolvers.prependhash('zip',archive) - resolvers.extendtexmfvariable(archive) - registeredfiles[archive]=z - resolvers.registerfilehash(archive,resolvers.registerzipfile(z,tree)) - resolvers.stoptiming() - elseif trace_locating then - report_zip("registering: unknown archive %a",archive) - end - elseif trace_locating then - report_zip("registering: archive %a not found",archive) - end + return notfound() end -function resolvers.registerzipfile(z,tree) +local function registerzipfile(z,tree) local names={} local files={} local remap={} local n=0 local filter=tree=="" and "^(.+)/(.-)$" or format("^%s/(.+)/(.-)$",tree) - local register=resolvers.registerfile if trace_locating then report_zip("registering: using filter %a",filter) end + starttiming() for i in traversezip(z) do local filename=i.filename local path,name=match(filename,filter) if not path then n=n+1 - register(names,filename,"") + registerfile(names,filename,"") local usedname=lower(filename) files[usedname]="" if usedname~=filename then @@ -24224,12 +24722,43 @@ function resolvers.registerzipfile(z,tree) else end end + stoptiming() report_zip("registering: %s files registered",n) return { files=files, remap=remap, } end +local function usezipfile(archive) + local specification=splitmethod(archive) + local archive=specification.filename + if archive and not registeredfiles[archive] then + local z=openarchive(archive) + if z then + local tree=urlquery(specification.query).tree or "" + if trace_locating then + report_zip("registering: archive %a",archive) + end + prependhash('zip',archive) + extendtexmf(archive) + registeredfiles[archive]=z + registerfilehash(archive,registerzipfile(z,tree)) + elseif trace_locating then + report_zip("registering: unknown archive %a",archive) + end + elseif trace_locating then + report_zip("registering: archive %a not found",archive) + end +end +resolvers.usezipfile=usezipfile +resolvers.registerzipfile=registerzipfile +function resolvers.hashers.zip(specification) + local archive=specification.filename + if trace_locating then + report_zip("loading file %a",archive) + end + usezipfile(specification.original) +end end -- of closure @@ -24238,7 +24767,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 8478, stripped down to: 5223 +-- original size: 10802, stripped down to: 6619 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -24247,180 +24776,255 @@ if not modules then modules={} end modules ['data-tre']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local type=type local find,gsub,lower=string.find,string.gsub,string.lower -local basename,dirname,joinname=file.basename,file.dirname,file .join +local basename,dirname,joinname=file.basename,file.dirname,file.join local globdir,isdir,isfile=dir.glob,lfs.isdir,lfs.isfile local P,lpegmatch=lpeg.P,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_trees=logs.reporter("resolvers","trees") local resolvers=resolvers -local resolveprefix=resolvers.resolve -local notfound=resolvers.finders.notfound -local lookup=resolvers.get_from_content -local collectors={} -local found={} -function resolvers.finders.tree(specification) - local spec=specification.filename - local okay=found[spec] - if okay==nil then - if spec~="" then - local path=dirname(spec) - local name=basename(spec) - if path=="" then - path="." - end - local names=collectors[path] - if not names then - local pattern=find(path,"/%*+$") and path or (path.."/*") - names=globdir(pattern) - collectors[path]=names - end - local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" - for i=1,#names do - local fullname=names[i] - if find(fullname,pattern) then - found[spec]=fullname - return fullname - end - end - local pattern=lower(pattern) - for i=1,#names do - local fullname=lower(names[i]) - if find(fullname,pattern) then - if isfile(fullname) then +local finders=resolvers.finders +local openers=resolvers.openers +local loaders=resolvers.loaders +local locators=resolvers.locators +local hashers=resolvers.hashers +local generators=resolvers.generators +do + local collectors={} + local found={} + local notfound=finders.notfound + function finders.tree(specification) + local spec=specification.filename + local okay=found[spec] + if okay==nil then + if spec~="" then + local path=dirname(spec) + local name=basename(spec) + if path=="" then + path="." + end + local names=collectors[path] + if not names then + local pattern=find(path,"/%*+$") and path or (path.."/*") + names=globdir(pattern) + collectors[path]=names + end + local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" + for i=1,#names do + local fullname=names[i] + if find(fullname,pattern) then found[spec]=fullname return fullname - else - break + end + end + local pattern=lower(pattern) + for i=1,#names do + local fullname=lower(names[i]) + if find(fullname,pattern) then + if isfile(fullname) then + found[spec]=fullname + return fullname + else + break + end end end end + okay=notfound() + found[spec]=okay end - okay=notfound() - found[spec]=okay + return okay end - return okay end -function resolvers.locators.tree(specification) - local name=specification.filename - local realname=resolveprefix(name) - if realname and realname~='' and isdir(realname) then - if trace_locating then - report_trees("locator %a found",realname) +do + local resolveprefix=resolvers.resolve + local appendhash=resolvers.appendhash + local function dolocate(specification) + local name=specification.filename + local realname=resolveprefix(name) + if realname and realname~='' and isdir(realname) then + if trace_locating then + report_trees("locator %a found",realname) + end + appendhash('tree',name,false) + elseif trace_locating then + report_trees("locator %a not found",name) end - resolvers.appendhash('tree',name,false) - elseif trace_locating then - report_trees("locator %a not found",name) end + locators.tree=dolocate + locators.dirlist=dolocate + locators.dirfile=dolocate end -function resolvers.hashers.tree(specification) - local name=specification.filename - if trace_locating then - report_trees("analyzing %a",name) - end - resolvers.methodhandler("hashers",name) - resolvers.generators.file(specification) -end -local collectors={} -local splitter=lpeg.splitat("/**/") -local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } -table.setmetatableindex(collectors,function(t,k) - local rootname=lpegmatch(stripper,k) - local dataname=joinname(rootname,"dirlist") - local content=caches.loadcontent(dataname,"files",dataname) - if not content then - content=resolvers.scanfiles(rootname,nil,nil,false,true) - caches.savecontent(dataname,"files",content,dataname) - end - t[k]=content - return content -end) -local function checked(root,p,n) - if p then - if type(p)=="table" then - for i=1,#p do - local fullname=joinname(root,p[i],n) +do + local filegenerator=generators.file + generators.dirlist=filegenerator + generators.dirfile=filegenerator +end +do + local filegenerator=generators.file + local methodhandler=resolvers.methodhandler + local function dohash(specification) + local name=specification.filename + if trace_locating then + report_trees("analyzing %a",name) + end + methodhandler("hashers",name) + filegenerator(specification) + end + hashers.tree=dohash + hashers.dirlist=dohash + hashers.dirfile=dohash +end +local resolve do + local collectors={} + local splitter=lpeg.splitat("/**/") + local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } + local loadcontent=caches.loadcontent + local savecontent=caches.savecontent + local notfound=finders.notfound + local scanfiles=resolvers.scanfiles + local lookup=resolvers.get_from_content + table.setmetatableindex(collectors,function(t,k) + local rootname=lpegmatch(stripper,k) + local dataname=joinname(rootname,"dirlist") + local content=loadcontent(dataname,"files",dataname) + if not content then + content=scanfiles(rootname,nil,nil,false,true) + savecontent(dataname,"files",content,dataname) + end + t[k]=content + return content + end) + local function checked(root,p,n) + if p then + if type(p)=="table" then + for i=1,#p do + local fullname=joinname(root,p[i],n) + if isfile(fullname) then + return fullname + end + end + else + local fullname=joinname(root,p,n) if isfile(fullname) then return fullname end end - else - local fullname=joinname(root,p,n) - if isfile(fullname) then - return fullname - end end + return notfound() end - return notfound() -end -local function resolve(specification) - local filename=specification.filename - if filename~="" then - local root,rest=lpegmatch(splitter,filename) - if root and rest then - local path,name=dirname(rest),basename(rest) - if name~=rest then - local content=collectors[root] - local p,n=lookup(content,name) - if not p then - return notfound() - end - local pattern=".*/"..path.."$" - local istable=type(p)=="table" - if istable then - for i=1,#p do - local pi=p[i] - if pi==path or find(pi,pattern) then - local fullname=joinname(root,pi,n) - if isfile(fullname) then - return fullname + resolve=function(specification) + local filename=specification.filename + if filename~="" then + local root,rest=lpegmatch(splitter,filename) + if root and rest then + local path,name=dirname(rest),basename(rest) + if name~=rest then + local content=collectors[root] + local p,n=lookup(content,name) + if not p then + return notfound() + end + local pattern=".*/"..path.."$" + local istable=type(p)=="table" + if istable then + for i=1,#p do + local pi=p[i] + if pi==path or find(pi,pattern) then + local fullname=joinname(root,pi,n) + if isfile(fullname) then + return fullname + end end end + elseif p==path or find(p,pattern) then + local fullname=joinname(root,p,n) + if isfile(fullname) then + return fullname + end end - elseif p==path or find(p,pattern) then - local fullname=joinname(root,p,n) - if isfile(fullname) then - return fullname + local queries=specification.queries + if queries and queries.option=="fileonly" then + return checked(root,p,n) + else + return notfound() end end - local queries=specification.queries - if queries and queries.option=="fileonly" then - return checked(root,p,n) - else - return notfound() - end + end + local path=dirname(filename) + local name=basename(filename) + local root=lpegmatch(stripper,path) + local content=collectors[path] + local p,n=lookup(content,name) + if p then + return checked(root,p,n) end end - local path,name=dirname(filename),basename(filename) - local root=lpegmatch(stripper,path) - local content=collectors[path] - local p,n=lookup(content,name) - if p then - return checked(root,p,n) + return notfound() + end + finders.dirlist=resolve + function finders.dirfile(specification) + local queries=specification.queries + if queries then + queries.option="fileonly" + else + specification.queries={ option="fileonly" } end + return resolve(specification) end - return notfound() end -resolvers.finders .dirlist=resolve -resolvers.locators .dirlist=resolvers.locators .tree -resolvers.hashers .dirlist=resolvers.hashers .tree -resolvers.generators.dirlist=resolvers.generators.file -resolvers.openers .dirlist=resolvers.openers .file -resolvers.loaders .dirlist=resolvers.loaders .file -function resolvers.finders.dirfile(specification) - local queries=specification.queries - if queries then - queries.option="fileonly" - else - specification.queries={ option="fileonly" } +do + local fileopener=openers.file + local fileloader=loaders.file + openers.dirlist=fileopener + loaders.dirlist=fileloader + openers.dirfile=fileopener + loaders.dirfile=fileloader +end +do + local hashfile="dirhash.lua" + local kind="HASH256" + local version=1.0 + local loadtable=table.load + local savetable=table.save + local loaddata=io.loaddata + function resolvers.dirstatus(patterns) + local t=type(patterns) + if t=="string" then + patterns={ patterns } + elseif t~="table" then + return false + end + local status=loadtable(hashfile) + if not status or status.version~=version or status.kind~=kind then + status={ + version=1.0, + kind=kind, + hashes={}, + } + end + local hashes=status.hashes + local changed={} + local action=sha2[kind] + local update={} + for i=1,#patterns do + local pattern=patterns[i] + local files=globdir(pattern) + for i=1,#files do + local name=files[i] + local hash=action(loaddata(name)) + if hashes[name]~=hash then + changed[#changed+1]=name + end + update[name]=hash + end + end + status.hashes=update + savetable(hashfile,status) + return #changed>0 and changed or false end - return resolve(specification) end -resolvers.locators .dirfile=resolvers.locators .dirlist -resolvers.hashers .dirfile=resolvers.hashers .dirlist -resolvers.generators.dirfile=resolvers.generators.dirlist -resolvers.openers .dirfile=resolvers.openers .dirlist -resolvers.loaders .dirfile=resolvers.loaders .dirlist end -- of closure @@ -24429,7 +25033,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6753, stripped down to: 5268 +-- original size: 6945, stripped down to: 5408 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -24439,8 +25043,11 @@ if not modules then modules={} end modules ['data-sch']={ license="see context related readme files" } local load,tonumber=load,tonumber -local gsub,concat,format=string.gsub,table.concat,string.format +local gsub,format=string.gsub,string.format +local sortedhash,concat=table.sortedhash,table.concat local finders,openers,loaders=resolvers.finders,resolvers.openers,resolvers.loaders +local addsuffix,suffix,splitbase=file.addsuffix,file.suffix,file.splitbase +local md5hex=md5.hex local trace_schemes=false trackers.register("resolvers.schemes",function(v) trace_schemes=v end) local report_schemes=logs.reporter("resolvers","schemes") local http=require("socket.http") @@ -24457,7 +25064,7 @@ function cleaners.none(specification) return specification.original end function cleaners.strip(specification) - local path,name=file.splitbase(specification.original) + local path,name=splitbase(specification.original) if path=="" then return (gsub(name,"[^%a%d%.]+","-")) else @@ -24465,7 +25072,7 @@ function cleaners.strip(specification) end end function cleaners.md5(specification) - return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) + return addsuffix(md5hex(specification.original),suffix(specification.path)) end local cleaner=cleaners.strip directives.register("schemes.cleanmethod",function(v) cleaner=cleaners[v] or cleaners.strip end) @@ -24485,7 +25092,7 @@ local runner=sandbox.registerrunner { name="curl resolver", method="execute", program="curl", - template="--silent --insecure --create-dirs --output %cachename% %original%", + template='--silent --insecure --create-dirs --output "%cachename%" "%original%"', checkers={ cachename="cache", original="url", @@ -24556,10 +25163,10 @@ end schemes.install=install local function http_handler(specification,cachename) local tempname=cachename..".tmp" - local f=io.open(tempname,"wb") + local handle=io.open(tempname,"wb") local status,message=http.request { url=specification.original, - sink=ltn12.sink.file(f) + sink=ltn12.sink.file(handle) } if not status then os.remove(tempname) @@ -24574,13 +25181,13 @@ install('https') install('ftp') statistics.register("scheme handling time",function() local l,r,nl,nr={},{},0,0 - for k,v in table.sortedhash(loaded) do + for k,v in sortedhash(loaded) do if v>0 then nl=nl+1 l[nl]=k..":"..v end end - for k,v in table.sortedhash(reused) do + for k,v in sortedhash(reused) do if v>0 then nr=nr+1 r[nr]=k..":"..v @@ -24588,10 +25195,10 @@ statistics.register("scheme handling time",function() end local n=nl+nr if n>0 then - l=nl>0 and concat(l) or "none" - r=nr>0 and concat(r) or "none" + if nl==0 then l={ "none" } end + if nr==0 then r={ "none" } end return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s", - statistics.elapsedtime(schemes),n,threshold,l,r) + statistics.elapsedtime(schemes),n,threshold,concat(l," "),concat(l," ")) else return nil end @@ -24624,7 +25231,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4207, stripped down to: 3041 +-- original size: 4227, stripped down to: 3049 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -24634,8 +25241,7 @@ if not modules then modules={} end modules ['data-lua']={ license="see context related readme files" } local package,lpeg=package,lpeg -local gsub=string.gsub -local concat=table.concat +local loadfile=loadfile local addsuffix=file.addsuffix local P,S,Cs,lpegmatch=lpeg.P,lpeg.S,lpeg.Cs,lpeg.match local luasuffixes={ 'tex','lua' } @@ -24646,6 +25252,8 @@ local helpers=package.helpers or {} local methods=helpers.methods or {} local resolvers=resolvers local resolveprefix=resolvers.resolve +local expandedpaths=resolvers.expandedpathlistfromvariable +local findfile=resolvers.findfile helpers.report=logs.reporter("resolvers","libraries") trackers.register("resolvers.libraries",function(v) helpers.trace=v end) trackers.register("resolvers.locating",function(v) helpers.trace=v end) @@ -24674,7 +25282,7 @@ local function getluaformatpaths() if not luaformatpaths then luaformatpaths={} for i=1,#luaformats do - registerpath("lua format","lua",luaformatpaths,resolvers.expandedpathlistfromvariable(luaformats[i])) + registerpath("lua format","lua",luaformatpaths,expandedpaths(luaformats[i])) end end return luaformatpaths @@ -24683,7 +25291,7 @@ local function getlibformatpaths() if not libformatpaths then libformatpaths={} for i=1,#libformats do - registerpath("lib format","lib",libformatpaths,resolvers.expandedpathlistfromvariable(libformats[i])) + registerpath("lib format","lib",libformatpaths,expandedpaths(libformats[i])) end end return libformatpaths @@ -24693,7 +25301,7 @@ local function loadedbyformat(name,rawname,suffixes,islib,what) local report=helpers.report for i=1,#suffixes do local format=suffixes[i] - local resolved=resolvers.findfile(name,format) or "" + local resolved=findfile(name,format) or "" if trace then report("%s format, identifying %a using format %a",what,name,format) end @@ -24731,7 +25339,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-aux"] = package.loaded["data-aux"] or true --- original size: 2452, stripped down to: 1877 +-- original size: 2610, stripped down to: 2019 if not modules then modules={} end modules ['data-aux']={ version=1.001, @@ -24742,24 +25350,28 @@ if not modules then modules={} end modules ['data-aux']={ } local find=string.find local type,next=type,next +local addsuffix,removesuffix=file.addsuffix,file.removesuffix +local loaddata,savedata=io.loaddata,io.savedata local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local resolvers=resolvers +local cleanpath=resolvers.cleanpath +local findfiles=resolvers.findfiles local report_scripts=logs.reporter("resolvers","scripts") function resolvers.updatescript(oldname,newname) local scriptpath="context/lua" - newname=file.addsuffix(newname,"lua") - local oldscript=resolvers.cleanpath(oldname) + local oldscript=cleanpath(oldname) + local newname=addsuffix(newname,"lua") + local newscripts=findfiles(newname) or {} if trace_locating then report_scripts("to be replaced old script %a",oldscript) end - local newscripts=resolvers.findfiles(newname) or {} if #newscripts==0 then if trace_locating then report_scripts("unable to locate new script") end else for i=1,#newscripts do - local newscript=resolvers.cleanpath(newscripts[i]) + local newscript=cleanpath(newscripts[i]) if trace_locating then report_scripts("checking new script %a",newscript) end @@ -24771,17 +25383,17 @@ function resolvers.updatescript(oldname,newname) if trace_locating then report_scripts("new script should come from %a",scriptpath) end - elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then + elseif not (find(oldscript,removesuffix(newname).."$") or find(oldscript,newname.."$")) then if trace_locating then report_scripts("invalid new script name") end else - local newdata=io.loaddata(newscript) + local newdata=loaddata(newscript) if newdata then if trace_locating then report_scripts("old script content replaced by new content: %s",oldscript) end - io.savedata(oldscript,newdata) + savedata(oldscript,newdata) break elseif trace_locating then report_scripts("unable to load new script") @@ -24854,7 +25466,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 1823, stripped down to: 1542 +-- original size: 2038, stripped down to: 1696 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -24864,15 +25476,22 @@ if not modules then modules={} end modules ['data-lst']={ license="see context related readme files" } local type=type -local concat,sortedhash=table.concat,table.sortedhash +local sortedhash=table.sortedhash +local isdir=lfs.isdir local resolvers=resolvers local listers=resolvers.listers or {} resolvers.listers=listers local resolveprefix=resolvers.resolve +local configurationfiles=resolvers.configurationfiles +local expandedpathfromlist=resolvers.expandedpathfromlist +local splitpath=resolvers.splitpath +local knownvariables=resolvers.knownvariables local report_lists=logs.reporter("resolvers","lists") local report_resolved=logs.reporter("system","resolved") local function tabstr(str) - if type(str)=='table' then + if not str then + return "unset" + elseif type(str)=='table' then return concat(str," | ") else return str @@ -24882,22 +25501,22 @@ function listers.variables(pattern) local result=resolvers.knownvariables(pattern) for key,value in sortedhash(result) do report_lists(key) - report_lists(" env: %s",tabstr(value.environment or "unset")) - report_lists(" var: %s",tabstr(value.variable or "unset")) - report_lists(" exp: %s",tabstr(value.expansion or "unset")) - report_lists(" res: %s",tabstr(value.resolved or "unset")) + report_lists(" env: %s",tabstr(value.environment)) + report_lists(" var: %s",tabstr(value.variable)) + report_lists(" exp: %s",tabstr(value.expansion)) + report_lists(" res: %s",tabstr(value.resolved)) end end function listers.configurations() - local configurations=resolvers.configurationfiles() + local configurations=configurationfiles() for i=1,#configurations do report_resolved("file : %s",resolveprefix(configurations[i])) end report_resolved("") - local list=resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) + local list=expandedpathfromlist(splitpath(resolvers.luacnfspec)) for i=1,#list do local li=resolveprefix(list[i]) - if lfs.isdir(li) then + if isdir(li) then report_resolved("path - %s",li) else report_resolved("path + %s",li) @@ -24910,318 +25529,147 @@ end -- of closure do -- create closure to overcome 200 locals limit -package.loaded["util-lib"] = package.loaded["util-lib"] or true +package.loaded["libs-ini"] = package.loaded["libs-ini"] or true --- original size: 16094, stripped down to: 8443 +-- original size: 5822, stripped down to: 3629 -if not modules then modules={} end modules ['util-lib']={ +if not modules then modules={} end modules ['libs-ini']={ version=1.001, comment="companion to luat-lib.mkiv", author="Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright="PRAGMA ADE / ConTeXt Development Team", - license="see context related readme files", + license="see context related readme files" } +local type,unpack=type,unpack local type=type -local next=next -local pcall=pcall -local gsub=string.gsub -local find=string.find -local sort=table.sort -local pathpart=file.pathpart local nameonly=file.nameonly local joinfile=file.join -local removesuffix=file.removesuffix local addsuffix=file.addsuffix -local findfile=resolvers.findfile -local findfiles=resolvers.findfiles -local expandpaths=resolvers.expandedpathlistfromvariable local qualifiedpath=file.is_qualified_path local isfile=lfs.isfile -local done=false -local function locate(required,version,trace,report,action) - if type(required)~="string" then - report("provide a proper library name") - return - end - if trace then - report("requiring library %a with version %a",required,version or "any") - end - local found_library=nil - local required_full=gsub(required,"%.","/") - local required_path=pathpart(required_full) - local required_base=nameonly(required_full) - if qualifiedpath(required) then - if isfile(addsuffix(required,os.libsuffix)) then - if trace then - report("qualified name %a found",required) +local findfile=resolvers.findfile +local expandpaths=resolvers.expandedpathlistfromvariable +local report=logs.reporter("resolvers","libraries") +local trace=false +trackers.register("resolvers.lib",function(v) trace=v end) +local function findlib(required) + local suffix=os.libsuffix or "so" + if not qualifiedpath(required) then + local list=directives.value("system.librarynames" ) + local only=nameonly(required) + if type(list)=="table" then + list=list[only] + if type(list)~="table" then + list={ only } end - found_library=required else - if trace then - report("qualified name %a not found",required) - end + list={ only } end - else - local required_name=required_base.."."..os.libsuffix - local version=type(version)=="string" and version~="" and version or false - local engine="luatex" - if trace and not done then - local list=expandpaths("lib") - for i=1,#list do - report("tds path %i: %s",i,list[i]) - end - end - local function found(locate,asked_library,how,...) - if trace then - report("checking %s: %a",how,asked_library) - end - return locate(asked_library,...) + if trace then + report("using lookup list for library %a: % | t",only,list) end - local function check(locate,...) - local found=nil - if version then - local asked_library=joinfile(required_path,version,required_name) - if trace then - report("checking %s: %a","with version",asked_library) - end - found=locate(asked_library,...) + for i=1,#list do + local name=list[i] + local found=findfile(name,"lib") + if not found then + found=findfile(addsuffix(name,suffix),"lib") end - if not found or found=="" then - local asked_library=joinfile(required_path,required_name) + if found then if trace then - report("checking %s: %a","with version",asked_library) + report("library %a resolved via %a path to %a",name,"tds lib",found) end - found=locate(asked_library,...) - end - return found and found~="" and found or false - end - local function attempt(checkpattern) - if trace then - report("checking tds lib paths strictly") - end - local found=findfile and check(findfile,"lib") - if found and (not checkpattern or find(found,checkpattern)) then return found end - if trace then - report("checking tds lib paths with wildcard") - end - local asked_library=joinfile(required_path,".*",required_name) - if trace then - report("checking %s: %a","latest version",asked_library) - end - local list=findfiles(asked_library,"lib",true) - if list and #list>0 then - sort(list) - local found=list[#list] - if found and (not checkpattern or find(found,checkpattern)) then - return found - end - end - if trace then - report("checking lib paths") - end - package.extralibpath(environment.ownpath) - local paths=package.libpaths() - local pattern="/[^/]+%."..os.libsuffix.."$" - for i=1,#paths do - required_path=gsub(paths[i],pattern,"") - local found=check(lfs.isfound) - if type(found)=="string" and (not checkpattern or find(found,checkpattern)) then - return found - end - end - return false end - if engine then - if trace then - report("attemp 1, engine %a",engine) - end - found_library=attempt("/"..engine.."/") - if not found_library then - if trace then - report("attemp 2, no engine",asked_library) + if expandpaths then + local list=expandpaths("PATH") + local base=addsuffix(only,suffix) + for i=1,#list do + local full=joinfile(list[i],base) + local found=isfile(full) and full + if found then + if trace then + report("library %a resolved via %a path to %a",name,"system",found) + end + return found end - found_library=attempt() end - else - found_library=attempt() end - end - if not found_library then + elseif isfile(addsuffix(required,suffix)) then if trace then - report("not found: %a",required) + report("library with qualified name %a %sfound",required,"") end - library=false + return required else if trace then - report("found: %a",found_library) - end - local result,message=action(found_library,required_base) - if result then - library=result - else - library=false - report("load error: message %a, library %a",tostring(message or "unknown"),found_library or "no library") + report("library with qualified name %a %sfound",required,"not ") end end - if trace then - if not library then - report("unknown library: %a",required) - else - report("stored library: %a",required) - end - end - return library or nil + return false end -do - local report_swiglib=logs.reporter("swiglib") - local trace_swiglib=false - local savedrequire=require - local loadedlibs={} - local loadlib=package.loadlib - local pushdir=dir.push - local popdir=dir.pop - trackers.register("resolvers.swiglib",function(v) trace_swiglib=v end) - function requireswiglib(required,version) - local library=loadedlibs[library] - if library==nil then - local trace_swiglib=trace_swiglib or package.helpers.trace - library=locate(required,version,trace_swiglib,report_swiglib,function(name,base) - pushdir(pathpart(name)) - local opener="luaopen_"..base - if trace_swiglib then - report_swiglib("opening: %a with %a",name,opener) - end - local library,message=loadlib(name,opener) - local libtype=type(library) - if libtype=="function" then - library=library() +local foundlibraries=table.setmetatableindex(function(t,k) + local v=findlib(k) + t[k]=v + return v +end) +function resolvers.findlib(required) + return foundlibraries[required] +end +local libraries={} +resolvers.libraries=libraries +local report=logs.reporter("optional") +if optional then optional.loaded={} end +function libraries.validoptional(name) + local thelib=optional and optional[name] + if not thelib then + elseif thelib.initialize then + return thelib + else + report("invalid optional library %a",libname) + end +end +function libraries.optionalloaded(name,libnames) + local thelib=optional and optional[name] + if not thelib then + report("no optional %a library found",name) + else + local thelib_initialize=thelib.initialize + if not thelib_initialize then + report("invalid optional library %a",name) + else + if type(libnames)=="string" then + libnames={ libnames } + end + if type(libnames)=="table" then + for i=1,#libnames do + local libname=libnames[i] + local filename=foundlibraries[libname] + if filename then + libnames[i]=filename + else + report("unable to locate library %a",libname) + return + end + end + local initialized=thelib_initialize(unpack(libnames)) + if initialized then + report("using library '% + t'",libnames) else - report_swiglib("load error: %a returns %a, message %a, library %a",opener,libtype,(string.gsub(message or "no message","[%s]+$","")),found_library or "no library") - library=false + report("unable to initialize library '% + t'",libnames) end - popdir() - return library - end) - loadedlibs[required]=library or false - end - return library - end - function require(name,version) - if find(name,"^swiglib%.") then - return requireswiglib(name,version) - else - return savedrequire(name) - end - end - local swiglibs={} - local initializer="core" - function swiglib(name,version) - local library=swiglibs[name] - if not library then - statistics.starttiming(swiglibs) - if trace_swiglib then - report_swiglib("loading %a",name) - end - if not find(name,"%."..initializer.."$") then - fullname="swiglib."..name.."."..initializer - else - fullname="swiglib."..name + return initialized end - library=requireswiglib(fullname,version) - swiglibs[name]=library - statistics.stoptiming(swiglibs) end - return library end - statistics.register("used swiglibs",function() - if next(swiglibs) then - return string.format("%s, initial load time %s seconds",table.concat(table.sortedkeys(swiglibs)," "),statistics.elapsedtime(swiglibs)) - end - end) end if FFISUPPORTED and ffi and ffi.load then - local report_ffilib=logs.reporter("ffilib") - local trace_ffilib=false - local savedffiload=ffi.load - trackers.register("resolvers.ffilib",function(v) trace_ffilib=v end) - local loaded={} - local function locateindeed(name) - name=removesuffix(name) - local l=loaded[name] - if l==nil then - local state,library=pcall(savedffiload,name) - if type(library)=="userdata" then - l=library - elseif type(state)=="userdata" then - l=state - else - l=false - end - loaded[name]=l - elseif trace_ffilib then - report_ffilib("reusing already loaded %a",name) - end - return l - end - local function getlist(required) - local list=directives.value("system.librarynames" ) - if type(list)=="table" then - list=list[required] - if type(list)=="table" then - if trace then - report("using lookup list for library %a: % | t",required,list) - end - return list - end - end - return { required } - end - function ffilib(name,version) - name=removesuffix(name) - local l=loaded[name] - if l~=nil then - if trace_ffilib then - report_ffilib("reusing already loaded %a",name) - end - return l - end - local list=getlist(name) - if version=="system" then - for i=1,#list do - local library=locateindeed(list[i]) - if type(library)=="userdata" then - return library - end - end - else - for i=1,#list do - local library=locate(list[i],version,trace_ffilib,report_ffilib,locateindeed) - if type(library)=="userdata" then - return library - end - end - end - end + local ffiload=ffi.load function ffi.load(name) - local list=getlist(name) - for i=1,#list do - local library=ffilib(list[i]) - if type(library)=="userdata" then - return library - end - end - if trace_ffilib then - report_ffilib("trying to load %a using normal loader",name) - end - for i=1,#list do - local state,library=pcall(savedffiload,list[i]) - if type(library)=="userdata" then - return library - elseif type(state)=="userdata" then - return library - end + local full=name and foundlibraries[name] + if full then + return ffiload(full) + else + return ffiload(name) end end end @@ -25336,7 +25784,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 9637, stripped down to: 7253 +-- original size: 13964, stripped down to: 10026 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -25350,16 +25798,14 @@ local concat=table.concat local quoted=string.quoted local luasuffixes=utilities.lua.suffixes local report_format=logs.reporter("resolvers","formats") -local function primaryflags() - local arguments=environment.arguments +local function primaryflags(arguments) local flags={} if arguments.silent then flags[#flags+1]="--interaction=batchmode" end return concat(flags," ") end -local function secondaryflags() - local arguments=environment.arguments +local function secondaryflags(arguments) local trackers=arguments.trackers local directives=arguments.directives local flags={} @@ -25381,6 +25827,9 @@ local function secondaryflags() if arguments.ansi then flags[#flags+1]="--c:ansi" end + if arguments.ansilog then + flags[#flags+1]="--c:ansilog" + end if arguments.strip then flags[#flags+1]="--c:strip" end @@ -25391,12 +25840,13 @@ local function secondaryflags() end local template=[[--ini %primaryflags% --lua=%luafile% %texfile% %secondaryflags% %dump% %redirect%]] local checkers={ - primaryflags="string", - secondaryflags="string", + primaryflags="verbose", + secondaryflags="verbose", luafile="readable", texfile="readable", redirect="string", dump="string", + binarypath="string", } local runners={ luatex=sandbox.registerrunner { @@ -25421,46 +25871,88 @@ local runners={ reporter=report_format, }, } -function environment.make_format(name,arguments) +local function validbinarypath() + if not environment.arguments.nobinarypath then + local path=environment.ownpath or file.dirname(environment.ownname) + if path and path~="" then + path=dir.expandname(path) + if path~="" and lfs.isdir(path) then + return path + end + end + end +end +function environment.make_format(formatname) + local arguments=environment.arguments local engine=environment.ownmain or "luatex" - local silent=environment.arguments.silent - local errors=environment.arguments.errors - local olddir=dir.current() - local path=caches.getwritablepath("formats",engine) or "" - if path~="" then - lfs.chdir(path) - end - report_format("using format path %a",dir.current()) - local texsourcename=file.addsuffix(name,"mkiv") - local fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + local silent=arguments.silent + local errors=arguments.errors + local texsourcename="" + local texsourcepath="" + local fulltexsourcename="" + if engine=="luametatex" then + texsourcename=file.addsuffix(formatname,"mkxl") + fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + end + if fulltexsourcename=="" then + texsourcename=file.addsuffix(formatname,"mkiv") + fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + end if fulltexsourcename=="" then - texsourcename=file.addsuffix(name,"tex") + texsourcename=file.addsuffix(formatname,"tex") fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" end if fulltexsourcename=="" then - report_format("no tex source file with name %a (mkiv or tex)",name) - lfs.chdir(olddir) + report_format("no tex source file with name %a (mkiv or tex)",formatname) return - else - report_format("using tex source file %a",fulltexsourcename) end - local texsourcepath=dir.expandname(file.dirname(fulltexsourcename)) - local specificationname=file.replacesuffix(fulltexsourcename,"lus") - local fullspecificationname=resolvers.findfile(specificationname,"tex") or "" - if fullspecificationname=="" then - specificationname=file.join(texsourcepath,"context.lus") - fullspecificationname=resolvers.findfile(specificationname,"tex") or "" + report_format("using tex source file %a",fulltexsourcename) + fulltexsourcename=dir.expandname(fulltexsourcename) + texsourcepath=file.dirname(fulltexsourcename) + if not lfs.isfile(fulltexsourcename) then + report_format("no accessible tex source file with name %a",fulltexsourcename) + return end + local specificationname="context.lus" + local specificationpath="" + local fullspecificationname=resolvers.findfile(specificationname) or "" if fullspecificationname=="" then - report_format("unknown stub specification %a",specificationname) - lfs.chdir(olddir) + report_format("unable to locate specification file %a",specificationname) + return + end + report_format("using specification file %a",fullspecificationname) + fullspecificationname=dir.expandname(fullspecificationname) + specificationpath=file.dirname(fullspecificationname) + if texsourcepath~=specificationpath then + report_format("tex source file and specification file are on different paths") + return + end + if not lfs.isfile(fulltexsourcename) then + report_format("no accessible tex source file with name %a",fulltexsourcename) + return + end + if not lfs.isfile(fullspecificationname) then + report_format("no accessible specification file with name %a",fulltexsourcename) + return + end + report_format("using tex source path %a",texsourcepath) + local validformatpath=caches.getwritablepath("formats",engine) or "" + local startupdir=dir.current() + if validformatpath=="" then + report_format("invalid format path, insufficient write access") + return + end + local binarypath=validbinarypath() + report_format("changing to format path %a",validformatpath) + lfs.chdir(validformatpath) + if dir.current()~=validformatpath then + report_format("unable to change to format path %a",validformatpath) return end - local specificationpath=file.dirname(fullspecificationname) local usedluastub=nil local usedlualibs=dofile(fullspecificationname) if type(usedlualibs)=="string" then - usedluastub=file.join(file.dirname(fullspecificationname),usedlualibs) + usedluastub=file.join(specificationpath,usedlualibs) elseif type(usedlualibs)=="table" then report_format("using stub specification %a",fullspecificationname) local texbasename=file.basename(name) @@ -25477,48 +25969,57 @@ function environment.make_format(name,arguments) end else report_format("invalid stub specification %a",fullspecificationname) - lfs.chdir(olddir) + lfs.chdir(startupdir) return end + local runner=runners[engine] + if not runner then + report_format("the format %a cannot be generated, no runner available for engine %a",name,engine) + lfs.chdir(startupdir) + return + end + local primaryflags=primaryflags(arguments) + local secondaryflags=secondaryflags(arguments) local specification={ - primaryflags=primaryflags(), - secondaryflags=secondaryflags(), + binarypath=binarypath, + primaryflags=primaryflags, + secondaryflags=secondaryflags, luafile=quoted(usedluastub), texfile=quoted(fulltexsourcename), dump=os.platform=="unix" and "\\\\dump" or "\\dump", } - local runner=runners[engine] - if not runner then - report_format("format %a cannot be generated, no runner available for engine %a",name,engine) - elseif silent then - statistics.starttiming() + if silent then specification.redirect="> temp.log" - local result=runner(specification) - local runtime=statistics.stoptiming() - if result~=0 then - print(format("%s silent make > fatal error when making format %q",engine,name)) - else - print(format("%s silent make > format %q made in %.3f seconds",engine,name,runtime)) - end + end + statistics.starttiming() + local result=runner(specification) + local runtime=statistics.stoptiming() + if silent then os.remove("temp.log") - else - runner(specification) end - local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" - local mp=dir.glob(pattern) - if mp then - for i=1,#mp do - local name=mp[i] - report_format("removing related mplib format %a",file.basename(name)) - os.remove(name) + report_format() + if binarypath and binarypath~="" then + report_format("binary path : %s",binarypath or "?") end - end - lfs.chdir(olddir) + report_format("format path : %s",validformatpath) + report_format("luatex engine : %s",engine) + report_format("lua startup file : %s",usedluastub) + if primaryflags~="" then + report_format("primary flags : %s",primaryflags) + end + if secondaryflags~="" then + report_format("secondary flags : %s",secondaryflags) + end + report_format("context file : %s",fulltexsourcename) + report_format("run time : %.3f seconds",runtime) + report_format("return value : %s",result==0 and "okay" or "error") + report_format() + lfs.chdir(startupdir) end -local template=[[%flags% --fmt=%fmtfile% --lua=%luafile% %texfile% %more%]] +local template=[[%primaryflags% --fmt=%fmtfile% --lua=%luafile% %texfile% %secondaryflags%]] local checkers={ - flags="string", - more="string", + primaryflags="verbose", + secondaryflags="verbose", fmtfile="readable", luafile="readable", texfile="readable", @@ -25531,6 +26032,13 @@ local runners={ checkers=checkers, reporter=report_format, }, + luametatex=sandbox.registerrunner { + name="run luametatex format", + program="luametatex", + template=template, + checkers=checkers, + reporter=report_format, + }, luajittex=sandbox.registerrunner { name="run luajittex format", program="luajittex", @@ -25539,51 +26047,78 @@ local runners={ reporter=report_format, }, } -function environment.run_format(name,data,more) - if name and name~="" then - local engine=environment.ownmain or "luatex" - local barename=file.removesuffix(name) - local fmtname=caches.getfirstreadablefile(file.addsuffix(barename,"fmt"),"formats",engine) - if fmtname=="" then - fmtname=resolvers.findfile(file.addsuffix(barename,"fmt")) or "" - end - fmtname=resolvers.cleanpath(fmtname) - if fmtname=="" then - report_format("no format with name %a",name) - else - local barename=file.removesuffix(name) - local luaname=file.addsuffix(barename,"luc") - if not lfs.isfile(luaname) then - luaname=file.addsuffix(barename,"lua") - end - if not lfs.isfile(luaname) then - report_format("using format name %a",fmtname) - report_format("no luc/lua file with name %a",barename) - else - local runner=runners[engine] - if not runner then - report_format("format %a cannot be run, no runner available for engine %a",name,engine) - else - runner { - flags=primaryflags(), - fmtfile=quoted(barename), - luafile=quoted(luaname), - texfile=quoted(data), - more=more, - } - end - end - end +function environment.run_format(formatname,scriptname,filename,primaryflags,secondaryflags,verbose) + local engine=environment.ownmain or "luatex" + if not formatname or formatname=="" then + report_format("missing format name") + return + end + if not scriptname or scriptname=="" then + report_format("missing script name") + return + end + if not lfs.isfile(formatname) or not lfs.isfile(scriptname) then + formatname,scriptname=resolvers.locateformat(formatname) end + if not formatname or formatname=="" then + report_format("invalid format name") + return + end + if not scriptname or scriptname=="" then + report_format("invalid script name") + return + end + local runner=runners[engine] + if not runner then + report_format("format %a cannot be run, no runner available for engine %a",file.nameonly(name),engine) + return + end + if not filename then + filename "" + end + local binarypath=validbinarypath() + local specification={ + binarypath=binarypath, + primaryflags=primaryflags or "", + secondaryflags=secondaryflags or "", + fmtfile=quoted(formatname), + luafile=quoted(scriptname), + texfile=filename~="" and quoted(filename) or "", + } + statistics.starttiming() + local result=runner(specification) + local runtime=statistics.stoptiming() + if verbose then + report_format() + if binarypath and binarypath~="" then + report_format("binary path : %s",binarypath) + end + report_format("luatex engine : %s",engine) + report_format("lua startup file : %s",scriptname) + report_format("tex format file : %s",formatname) + if filename~="" then + report_format("tex input file : %s",filename) + end + if primaryflags~="" then + report_format("primary flags : %s",primaryflags) + end + if secondaryflags~="" then + report_format("secondary flags : %s",secondaryflags) + end + report_format("run time : %.3f seconds",runtime) + report_format("return value : %s",result==0 and "okay" or "error") + report_format() + end + return result end end -- of closure --- used libraries : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua +-- used libraries : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua libs-ini.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 1019480 --- stripped bytes : 403728 +-- original bytes : 1038245 +-- stripped bytes : 409841 -- end library merge @@ -25697,7 +26232,7 @@ local ownlibs = { -- order can be made better 'data-tmf.lua', 'data-lst.lua', - 'util-lib.lua', -- swiglib + 'libs-ini.lua', 'luat-sta.lua', 'luat-fmt.lua', @@ -25846,7 +26381,7 @@ local helpinfo = [[ <category name="basic"> <subcategory> <flag name="script"><short>run an mtx script (lua prefered method) (<ref name="noquotes"/>), no script gives list</short></flag> - <flag name="evaluate"><short>run code passed on the commandline (between quotes)</short></flag> + <flag name="evaluate"><short>run code passed on the commandline (between quotes) (=loop) (exit|quit aborts)</short></flag> <flag name="execute"><short>run a script or program (texmfstart method) (<ref name="noquotes"/>)</short></flag> <flag name="resolve"><short>resolve prefixed arguments</short></flag> <flag name="ctxlua"><short>run internally (using preloaded libs)</short></flag> @@ -25865,6 +26400,7 @@ local helpinfo = [[ <flag name="stubpath" value="binpath"><short>paths where stubs wil be written</short></flag> <flag name="windows"><short>create windows (mswin) stubs</short></flag> <flag name="unix"><short>create unix (linux) stubs</short></flag> + <flag name="addbinarypath"><short>prepend the (found) binarypath to runners</short></flag> </subcategory> <subcategory> <flag name="verbose"><short>give a bit more info</short></flag> @@ -26086,8 +26622,7 @@ end report() io.flush() end - -- no os.exec because otherwise we get the wrong return value - local code = os.execute(command) -- maybe spawn + local code = os.execute(command) if code == 0 then return true else @@ -26130,7 +26665,7 @@ function runners.execute_program(fullname) report() report() io.flush() - local code = os.exec(command) -- (fullname,unpack(after)) does not work / maybe spawn + local code = os.execute(command) return code == 0 end end @@ -26517,16 +27052,22 @@ function runners.associate(filename) end function runners.evaluate(code,filename) -- for Luigi + local environment = table.setmetatableindex(_G) if code == "loop" then while true do - io.write("> ") + io.write("lua > ") local code = io.read() - if code ~= "" then + if code == "quit" or code == "exit" then + break + elseif code ~= "" then local temp = string.match(code,"^= (.*)$") if temp then - code = "print("..temp..")" + code = "inspect("..temp..")" + end + local compiled, message = load(code,"console","t",environment) + if type(compiled) ~= "function" then + compiled = load("inspect("..code..")","console","t",environment) end - local compiled, message = loadstring(code) if type(compiled) ~= "function" then io.write("! " .. (message or code).."\n") else @@ -26539,7 +27080,10 @@ function runners.evaluate(code,filename) -- for Luigi code = filename end if code ~= "" then - local compiled, message = loadstring(code) + local compiled, message = load(code,"console","t",environment) + if type(compiled) ~= "function" then + compiled = load("inspect("..code..")","console","t",environment) + end if type(compiled) ~= "function" then io.write("invalid lua code: " .. (message or code)) return @@ -26716,17 +27260,17 @@ do end -if e_argument("ansi") then +-- if e_argument("ansi") or e_argument("ansilog") then - logs.setformatters("ansi") +-- logs.setformatters(e_argument("ansi") and "ansi" or "ansilog") - local script = e_argument("script") or e_argument("scripts") +-- -- local script = e_argument("script") or e_argument("scripts") +-- -- +-- -- if type(script) == "string" then +-- -- logs.writer("]0;"..script.."") -- for Alan to test +-- -- end - if type(script) == "string" then - logs.writer("]0;"..script.."") -- for Alan to test - end - -end +-- end if e_argument("script") or e_argument("scripts") then @@ -26975,11 +27519,11 @@ elseif e_argument("format-path") then resolvers.load() report(caches.getwritablepath("format")) -elseif e_argument("pattern") then - - -- luatools - - runners.execute_ctx_script("mtx-base","--pattern='" .. e_argument("pattern") .. "'",filename) +-- elseif e_argument("pattern") then +-- +-- -- luatools +-- +-- runners.execute_ctx_script("mtx-base","--pattern='" .. e_argument("pattern") .. "'",filename) elseif e_argument("generate") then diff --git a/Master/texmf-dist/scripts/context/stubs/unix/mtxrun b/Master/texmf-dist/scripts/context/stubs/unix/mtxrun index 98fe7f752e3..5907fa33c66 100755 --- a/Master/texmf-dist/scripts/context/stubs/unix/mtxrun +++ b/Master/texmf-dist/scripts/context/stubs/unix/mtxrun @@ -194,7 +194,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 6330, stripped down to: 2831 +-- original size: 6529, stripped down to: 2933 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -313,6 +313,9 @@ elseif not ffi.number then end if LUAVERSION>5.3 then end +if status and os.setenv then + os.setenv("engine",string.lower(status.luatex_engine or "unknown")) +end end -- of closure @@ -321,7 +324,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-macro"] = package.loaded["l-macro"] or true --- original size: 10131, stripped down to: 5991 +-- original size: 10130, stripped down to: 5990 if not modules then modules={} end modules ['l-macros']={ version=1.001, @@ -546,7 +549,7 @@ end macros.loaded=loaded function required(name,trace) local filename=file.addsuffix(name,"lua") - local fullname=resolvers and resolvers.find_file(filename) or filename + local fullname=resolvers and resolvers.findfile(filename) or filename if not fullname or fullname=="" then return false end @@ -576,7 +579,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-sandbox"] = package.loaded["l-sandbox"] or true --- original size: 9747, stripped down to: 6313 +-- original size: 9604, stripped down to: 6394 if not modules then modules={} end modules ['l-sandbox']={ version=1.001, @@ -808,6 +811,9 @@ local function supported(library) return l end loadfile=register(loadfile,"loadfile") +if supported("lua") then + lua.openfile=register(lua.openfile,"lua.openfile") +end if supported("io") then io.open=register(io.open,"io.open") io.popen=register(io.popen,"io.popen") @@ -1168,7 +1174,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true --- original size: 38434, stripped down to: 19310 +-- original size: 38440, stripped down to: 19316 if not modules then modules={} end modules ['l-lpeg']={ version=1.001, @@ -1337,7 +1343,7 @@ patterns.propername=(uppercase+lowercase+underscore)*(uppercase+lowercase+unders patterns.somecontent=(anything-newline-space)^1 patterns.beginline=#(1-newline) patterns.longtostring=Cs(whitespace^0/""*((patterns.quoted+nonwhitespace^1+whitespace^1/""*(endofstring+Cc(" ")))^0)) -function anywhere(pattern) +local function anywhere(pattern) return (1-P(pattern))^0*P(pattern) end lpeg.anywhere=anywhere @@ -1967,7 +1973,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-string"] = package.loaded["l-string"] or true --- original size: 6461, stripped down to: 3255 +-- original size: 6644, stripped down to: 3410 if not modules then modules={} end modules ['l-string']={ version=1.001, @@ -2031,9 +2037,11 @@ function string.is_empty(str) end end local anything=patterns.anything -local allescapes=Cc("%")*S(".-+%?()[]*") -local someescapes=Cc("%")*S(".-+%()[]") -local matchescapes=Cc(".")*S("*?") +local moreescapes=Cc("%")*S(".-+%?()[]*$^{}") +local allescapes=Cc("%")*S(".-+%?()[]*") +local someescapes=Cc("%")*S(".-+%()[]") +local matchescapes=Cc(".")*S("*?") +local pattern_m=Cs ((moreescapes+anything )^0 ) local pattern_a=Cs ((allescapes+anything )^0 ) local pattern_b=Cs ((someescapes+matchescapes+anything )^0 ) local pattern_c=Cs (Cc("^")*(someescapes+matchescapes+anything )^0*Cc("$") ) @@ -2043,6 +2051,8 @@ end function string.topattern(str,lowercase,strict) if str=="" or type(str)~="string" then return ".*" + elseif strict=="all" then + str=lpegmatch(pattern_m,str) elseif strict then str=lpegmatch(pattern_c,str) else @@ -2092,7 +2102,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-table"] = package.loaded["l-table"] or true --- original size: 41332, stripped down to: 21508 +-- original size: 41758, stripped down to: 22643 if not modules then modules={} end modules ['l-table']={ version=1.001, @@ -2101,7 +2111,7 @@ if not modules then modules={} end modules ['l-table']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local type,next,tostring,tonumber,select=type,next,tostring,tonumber,select +local type,next,tostring,tonumber,select,rawget=type,next,tostring,tonumber,select,rawget local table,string=table,string local concat,sort=table.concat,table.sort local format,lower,dump=string.format,string.lower,string.dump @@ -2435,13 +2445,13 @@ function table.fromhash(t) end return hsh end -local noquotes,hexify,handle,compact,inline,functions,metacheck +local noquotes,hexify,handle,compact,inline,functions,metacheck,accurate local reserved=table.tohash { 'and','break','do','else','elseif','end','false','for','function','if', 'in','local','nil','not','or','repeat','return','then','true','until','while', - 'NaN','goto', + 'NaN','goto','const', } -local function is_simple_table(t,hexify) +local function is_simple_table(t,hexify,accurate) local nt=#t if nt>0 then local n=0 @@ -2460,6 +2470,8 @@ local function is_simple_table(t,hexify) if tv=="number" then if hexify then tt[i]=format("0x%X",v) + elseif accurate then + tt[i]=format("%q",v) else tt[i]=v end @@ -2480,6 +2492,8 @@ local function is_simple_table(t,hexify) if tv=="number" then if hexify then tt[i+1]=format("0x%X",v) + elseif accurate then + tt[i+1]=format("%q",v) else tt[i+1]=v end @@ -2551,6 +2565,8 @@ local function do_serialize(root,name,depth,level,indexed) if tv=="number" then if hexify then handle(format("%s 0x%X,",depth,v)) + elseif accurate then + handle(format("%s %q,",depth,v)) else handle(format("%s %s,",depth,v)) end @@ -2560,7 +2576,7 @@ local function do_serialize(root,name,depth,level,indexed) if next(v)==nil then handle(format("%s {},",depth)) elseif inline then - local st=is_simple_table(v,hexify) + local st=is_simple_table(v,hexify,accurate) if st then handle(format("%s { %s },",depth,concat(st,", "))) else @@ -2588,12 +2604,16 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]=0x%X,",depth,k,v)) + elseif accurate then + handle(format("%s [%s]=%q,",depth,k,v)) else handle(format("%s [%s]=%s,",depth,k,v)) end elseif tk=="boolean" then if hexify then handle(format("%s [%s]=0x%X,",depth,k and "true" or "false",v)) + elseif accurate then + handle(format("%s [%s]=%q,",depth,k and "true" or "false",v)) else handle(format("%s [%s]=%s,",depth,k and "true" or "false",v)) end @@ -2601,12 +2621,16 @@ local function do_serialize(root,name,depth,level,indexed) elseif noquotes and not reserved[k] and lpegmatch(propername,k) then if hexify then handle(format("%s %s=0x%X,",depth,k,v)) + elseif accurate then + handle(format("%s %s=%q,",depth,k,v)) else handle(format("%s %s=%s,",depth,k,v)) end else if hexify then handle(format("%s [%q]=0x%X,",depth,k,v)) + elseif accurate then + handle(format("%s [%q]=%q,",depth,k,v)) else handle(format("%s [%q]=%s,",depth,k,v)) end @@ -2615,6 +2639,8 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]=%q,",depth,k,v)) + elseif accurate then + handle(format("%s [%q]=%q,",depth,k,v)) else handle(format("%s [%s]=%q,",depth,k,v)) end @@ -2631,6 +2657,8 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]={},",depth,k)) + elseif accurate then + handle(format("%s [%q]={},",depth,k)) else handle(format("%s [%s]={},",depth,k)) end @@ -2643,11 +2671,13 @@ local function do_serialize(root,name,depth,level,indexed) handle(format("%s [%q]={},",depth,k)) end elseif inline then - local st=is_simple_table(v,hexify) + local st=is_simple_table(v,hexify,accurate) if st then if tk=="number" then if hexify then handle(format("%s [0x%X]={ %s },",depth,k,concat(st,", "))) + elseif accurate then + handle(format("%s [%q]={ %s },",depth,k,concat(st,", "))) else handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) end @@ -2669,6 +2699,8 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]=%s,",depth,k,v and "true" or "false")) + elseif accurate then + handle(format("%s [%q]=%s,",depth,k,v and "true" or "false")) else handle(format("%s [%s]=%s,",depth,k,v and "true" or "false")) end @@ -2688,6 +2720,8 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]=load(%q),",depth,k,f)) + elseif accurate then + handle(format("%s [%q]=load(%q),",depth,k,f)) else handle(format("%s [%s]=load(%q),",depth,k,f)) end @@ -2705,6 +2739,8 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]=%q,",depth,k,tostring(v))) + elseif accurate then + handle(format("%s [%q]=%q,",depth,k,tostring(v))) else handle(format("%s [%s]=%q,",depth,k,tostring(v))) end @@ -2728,6 +2764,7 @@ local function serialize(_handle,root,name,specification) if type(specification)=="table" then noquotes=specification.noquotes hexify=specification.hexify + accurate=specification.accurate handle=_handle or specification.handle or print functions=specification.functions compact=specification.compact @@ -3043,7 +3080,7 @@ end local function sequenced(t,sep,simple) if not t then return "" - elseif type(t)=="string" then + elseif type(t)~="table" then return t end local n=#t @@ -3082,7 +3119,11 @@ local function sequenced(t,sep,simple) end end end - return concat(s,sep or " | ") + if sep==true then + return "{ "..concat(s,", ").." }" + else + return concat(s,sep or " | ") + end end table.sequenced=sequenced function table.print(t,...) @@ -3213,7 +3254,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-io"] = package.loaded["l-io"] or true --- original size: 11823, stripped down to: 6325 +-- original size: 11829, stripped down to: 6331 if not modules then modules={} end modules ['l-io']={ version=1.001, @@ -3227,7 +3268,7 @@ local open,flush,write,read=io.open,io.flush,io.write,io.read local byte,find,gsub,format=string.byte,string.find,string.gsub,string.format local concat=table.concat local type=type -if string.find(os.getenv("PATH"),";",1,true) then +if string.find(os.getenv("PATH") or "",";",1,true) then io.fileseparator,io.pathseparator="\\",";" else io.fileseparator,io.pathseparator="/",":" @@ -3765,7 +3806,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-os"] = package.loaded["l-os"] or true --- original size: 18916, stripped down to: 10126 +-- original size: 19102, stripped down to: 10192 if not modules then modules={} end modules ['l-os']={ version=1.001, @@ -3779,7 +3820,7 @@ local date,time=os.date,os.time local find,format,gsub,upper,gmatch=string.find,string.format,string.gsub,string.upper,string.gmatch local concat=table.concat local random,ceil,randomseed=math.random,math.ceil,math.randomseed -local rawget,rawset,type,getmetatable,setmetatable,tonumber,tostring=rawget,rawset,type,getmetatable,setmetatable,tonumber,tostring +local type,setmetatable,tonumber,tostring=type,setmetatable,tonumber,tostring do local selfdir=os.selfdir if selfdir=="" then @@ -3930,7 +3971,8 @@ local launchers={ unix="xdg-open %s &> /dev/null &", } function os.launch(str) - execute(format(launchers[os.name] or launchers.unix,str)) + local command=format(launchers[os.name] or launchers.unix,str) + execute(command) end local gettimeofday=os.gettimeofday or os.clock os.gettimeofday=gettimeofday @@ -4185,6 +4227,12 @@ function os.validdate(year,month,day) end return year,month,day end +function os.date(fmt,...) + if not fmt then + fmt="%Y-%m-%d %H:%M" + end + return date(fmt,...) +end local osexit=os.exit local exitcode=nil function os.setexitcode(code) @@ -4207,7 +4255,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-file"] = package.loaded["l-file"] or true --- original size: 21984, stripped down to: 10148 +-- original size: 22175, stripped down to: 10302 if not modules then modules={} end modules ['l-file']={ version=1.001, @@ -4229,15 +4277,24 @@ local checkedsplit=string.checkedsplit local P,R,S,C,Cs,Cp,Cc,Ct=lpeg.P,lpeg.R,lpeg.S,lpeg.C,lpeg.Cs,lpeg.Cp,lpeg.Cc,lpeg.Ct local attributes=lfs.attributes function lfs.isdir(name) - return attributes(name,"mode")=="directory" + if name then + return attributes(name,"mode")=="directory" + end end function lfs.isfile(name) - local a=attributes(name,"mode") - return a=="file" or a=="link" or nil + if name then + local a=attributes(name,"mode") + return a=="file" or a=="link" or nil + end end function lfs.isfound(name) - local a=attributes(name,"mode") - return (a=="file" or a=="link") and name or nil + if name then + local a=attributes(name,"mode") + return (a=="file" or a=="link") and name or nil + end +end +function lfs.modification(name) + return name and attributes(name,"modification") or nil end if sandbox then sandbox.redefine(lfs.isfile,"lfs.isfile") @@ -4616,7 +4673,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-gzip"] = package.loaded["l-gzip"] or true --- original size: 1211, stripped down to: 951 +-- original size: 5115, stripped down to: 1699 if not modules then modules={} end modules ['l-gzip']={ version=1.001, @@ -4624,43 +4681,75 @@ if not modules then modules={} end modules ['l-gzip']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -if not gzip then - return -end -local suffix,suffixes=file.suffix,file.suffixes -function gzip.load(filename) - local f=io.open(filename,"rb") - if not f then - elseif suffix(filename)=="gz" then - f:close() - local g=gzip.open(filename,"rb") - if g then - local str=g:read("*all") - g:close() - return str +gzip=gzip or {} +if not zlib then + zlib=xzip +elseif not xzip then + xzip=zlib +end +if zlib then + local suffix=file.suffix + local suffixes=file.suffixes + local find=string.find + local openfile=io.open + local gzipwindow=15+16 + local gziplevel=3 + local identifier="^\x1F\x8B\x08" + local compress=zlib.compress + local decompress=zlib.decompress + function gzip.load(filename) + local f=openfile(filename,"rb") + if not f then + else + local data=f:read("*all") + f:close() + if data and data~="" then + if suffix(filename)=="gz" then + data=decompress(data,gzipwindow) + end + return data + end end - else - local str=f:read("*all") - f:close() - return str end -end -function gzip.save(filename,data) - if suffix(filename)~="gz" then - filename=filename..".gz" + function gzip.save(filename,data,level) + if suffix(filename)~="gz" then + filename=filename..".gz" + end + local f=openfile(filename,"wb") + if f then + data=compress(data or "",level or gziplevel,nil,gzipwindow) + f:write(data) + f:close() + return #data + end end - local f=io.open(filename,"wb") - if f then - local s=zlib.compress(data or "",9,nil,15+16) - f:write(s) - f:close() - return #s + function gzip.suffix(filename) + local suffix,extra=suffixes(filename) + local gzipped=extra=="gz" + return suffix,gzipped + end + function gzip.compressed(s) + return s and find(s,identifier) + end + function gzip.compress(s,level) + if s and not find(s,identifier) then + if not level then + level=gziplevel + elseif level<=0 then + return s + elseif level>9 then + level=9 + end + return compress(s,level or gziplevel,nil,gzipwindow) or s + end + end + function gzip.decompress(s) + if s and find(s,identifier) then + return decompress(s,gzipwindow) + else + return s + end end -end -function gzip.suffix(filename) - local suffix,extra=suffixes(filename) - local gzipped=extra=="gz" - return suffix,gzipped end @@ -4670,7 +4759,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-md5"] = package.loaded["l-md5"] or true --- original size: 3309, stripped down to: 2218 +-- original size: 3414, stripped down to: 2307 if not modules then modules={} end modules ['l-md5']={ version=1.001, @@ -4689,6 +4778,8 @@ if not md5 then end local md5,file=md5,file local gsub=string.gsub +local modification,isfile,touch=lfs.modification,lfs.isfile,lfs.touch +local loaddata,savedata=io.loaddata,io.savedata do local patterns=lpeg and lpeg.patterns if patterns then @@ -4704,10 +4795,11 @@ do md5.sumHEXA=md5.HEX end end +local md5HEX=md5.HEX function file.needsupdating(oldname,newname,threshold) - local oldtime=lfs.attributes(oldname,"modification") + local oldtime=modification(oldname) if oldtime then - local newtime=lfs.attributes(newname,"modification") + local newtime=modification(newname) if not newtime then return true elseif newtime>=oldtime then @@ -4723,31 +4815,32 @@ function file.needsupdating(oldname,newname,threshold) end file.needs_updating=file.needsupdating function file.syncmtimes(oldname,newname) - local oldtime=lfs.attributes(oldname,"modification") - if oldtime and lfs.isfile(newname) then - lfs.touch(newname,oldtime,oldtime) + local oldtime=modification(oldname) + if oldtime and isfile(newname) then + touch(newname,oldtime,oldtime) end end -function file.checksum(name) +local function checksum(name) if md5 then - local data=io.loaddata(name) + local data=loaddata(name) if data then - return md5.HEX(data) + return md5HEX(data) end end return nil end +file.checksum=checksum function file.loadchecksum(name) if md5 then - local data=io.loaddata(name..".md5") + local data=loaddata(name..".md5") return data and (gsub(data,"%s","")) end return nil end function file.savechecksum(name,checksum) - if not checksum then checksum=file.checksum(name) end + if not checksum then checksum=checksum(name) end if checksum then - io.savedata(name..".md5",checksum) + savedata(name..".md5",checksum) return checksum end return nil @@ -5636,7 +5729,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-unicode"] = package.loaded["l-unicode"] or true --- original size: 41047, stripped down to: 17171 +-- original size: 41281, stripped down to: 17261 if not modules then modules={} end modules ['l-unicode']={ version=1.001, @@ -6151,49 +6244,52 @@ end function utf.utf32_to_utf8_t(t,endian) return endian and utf32_to_utf8_be_t(t) or utf32_to_utf8_le_t(t) or t end -local function little(b) - if b<0x10000 then - return char(b%256,rshift(b,8)) - else - b=b-0x10000 - local b1=rshift(b,10)+0xD800 - local b2=b%1024+0xDC00 - return char(b1%256,rshift(b1,8),b2%256,rshift(b2,8)) +if bit32 then + local rshift=bit32.rshift + local function little(b) + if b<0x10000 then + return char(b%256,rshift(b,8)) + else + b=b-0x10000 + local b1=rshift(b,10)+0xD800 + local b2=b%1024+0xDC00 + return char(b1%256,rshift(b1,8),b2%256,rshift(b2,8)) + end end -end -local function big(b) - if b<0x10000 then - return char(rshift(b,8),b%256) - else - b=b-0x10000 - local b1=rshift(b,10)+0xD800 - local b2=b%1024+0xDC00 - return char(rshift(b1,8),b1%256,rshift(b2,8),b2%256) + local function big(b) + if b<0x10000 then + return char(rshift(b,8),b%256) + else + b=b-0x10000 + local b1=rshift(b,10)+0xD800 + local b2=b%1024+0xDC00 + return char(rshift(b1,8),b1%256,rshift(b2,8),b2%256) + end end -end -local l_remap=Cs((p_utf8byte/little+P(1)/"")^0) -local b_remap=Cs((p_utf8byte/big+P(1)/"")^0) -local function utf8_to_utf16_be(str,nobom) - if nobom then - return lpegmatch(b_remap,str) - else - return char(254,255)..lpegmatch(b_remap,str) + local l_remap=Cs((p_utf8byte/little+P(1)/"")^0) + local b_remap=Cs((p_utf8byte/big+P(1)/"")^0) + local function utf8_to_utf16_be(str,nobom) + if nobom then + return lpegmatch(b_remap,str) + else + return char(254,255)..lpegmatch(b_remap,str) + end end -end -local function utf8_to_utf16_le(str,nobom) - if nobom then - return lpegmatch(l_remap,str) - else - return char(255,254)..lpegmatch(l_remap,str) + local function utf8_to_utf16_le(str,nobom) + if nobom then + return lpegmatch(l_remap,str) + else + return char(255,254)..lpegmatch(l_remap,str) + end end -end -utf.utf8_to_utf16_be=utf8_to_utf16_be -utf.utf8_to_utf16_le=utf8_to_utf16_le -function utf.utf8_to_utf16(str,littleendian,nobom) - if littleendian then - return utf8_to_utf16_le(str,nobom) - else - return utf8_to_utf16_be(str,nobom) + utf.utf8_to_utf16_be=utf8_to_utf16_be + utf.utf8_to_utf16_le=utf8_to_utf16_le + function utf.utf8_to_utf16(str,littleendian,nobom) + if littleendian then + return utf8_to_utf16_le(str,nobom) + else + return utf8_to_utf16_be(str,nobom) + end end end local pattern=Cs ( @@ -6467,7 +6563,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-str"] = package.loaded["util-str"] or true --- original size: 43488, stripped down to: 21595 +-- original size: 45188, stripped down to: 22734 if not modules then modules={} end modules ['util-str']={ version=1.001, @@ -6797,6 +6893,13 @@ local template=[[ %s return function(%s) return %s end ]] +local pattern=Cs(Cc('"')*( + (1-S('"\\\n\r'))^1+P('"')/'\\"'+P('\\')/'\\\\'+P('\n')/'\\n'+P('\r')/'\\r' +)^0*Cc('"')) +patterns.escapedquotes=pattern +function string.escapedquotes(s) + return lpegmatch(pattern,s) +end local preamble="" local environment={ global=global or _G, @@ -6821,9 +6924,10 @@ local environment={ formattednumber=number.formatted, sparseexponent=number.sparseexponent, formattedfloat=number.formattedfloat, - stripzero=lpeg.patterns.stripzero, - stripzeros=lpeg.patterns.stripzeros, - FORMAT=string.f9, + stripzero=patterns.stripzero, + stripzeros=patterns.stripzeros, + escapedquotes=string.escapedquotes, + FORMAT=string.f6, } local arguments={ "a1" } setmetatable(arguments,{ __index=function(t,k) @@ -6874,13 +6978,16 @@ local format_left=function(f) return format("a%s..utfpadding(a%s,%i)",n,n,-f) end end -local format_q=function() +local format_q=JITSUPPORTED and function() n=n+1 return format("(a%s ~= nil and format('%%q',tostring(a%s)) or '')",n,n) +end or function() + n=n+1 + return format("(a%s ~= nil and format('%%q',a%s) or '')",n,n) end local format_Q=function() n=n+1 - return format("format('%%q',tostring(a%s))",n) + return format("escapedquotes(tostring(a%s))",n) end local format_i=function(f) n=n+1 @@ -7031,12 +7138,25 @@ local format_n=function() n=n+1 return format("((a%s %% 1 == 0) and format('%%i',a%s) or tostring(a%s))",n,n,n) end -local format_N=function(f) - n=n+1 - if not f or f=="" then - f=".9" - end - return format("(((a%s %% 1 == 0) and format('%%i',a%s)) or lpegmatch(stripzero,format('%%%sf',a%s)))",n,n,f,n) +local format_N if environment.FORMAT then + format_N=function(f) + n=n+1 + if not f or f=="" then + return format("FORMAT(a%s,'%%.9f')",n) + elseif f==".6" or f=="0.6" then + return format("FORMAT(a%s)",n) + else + return format("FORMAT(a%s,'%%%sf')",n,f) + end + end +else + format_N=function(f) + n=n+1 + if not f or f=="" then + f=".9" + end + return format("(((a%s %% 1 == 0) and format('%%i',a%s)) or lpegmatch(stripzero,format('%%%sf',a%s)))",n,n,f,n) + end end local format_a=function(f) n=n+1 @@ -7265,9 +7385,9 @@ patterns.xmlescape=Cs((P("<")/"<"+P(">")/">"+P("&")/"&"+P('"')/"" patterns.texescape=Cs((C(S("#$%\\{}"))/"\\%1"+anything)^0) patterns.luaescape=Cs(((1-S('"\n'))^1+P('"')/'\\"'+P('\n')/'\\n"')^0) patterns.luaquoted=Cs(Cc('"')*((1-S('"\n'))^1+P('"')/'\\"'+P('\n')/'\\n"')^0*Cc('"')) -add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],{ xmlescape=lpeg.patterns.xmlescape }) -add(formatters,"tex",[[lpegmatch(texescape,%s)]],{ texescape=lpeg.patterns.texescape }) -add(formatters,"lua",[[lpegmatch(luaescape,%s)]],{ luaescape=lpeg.patterns.luaescape }) +add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],{ xmlescape=patterns.xmlescape }) +add(formatters,"tex",[[lpegmatch(texescape,%s)]],{ texescape=patterns.texescape }) +add(formatters,"lua",[[lpegmatch(luaescape,%s)]],{ luaescape=patterns.luaescape }) local dquote=patterns.dquote local equote=patterns.escaped+dquote/'\\"'+1 local cquote=Cc('"') @@ -7299,6 +7419,27 @@ local f_16_16=formatters["%0.5N"] function number.to16dot16(n) return f_16_16(n/65536.0) end +if not string.explode then + local tsplitat=lpeg.tsplitat + local p_utf=patterns.utf8character + local p_check=C(p_utf)*(P("+")*Cc(true))^0 + local p_split=Ct(C(p_utf)^0) + local p_space=Ct((C(1-P(" ")^1)+P(" ")^1)^0) + function string.explode(str,symbol) + if symbol=="" then + return lpegmatch(p_split,str) + elseif symbol then + local a,b=lpegmatch(p_check,symbol) + if b then + return lpegmatch(tsplitat(P(a)^1),str) + else + return lpegmatch(tsplitat(a),str) + end + else + return lpegmatch(p_space,str) + end + end +end end -- of closure @@ -7307,7 +7448,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tab"] = package.loaded["util-tab"] or true --- original size: 28866, stripped down to: 16134 +-- original size: 32649, stripped down to: 18257 if not modules then modules={} end modules ['util-tab']={ version=1.001, @@ -7544,78 +7685,160 @@ function tables.encapsulate(core,capsule,protect) } ) end end -local f_hashed_string=formatters["[%q]=%q,"] -local f_hashed_number=formatters["[%q]=%s,"] -local f_hashed_boolean=formatters["[%q]=%l,"] -local f_hashed_table=formatters["[%q]="] -local f_indexed_string=formatters["[%s]=%q,"] -local f_indexed_number=formatters["[%s]=%s,"] -local f_indexed_boolean=formatters["[%s]=%l,"] -local f_indexed_table=formatters["[%s]="] -local f_ordered_string=formatters["%q,"] -local f_ordered_number=formatters["%s,"] -local f_ordered_boolean=formatters["%l,"] -function table.fastserialize(t,prefix) - local r={ type(prefix)=="string" and prefix or "return" } - local m=1 - local function fastserialize(t,outer) - local n=#t - m=m+1 - r[m]="{" - if n>0 then - for i=0,n do - local v=t[i] - local tv=type(v) - if tv=="string" then - m=m+1 r[m]=f_ordered_string(v) - elseif tv=="number" then - m=m+1 r[m]=f_ordered_number(v) - elseif tv=="table" then - fastserialize(v) - elseif tv=="boolean" then - m=m+1 r[m]=f_ordered_boolean(v) +if JITSUPPORTED then + local f_hashed_string=formatters["[%Q]=%Q,"] + local f_hashed_number=formatters["[%Q]=%s,"] + local f_hashed_boolean=formatters["[%Q]=%l,"] + local f_hashed_table=formatters["[%Q]="] + local f_indexed_string=formatters["[%s]=%Q,"] + local f_indexed_number=formatters["[%s]=%s,"] + local f_indexed_boolean=formatters["[%s]=%l,"] + local f_indexed_table=formatters["[%s]="] + local f_ordered_string=formatters["%Q,"] + local f_ordered_number=formatters["%s,"] + local f_ordered_boolean=formatters["%l,"] + function table.fastserialize(t,prefix) + local r={ type(prefix)=="string" and prefix or "return" } + local m=1 + local function fastserialize(t,outer) + local n=#t + m=m+1 + r[m]="{" + if n>0 then + local v=t[0] + if v then + local tv=type(v) + if tv=="string" then + m=m+1 r[m]=f_indexed_string(0,v) + elseif tv=="number" then + m=m+1 r[m]=f_indexed_number(0,v) + elseif tv=="table" then + m=m+1 r[m]=f_indexed_table(0) + fastserialize(v) + m=m+1 r[m]=f_indexed_table(0) + elseif tv=="boolean" then + m=m+1 r[m]=f_indexed_boolean(0,v) + end + end + for i=1,n do + local v=t[i] + local tv=type(v) + if tv=="string" then + m=m+1 r[m]=f_ordered_string(v) + elseif tv=="number" then + m=m+1 r[m]=f_ordered_number(v) + elseif tv=="table" then + fastserialize(v) + elseif tv=="boolean" then + m=m+1 r[m]=f_ordered_boolean(v) + end end end - end - for k,v in next,t do - local tk=type(k) - if tk=="number" then - if k>n or k<0 then + for k,v in next,t do + local tk=type(k) + if tk=="number" then + if k>n or k<0 then + local tv=type(v) + if tv=="string" then + m=m+1 r[m]=f_indexed_string(k,v) + elseif tv=="number" then + m=m+1 r[m]=f_indexed_number(k,v) + elseif tv=="table" then + m=m+1 r[m]=f_indexed_table(k) + fastserialize(v) + elseif tv=="boolean" then + m=m+1 r[m]=f_indexed_boolean(k,v) + end + end + else local tv=type(v) if tv=="string" then - m=m+1 r[m]=f_indexed_string(k,v) + m=m+1 r[m]=f_hashed_string(k,v) elseif tv=="number" then - m=m+1 r[m]=f_indexed_number(k,v) + m=m+1 r[m]=f_hashed_number(k,v) elseif tv=="table" then - m=m+1 r[m]=f_indexed_table(k) + m=m+1 r[m]=f_hashed_table(k) fastserialize(v) elseif tv=="boolean" then - m=m+1 r[m]=f_indexed_boolean(k,v) + m=m+1 r[m]=f_hashed_boolean(k,v) end end + end + m=m+1 + if outer then + r[m]="}" else - local tv=type(v) - if tv=="string" then - m=m+1 r[m]=f_hashed_string(k,v) - elseif tv=="number" then - m=m+1 r[m]=f_hashed_number(k,v) - elseif tv=="table" then - m=m+1 r[m]=f_hashed_table(k) - fastserialize(v) - elseif tv=="boolean" then - m=m+1 r[m]=f_hashed_boolean(k,v) - end + r[m]="}," end + return r end - m=m+1 - if outer then - r[m]="}" - else - r[m]="}," + return concat(fastserialize(t,true)) + end +else + local f_v=formatters["[%q]=%q,"] + local f_t=formatters["[%q]="] + local f_q=formatters["%q,"] + function table.fastserialize(t,prefix) + local r={ type(prefix)=="string" and prefix or "return" } + local m=1 + local function fastserialize(t,outer) + local n=#t + m=m+1 + r[m]="{" + if n>0 then + local v=t[0] + if v then + m=m+1 + r[m]="[0]='" + if type(v)=="table" then + fastserialize(v) + else + r[m]=format("%q,",v) + end + end + for i=1,n do + local v=t[i] + m=m+1 + if type(v)=="table" then + r[m]=format("[%i]=",i) + fastserialize(v) + else + r[m]=format("[%i]=%q,",i,v) + end + end + end + for k,v in next,t do + local tk=type(k) + if tk=="number" then + if k>n or k<0 then + m=m+1 + if type(v)=="table" then + r[m]=format("[%i]=",k) + fastserialize(v) + else + r[m]=format("[%i]=%q,",k,v) + end + end + else + m=m+1 + if type(v)=="table" then + r[m]=format("[%q]=",k) + fastserialize(v) + else + r[m]=format("[%q]=%q,",k,v) + end + end + end + m=m+1 + if outer then + r[m]="}" + else + r[m]="}," + end + return r end - return r + return concat(fastserialize(t,true)) end - return concat(fastserialize(t,true)) end function table.deserialize(str) if not str or str=="" then @@ -7709,28 +7932,28 @@ function table.twowaymapper(t) return t end local f_start_key_idx=formatters["%w{"] -local f_start_key_num=formatters["%w[%s]={"] +local f_start_key_num=JITSUPPORTED and formatters["%w[%s]={"] or formatters["%w[%q]={"] local f_start_key_str=formatters["%w[%q]={"] local f_start_key_boo=formatters["%w[%l]={"] local f_start_key_nop=formatters["%w{"] local f_stop=formatters["%w},"] -local f_key_num_value_num=formatters["%w[%s]=%s,"] -local f_key_str_value_num=formatters["%w[%q]=%s,"] -local f_key_boo_value_num=formatters["%w[%l]=%s,"] -local f_key_num_value_str=formatters["%w[%s]=%q,"] -local f_key_str_value_str=formatters["%w[%q]=%q,"] -local f_key_boo_value_str=formatters["%w[%l]=%q,"] -local f_key_num_value_boo=formatters["%w[%s]=%l,"] -local f_key_str_value_boo=formatters["%w[%q]=%l,"] +local f_key_num_value_num=JITSUPPORTED and formatters["%w[%s]=%s,"] or formatters["%w[%s]=%q,"] +local f_key_str_value_num=JITSUPPORTED and formatters["%w[%Q]=%s,"] or formatters["%w[%Q]=%q,"] +local f_key_boo_value_num=JITSUPPORTED and formatters["%w[%l]=%s,"] or formatters["%w[%l]=%q,"] +local f_key_num_value_str=JITSUPPORTED and formatters["%w[%s]=%Q,"] or formatters["%w[%q]=%Q,"] +local f_key_str_value_str=formatters["%w[%Q]=%Q,"] +local f_key_boo_value_str=formatters["%w[%l]=%Q,"] +local f_key_num_value_boo=JITSUPPORTED and formatters["%w[%s]=%l,"] or formatters["%w[%q]=%l,"] +local f_key_str_value_boo=formatters["%w[%Q]=%l,"] local f_key_boo_value_boo=formatters["%w[%l]=%l,"] -local f_key_num_value_not=formatters["%w[%s]={},"] -local f_key_str_value_not=formatters["%w[%q]={},"] +local f_key_num_value_not=JITSUPPORTED and formatters["%w[%s]={},"] or formatters["%w[%q]={},"] +local f_key_str_value_not=formatters["%w[%Q]={},"] local f_key_boo_value_not=formatters["%w[%l]={},"] -local f_key_num_value_seq=formatters["%w[%s]={ %, t },"] -local f_key_str_value_seq=formatters["%w[%q]={ %, t },"] +local f_key_num_value_seq=JITSUPPORTED and formatters["%w[%s]={ %, t },"] or formatters["%w[%q]={ %, t },"] +local f_key_str_value_seq=formatters["%w[%Q]={ %, t },"] local f_key_boo_value_seq=formatters["%w[%l]={ %, t },"] -local f_val_num=formatters["%w%s,"] -local f_val_str=formatters["%w%q,"] +local f_val_num=JITSUPPORTED and formatters["%w%s,"] or formatters["%w%q,"] +local f_val_str=formatters["%w%Q,"] local f_val_boo=formatters["%w%l,"] local f_val_not=formatters["%w{},"] local f_val_seq=formatters["%w{ %, t },"] @@ -7738,7 +7961,7 @@ local f_fin_seq=formatters[" %, t }"] local f_table_return=formatters["return {"] local f_table_name=formatters["%s={"] local f_table_direct=formatters["{"] -local f_table_entry=formatters["[%q]={"] +local f_table_entry=formatters["[%Q]={"] local f_table_finish=formatters["}"] local spaces=utilities.strings.newrepeater(" ") local original_serialize=table.serialize @@ -8339,7 +8562,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sac"] = package.loaded["util-sac"] or true --- original size: 11332, stripped down to: 8420 +-- original size: 12946, stripped down to: 9507 if not modules then modules={} end modules ['util-sac']={ version=1.001, @@ -8374,6 +8597,7 @@ end function streams.size(f) return f and f[3] or 0 end +streams.getsize=streams.size function streams.setposition(f,i) if f[4] then if i<=0 then @@ -8566,9 +8790,9 @@ function streams.readfixed2(f) f[2]=j+1 local a,b=byte(f[1],i,j) if a>=0x80 then - tonumber((a-0x100).."."..b) + return tonumber((a-0x100).."."..b) or 0 else - tonumber((a ).."."..b) + return tonumber((a ).."."..b) or 0 end end function streams.readfixed4(f) @@ -8577,9 +8801,9 @@ function streams.readfixed4(f) f[2]=j+1 local a,b,c,d=byte(f[1],i,j) if a>=0x80 then - tonumber((0x100*a+b-0x10000).."."..(0x100*c+d)) + return tonumber((0x100*a+b-0x10000).."."..(0x100*c+d)) or 0 else - tonumber((0x100*a+b ).."."..(0x100*c+d)) + return tonumber((0x100*a+b ).."."..(0x100*c+d)) or 0 end end if bit32 then @@ -8659,6 +8883,16 @@ if sio and sio.readcardinal2 then f[2]=i+4 return readinteger4(f[1],i) end + function streams.readfixed2(f) + local i=f[2] + f[2]=i+2 + return readfixed2(f[1],i) + end + function streams.readfixed4(f) + local i=f[2] + f[2]=i+4 + return readfixed4(f[1],i) + end function streams.read2dot4(f) local i=f[2] f[2]=i+2 @@ -8758,6 +8992,50 @@ else return t end end +do + local files=utilities.files + if files then + local openfile=files.open + local openstream=streams.open + local openstring=streams.openstring + local setmetatable=setmetatable + function io.newreader(str,method) + local f,m + if method=="string" then + f=openstring(str) + m=streams + elseif method=="stream" then + f=openstream(str) + m=streams + else + f=openfile(str,"rb") + m=files + end + if f then + local t={} + setmetatable(t,{ + __index=function(t,k) + local r=m[k] + if k=="close" then + if f then + m.close(f) + f=nil + end + return function() end + elseif r then + local v=function(_,a,b) return r(f,a,b) end + t[k]=v + return v + else + print("unknown key",k) + end + end + } ) + return t + end + end + end +end end -- of closure @@ -9801,7 +10079,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-soc-imp-copas"] = package.loaded["util-soc-imp-copas"] or true --- original size: 25844, stripped down to: 14821 +-- original size: 25959, stripped down to: 14893 local socket=socket or require("socket") @@ -9838,6 +10116,7 @@ local copas={ autoclose=true, running=false, report=report, + trace=false, } local function statushandler(status,...) if status then @@ -9847,7 +10126,9 @@ local function statushandler(status,...) if type(err)=="table" then err=err[1] end - report("error: %s",tostring(err)) + if copas.trace then + report("error: %s",tostring(err)) + end return nil,err end function socket.protect(func) @@ -9861,7 +10142,9 @@ function socket.newtry(finalizer) if not status then local detail=select(2,...) pcall(finalizer,detail) - report("error: %s",tostring(detail)) + if copas.trace then + report("error: %s",tostring(detail)) + end return end return... @@ -12429,7 +12712,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 13340, stripped down to: 8826 +-- original size: 13394, stripped down to: 8882 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -12439,8 +12722,9 @@ if not modules then modules={} end modules ['trac-set']={ license="see context related readme files" } local type,next,tostring,tonumber=type,next,tostring,tonumber +local print=print local concat,sortedhash=table.concat,table.sortedhash -local format,find,lower,gsub,topattern=string.format,string.find,string.lower,string.gsub,string.topattern +local formatters,find,lower,gsub,topattern=string.formatters,string.find,string.lower,string.gsub,string.topattern local is_boolean=string.is_boolean local settings_to_hash=utilities.parsers.settings_to_hash local allocate=utilities.storage.allocate @@ -12450,10 +12734,10 @@ local setters=utilities.setters or {} utilities.setters=setters local data={} local trace_initialize=false +local frozen=true function setters.initialize(filename,name,values) local setter=data[name] if setter then - frozen=true local data=setter.data if data then for key,newvalue in sortedhash(values) do @@ -12647,8 +12931,8 @@ function setters.show(t) end end local enable,disable,register,list,show=setters.enable,setters.disable,setters.register,setters.list,setters.show -function setters.report(setter,...) - print(format("%-15s : %s\n",setter.name,format(...))) +function setters.report(setter,fmt,...) + print(formatters["%-15s : %s\n"](setter.name,formatters[fmt](...))) end local function default(setter,name) local d=setter.data[name] @@ -12668,7 +12952,7 @@ function setters.new(name) disable=function(...) disable (setter,...) end, reset=function(...) reset (setter,...) end, register=function(...) register(setter,...) end, - list=function(...) list (setter,...) end, + list=function(...) return list (setter,...) end, show=function(...) show (setter,...) end, default=function(...) return default (setter,...) end, value=function(...) return value (setter,...) end, @@ -12771,7 +13055,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 32618, stripped down to: 20935 +-- original size: 33003, stripped down to: 21667 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -12816,14 +13100,30 @@ local function ignore() end setmetatableindex(logs,function(t,k) t[k]=ignore;return ignore end) local report,subreport,status,settarget,setformats,settranslations local direct,subdirect,writer,pushtarget,poptarget,setlogfile,settimedlog,setprocessor,setformatters,newline +local function ansisupported(specification) + if specification~="ansi" and specification~="ansilog" then + return false + elseif os and os.enableansi then + return os.enableansi() + else + return false + end +end if runningtex and texio then if texio.setescape then texio.setescape(0) end - if arg then + if arg and ansisupported then for k,v in next,arg do if v=="--ansi" or v=="--c:ansi" then - variant="ansi" + if ansisupported("ansi") then + variant="ansi" + end + break + elseif v=="--ansilog" or v=="--c:ansilog" then + if ansisupported("ansilog") then + variant="ansilog" + end break end end @@ -12928,6 +13228,10 @@ if runningtex and texio then }, } } + variants.ansilog={ + formats=variants.ansi.formats, + targets=variants.default.targets, + } logs.flush=io.flush writer=function(...) write_nl(target,...) @@ -13034,6 +13338,9 @@ if runningtex and texio then t=specification.targets f=specification.formats or specification else + if not ansisupported(specification) then + specification="default" + end local v=variants[specification] if v then t=v.targets @@ -13060,7 +13367,7 @@ if runningtex and texio then subdirect_nop=f.subdirect_nop status_yes=f.status_yes status_nop=f.status_nop - if variant=="ansi" then + if variant=="ansi" or variant=="ansilog" then useluawrites() end settarget(whereto) @@ -13153,6 +13460,9 @@ else if type(specification)=="table" then f=specification.formats or specification else + if not ansisupported(specification) then + specification="default" + end local v=variants[specification] if v then f=v.formats @@ -13408,12 +13718,6 @@ end local nesting=0 local verbose=false local hasscheme=url.hasscheme -function logs.show_open(name) -end -function logs.show_close(name) -end -function logs.show_load(name) -end local simple=logs.reporter("comment") logs.simple=simple logs.simpleline=simple @@ -13480,6 +13784,13 @@ local exporters={ logs.reporters=reporters logs.exporters=exporters function logs.application(t) + local arguments=environment and environment.arguments + if arguments then + local ansi=arguments.ansi or arguments.ansilog + if ansi then + logs.setformatters(arguments.ansi and "ansi" or "ansilog") + end + end t.name=t.name or "unknown" t.banner=t.banner t.moreinfo=moreinfo @@ -13553,8 +13864,6 @@ else print(format(...)) end end -io.stdout:setvbuf('no') -io.stderr:setvbuf('no') if package.helpers.report then package.helpers.report=logs.reporter("package loader") end @@ -13652,7 +13961,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-inf"] = package.loaded["trac-inf"] or true --- original size: 8966, stripped down to: 5972 +-- original size: 9973, stripped down to: 7492 if not modules then modules={} end modules ['trac-inf']={ version=1.001, @@ -13674,7 +13983,7 @@ statistics.enable=true statistics.threshold=0.01 local statusinfo,n,registered,timers={},0,{},{} setmetatableindex(timers,function(t,k) - local v={ timing=0,loadtime=0 } + local v={ timing=0,loadtime=0,offset=0 } t[k]=v return v end) @@ -13682,10 +13991,40 @@ local function hastiming(instance) return instance and timers[instance] end local function resettiming(instance) - timers[instance or "notimer"]={ timing=0,loadtime=0 } + timers[instance or "notimer"]={ timing=0,loadtime=0,offset=0 } end local ticks=clock local seconds=function(n) return n or 0 end +if lua.getpreciseticks then + ticks=lua.getpreciseticks + seconds=lua.getpreciseseconds +elseif FFISUPPORTED and ffi and os.type=="windows" then + local okay,kernel=pcall(ffi.load,"kernel32") + if kernel then + local tonumber=ffi.number or tonumber + ffi.cdef[[ + int QueryPerformanceFrequency(int64_t *lpFrequency); + int QueryPerformanceCounter(int64_t *lpPerformanceCount); + ]] + local target=ffi.new("__int64[1]") + ticks=function() + if kernel.QueryPerformanceCounter(target)==1 then + return tonumber(target[0]) + else + return 0 + end + end + local target=ffi.new("__int64[1]") + seconds=function(ticks) + if kernel.QueryPerformanceFrequency(target)==1 then + return ticks/tonumber(target[0]) + else + return 0 + end + end + end +else +end local function starttiming(instance,reset) local timer=timers[instance or "notimer"] local it=timer.timing @@ -13720,12 +14059,26 @@ local function stoptiming(instance) end return 0 end +local function benchmarktimer(instance) + local timer=timers[instance or "notimer"] + local it=timer.timing + if it>1 then + timer.timing=it-1 + else + local starttime=timer.starttime + if starttime and starttime>0 then + timer.offset=ticks()-starttime + else + timer.offset=0 + end + end +end local function elapsed(instance) if type(instance)=="number" then return instance else local timer=timers[instance or "notimer"] - return timer and seconds(timer.loadtime) or 0 + return timer and seconds(timer.loadtime-2*(timer.offset or 0)) or 0 end end local function currenttime(instance) @@ -13738,7 +14091,7 @@ local function currenttime(instance) else local starttime=timer.starttime if starttime and starttime>0 then - return seconds(timer.loadtime+ticks()-starttime) + return seconds(timer.loadtime+ticks()-starttime-2*(timer.offset or 0)) end end return 0 @@ -13764,6 +14117,7 @@ statistics.elapsed=elapsed statistics.elapsedtime=elapsedtime statistics.elapsedindeed=elapsedindeed statistics.elapsedseconds=elapsedseconds +statistics.benchmarktimer=benchmarktimer function statistics.register(tag,fnc) if statistics.enable and type(fnc)=="function" then local rt=registered[tag] or (#statusinfo+1) @@ -13780,10 +14134,17 @@ function statistics.show() return format("%s, type: %s, binary subtree: %s", os.platform or "unknown",os.type or "unknown",environment.texos or "unknown") end) - register("used engine",function() - return format("%s version %s with functionality level %s, banner: %s", - LUATEXENGINE,LUATEXVERSION,LUATEXFUNCTIONALITY,lower(status.banner)) - end) + if LUATEXENGINE=="luametatex" then + register("used engine",function() + return format("%s version %s, functionality level %s, format id %s", + LUATEXENGINE,LUATEXVERSION,LUATEXFUNCTIONALITY,LUATEXFORMATID) + end) + else + register("used engine",function() + return format("%s version %s with functionality level %s, banner: %s", + LUATEXENGINE,LUATEXVERSION,LUATEXFUNCTIONALITY,lower(status.banner)) + end) + end register("control sequences",function() return format("%s of %s + %s",status.cs_count,status.hash_size,status.hash_extra) end) @@ -13822,7 +14183,11 @@ function statistics.show() end function statistics.memused() local round=math.round or math.floor - return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000),round(status.luastate_bytes/1000000)) + return format("%s MB, ctx: %s MB, max: %s MB)", + round(collectgarbage("count")/1000), + round(status.luastate_bytes/1000000), + status.luastate_bytes_max and round(status.luastate_bytes_max/1000000) or "unknown" + ) end starttiming(statistics) function statistics.formatruntime(runtime) @@ -14013,7 +14378,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lua"] = package.loaded["util-lua"] or true --- original size: 6664, stripped down to: 4589 +-- original size: 7149, stripped down to: 4997 if not modules then modules={} end modules ['util-lua']={ version=1.001, @@ -14038,16 +14403,21 @@ luautilities.nofstrippedchunks=0 luautilities.nofstrippedbytes=0 local strippedchunks={} luautilities.strippedchunks=strippedchunks +if not LUATEXENGINE then + LUATEXENGINE=status.luatex_engine and string.lower(status.luatex_engine) + JITSUPPORTED=LUATEXENGINE=="luajittex" or jit + CONTEXTLMTXMODE=CONTEXTLMTXMODE or (LUATEXENGINE=="luametatex" and 1) or 0 +end luautilities.suffixes={ tma="tma", - tmc=jit and "tmb" or "tmc", + tmc=(CONTEXTLMTXMODE and CONTEXTLMTXMODE>0 and "tmd") or (jit and "tmb") or "tmc", lua="lua", - luc=jit and "lub" or "luc", + luc=(CONTEXTLMTXMODE and CONTEXTLMTXMODE>0 and "lud") or (jit and "lub") or "luc", lui="lui", luv="luv", luj="luj", tua="tua", - tuc="tuc", + tuc=(CONTEXTLMTXMODE and CONTEXTLMTXMODE>0 and "tud") or (jit and "tub") or "tuc", } local function register(name) if tracestripping then @@ -14186,7 +14556,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-deb"] = package.loaded["util-deb"] or true --- original size: 9955, stripped down to: 6693 +-- original size: 10136, stripped down to: 6832 if not modules then modules={} end modules ['util-deb']={ version=1.001, @@ -14210,7 +14580,13 @@ local dummycalls=10*1000 local nesting=0 local names={} local initialize=false -if not (FFISUPPORTED and ffi) then +if lua.getpreciseticks then + initialize=function() + ticks=lua.getpreciseticks + seconds=lua.getpreciseseconds + initialize=false + end +elseif not (FFISUPPORTED and ffi) then elseif os.type=="windows" then initialize=function() local kernel=ffilib("kernel32","system") @@ -14482,7 +14858,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tpl"] = package.loaded["util-tpl"] or true --- original size: 7112, stripped down to: 3887 +-- original size: 7722, stripped down to: 4212 if not modules then modules={} end modules ['util-tpl']={ version=1.001, @@ -14498,6 +14874,7 @@ local report_template=logs.reporter("template") local tostring,next=tostring,next local format,sub,byte=string.format,string.sub,string.byte local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns +local formatters=string.formatters local replacer local function replacekey(k,t,how,recursive) local v=t[k] @@ -14566,6 +14943,10 @@ local function replaceoptional(l,m,r,t,how,recurse) local v=t[l] return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" end +local function replaceformatted(l,m,r,t,how,recurse) + local v=t[r] + return v and formatters[l](v) +end local single=P("%") local double=P("%%") local lquoted=P("%[") @@ -14579,16 +14960,19 @@ local nolquoted=lquoted/'' local norquoted=rquoted/'' local nolquotedq=lquotedq/'' local norquotedq=rquotedq/'' +local nolformatted=P(":")/"%%" +local norformatted=P(":")/"" local noloptional=P("%?")/'' local noroptional=P("?%")/'' local nomoptional=P(":")/'' local args=Carg(1)*Carg(2)*Carg(3) -local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle -local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq -local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted +local key=nosingle*((C((1-nosingle)^1)*args)/replacekey)*nosingle +local quoted=nolquotedq*((C((1-norquotedq)^1)*args)/replacekeyquoted)*norquotedq +local unquoted=nolquoted*((C((1-norquoted)^1)*args)/replacekeyunquoted)*norquoted local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional +local formatted=nosingle*((Cs(nolformatted*(1-norformatted )^1)*norformatted*C((1-nosingle)^1)*args)/replaceformatted)*nosingle local any=P(1) - replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) + replacer=Cs((unquoted+quoted+formatted+escape+optional+key+any)^0) local function replace(str,mapping,how,recurse) if mapping and str then return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str @@ -14627,7 +15011,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sbx"] = package.loaded["util-sbx"] or true --- original size: 20393, stripped down to: 13121 +-- original size: 21084, stripped down to: 13214 if not modules then modules={} end modules ['util-sbx']={ version=1.001, @@ -14873,37 +15257,50 @@ local iopopen=sandbox.original(io.popen) local reported={} local function validcommand(name,program,template,checkers,defaults,variables,reporter,strict) if validbinaries~=false and (validbinaries==true or validbinaries[program]) then + local binpath=nil if variables then for variable,value in next,variables do - local checker=validators[checkers[variable]] - if checker then - value=checker(unquoted(value),strict) - if value then - variables[variable]=optionalquoted(value) + local chktype=checkers[variable] + if chktype=="verbose" then + else + local checker=validators[chktype] + if checker then + value=checker(unquoted(value),strict) + if value then + variables[variable]=optionalquoted(value) + else + report("variable %a with value %a fails the check",variable,value) + return + end else - report("variable %a with value %a fails the check",variable,value) + report("variable %a has no checker",variable) return end - else - report("variable %a has no checker",variable) - return end end for variable,default in next,defaults do local value=variables[variable] if not value or value=="" then - local checker=validators[checkers[variable]] - if checker then - default=checker(unquoted(default),strict) - if default then - variables[variable]=optionalquoted(default) - else - report("variable %a with default %a fails the check",variable,default) - return + local chktype=checkers[variable] + if chktype=="verbose" then + else + local checker=validators[chktype] + if checker then + default=checker(unquoted(default),strict) + if default then + variables[variable]=optionalquoted(default) + else + report("variable %a with default %a fails the check",variable,default) + return + end end end end end + binpath=variables.binarypath + end + if type(binpath)=="string" and binpath~="" then + program=binpath.."/"..program end local command=program.." "..replace(template,variables) if reporter then @@ -14938,7 +15335,8 @@ local runners={ if trace then report("execute: %s",command) end - return osexecute(command) + local okay=osexecute(command) + return okay end end, pipeto=function(...) @@ -14972,7 +15370,7 @@ function sandbox.registerrunner(specification) return end if validrunners[name] then - report("invalid name, runner %a already defined") + report("invalid name, runner %a already defined",name) return end local program=specification.program @@ -15090,8 +15488,8 @@ if io then end if os then overload(os.execute,binaryrunner,"os.execute") - overload(os.spawn,dummyrunner,"os.spawn") - overload(os.exec,dummyrunner,"os.exec") + overload(os.spawn,dummyrunner,"os.spawn") + overload(os.exec,dummyrunner,"os.exec") overload(os.resultof,binaryrunner,"os.resultof") overload(os.pipeto,binaryrunner,"os.pipeto") overload(os.rename,filehandlertwo,"os.rename") @@ -15116,13 +15514,6 @@ end if zip then zip.open=register(zip.open,filehandlerone,"zip.open") end -if fontloader then - fontloader.open=register(fontloader.open,filehandlerone,"fontloader.open") - fontloader.info=register(fontloader.info,filehandlerone,"fontloader.info") -end -if epdf then - epdf.open=register(epdf.open,filehandlerone,"epdf.open") -end sandbox.registerroot=registerroot sandbox.registerbinary=registerbinary sandbox.registerlibrary=registerlibrary @@ -15528,7 +15919,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-env"] = package.loaded["luat-env"] or true --- original size: 6134, stripped down to: 4118 +-- original size: 6551, stripped down to: 4315 if not modules then modules={} end modules ['luat-env']={ version=1.001, @@ -15537,7 +15928,7 @@ package.loaded["luat-env"] = package.loaded["luat-env"] or true copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local rawset,rawget,loadfile=rawset,rawget,loadfile +local rawset,loadfile=rawset,loadfile local gsub=string.gsub local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_lua=logs.reporter("resolvers","lua") @@ -15584,6 +15975,12 @@ function environment.texfile(filename) return resolvers.findfile(filename,'tex') end function environment.luafile(filename) + if CONTEXTLMTXMODE and CONTEXTLMTXMODE>0 and file.suffix(filename)=="lua" then + local resolved=resolvers.findfile(file.replacesuffix(filename,"lmt")) or "" + if resolved~="" then + return resolved + end + end local resolved=resolvers.findfile(filename,'tex') or "" if resolved~="" then return resolved @@ -15695,7 +16092,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-zip"] = package.loaded["util-zip"] or true --- original size: 18645, stripped down to: 11291 +-- original size: 19496, stripped down to: 10858 if not modules then modules={} end modules ['util-zip']={ version=1.001, @@ -15706,7 +16103,7 @@ if not modules then modules={} end modules ['util-zip']={ local type,tostring,tonumber=type,tostring,tonumber local sort=table.sort local find,format,sub,gsub=string.find,string.format,string.sub,string.gsub -local osdate,ostime=os.date,os.time +local osdate,ostime,osclock=os.date,os.time,os.clock local ioopen=io.open local loaddata,savedata=io.loaddata,io.savedata local filejoin,isdir,dirname,mkdirs=file.join,lfs.isdir,file.dirname,dir.mkdirs @@ -15721,25 +16118,19 @@ local getposition=files.getposition local band=bit32.band local rshift=bit32.rshift local lshift=bit32.lshift -local decompress,calculatecrc -if flate then - decompress=flate.flate_decompress - calculatecrc=flate.update_crc32 -else +local decompress,expandsize,calculatecrc local zlibdecompress=zlib.decompress + local zlibexpandsize=zlib.expandsize local zlibchecksum=zlib.crc32 - decompress=function(source,targetsize) - local target=zlibdecompress(source,-15) - if target then - return target - else - return false,1 - end + decompress=function(source) + return zlibdecompress(source,-15) end + expandsize=zlibexpandsize and function(source,targetsize) + return zlibexpandsize(source,targetsize,-15) + end or decompress calculatecrc=function(buffer,initial) return zlibchecksum(initial or 0,buffer) end -end local zipfiles={} utilities.zipfiles=zipfiles local openzipfile,closezipfile,unzipfile,foundzipfile,getziphash,getziplist do @@ -15864,7 +16255,11 @@ local openzipfile,closezipfile,unzipfile,foundzipfile,getziphash,getziplist do setposition(handle,position) local result=readstring(handle,compressed) if data.method==8 then - result=decompress(result,data.uncompressed) + if expandsize then + result=expandsize(result,data.uncompressed) + else + result=decompress(result) + end end if check and data.crc32~=calculatecrc(result) then print("checksum mismatch") @@ -15883,14 +16278,14 @@ local openzipfile,closezipfile,unzipfile,foundzipfile,getziphash,getziplist do zipfiles.list=getziplist zipfiles.found=foundzipfile end -if flate then do +if xzip then local writecardinal1=files.writebyte local writecardinal2=files.writecardinal2le local writecardinal4=files.writecardinal4le local logwriter=logs.writer local globpattern=dir.globpattern - local compress=flate.flate_compress - local checksum=flate.update_crc32 + local compress=xzip.compress + local checksum=xzip.crc32 local function fromdostime(dostime,dosdate) return ostime { year=rshift(dosdate,9)+1980, @@ -16087,27 +16482,33 @@ if flate then do local count=#list local step=number.idiv(count,10) local done=0 + local steps=verbose=="steps" + local time=steps and osclock() for i=1,count do local l=list[i] local n=l.filename local d=unzipfile(z,n) - local p=filejoin(path,n) - if mkdirs(dirname(p)) then - if verbose=="steps" then - total=total+#d - done=done+1 - if done>=step then - done=0 - logwriter(format("%4i files of %4i done, %10i bytes",i,count,total)) + if d then + local p=filejoin(path,n) + if mkdirs(dirname(p)) then + if steps then + total=total+#d + done=done+1 + if done>=step then + done=0 + logwriter(format("%4i files of %4i done, %10i bytes, %0.3f seconds",i,count,total,osclock()-time)) + end + elseif verbose then + logwriter(n) end - elseif verbose then - logwriter(n) + savedata(p,d) end - savedata(p,d) + else + logwriter(format("problem with file %s",n)) end end - if verbose=="steps" then - logwriter(format("%4i files of %4i done, %10i bytes",count,count,total)) + if steps then + logwriter(format("%4i files of %4i done, %10i bytes, %0.3f seconds",count,count,total,osclock()-time)) end closezipfile(z) return true @@ -16118,37 +16519,8 @@ if flate then do end zipfiles.zipdir=zipdir zipfiles.unzipdir=unzipdir -end end -if flate then - local streams=utilities.streams - local openfile=streams.open - local closestream=streams.close - local setposition=streams.setposition - local getsize=streams.size - local readcardinal4=streams.readcardinal4le - local getstring=streams.getstring - local decompress=flate.gz_decompress - function zipfiles.gunzipfile(filename) - local strm=openfile(filename) - if strm then - setposition(strm,getsize(strm)-4+1) - local size=readcardinal4(strm) - local data=decompress(getstring(strm),size) - closestream(strm) - return data - end - end -elseif gzip then - local openfile=gzip.open - function zipfiles.gunzipfile(filename) - local g=openfile(filename,"rb") - if g then - local d=g:read("*a") - d:close() - return d - end - end end +zipfiles.gunzipfile=gzip.load end -- of closure @@ -16157,7 +16529,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-tab"] = package.loaded["lxml-tab"] or true --- original size: 60383, stripped down to: 35698 +-- original size: 61191, stripped down to: 35864 if not modules then modules={} end modules ['lxml-tab']={ version=1.001, @@ -16881,7 +17253,10 @@ local slash=P('/') local colon=P(':') local semicolon=P(';') local ampersand=P('&') -local valid=R('az','AZ','09')+S('_-.') +local valid_0=R("\128\255") +local valid_1=R('az','AZ')+S('_')+valid_0 +local valid_2=valid_1+R('09')+S('-.') +local valid=valid_1*valid_2^0 local name_yes=C(valid^1)*colon*C(valid^1) local name_nop=C(P(true))*C(valid^1) local name=name_yes+name_nop @@ -16917,8 +17292,9 @@ end local function entityfile(pattern,k,v,n) if n then local okay,data - if resolvers then - okay,data=resolvers.loadbinfile(n) + local loadbinfile=resolvers and resolvers.loadbinfile + if loadbinfile then + okay,data=loadbinfile(n) else data=io.loaddata(n) okay=data and data~="" @@ -17027,12 +17403,14 @@ publicentityfile+publicdoctype+systemdoctype+definitiondoctype+simpledoctype)*op } return grammar_parsed_text_one,grammar_parsed_text_two,grammar_unparsed_text end -grammar_parsed_text_one_nop, -grammar_parsed_text_two_nop, -grammar_unparsed_text_nop=install(space,spacing,anything) -grammar_parsed_text_one_yes, -grammar_parsed_text_two_yes, -grammar_unparsed_text_yes=install(space_nl,spacing_nl,anything_nl) +local + grammar_parsed_text_one_nop, + grammar_parsed_text_two_nop, + grammar_unparsed_text_nop=install(space,spacing,anything) +local + grammar_parsed_text_one_yes, + grammar_parsed_text_two_yes, + grammar_unparsed_text_yes=install(space_nl,spacing_nl,anything_nl) local function _xmlconvert_(data,settings,detail) settings=settings or {} preparexmlstate(settings) @@ -17613,7 +17991,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-lpt"] = package.loaded["lxml-lpt"] or true --- original size: 55145, stripped down to: 30992 +-- original size: 54626, stripped down to: 31255 if not modules then modules={} end modules ['lxml-lpt']={ version=1.001, @@ -18095,6 +18473,8 @@ local builtin={ lastindex="(#ll.__p__.dt or 1)", lastelement="(ll.__p__.en or 1)", last="#list", + list="list", + self="ll", rootposition="order", order="order", element="(ll.ei or 1)", @@ -18203,7 +18583,8 @@ local function register_selector(specification) end local function register_expression(expression) local converted=lpegmatch(converter,expression) - local runner=load(format(template_e,converted)) + local wrapped=format(template_e,converted) + local runner=load(wrapped) runner=(runner and runner()) or function() errorrunner_e(expression,converted) end return { kind="expression",expression=expression,converted=converted,evaluator=runner } end @@ -18575,6 +18956,20 @@ expressions.count=function(e,pattern) local collected=applylpath(e,pattern) return pattern and (collected and #collected) or 0 end +expressions.attribute=function(e,name,value) + if type(e)=="table" and name then + local a=e.at + if a then + local v=a[name] + if value then + return v==value + else + return v + end + end + end + return nil +end expressions.oneof=function(s,...) for i=1,select("#",...) do if s==select(i,...) then @@ -18621,7 +19016,7 @@ function expressions.contains(str,pattern) end return false end -function xml.expressions.idstring(str) +function expressions.idstring(str) return type(str)=="string" and gsub(str,"^#","") or "" end local function traverse(root,pattern,handle) @@ -20394,7 +20789,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 11099, stripped down to: 7152 +-- original size: 11019, stripped down to: 7086 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -20407,9 +20802,9 @@ local next,type,getmetatable,rawset=next,type,getmetatable,rawset local gsub,find,gmatch,char=string.gsub,string.find,string.gmatch,string.char local filedirname,filebasename,filejoin=file.dirname,file.basename,file.join local ostype,osname,osuname,ossetenv,osgetenv=os.type,os.name,os.uname,os.setenv,os.getenv +local sortedpairs=table.sortedpairs local P,S,R,C,Cs,Cc,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) -local trace_detail=false trackers.register("resolvers.details",function(v) trace_detail=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) local report_initialization=logs.reporter("resolvers","initialization") resolvers=resolvers or {} @@ -20618,7 +21013,7 @@ if ostype=="unix" then rawset(t,k,v) end local colon=P(":") - for k,v in table.sortedpairs(prefixes) do + for k,v in sortedpairs(prefixes) do if p then p=P(k)+p else @@ -20645,7 +21040,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 18154, stripped down to: 10416 +-- original size: 18179, stripped down to: 10432 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -20929,7 +21324,7 @@ local function scan(files,remap,spec,path,n,m,r,onlyone,tolerant) scancache[sub(full,1,-2)]=files return files,remap,n,m,r end -function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) +local function scanfiles(path,branch,usecache,onlyonce,tolerant) local realpath=resolveprefix(path) if usecache then local content=fullcache[realpath] @@ -20984,8 +21379,9 @@ function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) statistics.stoptiming(timer) return content end +resolvers.scanfiles=scanfiles function resolvers.simplescanfiles(path,branch,usecache) - return resolvers.scanfiles(path,branch,usecache,true,true) + return scanfiles(path,branch,usecache,true,true) end function resolvers.scandata() table.sort(scanned) @@ -21052,7 +21448,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 9360, stripped down to: 6312 +-- original size: 9400, stripped down to: 6347 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -21062,7 +21458,7 @@ if not modules then modules={} end modules ['data-env']={ license="see context related readme files", } local lower,gsub=string.lower,string.gsub -local next=next +local next,rawget=next,rawget local resolvers=resolvers local allocate=utilities.storage.allocate local setmetatableindex=table.setmetatableindex @@ -21143,13 +21539,13 @@ local relations=allocate { mp={ names={ "mp" }, variable='MPINPUTS', - suffixes={ 'mp','mpvi','mpiv','mpii' }, + suffixes={ 'mp','mpvi','mpiv','mpxl','mpii' }, usertype=true, }, tex={ names={ "tex" }, variable='TEXINPUTS', - suffixes={ "tex","mkvi","mkiv","mkii","cld","lfg","xml" }, + suffixes={ "tex","mkiv","mkvi","mkxl","mklx","mkii","cld","lfg","xml" }, usertype=true, }, icc={ @@ -21337,7 +21733,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 16284, stripped down to: 10938 +-- original size: 16099, stripped down to: 11379 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -21346,48 +21742,83 @@ if not modules then modules={} end modules ['data-tmp']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local format,lower,gsub,concat=string.format,string.lower,string.gsub,table.concat -local concat=table.concat -local mkdirs,isdir,isfile=dir.mkdirs,lfs.isdir,lfs.isfile -local addsuffix,is_writable,is_readable=file.addsuffix,file.is_writable,file.is_readable -local formatters=string.formatters local next,type=next,type +local pcall,loadfile,collectgarbage=pcall,loadfile,collectgarbage +local format,lower,gsub=string.format,string.lower,string.gsub +local concat,serialize,fastserialize,serializetofile=table.concat,table.serialize,table.fastserialize,table.tofile +local mkdirs,expanddirname,isdir,isfile=dir.mkdirs,dir.expandname,lfs.isdir,lfs.isfile +local is_writable,is_readable=file.is_writable,file.is_readable +local collapsepath,joinfile,addsuffix,dirname=file.collapsepath,file.join,file.addsuffix,file.dirname +local savedata=file.savedata +local formatters=string.formatters +local osexit,osdate,osuuid=os.exit,os.date,os.uuid +local removefile=os.remove +local md5hex=md5.hex local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) local report_caches=logs.reporter("resolvers","caches") local report_resolvers=logs.reporter("resolvers","caching") local resolvers=resolvers local cleanpath=resolvers.cleanpath -local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) -local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) -local compile=utilities.lua.compile -function utilities.lua.compile(luafile,lucfile,cleanup,strip) - if cleanup==nil then cleanup=directive_cleanup end - if strip==nil then strip=directive_strip end - return compile(luafile,lucfile,cleanup,strip) +local resolvepath=resolvers.resolve +local luautilities=utilities.lua +do + local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) + local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) + local compilelua=luautilities.compile + function luautilities.compile(luafile,lucfile,cleanup,strip) + if cleanup==nil then cleanup=directive_cleanup end + if strip==nil then strip=directive_strip end + return compilelua(luafile,lucfile,cleanup,strip) + end end caches=caches or {} local caches=caches -local luasuffixes=utilities.lua.suffixes -caches.base=caches.base or "luatex-cache" -caches.more=caches.more or "context" -caches.direct=false -caches.tree=false -caches.force=true -caches.ask=false -caches.relocate=false +local writable=nil +local readables={} +local usedreadables={} +local compilelua=luautilities.compile +local luasuffixes=luautilities.suffixes +caches.base=caches.base or "luatex-cache" +caches.more=caches.more or "context" caches.defaults={ "TMPDIR","TEMPDIR","TMP","TEMP","HOME","HOMEPATH" } -directives.register("system.caches.fast",function(v) caches.fast=true end) -local writable,readables,usedreadables=nil,{},{} +local direct_cache=false +local fast_cache=false +local cache_tree=false +directives.register("system.caches.direct",function(v) direct_cache=true end) +directives.register("system.caches.fast",function(v) fast_cache=true end) +local function configfiles() + return concat(resolvers.configurationfiles(),";") +end +local function hashed(tree) + tree=gsub(tree,"[\\/]+$","") + tree=lower(tree) + local hash=md5hex(tree) + if trace_cache or trace_locating then + report_caches("hashing tree %a, hash %a",tree,hash) + end + return hash +end +local function treehash() + local tree=configfiles() + if not tree or tree=="" then + return false + else + return hashed(tree) + end +end +caches.hashed=hashed +caches.treehash=treehash +caches.configfiles=configfiles local function identify() local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") if texmfcaches then for k=1,#texmfcaches do local cachepath=texmfcaches[k] if cachepath~="" then - cachepath=resolvers.resolve(cachepath) - cachepath=resolvers.cleanpath(cachepath) - cachepath=file.collapsepath(cachepath) + cachepath=resolvepath(cachepath) + cachepath=cleanpath(cachepath) + cachepath=collapsepath(cachepath) local valid=isdir(cachepath) if valid then if is_readable(cachepath) then @@ -21396,16 +21827,14 @@ local function identify() writable=cachepath end end - elseif not writable and caches.force then - local cacheparent=file.dirname(cachepath) - if is_writable(cacheparent) and true then - if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath),"no",{ "yes","no" })=="yes" then - mkdirs(cachepath) - if isdir(cachepath) and is_writable(cachepath) then - report_caches("path %a created",cachepath) - writable=cachepath - readables[#readables+1]=cachepath - end + elseif not writable then + local cacheparent=dirname(cachepath) + if is_writable(cacheparent) then + mkdirs(cachepath) + if isdir(cachepath) and is_writable(cachepath) then + report_caches("path %a created",cachepath) + writable=cachepath + readables[#readables+1]=cachepath end end end @@ -21418,8 +21847,8 @@ local function identify() local cachepath=texmfcaches[k] cachepath=resolvers.expansion(cachepath) if cachepath~="" then - cachepath=resolvers.resolve(cachepath) - cachepath=resolvers.cleanpath(cachepath) + cachepath=resolvepath(cachepath) + cachepath=cleanpath(cachepath) local valid=isdir(cachepath) if valid and is_readable(cachepath) then if not writable and is_writable(cachepath) then @@ -21433,23 +21862,25 @@ local function identify() end if not writable then report_caches("fatal error: there is no valid writable cache path defined") - os.exit() + osexit() elseif #readables==0 then report_caches("fatal error: there is no valid readable cache path defined") - os.exit() + osexit() end - writable=dir.expandname(resolvers.cleanpath(writable)) - local base,more,tree=caches.base,caches.more,caches.tree or caches.treehash() + writable=expanddirname(cleanpath(writable)) + local base=caches.base + local more=caches.more + local tree=cache_tree or treehash() if tree then - caches.tree=tree + cache_tree=tree writable=mkdirs(writable,base,more,tree) for i=1,#readables do - readables[i]=file.join(readables[i],base,more,tree) + readables[i]=joinfile(readables[i],base,more,tree) end else writable=mkdirs(writable,base,more) for i=1,#readables do - readables[i]=file.join(readables[i],base,more) + readables[i]=joinfile(readables[i],base,more) end end if trace_cache then @@ -21486,27 +21917,8 @@ function caches.usedpaths(separator) return writable or "?" end end -function caches.configfiles() - return concat(resolvers.configurationfiles(),";") -end -function caches.hashed(tree) - tree=gsub(tree,"[\\/]+$","") - tree=lower(tree) - local hash=md5.hex(tree) - if trace_cache or trace_locating then - report_caches("hashing tree %a, hash %a",tree,hash) - end - return hash -end -function caches.treehash() - local tree=caches.configfiles() - if not tree or tree=="" then - return false - else - return caches.hashed(tree) - end -end -local r_cache,w_cache={},{} +local r_cache={} +local w_cache={} local function getreadablepaths(...) local tags={... } local hash=concat(tags,"/") @@ -21516,7 +21928,7 @@ local function getreadablepaths(...) if #tags>0 then done={} for i=1,#readables do - done[i]=file.join(readables[i],...) + done[i]=joinfile(readables[i],...) end else done=readables @@ -21540,17 +21952,25 @@ local function getwritablepath(...) end return done end -caches.getreadablepaths=getreadablepaths -caches.getwritablepath=getwritablepath -function caches.getfirstreadablefile(filename,...) - local fullname,path=caches.setfirstwritablefile(filename,...) +local function setfirstwritablefile(filename,...) + local wr=getwritablepath(...) + local fullname=joinfile(wr,filename) + return fullname,wr +end +local function setluanames(path,name) + return + format("%s/%s.%s",path,name,luasuffixes.tma), + format("%s/%s.%s",path,name,luasuffixes.tmc) +end +local function getfirstreadablefile(filename,...) + local fullname,path=setfirstwritablefile(filename,...) if is_readable(fullname) then return fullname,path end local rd=getreadablepaths(...) for i=1,#rd do local path=rd[i] - local fullname=file.join(path,filename) + local fullname=joinfile(path,filename) if is_readable(fullname) then usedreadables[i]=true return fullname,path @@ -21558,19 +21978,11 @@ function caches.getfirstreadablefile(filename,...) end return fullname,path end -function caches.setfirstwritablefile(filename,...) - local wr=getwritablepath(...) - local fullname=file.join(wr,filename) - return fullname,wr -end -function caches.define(category,subcategory) - return function() - return getwritablepath(category,subcategory) - end -end -function caches.setluanames(path,name) - return format("%s/%s.%s",path,name,luasuffixes.tma),format("%s/%s.%s",path,name,luasuffixes.tmc) -end +caches.getreadablepaths=getreadablepaths +caches.getwritablepath=getwritablepath +caches.setfirstwritablefile=setfirstwritablefile +caches.getfirstreadablefile=getfirstreadablefile +caches.setluanames=setluanames function caches.loaddata(readables,name,writable) if type(readables)=="string" then readables={ readables } @@ -21578,21 +21990,22 @@ function caches.loaddata(readables,name,writable) for i=1,#readables do local path=readables[i] local loader=false - local tmaname,tmcname=caches.setluanames(path,name) + local state=false + local tmaname,tmcname=setluanames(path,name) if isfile(tmcname) then - loader=loadfile(tmcname) + state,loader=pcall(loadfile,tmcname) end if not loader and isfile(tmaname) then - local tmacrap,tmcname=caches.setluanames(writable,name) + local tmacrap,tmcname=setluanames(writable,name) if isfile(tmcname) then - loader=loadfile(tmcname) + state,loader=pcall(loadfile,tmcname) end - utilities.lua.compile(tmaname,tmcname) + compilelua(tmaname,tmcname) if isfile(tmcname) then - loader=loadfile(tmcname) + state,loader=pcall(loadfile,tmcname) end if not loader then - loader=loadfile(tmaname) + state,loader=pcall(loadfile,tmaname) end end if loader then @@ -21604,21 +22017,21 @@ function caches.loaddata(readables,name,writable) return false end function caches.is_writable(filepath,filename) - local tmaname,tmcname=caches.setluanames(filepath,filename) + local tmaname,tmcname=setluanames(filepath,filename) return is_writable(tmaname) end -local saveoptions={ compact=true } -function caches.savedata(filepath,filename,data,raw) - local tmaname,tmcname=caches.setluanames(filepath,filename) - data.cache_uuid=os.uuid() - if caches.fast then - file.savedata(tmaname,table.fastserialize(data,true)) - elseif caches.direct then - file.savedata(tmaname,table.serialize(data,true,saveoptions)) +local saveoptions={ compact=true,accurate=not JITSUPPORTED } +function caches.savedata(filepath,filename,data,fast) + local tmaname,tmcname=setluanames(filepath,filename) + data.cache_uuid=osuuid() + if fast or fast_cache then + savedata(tmaname,fastserialize(data,true)) + elseif direct_cache then + savedata(tmaname,serialize(data,true,saveoptions)) else - table.tofile(tmaname,data,true,saveoptions) + serializetofile(tmaname,data,true,saveoptions) end - utilities.lua.compile(tmaname,tmcname) + compilelua(tmaname,tmcname) end local content_state={} function caches.contentstate() @@ -21626,11 +22039,14 @@ function caches.contentstate() end function caches.loadcontent(cachename,dataname,filename) if not filename then - local name=caches.hashed(cachename) - local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") - filename=file.join(path,name) + local name=hashed(cachename) + local full,path=getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=joinfile(path,name) + end + local state,blob=pcall(loadfile,addsuffix(filename,luasuffixes.luc)) + if not blob then + state,blob=pcall(loadfile,addsuffix(filename,luasuffixes.lua)) end - local blob=loadfile(addsuffix(filename,luasuffixes.luc)) or loadfile(addsuffix(filename,luasuffixes.lua)) if blob then local data=blob() if data and data.content then @@ -21663,9 +22079,9 @@ function caches.collapsecontent(content) end function caches.savecontent(cachename,dataname,content,filename) if not filename then - local name=caches.hashed(cachename) - local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") - filename=file.join(path,name) + local name=hashed(cachename) + local full,path=setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=joinfile(path,name) end local luaname=addsuffix(filename,luasuffixes.lua) local lucname=addsuffix(filename,luasuffixes.luc) @@ -21676,17 +22092,17 @@ function caches.savecontent(cachename,dataname,content,filename) type=dataname, root=cachename, version=resolvers.cacheversion, - date=os.date("%Y-%m-%d"), - time=os.date("%H:%M:%S"), + date=osdate("%Y-%m-%d"), + time=osdate("%H:%M:%S"), content=content, - uuid=os.uuid(), + uuid=osuuid(), } - local ok=io.savedata(luaname,table.serialize(data,true)) + local ok=savedata(luaname,serialize(data,true)) if ok then if trace_locating then report_resolvers("category %a, cachename %a saved in %a",dataname,cachename,luaname) end - if utilities.lua.compile(luaname,lucname) then + if compilelua(luaname,lucname) then if trace_locating then report_resolvers("%a compiled to %a",dataname,lucname) end @@ -21695,7 +22111,7 @@ function caches.savecontent(cachename,dataname,content,filename) if trace_locating then report_resolvers("compiling failed for %a, deleting file %a",dataname,lucname) end - os.remove(lucname) + removefile(lucname) end elseif trace_locating then report_resolvers("unable to save %a in %a (access error)",dataname,luaname) @@ -21709,7 +22125,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5310, stripped down to: 3784 +-- original size: 5518, stripped down to: 3854 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -21718,31 +22134,45 @@ if not modules then modules={} end modules ['data-met']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local find,format=string.find,string.format -local sequenced=table.sequenced +local type=type +local find=string.find local addurlscheme,urlhashed=url.addscheme,url.hashed +local collapsepath,joinfile=file.collapsepath,file.join +local report_methods=logs.reporter("resolvers","methods") local trace_locating=false local trace_methods=false trackers.register("resolvers.locating",function(v) trace_methods=v end) trackers.register("resolvers.methods",function(v) trace_methods=v end) -local report_methods=logs.reporter("resolvers","methods") local allocate=utilities.storage.allocate local resolvers=resolvers local registered={} local function splitmethod(filename) if not filename then - return { scheme="unknown",original=filename } + return { + scheme="unknown", + original=filename, + } end if type(filename)=="table" then return filename end - filename=file.collapsepath(filename,".") + filename=collapsepath(filename,".") if not find(filename,"://",1,true) then - return { scheme="file",path=filename,original=filename,filename=filename } + return { + scheme="file", + path=filename, + original=filename, + filename=filename, + } end - local specification=url.hashed(filename) + local specification=urlhashed(filename) if not specification.scheme or specification.scheme=="" then - return { scheme="file",path=filename,original=filename,filename=filename } + return { + scheme="file", + path=filename, + original=filename, + filename=filename, + } else return specification end @@ -21751,7 +22181,8 @@ resolvers.splitmethod=splitmethod local function methodhandler(what,first,...) local method=registered[what] if method then - local how,namespace=method.how,method.namespace + local how=method.how + local namespace=method.namespace if how=="uri" or how=="url" then local specification=splitmethod(first) local scheme=specification.scheme @@ -21797,7 +22228,10 @@ local function methodhandler(what,first,...) end resolvers.methodhandler=methodhandler function resolvers.registermethod(name,namespace,how) - registered[name]={ how=how or "tag",namespace=namespace } + registered[name]={ + how=how or "tag", + namespace=namespace + } namespace["byscheme"]=function(scheme,filename,...) if scheme=="file" then return methodhandler(name,filename,...) @@ -21806,7 +22240,7 @@ function resolvers.registermethod(name,namespace,how) end end end -local concatinators=allocate { notfound=file.join } +local concatinators=allocate { notfound=joinfile } local locators=allocate { notfound=function() end } local hashers=allocate { notfound=function() end } local generators=allocate { notfound=function() end } @@ -21827,7 +22261,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 68195, stripped down to: 43680 +-- original size: 69576, stripped down to: 44470 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -21838,7 +22272,8 @@ if not modules then modules={} end modules ['data-res']={ } local gsub,find,lower,upper,match,gmatch=string.gsub,string.find,string.lower,string.upper,string.match,string.gmatch local concat,insert,remove=table.concat,table.insert,table.remove -local next,type,rawget=next,type,rawget +local next,type,rawget,loadfile=next,type,rawget,loadfile +local mergedtable=table.merged local os=os local P,S,R,C,Cc,Cs,Ct,Carg=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Carg local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns @@ -21854,13 +22289,14 @@ local joinpath=file.joinpath local is_qualified_path=file.is_qualified_path local allocate=utilities.storage.allocate local settings_to_array=utilities.parsers.settings_to_array +local urlhasscheme=url.hasscheme local getcurrentdir=lfs.currentdir local isfile=lfs.isfile local isdir=lfs.isdir local setmetatableindex=table.setmetatableindex local luasuffixes=utilities.lua.suffixes local trace_locating=false trackers .register("resolvers.locating",function(v) trace_locating=v end) -local trace_detail=false trackers .register("resolvers.details",function(v) trace_detail=v end) +local trace_details=false trackers .register("resolvers.details",function(v) trace_details=v end) local trace_expansions=false trackers .register("resolvers.expansions",function(v) trace_expansions=v end) local trace_paths=false trackers .register("resolvers.paths",function(v) trace_paths=v end) local resolve_otherwise=true directives.register("resolvers.otherwise",function(v) resolve_otherwise=v end) @@ -21879,10 +22315,17 @@ local ostype,osname,osenv,ossetenv,osgetenv=os.type,os.name,os.env,os.setenv,os. resolvers.cacheversion="1.100" resolvers.configbanner="" resolvers.homedir=environment.homedir -resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } resolvers.luacnfname="texmfcnf.lua" resolvers.luacnffallback="contextcnf.lua" resolvers.luacnfstate="unknown" +local criticalvars={ + "SELFAUTOLOC", + "SELFAUTODIR", + "SELFAUTOPARENT", + "TEXMFCNF", + "TEXMF", + "TEXOS", +} if environment.default_texmfcnf then resolvers.luacnfspec="home:texmf/web2c;"..environment.default_texmfcnf else @@ -21902,13 +22345,20 @@ local dangerous=resolvers.dangerous local suffixmap=resolvers.suffixmap resolvers.defaultsuffixes={ "tex" } local instance=nil -function resolvers.setenv(key,value,raw) +local variable +local expansion +local setenv +local getenv +local formatofsuffix=resolvers.formatofsuffix +local splitpath=resolvers.splitpath +local splitmethod=resolvers.splitmethod +setenv=function(key,value,raw) if instance then instance.environment[key]=value ossetenv(key,raw and value or resolveprefix(value)) end end -local function getenv(key) +getenv=function(key) local value=rawget(instance.environment,key) if value and value~="" then return value @@ -21918,94 +22368,97 @@ local function getenv(key) end end resolvers.getenv=getenv -resolvers.env=getenv -local function resolvevariable(k) - return instance.expansions[k] -end +resolvers.setenv=setenv local dollarstripper=lpeg.stripper("$") local inhibitstripper=P("!")^0*Cs(P(1)^0) -local somevariable=P("$")/"" -local somekey=C(R("az","AZ","09","__","--")^1) -local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) -local variableexpander=Cs((somevariable*(somekey/resolvevariable)+somethingelse)^1 ) -local cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" -local variablecleaner=Cs((cleaner+P(1))^0) -local somevariable=R("az","AZ","09","__","--")^1/resolvevariable -local variable=(P("$")/"")*(somevariable+(P("{")/"")*somevariable*(P("}")/"")) -local variableresolver=Cs((variable+P(1))^0) -local function expandedvariable(var) - return lpegmatch(variableexpander,var) or var -end -function resolvers.reset() - if trace_locating then - report_resolving("creating instance") - end - local environment={} - local variables={} - local expansions={} - local order={} - instance={ - environment=environment, - variables=variables, - expansions=expansions, - order=order, - files={}, - setups={}, - found={}, - foundintrees={}, - hashes={}, - hashed={}, - pathlists=false, - specification={}, - lists={}, - data={}, - fakepaths={}, - remember=true, - diskcache=true, - renewcache=false, - renewtree=false, - loaderror=false, - savelists=true, - pattern=nil, - force_suffixes=true, - pathstack={}, - } - setmetatableindex(variables,function(t,k) - local v - for i=1,#order do - v=order[i][k] +local expandedvariable,resolvedvariable do + local function resolveinstancevariable(k) + return instance.expansions[k] + end + local p_variable=P("$")/"" + local p_key=C(R("az","AZ","09","__","--")^1) + local p_whatever=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) + local variableexpander=Cs((p_variable*(p_key/resolveinstancevariable)+p_whatever)^1 ) + local p_cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" + local variablecleaner=Cs((p_cleaner+P(1))^0) + local p_variable=R("az","AZ","09","__","--")^1/resolveinstancevariable + local p_variable=(P("$")/"")*(p_variable+(P("{")/"")*p_variable*(P("}")/"")) + local variableresolver=Cs((p_variable+P(1))^0) + expandedvariable=function(var) + return lpegmatch(variableexpander,var) or var + end + function resolvers.reset() + if trace_locating then + report_resolving("creating instance") + end + local environment={} + local variables={} + local expansions={} + local order={} + instance={ + environment=environment, + variables=variables, + expansions=expansions, + order=order, + files={}, + setups={}, + found={}, + foundintrees={}, + hashes={}, + hashed={}, + pathlists=false, + specification={}, + lists={}, + data={}, + fakepaths={}, + remember=true, + diskcache=true, + renewcache=false, + renewtree=false, + loaderror=false, + savelists=true, + pattern=nil, + force_suffixes=true, + pathstack={}, + } + setmetatableindex(variables,function(t,k) + local v + for i=1,#order do + v=order[i][k] + if v~=nil then + t[k]=v + return v + end + end + if v==nil then + v="" + end + t[k]=v + return v + end) + local repath=resolvers.repath + setmetatableindex(environment,function(t,k) + local v=osgetenv(k) + if v==nil then + v=variables[k] + end if v~=nil then - t[k]=v - return v + v=checkedvariable(v) or "" end - end - if v==nil then - v="" - end - t[k]=v - return v - end) - setmetatableindex(environment,function(t,k) - local v=osgetenv(k) - if v==nil then - v=variables[k] - end - if v~=nil then - v=checkedvariable(v) or "" - end - v=resolvers.repath(v) - t[k]=v - return v - end) - setmetatableindex(expansions,function(t,k) - local v=environment[k] - if type(v)=="string" then - v=lpegmatch(variableresolver,v) - v=lpegmatch(variablecleaner,v) - end - t[k]=v - return v - end) + v=repath(v) + t[k]=v + return v + end) + setmetatableindex(expansions,function(t,k) + local v=environment[k] + if type(v)=="string" then + v=lpegmatch(variableresolver,v) + v=lpegmatch(variablecleaner,v) + end + t[k]=v + return v + end) + end end function resolvers.initialized() return instance~=nil @@ -22019,31 +22472,33 @@ local function reset_caches() instance.lists={} instance.pathlists=false end -local slash=P("/") -local pathexpressionpattern=Cs ( - Cc("^")*( - Cc("%")*S(".-")+slash^2*P(-1)/"/.*" +local makepathexpression do + local slash=P("/") + local pathexpressionpattern=Cs ( + Cc("^")*( + Cc("%")*S(".-")+slash^2*P(-1)/"/.*" +slash^2/"/"+(1-slash)*P(-1)*Cc("/")+P(1) - )^1*Cc("$") -) -local cache={} -local function makepathexpression(str) - if str=="." then - return "^%./$" - else - local c=cache[str] - if not c then - c=lpegmatch(pathexpressionpattern,str) - cache[str]=c + )^1*Cc("$") + ) + local cache={} + makepathexpression=function(str) + if str=="." then + return "^%./$" + else + local c=cache[str] + if not c then + c=lpegmatch(pathexpressionpattern,str) + cache[str]=c + end + return c end - return c end end local function reportcriticalvariables(cnfspec) if trace_locating then - for i=1,#resolvers.criticalvars do - local k=resolvers.criticalvars[i] - local v=resolvers.getenv(k) or "unknown" + for i=1,#criticalvars do + local k=criticalvars[i] + local v=getenv(k) or "unknown" report_resolving("variable %a set to %a",k,v) end report_resolving() @@ -22065,7 +22520,7 @@ local function identify_configuration_files() resolvers.luacnfstate="environment" end reportcriticalvariables(cnfspec) - local cnfpaths=expandedpathfromlist(resolvers.splitpath(cnfspec)) + local cnfpaths=expandedpathfromlist(splitpath(cnfspec)) local function locatecnf(luacnfname,kind) for i=1,#cnfpaths do local filepath=cnfpaths[i] @@ -22098,6 +22553,8 @@ local function identify_configuration_files() end local function load_configuration_files() local specification=instance.specification + local setups=instance.setups + local order=instance.order if #specification>0 then local luacnfname=resolvers.luacnfname for i=1,#specification do @@ -22107,7 +22564,6 @@ local function load_configuration_files() local realname=resolveprefix(filename) local blob=loadfile(realname) if blob then - local setups=instance.setups local data=blob() local parent=data and data.parent if parent then @@ -22118,7 +22574,7 @@ local function load_configuration_files() local parentdata=blob() if parentdata then report_resolving("loading configuration file %a",filename) - data=table.merged(parentdata,data) + data=mergedtable(parentdata,data) end end end @@ -22150,7 +22606,7 @@ local function load_configuration_files() if trace_locating then report_resolving("reloading configuration due to TEXMF redefinition") end - resolvers.setenv("TEXMFCNF",cnfspec) + setenv("TEXMFCNF",cnfspec) instance.specification={} identify_configuration_files() load_configuration_files() @@ -22168,7 +22624,7 @@ local function load_configuration_files() elseif trace_locating then report_resolving("skipping configuration file %a (no valid format)",filename) end - instance.order[#instance.order+1]=instance.setups[pathname] + order[#order+1]=setups[pathname] if instance.loaderror then break end @@ -22177,6 +22633,8 @@ local function load_configuration_files() report_resolving("warning: no lua configuration files found") end end +local expandedpathlist +local unexpandedpathlist function resolvers.configurationfiles() return instance.specification or {} end @@ -22193,7 +22651,7 @@ local function load_file_databases() end end local function locate_file_databases() - local texmfpaths=resolvers.expandedpathlist("TEXMF") + local texmfpaths=expandedpathlist("TEXMF") if #texmfpaths>0 then for i=1,#texmfpaths do local path=collapsepath(texmfpaths[i]) @@ -22202,7 +22660,7 @@ local function locate_file_databases() if stripped~="" then local runtime=stripped==path path=cleanpath(path) - local spec=resolvers.splitmethod(stripped) + local spec=splitmethod(stripped) if runtime and (spec.noscheme or spec.scheme=="file") then stripped="tree:///"..stripped elseif spec.scheme=="cache" or spec.scheme=="file" then @@ -22236,11 +22694,13 @@ local function generate_file_databases() end end local function save_file_databases() - for i=1,#instance.hashes do - local hash=instance.hashes[i] + local hashes=instance.hashes + local files=instance.files + for i=1,#hashes do + local hash=hashes[i] local cachename=hash.name if hash.cache then - local content=instance.files[cachename] + local content=files[cachename] caches.collapsecontent(content) if trace_locating then report_resolving("saving tree %a",cachename) @@ -22252,8 +22712,9 @@ local function save_file_databases() end end function resolvers.renew(hashname) + local files=instance.files if hashname and hashname~="" then - local expanded=resolvers.expansion(hashname) or "" + local expanded=expansion(hashname) or "" if expanded~="" then if trace_locating then report_resolving("identifying tree %a from %a",expanded,hashname) @@ -22270,7 +22731,7 @@ function resolvers.renew(hashname) report_resolving("using path %a",realpath) end methodhandler('generators',hashname) - local content=instance.files[hashname] + local content=files[hashname] caches.collapsecontent(content) if trace_locating then report_resolving("saving tree %a",hashname) @@ -22297,38 +22758,46 @@ local function load_databases() end end function resolvers.appendhash(type,name,cache) - if not instance.hashed[name] then + local hashed=instance.hashed + local hashes=instance.hashes + if hashed[name] then + else if trace_locating then report_resolving("hash %a appended",name) end - insert(instance.hashes,{ type=type,name=name,cache=cache } ) - instance.hashed[name]=cache + insert(hashes,{ type=type,name=name,cache=cache } ) + hashed[name]=cache end end function resolvers.prependhash(type,name,cache) - if not instance.hashed[name] then + local hashed=instance.hashed + local hashes=instance.hashes + if hashed[name] then + else if trace_locating then report_resolving("hash %a prepended",name) end - insert(instance.hashes,1,{ type=type,name=name,cache=cache } ) - instance.hashed[name]=cache + insert(hashes,1,{ type=type,name=name,cache=cache } ) + hashed[name]=cache end end function resolvers.extendtexmfvariable(specification) - local t=resolvers.splitpath(getenv("TEXMF")) - insert(t,1,specification) - local newspec=concat(t,",") - if instance.environment["TEXMF"] then - instance.environment["TEXMF"]=newspec - elseif instance.variables["TEXMF"] then - instance.variables["TEXMF"]=newspec + local environment=instance.environment + local variables=instance.variables + local texmftrees=splitpath(getenv("TEXMF")) + insert(texmftrees,1,specification) + texmftrees=concat(texmftrees,",") + if environment["TEXMF"] then + environment["TEXMF"]=texmftrees + elseif variables["TEXMF"] then + variables["TEXMF"]=texmftrees else end reset_hashes() end function resolvers.splitexpansions() - local ie=instance.expansions - for k,v in next,ie do + local expansions=instance.expansions + for k,v in next,expansions do local t,tn,h,p={},0,{},splitconfigurationpath(v) for kk=1,#p do local vv=p[kk] @@ -22338,33 +22807,37 @@ function resolvers.splitexpansions() h[vv]=true end end - if #t>1 then - ie[k]=t + if tn>1 then + expansions[k]=t else - ie[k]=t[1] + expansions[k]=t[1] end end end function resolvers.datastate() return caches.contentstate() end -function resolvers.variable(name) +variable=function(name) + local variables=instance.variables local name=name and lpegmatch(dollarstripper,name) - local result=name and instance.variables[name] + local result=name and variables[name] return result~=nil and result or "" end -function resolvers.expansion(name) +expansion=function(name) + local expansions=instance.expansions local name=name and lpegmatch(dollarstripper,name) - local result=name and instance.expansions[name] + local result=name and expansions[name] return result~=nil and result or "" end -function resolvers.unexpandedpathlist(str) - local pth=resolvers.variable(str) - local lst=resolvers.splitpath(pth) +resolvers.variable=variable +resolvers.expansion=expansion +unexpandedpathlist=function(str) + local pth=variable(str) + local lst=splitpath(pth) return expandedpathfromlist(lst) end function resolvers.unexpandedpath(str) - return joinpath(resolvers.unexpandedpathlist(str)) + return joinpath(unexpandedpathlist(str)) end function resolvers.pushpath(name) local pathstack=instance.pathstack @@ -22394,8 +22867,8 @@ function resolvers.stackpath() end local done={} function resolvers.resetextrapaths() - local ep=instance.extra_paths - if not ep then + local extra_paths=instance.extra_paths + if not extra_paths then done={} instance.extra_paths={} elseif #ep>0 then @@ -22416,8 +22889,8 @@ function resolvers.registerextrapath(paths,subpaths) end local paths=settings_to_array(paths) local subpaths=settings_to_array(subpaths) - local ep=instance.extra_paths or {} - local oldn=#ep + local extra_paths=instance.extra_paths or {} + local oldn=#extra_paths local newn=oldn local nofpaths=#paths local nofsubpaths=#subpaths @@ -22430,7 +22903,7 @@ function resolvers.registerextrapath(paths,subpaths) local ps=p.."/"..s if not done[ps] then newn=newn+1 - ep[newn]=cleanpath(ps) + extra_paths[newn]=cleanpath(ps) done[ps]=true end end @@ -22440,7 +22913,7 @@ function resolvers.registerextrapath(paths,subpaths) local p=paths[i] if not done[p] then newn=newn+1 - ep[newn]=cleanpath(p) + extra_paths[newn]=cleanpath(p) done[p]=true end end @@ -22449,17 +22922,17 @@ function resolvers.registerextrapath(paths,subpaths) for i=1,oldn do for j=1,nofsubpaths do local s=subpaths[j] - local ps=ep[i].."/"..s + local ps=extra_paths[i].."/"..s if not done[ps] then newn=newn+1 - ep[newn]=cleanpath(ps) + extra_paths[newn]=cleanpath(ps) done[ps]=true end end end end if newn>0 then - instance.extra_paths=ep + instance.extra_paths=extra_paths end if newn~=oldn then reset_caches() @@ -22467,17 +22940,19 @@ function resolvers.registerextrapath(paths,subpaths) end function resolvers.pushextrapath(path) local paths=settings_to_array(path) - if instance.extra_stack then - insert(instance.extra_stack,1,paths) + local extra_stack=instance.extra_stack + if extra_stack then + insert(extra_stack,1,paths) else instance.extra_stack={ paths } end reset_caches() end function resolvers.popextrapath() - if instance.extra_stack then + local extra_stack=instance.extra_stack + if extra_stack then reset_caches() - return remove(instance.extra_stack,1) + return remove(extra_stack,1) end end local function made_list(instance,list,extra_too) @@ -22506,33 +22981,21 @@ local function made_list(instance,list,extra_too) end end if extra_too then - local es=instance.extra_stack - if es and #es>0 then - for k=1,#es do - add(es[k]) + local extra_stack=instance.extra_stack + local extra_paths=instance.extra_paths + if extra_stack and #extra_stack>0 then + for k=1,#extra_stack do + add(extra_stack[k]) end end - local ep=instance.extra_paths - if ep and #ep>0 then - add(ep) + if extra_paths and #extra_paths>0 then + add(extra_paths) end end add(list) return new end -function resolvers.cleanpathlist(str) - local t=resolvers.expandedpathlist(str) - if t then - for i=1,#t do - t[i]=collapsepath(cleanpath(t[i])) - end - end - return t -end -function resolvers.expandpath(str) - return joinpath(resolvers.expandedpathlist(str)) -end -function resolvers.expandedpathlist(str,extra_too) +expandedpathlist=function(str,extra_too) if not str then return {} elseif instance.savelists then @@ -22540,40 +23003,56 @@ function resolvers.expandedpathlist(str,extra_too) local lists=instance.lists local lst=lists[str] if not lst then - local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str)),extra_too) + local l=made_list(instance,splitpath(expansion(str)),extra_too) lst=expandedpathfromlist(l) lists[str]=lst end return lst else - local lst=resolvers.splitpath(resolvers.expansion(str)) + local lst=splitpath(expansion(str)) return made_list(instance,expandedpathfromlist(lst),extra_too) end end -function resolvers.expandedpathlistfromvariable(str) +resolvers.expandedpathlist=expandedpathlist +resolvers.unexpandedpathlist=unexpandedpathlist +function resolvers.cleanpathlist(str) + local t=expandedpathlist(str) + if t then + for i=1,#t do + t[i]=collapsepath(cleanpath(t[i])) + end + end + return t +end +function resolvers.expandpath(str) + return joinpath(expandedpathlist(str)) +end +local function expandedpathlistfromvariable(str) str=lpegmatch(dollarstripper,str) local tmp=resolvers.variableofformatorsuffix(str) - return resolvers.expandedpathlist(tmp~="" and tmp or str) + return expandedpathlist(tmp~="" and tmp or str) end function resolvers.expandpathfromvariable(str) - return joinpath(resolvers.expandedpathlistfromvariable(str)) + return joinpath(expandedpathlistfromvariable(str)) end +resolvers.expandedpathlistfromvariable=expandedpathlistfromvariable function resolvers.cleanedpathlist(v) - local t=resolvers.expandedpathlist(v) + local t=expandedpathlist(v) for i=1,#t do - t[i]=resolvers.resolve(resolvers.cleanpath(t[i])) + t[i]=resolveprefix(cleanpath(t[i])) end return t end function resolvers.expandbraces(str) - local pth=expandedpathfromlist(resolvers.splitpath(str)) + local pth=expandedpathfromlist(splitpath(str)) return joinpath(pth) end function resolvers.registerfilehash(name,content,someerror) + local files=instance.files if content then - instance.files[name]=content + files[name]=content else - instance.files[name]={} + files[name]={} if somerror==true then instance.loaderror=someerror end @@ -22592,7 +23071,7 @@ function resolvers.renewcache() end local function isreadable(name) local readable=isfile(name) - if trace_detail then + if trace_details then if readable then report_resolving("file %a is readable",name) else @@ -22609,7 +23088,7 @@ local function collect_files(names) local variant=hash.type local search=filejoin(root,path,name) local result=methodhandler('concatinators',variant,root,path,name) - if trace_detail then + if trace_details then report_resolving("match: variant %a, search %a, result %a",variant,search,result) end noffiles=noffiles+1 @@ -22618,7 +23097,7 @@ local function collect_files(names) end for k=1,#names do local filename=names[k] - if trace_detail then + if trace_details then report_resolving("checking name %a",filename) end local basename=filebasename(filename) @@ -22630,12 +23109,13 @@ local function collect_files(names) pathname="/"..pathname.."$" end local hashes=instance.hashes + local files=instance.files for h=1,#hashes do local hash=hashes[h] local hashname=hash.name - local content=hashname and instance.files[hashname] + local content=hashname and files[hashname] if content then - if trace_detail then + if trace_details then report_resolving("deep checking %a, base %a, pattern %a",hashname,basename,pathname) end local path,name=lookup(content,basename) @@ -22703,7 +23183,6 @@ local function find_analyze(filename,askedformat,allresults) if askedformat=="" then if filesuffix=="" or not suffixmap[filesuffix] then local defaultsuffixes=resolvers.defaultsuffixes - local formatofsuffix=resolvers.formatofsuffix for i=1,#defaultsuffixes do local forcedname=filename..'.'..defaultsuffixes[i] wantedfiles[#wantedfiles+1]=forcedname @@ -22713,7 +23192,7 @@ local function find_analyze(filename,askedformat,allresults) end end else - filetype=resolvers.formatofsuffix(filename) + filetype=formatofsuffix(filename) if trace_locating then report_resolving("using suffix based filetype %a",filetype) end @@ -22736,7 +23215,7 @@ local function find_analyze(filename,askedformat,allresults) end local function find_direct(filename,allresults) if not dangerous[askedformat] and isreadable(filename) then - if trace_detail then + if trace_details then report_resolving("file %a found directly",filename) end return "direct",{ filename } @@ -22761,12 +23240,12 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) report_resolving("checking qualified name %a",filename) end if isreadable(filename) then - if trace_detail then + if trace_details then report_resolving("qualified file %a found",filename) end return "qualified",{ filename } end - if trace_detail then + if trace_details then report_resolving("locating qualified file %a",filename) end local forcedname,suffix="",suffixonly(filename) @@ -22774,11 +23253,11 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) local format_suffixes=askedformat=="" and resolvers.defaultsuffixes or suffixes[askedformat] if format_suffixes then for i=1,#format_suffixes do - local s=format_suffixes[i] - forcedname=filename.."."..s + local suffix=format_suffixes[i] + forcedname=filename.."."..suffix if isreadable(forcedname) then if trace_locating then - report_resolving("no suffix, forcing format filetype %a",s) + report_resolving("no suffix, forcing format filetype %a",suffix) end return "qualified",{ forcedname } end @@ -22791,7 +23270,7 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) local savedformat=askedformat local format=savedformat or "" if format=="" then - askedformat=resolvers.formatofsuffix(suffix) + askedformat=formatofsuffix(suffix) end if not format then askedformat="othertextfiles" @@ -22822,7 +23301,7 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) end local function check_subpath(fname) if isreadable(fname) then - if trace_detail then + if trace_details then report_resolving("found %a by deep scanning",fname) end return fname @@ -22830,7 +23309,7 @@ local function check_subpath(fname) end local function makepathlist(list,filetype) local typespec=resolvers.variableofformat(filetype) - local pathlist=resolvers.expandedpathlist(typespec,filetype and usertypes[filetype]) + local pathlist=expandedpathlist(typespec,filetype and usertypes[filetype]) local entry={} if pathlist and #pathlist>0 then for k=1,#pathlist do @@ -22841,7 +23320,7 @@ local function makepathlist(list,filetype) local expression=makepathexpression(pathname) local barename=gsub(pathname,"/+$","") barename=resolveprefix(barename) - local scheme=url.hasscheme(barename) + local scheme=urlhasscheme(barename) local schemename=gsub(barename,"%.%*$",'') entry[k]={ path=path, @@ -22878,7 +23357,7 @@ local function find_intree(filename,filetype,wantedfiles,allresults) dirlist[i]=filedirname(filelist[i][3]).."/" end end - if trace_detail then + if trace_details then report_resolving("checking filename %a in tree",filename) end for k=1,#pathlist do @@ -22888,7 +23367,7 @@ local function find_intree(filename,filetype,wantedfiles,allresults) local done=false if filelist then local expression=entry.expression - if trace_detail then + if trace_details then report_resolving("using pattern %a for path %a",expression,pathname) end for k=1,#filelist do @@ -22899,16 +23378,16 @@ local function find_intree(filename,filetype,wantedfiles,allresults) result[#result+1]=resolveprefix(fl[3]) done=true if allresults then - if trace_detail then + if trace_details then report_resolving("match to %a in hash for file %a and path %a, continue scanning",expression,f,d) end else - if trace_detail then + if trace_details then report_resolving("match to %a in hash for file %a and path %a, quit scanning",expression,f,d) end break end - elseif trace_detail then + elseif trace_details then report_resolving("no match to %a in hash for file %a and path %a",expression,f,d) end end @@ -22923,7 +23402,7 @@ local function find_intree(filename,filetype,wantedfiles,allresults) if not find(pname,"*",1,true) then if can_be_dir(pname) then if not done and not entry.prescanned then - if trace_detail then + if trace_details then report_resolving("quick root scan for %a",pname) end for k=1,#wantedfiles do @@ -22938,7 +23417,7 @@ local function find_intree(filename,filetype,wantedfiles,allresults) end end if not done and entry.recursive then - if trace_detail then + if trace_details then report_resolving("scanning filesystem for %a",pname) end local files=resolvers.simplescanfiles(pname,false,true) @@ -23004,7 +23483,7 @@ local function find_intree(filename,filetype,wantedfiles,allresults) end end local function find_onpath(filename,filetype,wantedfiles,allresults) - if trace_detail then + if trace_details then report_resolving("checking filename %a, filetype %a, wanted files %a",filename,filetype,concat(wantedfiles," | ")) end local result={} @@ -23046,7 +23525,9 @@ collect_instance_files=function(filename,askedformat,allresults) { find_onpath (filename,filetype,wantedfiles,true) }, { find_otherwise(filename,filetype,wantedfiles,true) }, } - local result,status,done={},{},{} + local result={} + local status={} + local done={} for k,r in next,results do local method,list=r[1],r[2] if method and list then @@ -23060,7 +23541,7 @@ collect_instance_files=function(filename,askedformat,allresults) end end end - if trace_detail then + if trace_details then report_resolving("lookup status: %s",table.serialize(status,filename)) end return result,status @@ -23117,6 +23598,9 @@ local function findfiles(filename,filetype,allresults) if not filename or filename=="" then return {} end + if allresults==nil then + allresults=true + end local result,status=collect_instance_files(filename,filetype or "",allresults) if not result or #result==0 then local lowered=lower(filename) @@ -23126,27 +23610,25 @@ local function findfiles(filename,filetype,allresults) end return result or {},status end -function resolvers.findfiles(filename,filetype) - if not filename or filename=="" then - return "" - else - return findfiles(filename,filetype,true) - end -end -function resolvers.findfile(filename,filetype) +local function findfile(filename,filetype) if not filename or filename=="" then return "" else return findfiles(filename,filetype,false)[1] or "" end end +resolvers.findfiles=findfiles +resolvers.findfile=findfile +resolvers.find_file=findfile +resolvers.find_files=findfiles function resolvers.findpath(filename,filetype) return filedirname(findfiles(filename,filetype,false)[1] or "") end local function findgivenfiles(filename,allresults) + local hashes=instance.hashes + local files=instance.files local base=filebasename(filename) local result={} - local hashes=instance.hashes local function okay(hash,path,name) local found=methodhandler('concatinators',hash.type,hash.name,path,name) if found and found~="" then @@ -23156,7 +23638,7 @@ local function findgivenfiles(filename,allresults) end for k=1,#hashes do local hash=hashes[k] - local content=instance.files[hash.name] + local content=files[hash.name] if content then local path,name=lookup(content,base) if not path then @@ -23188,14 +23670,14 @@ function resolvers.wildcardpattern(pattern) return lpegmatch(makewildcard,pattern) or pattern end local function findwildcardfiles(filename,allresults,result) + local files=instance.files + local hashes=instance.hashes local result=result or {} local base=filebasename(filename) local dirn=filedirname(filename) local path=lower(lpegmatch(makewildcard,dirn) or dirn) local name=lower(lpegmatch(makewildcard,base) or base) - local files=instance.files if find(name,"*",1,true) then - local hashes=instance.hashes local function okay(found,path,base,hashname,hashtype) if find(found,path) then local full=methodhandler('concatinators',hashtype,hashname,found,base) @@ -23235,7 +23717,6 @@ local function findwildcardfiles(filename,allresults,result) end end end - local hashes=instance.hashes for k=1,#hashes do local hash=hashes[k] local hashname=hash.name @@ -23265,13 +23746,21 @@ end function resolvers.findwildcardfile(filename) return findwildcardfiles(filename,false)[1] or "" end -function resolvers.automount() -end -function resolvers.starttiming() - statistics.starttiming(instance) +do + local starttiming=statistics.starttiming + local stoptiming=statistics.stoptiming + local elapsedtime=statistics.elapsedtime + function resolvers.starttiming() + starttiming(instance) + end + function resolvers.stoptiming() + stoptiming(instance) + end + function resolvers.loadtime() + return elapsedtime(instance) + end end -function resolvers.stoptiming() - statistics.stoptiming(instance) +function resolvers.automount() end function resolvers.load(option) resolvers.starttiming() @@ -23285,9 +23774,6 @@ function resolvers.load(option) local files=instance.files return files and next(files) and true end -function resolvers.loadtime() - return statistics.elapsedtime(instance) -end local function report(str) if trace_locating then report_resolving(str) @@ -23317,7 +23803,7 @@ function resolvers.dowithfilesandreport(command,files,...) end end function resolvers.showpath(str) - return joinpath(resolvers.expandedpathlist(resolvers.formatofvariable(str))) + return joinpath(expandedpathlist(resolvers.formatofvariable(str))) end function resolvers.registerfile(files,name,path) if files[name] then @@ -23331,7 +23817,7 @@ function resolvers.registerfile(files,name,path) end end function resolvers.dowithpath(name,func) - local pathlist=resolvers.expandedpathlist(name) + local pathlist=expandedpathlist(name) for i=1,#pathlist do func("^"..cleanpath(pathlist[i])) end @@ -23341,11 +23827,11 @@ function resolvers.dowithvariable(name,func) end function resolvers.locateformat(name) local engine=environment.ownmain or "luatex" - local barename=removesuffix(name) + local barename=removesuffix(file.basename(name)) local fullname=addsuffix(barename,"fmt") local fmtname=caches.getfirstreadablefile(fullname,"formats",engine) or "" if fmtname=="" then - fmtname=resolvers.findfile(fullname) + fmtname=findfile(fullname) fmtname=cleanpath(fmtname) end if fmtname~="" then @@ -23354,17 +23840,17 @@ function resolvers.locateformat(name) local lucname=addsuffix(barename,luasuffixes.luc) local luiname=addsuffix(barename,luasuffixes.lui) if isfile(luiname) then - return barename,luiname + return fmtname,luiname elseif isfile(lucname) then - return barename,lucname + return fmtname,lucname elseif isfile(luaname) then - return barename,luaname + return fmtname,luaname end end return nil,nil end function resolvers.booleanvariable(str,default) - local b=resolvers.expansion(str) + local b=expansion(str) if b=="" then return default else @@ -23374,6 +23860,7 @@ function resolvers.booleanvariable(str,default) end function resolvers.dowithfilesintree(pattern,handle,before,after) local hashes=instance.hashes + local files=instance.files for i=1,#hashes do local hash=hashes[i] local blobtype=hash.type @@ -23385,7 +23872,7 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) if before then before(blobtype,blobpath,pattern) end - for path,name in filtered(instance.files[blobpath],pattern) do + for path,name in filtered(files[blobpath],pattern) do if type(path)=="string" then checked=checked+1 if handle(blobtype,blobpath,path,name) then @@ -23406,10 +23893,6 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) end end end -local obsolete=resolvers.obsolete or {} -resolvers.obsolete=obsolete -resolvers.find_file=resolvers.findfile obsolete.find_file=resolvers.findfile -resolvers.find_files=resolvers.findfiles obsolete.find_files=resolvers.findfiles function resolvers.knownvariables(pattern) if instance then local environment=instance.environment @@ -23443,7 +23926,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 4854, stripped down to: 2889 +-- original size: 5088, stripped down to: 3144 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -23452,6 +23935,7 @@ if not modules then modules={} end modules ['data-pre']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local insert,remove=table.insert,table.remove local resolvers=resolvers local prefixes=resolvers.prefixes local cleanpath=resolvers.cleanpath @@ -23529,8 +24013,9 @@ prefixes.kpse=prefixes.locate prefixes.full=prefixes.locate prefixes.file=prefixes.filename prefixes.path=prefixes.pathname +local inputstack={} +local stackpath=resolvers.stackpath local function toppath() - local inputstack=resolvers.inputstack if not inputstack then return "." end @@ -23542,15 +24027,23 @@ local function toppath() end end local function jobpath() - local path=resolvers.stackpath() + local path=stackpath() if not path or path=="" then return "." else return path end end +local function pushinputname(name) + insert(inputstack,name) +end +local function popinputname(name) + return remove(inputstack) +end resolvers.toppath=toppath resolvers.jobpath=jobpath +resolvers.pushinputname=pushinputname +resolvers.popinputname=popinputname prefixes.toppath=function(str) return cleanpath(joinpath(toppath(),str)) end prefixes.jobpath=function(str) return cleanpath(joinpath(jobpath(),str)) end resolvers.setdynamic("toppath") @@ -23593,7 +24086,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-out"] = package.loaded["data-out"] or true --- original size: 530, stripped down to: 470 +-- original size: 551, stripped down to: 470 if not modules then modules={} end modules ['data-out']={ version=1.001, @@ -23616,7 +24109,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3863, stripped down to: 3170 +-- original size: 4365, stripped down to: 3588 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -23625,39 +24118,46 @@ if not modules then modules={} end modules ['data-fil']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local ioopen=io.open +local isdir=lfs.isdir local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_files=logs.reporter("resolvers","files") local resolvers=resolvers local resolveprefix=resolvers.resolve -local finders,openers,loaders,savers=resolvers.finders,resolvers.openers,resolvers.loaders,resolvers.savers -local locators,hashers,generators,concatinators=resolvers.locators,resolvers.hashers,resolvers.generators,resolvers.concatinators +local findfile=resolvers.findfile +local scanfiles=resolvers.scanfiles +local registerfilehash=resolvers.registerfilehash +local appendhash=resolvers.appendhash +local loadcachecontent=caches.loadcontent local checkgarbage=utilities.garbagecollector and utilities.garbagecollector.check -function locators.file(specification) +function resolvers.locators.file(specification) local filename=specification.filename local realname=resolveprefix(filename) - if realname and realname~='' and lfs.isdir(realname) then + if realname and realname~='' and isdir(realname) then if trace_locating then report_files("file locator %a found as %a",filename,realname) end - resolvers.appendhash('file',filename,true) + appendhash('file',filename,true) elseif trace_locating then report_files("file locator %a not found",filename) end end -function hashers.file(specification) +function resolvers.hashers.file(specification) local pathname=specification.filename - local content=caches.loadcontent(pathname,'files') - resolvers.registerfilehash(pathname,content,content==nil) + local content=loadcachecontent(pathname,'files') + registerfilehash(pathname,content,content==nil) end -function generators.file(specification) +function resolvers.generators.file(specification) local pathname=specification.filename - local content=resolvers.scanfiles(pathname,false,true) - resolvers.registerfilehash(pathname,content,true) + local content=scanfiles(pathname,false,true) + registerfilehash(pathname,content,true) end -concatinators.file=file.join +resolvers.concatinators.file=file.join +local finders=resolvers.finders +local notfound=finders.notfound function finders.file(specification,filetype) local filename=specification.filename - local foundname=resolvers.findfile(filename,filetype) + local foundname=findfile(filename,filetype) if foundname and foundname~="" then if trace_locating then report_files("file finder: %a found",filename) @@ -23667,37 +24167,55 @@ function finders.file(specification,filetype) if trace_locating then report_files("file finder: %a not found",filename) end - return finders.notfound() + return notfound() end end -function openers.helpers.textopener(tag,filename,f) +local openers=resolvers.openers +local notfound=openers.notfound +local overloaded=false +local function textopener(tag,filename,f) return { - reader=function() return f:read () end, - close=function() logs.show_close(filename) return f:close() end, + reader=function() return f:read () end, + close=function() return f:close() end, } end +function openers.helpers.textopener(...) + return textopener(...) +end +function openers.helpers.settextopener(opener) + if overloaded then + report_files("file opener: %s overloaded","already") + else + if trace_locating then + report_files("file opener: %s overloaded","once") + end + overloaded=true + textopener=opener + end +end function openers.file(specification,filetype) local filename=specification.filename if filename and filename~="" then - local f=io.open(filename,"r") + local f=ioopen(filename,"r") if f then if trace_locating then report_files("file opener: %a opened",filename) end - return openers.helpers.textopener("file",filename,f) + return textopener("file",filename,f) end end if trace_locating then report_files("file opener: %a not found",filename) end - return openers.notfound() + return notfound() end +local loaders=resolvers.loaders +local notfound=loaders.notfound function loaders.file(specification,filetype) local filename=specification.filename if filename and filename~="" then - local f=io.open(filename,"rb") + local f=ioopen(filename,"rb") if f then - logs.show_load(filename) if trace_locating then report_files("file loader: %a loaded",filename) end @@ -23714,7 +24232,7 @@ function loaders.file(specification,filetype) if trace_locating then report_files("file loader: %a not found",filename) end - return loaders.notfound() + return notfound() end @@ -23724,7 +24242,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-con"] = package.loaded["data-con"] or true --- original size: 5029, stripped down to: 3432 +-- original size: 5388, stripped down to: 3685 if not modules then modules={} end modules ['data-con']={ version=1.100, @@ -23733,6 +24251,7 @@ if not modules then modules={} end modules ['data-con']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local setmetatable=setmetatable local format,lower,gsub=string.format,string.lower,string.gsub local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) local trace_containers=false trackers.register("resolvers.containers",function(v) trace_containers=v end) @@ -23740,16 +24259,21 @@ local trace_storage=false trackers.register("resolvers.storage",function(v) tra containers=containers or {} local containers=containers containers.usecache=true +local getwritablepath=caches.getwritablepath +local getreadablepaths=caches.getreadablepaths +local cacheiswritable=caches.is_writable +local loaddatafromcache=caches.loaddata +local savedataincache=caches.savedata local report_containers=logs.reporter("resolvers","containers") local allocated={} local mt={ __index=function(t,k) if k=="writable" then - local writable=caches.getwritablepath(t.category,t.subcategory) or { "." } + local writable=getwritablepath(t.category,t.subcategory) or { "." } t.writable=writable return writable elseif k=="readables" then - local readables=caches.getreadablepaths(t.category,t.subcategory) or { "." } + local readables=getreadablepaths(t.category,t.subcategory) or { "." } t.readables=readables return readables end @@ -23780,7 +24304,7 @@ function containers.define(category,subcategory,version,enabled) end end function containers.is_usable(container,name) - return container.enabled and caches and caches.is_writable(container.writable,name) + return container.enabled and caches and cacheiswritable(container.writable,name) end function containers.is_valid(container,name) if name and name~="" then @@ -23794,7 +24318,7 @@ function containers.read(container,name) local storage=container.storage local stored=storage[name] if not stored and container.enabled and caches and containers.usecache then - stored=caches.loaddata(container.readables,name,container.writable) + stored=loaddatafromcache(container.readables,name,container.writable) if stored and stored.cache_version==container.version then if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","load",container.subcategory,name) @@ -23810,17 +24334,20 @@ function containers.read(container,name) end return stored end -function containers.write(container,name,data) +function containers.write(container,name,data,fast) if data then data.cache_version=container.version if container.enabled and caches then - local unique,shared=data.unique,data.shared - data.unique,data.shared=nil,nil - caches.savedata(container.writable,name,data) + local unique=data.unique + local shared=data.shared + data.unique=nil + data.shared=nil + savedataincache(container.writable,name,data,fast) if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","save",container.subcategory,name) end - data.unique,data.shared=unique,shared + data.unique=unique + data.shared=shared end if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","store",container.subcategory,name) @@ -23843,7 +24370,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-use"] = package.loaded["data-use"] or true --- original size: 4434, stripped down to: 3180 +-- original size: 5790, stripped down to: 2910 if not modules then modules={} end modules ['data-use']={ version=1.001, @@ -23852,40 +24379,11 @@ if not modules then modules={} end modules ['data-use']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local format,lower,gsub,find=string.format,string.lower,string.gsub,string.find +local format=string.format local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_mounts=logs.reporter("resolvers","mounts") local resolvers=resolvers -resolvers.automounted=resolvers.automounted or {} -function resolvers.automount(usecache) - local mountpaths=resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) - if (not mountpaths or #mountpaths==0) and usecache then - mountpaths=caches.getreadablepaths("mount") - end - if mountpaths and #mountpaths>0 then - resolvers.starttiming() - for k=1,#mountpaths do - local root=mountpaths[k] - local f=io.open(root.."/url.tmi") - if f then - for line in f:lines() do - if line then - if find(line,"^[%%#%-]") then - elseif find(line,"^zip://") then - if trace_locating then - report_mounts("mounting %a",line) - end - table.insert(resolvers.automounted,line) - resolvers.usezipfile(line) - end - end - end - f:close() - end - end - resolvers.stoptiming() - end -end +local findfile=resolvers.findfile statistics.register("used config file",function() return caches.configfiles() end) statistics.register("used cache path",function() return caches.usedpaths() end) function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) @@ -23895,9 +24393,11 @@ function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) local luvdata={ enginebanner=enginebanner, formatbanner=formatbanner, - sourcehash=md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), + sourcehash=md5.hex(io.loaddata(findfile(sourcefile)) or "unknown"), sourcefile=sourcefile, luaversion=LUAVERSION, + formatid=LUATEXFORMATID, + functionality=LUATEXFUNCTIONALITY, } io.savedata(luvname,table.serialize(luvdata,true)) lua.registerfinalizer(function() @@ -23917,7 +24417,7 @@ function statistics.checkfmtstatus(texname) if lfs.isfile(luvname) then local luv=dofile(luvname) if luv and luv.sourcefile then - local sourcehash=md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") + local sourcehash=md5.hex(io.loaddata(findfile(luv.sourcefile)) or "unknown") local luvbanner=luv.enginebanner or "?" if luvbanner~=enginebanner then return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) @@ -23927,8 +24427,19 @@ function statistics.checkfmtstatus(texname) return format("source mismatch (luv: %s <> bin: %s)",luvhash,sourcehash) end local luvluaversion=luv.luaversion or 0 - if luvluaversion~=LUAVERSION then - return format("lua mismatch (luv: %s <> bin: %s)",luvluaversion,LUAVERSION) + local engluaversion=LUAVERSION or 0 + if luvluaversion~=engluaversion then + return format("lua mismatch (luv: %s <> bin: %s)",luvluaversion,engluaversion) + end + local luvfunctionality=luv.functionality or 0 + local engfunctionality=status.development_id or 0 + if luvfunctionality~=engfunctionality then + return format("functionality mismatch (luv: %s <> bin: %s)",luvfunctionality,engfunctionality) + end + local luvformatid=luv.formatid or 0 + local engformatid=status.format_id or 0 + if luvformatid~=engformatid then + return format("formatid mismatch (luv: %s <> bin: %s)",luvformatid,engformatid) end else return "invalid status file" @@ -23947,7 +24458,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 10263, stripped down to: 7556 +-- original size: 10725, stripped down to: 7949 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -23960,6 +24471,14 @@ local format,find,match=string.format,string.find,string.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_zip=logs.reporter("resolvers","zip") local resolvers=resolvers +local findfile=resolvers.findfile +local registerfile=resolvers.registerfile +local splitmethod=resolvers.splitmethod +local prependhash=resolvers.prependhash +local starttiming=resolvers.starttiming +local extendtexmf=resolvers.extendtexmfvariable +local stoptiming=resolvers.stoptiming +local urlquery=url.query zip=zip or {} local zip=zip local archives=zip.archives or {} @@ -23974,8 +24493,9 @@ if zipfiles then closezip=zipfiles.close validfile=zipfiles.found wholefile=zipfiles.unzip + local listzip=zipfiles.list traversezip=function(zfile) - return ipairs(zipfiles.list(zfile)) + return ipairs(listzip(zfile)) end local streams=utilities.streams local openstream=streams.open @@ -24032,28 +24552,30 @@ local function validzip(str) return str end end -function zip.openarchive(name) +local function openarchive(name) if not name or name=="" then return nil else local arch=archives[name] if not arch then - local full=resolvers.findfile(name) or "" + local full=findfile(name) or "" arch=full~="" and openzip(full) or false archives[name]=arch end return arch end end -function zip.closearchive(name) +local function closearchive(name) if not name or (name=="" and archives[name]) then closezip(archives[name]) archives[name]=nil end end +zip.openarchive=openarchive +zip.closearchive=closearchive function resolvers.locators.zip(specification) local archive=specification.filename - local zipfile=archive and archive~="" and zip.openarchive(archive) + local zipfile=archive and archive~="" and openarchive(archive) if trace_locating then if zipfile then report_zip("locator: archive %a found",archive) @@ -24062,13 +24584,6 @@ function resolvers.locators.zip(specification) end end end -function resolvers.hashers.zip(specification) - local archive=specification.filename - if trace_locating then - report_zip("loading file %a",archive) - end - resolvers.usezipfile(specification.original) -end function resolvers.concatinators.zip(zipfile,path,name) if not path or path=="" then return format('%s?name=%s',zipfile,name) @@ -24076,14 +24591,16 @@ function resolvers.concatinators.zip(zipfile,path,name) return format('%s?name=%s/%s',zipfile,path,name) end end -function resolvers.finders.zip(specification) +local finders=resolvers.finders +local notfound=finders.notfound +function finders.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("finder: archive %a found",archive) @@ -24104,16 +24621,19 @@ function resolvers.finders.zip(specification) if trace_locating then report_zip("finder: %a not found",original) end - return resolvers.finders.notfound() + return notfound() end -function resolvers.openers.zip(specification) +local openers=resolvers.openers +local notfound=openers.notfound +local textopener=openers.helpers.textopener +function openers.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("opener; archive %a opened",archive) @@ -24123,7 +24643,7 @@ function resolvers.openers.zip(specification) if trace_locating then report_zip("opener: file %a found",queryname) end - return resolvers.openers.helpers.textopener('zip',original,handle) + return textopener('zip',original,handle) elseif trace_locating then report_zip("opener: file %a not found",queryname) end @@ -24135,23 +24655,24 @@ function resolvers.openers.zip(specification) if trace_locating then report_zip("opener: %a not found",original) end - return resolvers.openers.notfound() + return notfound() end -function resolvers.loaders.zip(specification) +local loaders=resolvers.loaders +local notfound=loaders.notfound +function loaders.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("loader: archive %a opened",archive) end local data=wholefile(zfile,queryname) if data then - logs.show_load(original) if trace_locating then report_zip("loader; file %a loaded",original) end @@ -24167,47 +24688,24 @@ function resolvers.loaders.zip(specification) if trace_locating then report_zip("loader: %a not found",original) end - return resolvers.openers.notfound() -end -function resolvers.usezipfile(archive) - local specification=resolvers.splitmethod(archive) - local archive=specification.filename - if archive and not registeredfiles[archive] then - local z=zip.openarchive(archive) - if z then - local tree=url.query(specification.query).tree or "" - if trace_locating then - report_zip("registering: archive %a",archive) - end - resolvers.starttiming() - resolvers.prependhash('zip',archive) - resolvers.extendtexmfvariable(archive) - registeredfiles[archive]=z - resolvers.registerfilehash(archive,resolvers.registerzipfile(z,tree)) - resolvers.stoptiming() - elseif trace_locating then - report_zip("registering: unknown archive %a",archive) - end - elseif trace_locating then - report_zip("registering: archive %a not found",archive) - end + return notfound() end -function resolvers.registerzipfile(z,tree) +local function registerzipfile(z,tree) local names={} local files={} local remap={} local n=0 local filter=tree=="" and "^(.+)/(.-)$" or format("^%s/(.+)/(.-)$",tree) - local register=resolvers.registerfile if trace_locating then report_zip("registering: using filter %a",filter) end + starttiming() for i in traversezip(z) do local filename=i.filename local path,name=match(filename,filter) if not path then n=n+1 - register(names,filename,"") + registerfile(names,filename,"") local usedname=lower(filename) files[usedname]="" if usedname~=filename then @@ -24224,12 +24722,43 @@ function resolvers.registerzipfile(z,tree) else end end + stoptiming() report_zip("registering: %s files registered",n) return { files=files, remap=remap, } end +local function usezipfile(archive) + local specification=splitmethod(archive) + local archive=specification.filename + if archive and not registeredfiles[archive] then + local z=openarchive(archive) + if z then + local tree=urlquery(specification.query).tree or "" + if trace_locating then + report_zip("registering: archive %a",archive) + end + prependhash('zip',archive) + extendtexmf(archive) + registeredfiles[archive]=z + registerfilehash(archive,registerzipfile(z,tree)) + elseif trace_locating then + report_zip("registering: unknown archive %a",archive) + end + elseif trace_locating then + report_zip("registering: archive %a not found",archive) + end +end +resolvers.usezipfile=usezipfile +resolvers.registerzipfile=registerzipfile +function resolvers.hashers.zip(specification) + local archive=specification.filename + if trace_locating then + report_zip("loading file %a",archive) + end + usezipfile(specification.original) +end end -- of closure @@ -24238,7 +24767,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 8478, stripped down to: 5223 +-- original size: 10802, stripped down to: 6619 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -24247,180 +24776,255 @@ if not modules then modules={} end modules ['data-tre']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local type=type local find,gsub,lower=string.find,string.gsub,string.lower -local basename,dirname,joinname=file.basename,file.dirname,file .join +local basename,dirname,joinname=file.basename,file.dirname,file.join local globdir,isdir,isfile=dir.glob,lfs.isdir,lfs.isfile local P,lpegmatch=lpeg.P,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_trees=logs.reporter("resolvers","trees") local resolvers=resolvers -local resolveprefix=resolvers.resolve -local notfound=resolvers.finders.notfound -local lookup=resolvers.get_from_content -local collectors={} -local found={} -function resolvers.finders.tree(specification) - local spec=specification.filename - local okay=found[spec] - if okay==nil then - if spec~="" then - local path=dirname(spec) - local name=basename(spec) - if path=="" then - path="." - end - local names=collectors[path] - if not names then - local pattern=find(path,"/%*+$") and path or (path.."/*") - names=globdir(pattern) - collectors[path]=names - end - local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" - for i=1,#names do - local fullname=names[i] - if find(fullname,pattern) then - found[spec]=fullname - return fullname - end - end - local pattern=lower(pattern) - for i=1,#names do - local fullname=lower(names[i]) - if find(fullname,pattern) then - if isfile(fullname) then +local finders=resolvers.finders +local openers=resolvers.openers +local loaders=resolvers.loaders +local locators=resolvers.locators +local hashers=resolvers.hashers +local generators=resolvers.generators +do + local collectors={} + local found={} + local notfound=finders.notfound + function finders.tree(specification) + local spec=specification.filename + local okay=found[spec] + if okay==nil then + if spec~="" then + local path=dirname(spec) + local name=basename(spec) + if path=="" then + path="." + end + local names=collectors[path] + if not names then + local pattern=find(path,"/%*+$") and path or (path.."/*") + names=globdir(pattern) + collectors[path]=names + end + local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" + for i=1,#names do + local fullname=names[i] + if find(fullname,pattern) then found[spec]=fullname return fullname - else - break + end + end + local pattern=lower(pattern) + for i=1,#names do + local fullname=lower(names[i]) + if find(fullname,pattern) then + if isfile(fullname) then + found[spec]=fullname + return fullname + else + break + end end end end + okay=notfound() + found[spec]=okay end - okay=notfound() - found[spec]=okay + return okay end - return okay end -function resolvers.locators.tree(specification) - local name=specification.filename - local realname=resolveprefix(name) - if realname and realname~='' and isdir(realname) then - if trace_locating then - report_trees("locator %a found",realname) +do + local resolveprefix=resolvers.resolve + local appendhash=resolvers.appendhash + local function dolocate(specification) + local name=specification.filename + local realname=resolveprefix(name) + if realname and realname~='' and isdir(realname) then + if trace_locating then + report_trees("locator %a found",realname) + end + appendhash('tree',name,false) + elseif trace_locating then + report_trees("locator %a not found",name) end - resolvers.appendhash('tree',name,false) - elseif trace_locating then - report_trees("locator %a not found",name) end + locators.tree=dolocate + locators.dirlist=dolocate + locators.dirfile=dolocate end -function resolvers.hashers.tree(specification) - local name=specification.filename - if trace_locating then - report_trees("analyzing %a",name) - end - resolvers.methodhandler("hashers",name) - resolvers.generators.file(specification) -end -local collectors={} -local splitter=lpeg.splitat("/**/") -local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } -table.setmetatableindex(collectors,function(t,k) - local rootname=lpegmatch(stripper,k) - local dataname=joinname(rootname,"dirlist") - local content=caches.loadcontent(dataname,"files",dataname) - if not content then - content=resolvers.scanfiles(rootname,nil,nil,false,true) - caches.savecontent(dataname,"files",content,dataname) - end - t[k]=content - return content -end) -local function checked(root,p,n) - if p then - if type(p)=="table" then - for i=1,#p do - local fullname=joinname(root,p[i],n) +do + local filegenerator=generators.file + generators.dirlist=filegenerator + generators.dirfile=filegenerator +end +do + local filegenerator=generators.file + local methodhandler=resolvers.methodhandler + local function dohash(specification) + local name=specification.filename + if trace_locating then + report_trees("analyzing %a",name) + end + methodhandler("hashers",name) + filegenerator(specification) + end + hashers.tree=dohash + hashers.dirlist=dohash + hashers.dirfile=dohash +end +local resolve do + local collectors={} + local splitter=lpeg.splitat("/**/") + local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } + local loadcontent=caches.loadcontent + local savecontent=caches.savecontent + local notfound=finders.notfound + local scanfiles=resolvers.scanfiles + local lookup=resolvers.get_from_content + table.setmetatableindex(collectors,function(t,k) + local rootname=lpegmatch(stripper,k) + local dataname=joinname(rootname,"dirlist") + local content=loadcontent(dataname,"files",dataname) + if not content then + content=scanfiles(rootname,nil,nil,false,true) + savecontent(dataname,"files",content,dataname) + end + t[k]=content + return content + end) + local function checked(root,p,n) + if p then + if type(p)=="table" then + for i=1,#p do + local fullname=joinname(root,p[i],n) + if isfile(fullname) then + return fullname + end + end + else + local fullname=joinname(root,p,n) if isfile(fullname) then return fullname end end - else - local fullname=joinname(root,p,n) - if isfile(fullname) then - return fullname - end end + return notfound() end - return notfound() -end -local function resolve(specification) - local filename=specification.filename - if filename~="" then - local root,rest=lpegmatch(splitter,filename) - if root and rest then - local path,name=dirname(rest),basename(rest) - if name~=rest then - local content=collectors[root] - local p,n=lookup(content,name) - if not p then - return notfound() - end - local pattern=".*/"..path.."$" - local istable=type(p)=="table" - if istable then - for i=1,#p do - local pi=p[i] - if pi==path or find(pi,pattern) then - local fullname=joinname(root,pi,n) - if isfile(fullname) then - return fullname + resolve=function(specification) + local filename=specification.filename + if filename~="" then + local root,rest=lpegmatch(splitter,filename) + if root and rest then + local path,name=dirname(rest),basename(rest) + if name~=rest then + local content=collectors[root] + local p,n=lookup(content,name) + if not p then + return notfound() + end + local pattern=".*/"..path.."$" + local istable=type(p)=="table" + if istable then + for i=1,#p do + local pi=p[i] + if pi==path or find(pi,pattern) then + local fullname=joinname(root,pi,n) + if isfile(fullname) then + return fullname + end end end + elseif p==path or find(p,pattern) then + local fullname=joinname(root,p,n) + if isfile(fullname) then + return fullname + end end - elseif p==path or find(p,pattern) then - local fullname=joinname(root,p,n) - if isfile(fullname) then - return fullname + local queries=specification.queries + if queries and queries.option=="fileonly" then + return checked(root,p,n) + else + return notfound() end end - local queries=specification.queries - if queries and queries.option=="fileonly" then - return checked(root,p,n) - else - return notfound() - end + end + local path=dirname(filename) + local name=basename(filename) + local root=lpegmatch(stripper,path) + local content=collectors[path] + local p,n=lookup(content,name) + if p then + return checked(root,p,n) end end - local path,name=dirname(filename),basename(filename) - local root=lpegmatch(stripper,path) - local content=collectors[path] - local p,n=lookup(content,name) - if p then - return checked(root,p,n) + return notfound() + end + finders.dirlist=resolve + function finders.dirfile(specification) + local queries=specification.queries + if queries then + queries.option="fileonly" + else + specification.queries={ option="fileonly" } end + return resolve(specification) end - return notfound() end -resolvers.finders .dirlist=resolve -resolvers.locators .dirlist=resolvers.locators .tree -resolvers.hashers .dirlist=resolvers.hashers .tree -resolvers.generators.dirlist=resolvers.generators.file -resolvers.openers .dirlist=resolvers.openers .file -resolvers.loaders .dirlist=resolvers.loaders .file -function resolvers.finders.dirfile(specification) - local queries=specification.queries - if queries then - queries.option="fileonly" - else - specification.queries={ option="fileonly" } +do + local fileopener=openers.file + local fileloader=loaders.file + openers.dirlist=fileopener + loaders.dirlist=fileloader + openers.dirfile=fileopener + loaders.dirfile=fileloader +end +do + local hashfile="dirhash.lua" + local kind="HASH256" + local version=1.0 + local loadtable=table.load + local savetable=table.save + local loaddata=io.loaddata + function resolvers.dirstatus(patterns) + local t=type(patterns) + if t=="string" then + patterns={ patterns } + elseif t~="table" then + return false + end + local status=loadtable(hashfile) + if not status or status.version~=version or status.kind~=kind then + status={ + version=1.0, + kind=kind, + hashes={}, + } + end + local hashes=status.hashes + local changed={} + local action=sha2[kind] + local update={} + for i=1,#patterns do + local pattern=patterns[i] + local files=globdir(pattern) + for i=1,#files do + local name=files[i] + local hash=action(loaddata(name)) + if hashes[name]~=hash then + changed[#changed+1]=name + end + update[name]=hash + end + end + status.hashes=update + savetable(hashfile,status) + return #changed>0 and changed or false end - return resolve(specification) end -resolvers.locators .dirfile=resolvers.locators .dirlist -resolvers.hashers .dirfile=resolvers.hashers .dirlist -resolvers.generators.dirfile=resolvers.generators.dirlist -resolvers.openers .dirfile=resolvers.openers .dirlist -resolvers.loaders .dirfile=resolvers.loaders .dirlist end -- of closure @@ -24429,7 +25033,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6753, stripped down to: 5268 +-- original size: 6945, stripped down to: 5408 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -24439,8 +25043,11 @@ if not modules then modules={} end modules ['data-sch']={ license="see context related readme files" } local load,tonumber=load,tonumber -local gsub,concat,format=string.gsub,table.concat,string.format +local gsub,format=string.gsub,string.format +local sortedhash,concat=table.sortedhash,table.concat local finders,openers,loaders=resolvers.finders,resolvers.openers,resolvers.loaders +local addsuffix,suffix,splitbase=file.addsuffix,file.suffix,file.splitbase +local md5hex=md5.hex local trace_schemes=false trackers.register("resolvers.schemes",function(v) trace_schemes=v end) local report_schemes=logs.reporter("resolvers","schemes") local http=require("socket.http") @@ -24457,7 +25064,7 @@ function cleaners.none(specification) return specification.original end function cleaners.strip(specification) - local path,name=file.splitbase(specification.original) + local path,name=splitbase(specification.original) if path=="" then return (gsub(name,"[^%a%d%.]+","-")) else @@ -24465,7 +25072,7 @@ function cleaners.strip(specification) end end function cleaners.md5(specification) - return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) + return addsuffix(md5hex(specification.original),suffix(specification.path)) end local cleaner=cleaners.strip directives.register("schemes.cleanmethod",function(v) cleaner=cleaners[v] or cleaners.strip end) @@ -24485,7 +25092,7 @@ local runner=sandbox.registerrunner { name="curl resolver", method="execute", program="curl", - template="--silent --insecure --create-dirs --output %cachename% %original%", + template='--silent --insecure --create-dirs --output "%cachename%" "%original%"', checkers={ cachename="cache", original="url", @@ -24556,10 +25163,10 @@ end schemes.install=install local function http_handler(specification,cachename) local tempname=cachename..".tmp" - local f=io.open(tempname,"wb") + local handle=io.open(tempname,"wb") local status,message=http.request { url=specification.original, - sink=ltn12.sink.file(f) + sink=ltn12.sink.file(handle) } if not status then os.remove(tempname) @@ -24574,13 +25181,13 @@ install('https') install('ftp') statistics.register("scheme handling time",function() local l,r,nl,nr={},{},0,0 - for k,v in table.sortedhash(loaded) do + for k,v in sortedhash(loaded) do if v>0 then nl=nl+1 l[nl]=k..":"..v end end - for k,v in table.sortedhash(reused) do + for k,v in sortedhash(reused) do if v>0 then nr=nr+1 r[nr]=k..":"..v @@ -24588,10 +25195,10 @@ statistics.register("scheme handling time",function() end local n=nl+nr if n>0 then - l=nl>0 and concat(l) or "none" - r=nr>0 and concat(r) or "none" + if nl==0 then l={ "none" } end + if nr==0 then r={ "none" } end return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s", - statistics.elapsedtime(schemes),n,threshold,l,r) + statistics.elapsedtime(schemes),n,threshold,concat(l," "),concat(l," ")) else return nil end @@ -24624,7 +25231,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4207, stripped down to: 3041 +-- original size: 4227, stripped down to: 3049 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -24634,8 +25241,7 @@ if not modules then modules={} end modules ['data-lua']={ license="see context related readme files" } local package,lpeg=package,lpeg -local gsub=string.gsub -local concat=table.concat +local loadfile=loadfile local addsuffix=file.addsuffix local P,S,Cs,lpegmatch=lpeg.P,lpeg.S,lpeg.Cs,lpeg.match local luasuffixes={ 'tex','lua' } @@ -24646,6 +25252,8 @@ local helpers=package.helpers or {} local methods=helpers.methods or {} local resolvers=resolvers local resolveprefix=resolvers.resolve +local expandedpaths=resolvers.expandedpathlistfromvariable +local findfile=resolvers.findfile helpers.report=logs.reporter("resolvers","libraries") trackers.register("resolvers.libraries",function(v) helpers.trace=v end) trackers.register("resolvers.locating",function(v) helpers.trace=v end) @@ -24674,7 +25282,7 @@ local function getluaformatpaths() if not luaformatpaths then luaformatpaths={} for i=1,#luaformats do - registerpath("lua format","lua",luaformatpaths,resolvers.expandedpathlistfromvariable(luaformats[i])) + registerpath("lua format","lua",luaformatpaths,expandedpaths(luaformats[i])) end end return luaformatpaths @@ -24683,7 +25291,7 @@ local function getlibformatpaths() if not libformatpaths then libformatpaths={} for i=1,#libformats do - registerpath("lib format","lib",libformatpaths,resolvers.expandedpathlistfromvariable(libformats[i])) + registerpath("lib format","lib",libformatpaths,expandedpaths(libformats[i])) end end return libformatpaths @@ -24693,7 +25301,7 @@ local function loadedbyformat(name,rawname,suffixes,islib,what) local report=helpers.report for i=1,#suffixes do local format=suffixes[i] - local resolved=resolvers.findfile(name,format) or "" + local resolved=findfile(name,format) or "" if trace then report("%s format, identifying %a using format %a",what,name,format) end @@ -24731,7 +25339,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-aux"] = package.loaded["data-aux"] or true --- original size: 2452, stripped down to: 1877 +-- original size: 2610, stripped down to: 2019 if not modules then modules={} end modules ['data-aux']={ version=1.001, @@ -24742,24 +25350,28 @@ if not modules then modules={} end modules ['data-aux']={ } local find=string.find local type,next=type,next +local addsuffix,removesuffix=file.addsuffix,file.removesuffix +local loaddata,savedata=io.loaddata,io.savedata local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local resolvers=resolvers +local cleanpath=resolvers.cleanpath +local findfiles=resolvers.findfiles local report_scripts=logs.reporter("resolvers","scripts") function resolvers.updatescript(oldname,newname) local scriptpath="context/lua" - newname=file.addsuffix(newname,"lua") - local oldscript=resolvers.cleanpath(oldname) + local oldscript=cleanpath(oldname) + local newname=addsuffix(newname,"lua") + local newscripts=findfiles(newname) or {} if trace_locating then report_scripts("to be replaced old script %a",oldscript) end - local newscripts=resolvers.findfiles(newname) or {} if #newscripts==0 then if trace_locating then report_scripts("unable to locate new script") end else for i=1,#newscripts do - local newscript=resolvers.cleanpath(newscripts[i]) + local newscript=cleanpath(newscripts[i]) if trace_locating then report_scripts("checking new script %a",newscript) end @@ -24771,17 +25383,17 @@ function resolvers.updatescript(oldname,newname) if trace_locating then report_scripts("new script should come from %a",scriptpath) end - elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then + elseif not (find(oldscript,removesuffix(newname).."$") or find(oldscript,newname.."$")) then if trace_locating then report_scripts("invalid new script name") end else - local newdata=io.loaddata(newscript) + local newdata=loaddata(newscript) if newdata then if trace_locating then report_scripts("old script content replaced by new content: %s",oldscript) end - io.savedata(oldscript,newdata) + savedata(oldscript,newdata) break elseif trace_locating then report_scripts("unable to load new script") @@ -24854,7 +25466,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 1823, stripped down to: 1542 +-- original size: 2038, stripped down to: 1696 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -24864,15 +25476,22 @@ if not modules then modules={} end modules ['data-lst']={ license="see context related readme files" } local type=type -local concat,sortedhash=table.concat,table.sortedhash +local sortedhash=table.sortedhash +local isdir=lfs.isdir local resolvers=resolvers local listers=resolvers.listers or {} resolvers.listers=listers local resolveprefix=resolvers.resolve +local configurationfiles=resolvers.configurationfiles +local expandedpathfromlist=resolvers.expandedpathfromlist +local splitpath=resolvers.splitpath +local knownvariables=resolvers.knownvariables local report_lists=logs.reporter("resolvers","lists") local report_resolved=logs.reporter("system","resolved") local function tabstr(str) - if type(str)=='table' then + if not str then + return "unset" + elseif type(str)=='table' then return concat(str," | ") else return str @@ -24882,22 +25501,22 @@ function listers.variables(pattern) local result=resolvers.knownvariables(pattern) for key,value in sortedhash(result) do report_lists(key) - report_lists(" env: %s",tabstr(value.environment or "unset")) - report_lists(" var: %s",tabstr(value.variable or "unset")) - report_lists(" exp: %s",tabstr(value.expansion or "unset")) - report_lists(" res: %s",tabstr(value.resolved or "unset")) + report_lists(" env: %s",tabstr(value.environment)) + report_lists(" var: %s",tabstr(value.variable)) + report_lists(" exp: %s",tabstr(value.expansion)) + report_lists(" res: %s",tabstr(value.resolved)) end end function listers.configurations() - local configurations=resolvers.configurationfiles() + local configurations=configurationfiles() for i=1,#configurations do report_resolved("file : %s",resolveprefix(configurations[i])) end report_resolved("") - local list=resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) + local list=expandedpathfromlist(splitpath(resolvers.luacnfspec)) for i=1,#list do local li=resolveprefix(list[i]) - if lfs.isdir(li) then + if isdir(li) then report_resolved("path - %s",li) else report_resolved("path + %s",li) @@ -24910,318 +25529,147 @@ end -- of closure do -- create closure to overcome 200 locals limit -package.loaded["util-lib"] = package.loaded["util-lib"] or true +package.loaded["libs-ini"] = package.loaded["libs-ini"] or true --- original size: 16094, stripped down to: 8443 +-- original size: 5822, stripped down to: 3629 -if not modules then modules={} end modules ['util-lib']={ +if not modules then modules={} end modules ['libs-ini']={ version=1.001, comment="companion to luat-lib.mkiv", author="Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright="PRAGMA ADE / ConTeXt Development Team", - license="see context related readme files", + license="see context related readme files" } +local type,unpack=type,unpack local type=type -local next=next -local pcall=pcall -local gsub=string.gsub -local find=string.find -local sort=table.sort -local pathpart=file.pathpart local nameonly=file.nameonly local joinfile=file.join -local removesuffix=file.removesuffix local addsuffix=file.addsuffix -local findfile=resolvers.findfile -local findfiles=resolvers.findfiles -local expandpaths=resolvers.expandedpathlistfromvariable local qualifiedpath=file.is_qualified_path local isfile=lfs.isfile -local done=false -local function locate(required,version,trace,report,action) - if type(required)~="string" then - report("provide a proper library name") - return - end - if trace then - report("requiring library %a with version %a",required,version or "any") - end - local found_library=nil - local required_full=gsub(required,"%.","/") - local required_path=pathpart(required_full) - local required_base=nameonly(required_full) - if qualifiedpath(required) then - if isfile(addsuffix(required,os.libsuffix)) then - if trace then - report("qualified name %a found",required) +local findfile=resolvers.findfile +local expandpaths=resolvers.expandedpathlistfromvariable +local report=logs.reporter("resolvers","libraries") +local trace=false +trackers.register("resolvers.lib",function(v) trace=v end) +local function findlib(required) + local suffix=os.libsuffix or "so" + if not qualifiedpath(required) then + local list=directives.value("system.librarynames" ) + local only=nameonly(required) + if type(list)=="table" then + list=list[only] + if type(list)~="table" then + list={ only } end - found_library=required else - if trace then - report("qualified name %a not found",required) - end + list={ only } end - else - local required_name=required_base.."."..os.libsuffix - local version=type(version)=="string" and version~="" and version or false - local engine="luatex" - if trace and not done then - local list=expandpaths("lib") - for i=1,#list do - report("tds path %i: %s",i,list[i]) - end - end - local function found(locate,asked_library,how,...) - if trace then - report("checking %s: %a",how,asked_library) - end - return locate(asked_library,...) + if trace then + report("using lookup list for library %a: % | t",only,list) end - local function check(locate,...) - local found=nil - if version then - local asked_library=joinfile(required_path,version,required_name) - if trace then - report("checking %s: %a","with version",asked_library) - end - found=locate(asked_library,...) + for i=1,#list do + local name=list[i] + local found=findfile(name,"lib") + if not found then + found=findfile(addsuffix(name,suffix),"lib") end - if not found or found=="" then - local asked_library=joinfile(required_path,required_name) + if found then if trace then - report("checking %s: %a","with version",asked_library) + report("library %a resolved via %a path to %a",name,"tds lib",found) end - found=locate(asked_library,...) - end - return found and found~="" and found or false - end - local function attempt(checkpattern) - if trace then - report("checking tds lib paths strictly") - end - local found=findfile and check(findfile,"lib") - if found and (not checkpattern or find(found,checkpattern)) then return found end - if trace then - report("checking tds lib paths with wildcard") - end - local asked_library=joinfile(required_path,".*",required_name) - if trace then - report("checking %s: %a","latest version",asked_library) - end - local list=findfiles(asked_library,"lib",true) - if list and #list>0 then - sort(list) - local found=list[#list] - if found and (not checkpattern or find(found,checkpattern)) then - return found - end - end - if trace then - report("checking lib paths") - end - package.extralibpath(environment.ownpath) - local paths=package.libpaths() - local pattern="/[^/]+%."..os.libsuffix.."$" - for i=1,#paths do - required_path=gsub(paths[i],pattern,"") - local found=check(lfs.isfound) - if type(found)=="string" and (not checkpattern or find(found,checkpattern)) then - return found - end - end - return false end - if engine then - if trace then - report("attemp 1, engine %a",engine) - end - found_library=attempt("/"..engine.."/") - if not found_library then - if trace then - report("attemp 2, no engine",asked_library) + if expandpaths then + local list=expandpaths("PATH") + local base=addsuffix(only,suffix) + for i=1,#list do + local full=joinfile(list[i],base) + local found=isfile(full) and full + if found then + if trace then + report("library %a resolved via %a path to %a",name,"system",found) + end + return found end - found_library=attempt() end - else - found_library=attempt() end - end - if not found_library then + elseif isfile(addsuffix(required,suffix)) then if trace then - report("not found: %a",required) + report("library with qualified name %a %sfound",required,"") end - library=false + return required else if trace then - report("found: %a",found_library) - end - local result,message=action(found_library,required_base) - if result then - library=result - else - library=false - report("load error: message %a, library %a",tostring(message or "unknown"),found_library or "no library") + report("library with qualified name %a %sfound",required,"not ") end end - if trace then - if not library then - report("unknown library: %a",required) - else - report("stored library: %a",required) - end - end - return library or nil + return false end -do - local report_swiglib=logs.reporter("swiglib") - local trace_swiglib=false - local savedrequire=require - local loadedlibs={} - local loadlib=package.loadlib - local pushdir=dir.push - local popdir=dir.pop - trackers.register("resolvers.swiglib",function(v) trace_swiglib=v end) - function requireswiglib(required,version) - local library=loadedlibs[library] - if library==nil then - local trace_swiglib=trace_swiglib or package.helpers.trace - library=locate(required,version,trace_swiglib,report_swiglib,function(name,base) - pushdir(pathpart(name)) - local opener="luaopen_"..base - if trace_swiglib then - report_swiglib("opening: %a with %a",name,opener) - end - local library,message=loadlib(name,opener) - local libtype=type(library) - if libtype=="function" then - library=library() +local foundlibraries=table.setmetatableindex(function(t,k) + local v=findlib(k) + t[k]=v + return v +end) +function resolvers.findlib(required) + return foundlibraries[required] +end +local libraries={} +resolvers.libraries=libraries +local report=logs.reporter("optional") +if optional then optional.loaded={} end +function libraries.validoptional(name) + local thelib=optional and optional[name] + if not thelib then + elseif thelib.initialize then + return thelib + else + report("invalid optional library %a",libname) + end +end +function libraries.optionalloaded(name,libnames) + local thelib=optional and optional[name] + if not thelib then + report("no optional %a library found",name) + else + local thelib_initialize=thelib.initialize + if not thelib_initialize then + report("invalid optional library %a",name) + else + if type(libnames)=="string" then + libnames={ libnames } + end + if type(libnames)=="table" then + for i=1,#libnames do + local libname=libnames[i] + local filename=foundlibraries[libname] + if filename then + libnames[i]=filename + else + report("unable to locate library %a",libname) + return + end + end + local initialized=thelib_initialize(unpack(libnames)) + if initialized then + report("using library '% + t'",libnames) else - report_swiglib("load error: %a returns %a, message %a, library %a",opener,libtype,(string.gsub(message or "no message","[%s]+$","")),found_library or "no library") - library=false + report("unable to initialize library '% + t'",libnames) end - popdir() - return library - end) - loadedlibs[required]=library or false - end - return library - end - function require(name,version) - if find(name,"^swiglib%.") then - return requireswiglib(name,version) - else - return savedrequire(name) - end - end - local swiglibs={} - local initializer="core" - function swiglib(name,version) - local library=swiglibs[name] - if not library then - statistics.starttiming(swiglibs) - if trace_swiglib then - report_swiglib("loading %a",name) - end - if not find(name,"%."..initializer.."$") then - fullname="swiglib."..name.."."..initializer - else - fullname="swiglib."..name + return initialized end - library=requireswiglib(fullname,version) - swiglibs[name]=library - statistics.stoptiming(swiglibs) end - return library end - statistics.register("used swiglibs",function() - if next(swiglibs) then - return string.format("%s, initial load time %s seconds",table.concat(table.sortedkeys(swiglibs)," "),statistics.elapsedtime(swiglibs)) - end - end) end if FFISUPPORTED and ffi and ffi.load then - local report_ffilib=logs.reporter("ffilib") - local trace_ffilib=false - local savedffiload=ffi.load - trackers.register("resolvers.ffilib",function(v) trace_ffilib=v end) - local loaded={} - local function locateindeed(name) - name=removesuffix(name) - local l=loaded[name] - if l==nil then - local state,library=pcall(savedffiload,name) - if type(library)=="userdata" then - l=library - elseif type(state)=="userdata" then - l=state - else - l=false - end - loaded[name]=l - elseif trace_ffilib then - report_ffilib("reusing already loaded %a",name) - end - return l - end - local function getlist(required) - local list=directives.value("system.librarynames" ) - if type(list)=="table" then - list=list[required] - if type(list)=="table" then - if trace then - report("using lookup list for library %a: % | t",required,list) - end - return list - end - end - return { required } - end - function ffilib(name,version) - name=removesuffix(name) - local l=loaded[name] - if l~=nil then - if trace_ffilib then - report_ffilib("reusing already loaded %a",name) - end - return l - end - local list=getlist(name) - if version=="system" then - for i=1,#list do - local library=locateindeed(list[i]) - if type(library)=="userdata" then - return library - end - end - else - for i=1,#list do - local library=locate(list[i],version,trace_ffilib,report_ffilib,locateindeed) - if type(library)=="userdata" then - return library - end - end - end - end + local ffiload=ffi.load function ffi.load(name) - local list=getlist(name) - for i=1,#list do - local library=ffilib(list[i]) - if type(library)=="userdata" then - return library - end - end - if trace_ffilib then - report_ffilib("trying to load %a using normal loader",name) - end - for i=1,#list do - local state,library=pcall(savedffiload,list[i]) - if type(library)=="userdata" then - return library - elseif type(state)=="userdata" then - return library - end + local full=name and foundlibraries[name] + if full then + return ffiload(full) + else + return ffiload(name) end end end @@ -25336,7 +25784,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 9637, stripped down to: 7253 +-- original size: 13964, stripped down to: 10026 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -25350,16 +25798,14 @@ local concat=table.concat local quoted=string.quoted local luasuffixes=utilities.lua.suffixes local report_format=logs.reporter("resolvers","formats") -local function primaryflags() - local arguments=environment.arguments +local function primaryflags(arguments) local flags={} if arguments.silent then flags[#flags+1]="--interaction=batchmode" end return concat(flags," ") end -local function secondaryflags() - local arguments=environment.arguments +local function secondaryflags(arguments) local trackers=arguments.trackers local directives=arguments.directives local flags={} @@ -25381,6 +25827,9 @@ local function secondaryflags() if arguments.ansi then flags[#flags+1]="--c:ansi" end + if arguments.ansilog then + flags[#flags+1]="--c:ansilog" + end if arguments.strip then flags[#flags+1]="--c:strip" end @@ -25391,12 +25840,13 @@ local function secondaryflags() end local template=[[--ini %primaryflags% --lua=%luafile% %texfile% %secondaryflags% %dump% %redirect%]] local checkers={ - primaryflags="string", - secondaryflags="string", + primaryflags="verbose", + secondaryflags="verbose", luafile="readable", texfile="readable", redirect="string", dump="string", + binarypath="string", } local runners={ luatex=sandbox.registerrunner { @@ -25421,46 +25871,88 @@ local runners={ reporter=report_format, }, } -function environment.make_format(name,arguments) +local function validbinarypath() + if not environment.arguments.nobinarypath then + local path=environment.ownpath or file.dirname(environment.ownname) + if path and path~="" then + path=dir.expandname(path) + if path~="" and lfs.isdir(path) then + return path + end + end + end +end +function environment.make_format(formatname) + local arguments=environment.arguments local engine=environment.ownmain or "luatex" - local silent=environment.arguments.silent - local errors=environment.arguments.errors - local olddir=dir.current() - local path=caches.getwritablepath("formats",engine) or "" - if path~="" then - lfs.chdir(path) - end - report_format("using format path %a",dir.current()) - local texsourcename=file.addsuffix(name,"mkiv") - local fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + local silent=arguments.silent + local errors=arguments.errors + local texsourcename="" + local texsourcepath="" + local fulltexsourcename="" + if engine=="luametatex" then + texsourcename=file.addsuffix(formatname,"mkxl") + fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + end + if fulltexsourcename=="" then + texsourcename=file.addsuffix(formatname,"mkiv") + fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + end if fulltexsourcename=="" then - texsourcename=file.addsuffix(name,"tex") + texsourcename=file.addsuffix(formatname,"tex") fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" end if fulltexsourcename=="" then - report_format("no tex source file with name %a (mkiv or tex)",name) - lfs.chdir(olddir) + report_format("no tex source file with name %a (mkiv or tex)",formatname) return - else - report_format("using tex source file %a",fulltexsourcename) end - local texsourcepath=dir.expandname(file.dirname(fulltexsourcename)) - local specificationname=file.replacesuffix(fulltexsourcename,"lus") - local fullspecificationname=resolvers.findfile(specificationname,"tex") or "" - if fullspecificationname=="" then - specificationname=file.join(texsourcepath,"context.lus") - fullspecificationname=resolvers.findfile(specificationname,"tex") or "" + report_format("using tex source file %a",fulltexsourcename) + fulltexsourcename=dir.expandname(fulltexsourcename) + texsourcepath=file.dirname(fulltexsourcename) + if not lfs.isfile(fulltexsourcename) then + report_format("no accessible tex source file with name %a",fulltexsourcename) + return end + local specificationname="context.lus" + local specificationpath="" + local fullspecificationname=resolvers.findfile(specificationname) or "" if fullspecificationname=="" then - report_format("unknown stub specification %a",specificationname) - lfs.chdir(olddir) + report_format("unable to locate specification file %a",specificationname) + return + end + report_format("using specification file %a",fullspecificationname) + fullspecificationname=dir.expandname(fullspecificationname) + specificationpath=file.dirname(fullspecificationname) + if texsourcepath~=specificationpath then + report_format("tex source file and specification file are on different paths") + return + end + if not lfs.isfile(fulltexsourcename) then + report_format("no accessible tex source file with name %a",fulltexsourcename) + return + end + if not lfs.isfile(fullspecificationname) then + report_format("no accessible specification file with name %a",fulltexsourcename) + return + end + report_format("using tex source path %a",texsourcepath) + local validformatpath=caches.getwritablepath("formats",engine) or "" + local startupdir=dir.current() + if validformatpath=="" then + report_format("invalid format path, insufficient write access") + return + end + local binarypath=validbinarypath() + report_format("changing to format path %a",validformatpath) + lfs.chdir(validformatpath) + if dir.current()~=validformatpath then + report_format("unable to change to format path %a",validformatpath) return end - local specificationpath=file.dirname(fullspecificationname) local usedluastub=nil local usedlualibs=dofile(fullspecificationname) if type(usedlualibs)=="string" then - usedluastub=file.join(file.dirname(fullspecificationname),usedlualibs) + usedluastub=file.join(specificationpath,usedlualibs) elseif type(usedlualibs)=="table" then report_format("using stub specification %a",fullspecificationname) local texbasename=file.basename(name) @@ -25477,48 +25969,57 @@ function environment.make_format(name,arguments) end else report_format("invalid stub specification %a",fullspecificationname) - lfs.chdir(olddir) + lfs.chdir(startupdir) return end + local runner=runners[engine] + if not runner then + report_format("the format %a cannot be generated, no runner available for engine %a",name,engine) + lfs.chdir(startupdir) + return + end + local primaryflags=primaryflags(arguments) + local secondaryflags=secondaryflags(arguments) local specification={ - primaryflags=primaryflags(), - secondaryflags=secondaryflags(), + binarypath=binarypath, + primaryflags=primaryflags, + secondaryflags=secondaryflags, luafile=quoted(usedluastub), texfile=quoted(fulltexsourcename), dump=os.platform=="unix" and "\\\\dump" or "\\dump", } - local runner=runners[engine] - if not runner then - report_format("format %a cannot be generated, no runner available for engine %a",name,engine) - elseif silent then - statistics.starttiming() + if silent then specification.redirect="> temp.log" - local result=runner(specification) - local runtime=statistics.stoptiming() - if result~=0 then - print(format("%s silent make > fatal error when making format %q",engine,name)) - else - print(format("%s silent make > format %q made in %.3f seconds",engine,name,runtime)) - end + end + statistics.starttiming() + local result=runner(specification) + local runtime=statistics.stoptiming() + if silent then os.remove("temp.log") - else - runner(specification) end - local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" - local mp=dir.glob(pattern) - if mp then - for i=1,#mp do - local name=mp[i] - report_format("removing related mplib format %a",file.basename(name)) - os.remove(name) + report_format() + if binarypath and binarypath~="" then + report_format("binary path : %s",binarypath or "?") end - end - lfs.chdir(olddir) + report_format("format path : %s",validformatpath) + report_format("luatex engine : %s",engine) + report_format("lua startup file : %s",usedluastub) + if primaryflags~="" then + report_format("primary flags : %s",primaryflags) + end + if secondaryflags~="" then + report_format("secondary flags : %s",secondaryflags) + end + report_format("context file : %s",fulltexsourcename) + report_format("run time : %.3f seconds",runtime) + report_format("return value : %s",result==0 and "okay" or "error") + report_format() + lfs.chdir(startupdir) end -local template=[[%flags% --fmt=%fmtfile% --lua=%luafile% %texfile% %more%]] +local template=[[%primaryflags% --fmt=%fmtfile% --lua=%luafile% %texfile% %secondaryflags%]] local checkers={ - flags="string", - more="string", + primaryflags="verbose", + secondaryflags="verbose", fmtfile="readable", luafile="readable", texfile="readable", @@ -25531,6 +26032,13 @@ local runners={ checkers=checkers, reporter=report_format, }, + luametatex=sandbox.registerrunner { + name="run luametatex format", + program="luametatex", + template=template, + checkers=checkers, + reporter=report_format, + }, luajittex=sandbox.registerrunner { name="run luajittex format", program="luajittex", @@ -25539,51 +26047,78 @@ local runners={ reporter=report_format, }, } -function environment.run_format(name,data,more) - if name and name~="" then - local engine=environment.ownmain or "luatex" - local barename=file.removesuffix(name) - local fmtname=caches.getfirstreadablefile(file.addsuffix(barename,"fmt"),"formats",engine) - if fmtname=="" then - fmtname=resolvers.findfile(file.addsuffix(barename,"fmt")) or "" - end - fmtname=resolvers.cleanpath(fmtname) - if fmtname=="" then - report_format("no format with name %a",name) - else - local barename=file.removesuffix(name) - local luaname=file.addsuffix(barename,"luc") - if not lfs.isfile(luaname) then - luaname=file.addsuffix(barename,"lua") - end - if not lfs.isfile(luaname) then - report_format("using format name %a",fmtname) - report_format("no luc/lua file with name %a",barename) - else - local runner=runners[engine] - if not runner then - report_format("format %a cannot be run, no runner available for engine %a",name,engine) - else - runner { - flags=primaryflags(), - fmtfile=quoted(barename), - luafile=quoted(luaname), - texfile=quoted(data), - more=more, - } - end - end - end +function environment.run_format(formatname,scriptname,filename,primaryflags,secondaryflags,verbose) + local engine=environment.ownmain or "luatex" + if not formatname or formatname=="" then + report_format("missing format name") + return + end + if not scriptname or scriptname=="" then + report_format("missing script name") + return + end + if not lfs.isfile(formatname) or not lfs.isfile(scriptname) then + formatname,scriptname=resolvers.locateformat(formatname) end + if not formatname or formatname=="" then + report_format("invalid format name") + return + end + if not scriptname or scriptname=="" then + report_format("invalid script name") + return + end + local runner=runners[engine] + if not runner then + report_format("format %a cannot be run, no runner available for engine %a",file.nameonly(name),engine) + return + end + if not filename then + filename "" + end + local binarypath=validbinarypath() + local specification={ + binarypath=binarypath, + primaryflags=primaryflags or "", + secondaryflags=secondaryflags or "", + fmtfile=quoted(formatname), + luafile=quoted(scriptname), + texfile=filename~="" and quoted(filename) or "", + } + statistics.starttiming() + local result=runner(specification) + local runtime=statistics.stoptiming() + if verbose then + report_format() + if binarypath and binarypath~="" then + report_format("binary path : %s",binarypath) + end + report_format("luatex engine : %s",engine) + report_format("lua startup file : %s",scriptname) + report_format("tex format file : %s",formatname) + if filename~="" then + report_format("tex input file : %s",filename) + end + if primaryflags~="" then + report_format("primary flags : %s",primaryflags) + end + if secondaryflags~="" then + report_format("secondary flags : %s",secondaryflags) + end + report_format("run time : %.3f seconds",runtime) + report_format("return value : %s",result==0 and "okay" or "error") + report_format() + end + return result end end -- of closure --- used libraries : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua +-- used libraries : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua libs-ini.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 1019480 --- stripped bytes : 403728 +-- original bytes : 1038245 +-- stripped bytes : 409841 -- end library merge @@ -25697,7 +26232,7 @@ local ownlibs = { -- order can be made better 'data-tmf.lua', 'data-lst.lua', - 'util-lib.lua', -- swiglib + 'libs-ini.lua', 'luat-sta.lua', 'luat-fmt.lua', @@ -25846,7 +26381,7 @@ local helpinfo = [[ <category name="basic"> <subcategory> <flag name="script"><short>run an mtx script (lua prefered method) (<ref name="noquotes"/>), no script gives list</short></flag> - <flag name="evaluate"><short>run code passed on the commandline (between quotes)</short></flag> + <flag name="evaluate"><short>run code passed on the commandline (between quotes) (=loop) (exit|quit aborts)</short></flag> <flag name="execute"><short>run a script or program (texmfstart method) (<ref name="noquotes"/>)</short></flag> <flag name="resolve"><short>resolve prefixed arguments</short></flag> <flag name="ctxlua"><short>run internally (using preloaded libs)</short></flag> @@ -25865,6 +26400,7 @@ local helpinfo = [[ <flag name="stubpath" value="binpath"><short>paths where stubs wil be written</short></flag> <flag name="windows"><short>create windows (mswin) stubs</short></flag> <flag name="unix"><short>create unix (linux) stubs</short></flag> + <flag name="addbinarypath"><short>prepend the (found) binarypath to runners</short></flag> </subcategory> <subcategory> <flag name="verbose"><short>give a bit more info</short></flag> @@ -26086,8 +26622,7 @@ end report() io.flush() end - -- no os.exec because otherwise we get the wrong return value - local code = os.execute(command) -- maybe spawn + local code = os.execute(command) if code == 0 then return true else @@ -26130,7 +26665,7 @@ function runners.execute_program(fullname) report() report() io.flush() - local code = os.exec(command) -- (fullname,unpack(after)) does not work / maybe spawn + local code = os.execute(command) return code == 0 end end @@ -26517,16 +27052,22 @@ function runners.associate(filename) end function runners.evaluate(code,filename) -- for Luigi + local environment = table.setmetatableindex(_G) if code == "loop" then while true do - io.write("> ") + io.write("lua > ") local code = io.read() - if code ~= "" then + if code == "quit" or code == "exit" then + break + elseif code ~= "" then local temp = string.match(code,"^= (.*)$") if temp then - code = "print("..temp..")" + code = "inspect("..temp..")" + end + local compiled, message = load(code,"console","t",environment) + if type(compiled) ~= "function" then + compiled = load("inspect("..code..")","console","t",environment) end - local compiled, message = loadstring(code) if type(compiled) ~= "function" then io.write("! " .. (message or code).."\n") else @@ -26539,7 +27080,10 @@ function runners.evaluate(code,filename) -- for Luigi code = filename end if code ~= "" then - local compiled, message = loadstring(code) + local compiled, message = load(code,"console","t",environment) + if type(compiled) ~= "function" then + compiled = load("inspect("..code..")","console","t",environment) + end if type(compiled) ~= "function" then io.write("invalid lua code: " .. (message or code)) return @@ -26716,17 +27260,17 @@ do end -if e_argument("ansi") then +-- if e_argument("ansi") or e_argument("ansilog") then - logs.setformatters("ansi") +-- logs.setformatters(e_argument("ansi") and "ansi" or "ansilog") - local script = e_argument("script") or e_argument("scripts") +-- -- local script = e_argument("script") or e_argument("scripts") +-- -- +-- -- if type(script) == "string" then +-- -- logs.writer("]0;"..script.."") -- for Alan to test +-- -- end - if type(script) == "string" then - logs.writer("]0;"..script.."") -- for Alan to test - end - -end +-- end if e_argument("script") or e_argument("scripts") then @@ -26975,11 +27519,11 @@ elseif e_argument("format-path") then resolvers.load() report(caches.getwritablepath("format")) -elseif e_argument("pattern") then - - -- luatools - - runners.execute_ctx_script("mtx-base","--pattern='" .. e_argument("pattern") .. "'",filename) +-- elseif e_argument("pattern") then +-- +-- -- luatools +-- +-- runners.execute_ctx_script("mtx-base","--pattern='" .. e_argument("pattern") .. "'",filename) elseif e_argument("generate") then diff --git a/Master/texmf-dist/scripts/context/stubs/win64/mtxrun.lua b/Master/texmf-dist/scripts/context/stubs/win64/mtxrun.lua index 98fe7f752e3..5907fa33c66 100644 --- a/Master/texmf-dist/scripts/context/stubs/win64/mtxrun.lua +++ b/Master/texmf-dist/scripts/context/stubs/win64/mtxrun.lua @@ -194,7 +194,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 6330, stripped down to: 2831 +-- original size: 6529, stripped down to: 2933 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -313,6 +313,9 @@ elseif not ffi.number then end if LUAVERSION>5.3 then end +if status and os.setenv then + os.setenv("engine",string.lower(status.luatex_engine or "unknown")) +end end -- of closure @@ -321,7 +324,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-macro"] = package.loaded["l-macro"] or true --- original size: 10131, stripped down to: 5991 +-- original size: 10130, stripped down to: 5990 if not modules then modules={} end modules ['l-macros']={ version=1.001, @@ -546,7 +549,7 @@ end macros.loaded=loaded function required(name,trace) local filename=file.addsuffix(name,"lua") - local fullname=resolvers and resolvers.find_file(filename) or filename + local fullname=resolvers and resolvers.findfile(filename) or filename if not fullname or fullname=="" then return false end @@ -576,7 +579,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-sandbox"] = package.loaded["l-sandbox"] or true --- original size: 9747, stripped down to: 6313 +-- original size: 9604, stripped down to: 6394 if not modules then modules={} end modules ['l-sandbox']={ version=1.001, @@ -808,6 +811,9 @@ local function supported(library) return l end loadfile=register(loadfile,"loadfile") +if supported("lua") then + lua.openfile=register(lua.openfile,"lua.openfile") +end if supported("io") then io.open=register(io.open,"io.open") io.popen=register(io.popen,"io.popen") @@ -1168,7 +1174,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true --- original size: 38434, stripped down to: 19310 +-- original size: 38440, stripped down to: 19316 if not modules then modules={} end modules ['l-lpeg']={ version=1.001, @@ -1337,7 +1343,7 @@ patterns.propername=(uppercase+lowercase+underscore)*(uppercase+lowercase+unders patterns.somecontent=(anything-newline-space)^1 patterns.beginline=#(1-newline) patterns.longtostring=Cs(whitespace^0/""*((patterns.quoted+nonwhitespace^1+whitespace^1/""*(endofstring+Cc(" ")))^0)) -function anywhere(pattern) +local function anywhere(pattern) return (1-P(pattern))^0*P(pattern) end lpeg.anywhere=anywhere @@ -1967,7 +1973,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-string"] = package.loaded["l-string"] or true --- original size: 6461, stripped down to: 3255 +-- original size: 6644, stripped down to: 3410 if not modules then modules={} end modules ['l-string']={ version=1.001, @@ -2031,9 +2037,11 @@ function string.is_empty(str) end end local anything=patterns.anything -local allescapes=Cc("%")*S(".-+%?()[]*") -local someescapes=Cc("%")*S(".-+%()[]") -local matchescapes=Cc(".")*S("*?") +local moreescapes=Cc("%")*S(".-+%?()[]*$^{}") +local allescapes=Cc("%")*S(".-+%?()[]*") +local someescapes=Cc("%")*S(".-+%()[]") +local matchescapes=Cc(".")*S("*?") +local pattern_m=Cs ((moreescapes+anything )^0 ) local pattern_a=Cs ((allescapes+anything )^0 ) local pattern_b=Cs ((someescapes+matchescapes+anything )^0 ) local pattern_c=Cs (Cc("^")*(someescapes+matchescapes+anything )^0*Cc("$") ) @@ -2043,6 +2051,8 @@ end function string.topattern(str,lowercase,strict) if str=="" or type(str)~="string" then return ".*" + elseif strict=="all" then + str=lpegmatch(pattern_m,str) elseif strict then str=lpegmatch(pattern_c,str) else @@ -2092,7 +2102,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-table"] = package.loaded["l-table"] or true --- original size: 41332, stripped down to: 21508 +-- original size: 41758, stripped down to: 22643 if not modules then modules={} end modules ['l-table']={ version=1.001, @@ -2101,7 +2111,7 @@ if not modules then modules={} end modules ['l-table']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local type,next,tostring,tonumber,select=type,next,tostring,tonumber,select +local type,next,tostring,tonumber,select,rawget=type,next,tostring,tonumber,select,rawget local table,string=table,string local concat,sort=table.concat,table.sort local format,lower,dump=string.format,string.lower,string.dump @@ -2435,13 +2445,13 @@ function table.fromhash(t) end return hsh end -local noquotes,hexify,handle,compact,inline,functions,metacheck +local noquotes,hexify,handle,compact,inline,functions,metacheck,accurate local reserved=table.tohash { 'and','break','do','else','elseif','end','false','for','function','if', 'in','local','nil','not','or','repeat','return','then','true','until','while', - 'NaN','goto', + 'NaN','goto','const', } -local function is_simple_table(t,hexify) +local function is_simple_table(t,hexify,accurate) local nt=#t if nt>0 then local n=0 @@ -2460,6 +2470,8 @@ local function is_simple_table(t,hexify) if tv=="number" then if hexify then tt[i]=format("0x%X",v) + elseif accurate then + tt[i]=format("%q",v) else tt[i]=v end @@ -2480,6 +2492,8 @@ local function is_simple_table(t,hexify) if tv=="number" then if hexify then tt[i+1]=format("0x%X",v) + elseif accurate then + tt[i+1]=format("%q",v) else tt[i+1]=v end @@ -2551,6 +2565,8 @@ local function do_serialize(root,name,depth,level,indexed) if tv=="number" then if hexify then handle(format("%s 0x%X,",depth,v)) + elseif accurate then + handle(format("%s %q,",depth,v)) else handle(format("%s %s,",depth,v)) end @@ -2560,7 +2576,7 @@ local function do_serialize(root,name,depth,level,indexed) if next(v)==nil then handle(format("%s {},",depth)) elseif inline then - local st=is_simple_table(v,hexify) + local st=is_simple_table(v,hexify,accurate) if st then handle(format("%s { %s },",depth,concat(st,", "))) else @@ -2588,12 +2604,16 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]=0x%X,",depth,k,v)) + elseif accurate then + handle(format("%s [%s]=%q,",depth,k,v)) else handle(format("%s [%s]=%s,",depth,k,v)) end elseif tk=="boolean" then if hexify then handle(format("%s [%s]=0x%X,",depth,k and "true" or "false",v)) + elseif accurate then + handle(format("%s [%s]=%q,",depth,k and "true" or "false",v)) else handle(format("%s [%s]=%s,",depth,k and "true" or "false",v)) end @@ -2601,12 +2621,16 @@ local function do_serialize(root,name,depth,level,indexed) elseif noquotes and not reserved[k] and lpegmatch(propername,k) then if hexify then handle(format("%s %s=0x%X,",depth,k,v)) + elseif accurate then + handle(format("%s %s=%q,",depth,k,v)) else handle(format("%s %s=%s,",depth,k,v)) end else if hexify then handle(format("%s [%q]=0x%X,",depth,k,v)) + elseif accurate then + handle(format("%s [%q]=%q,",depth,k,v)) else handle(format("%s [%q]=%s,",depth,k,v)) end @@ -2615,6 +2639,8 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]=%q,",depth,k,v)) + elseif accurate then + handle(format("%s [%q]=%q,",depth,k,v)) else handle(format("%s [%s]=%q,",depth,k,v)) end @@ -2631,6 +2657,8 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]={},",depth,k)) + elseif accurate then + handle(format("%s [%q]={},",depth,k)) else handle(format("%s [%s]={},",depth,k)) end @@ -2643,11 +2671,13 @@ local function do_serialize(root,name,depth,level,indexed) handle(format("%s [%q]={},",depth,k)) end elseif inline then - local st=is_simple_table(v,hexify) + local st=is_simple_table(v,hexify,accurate) if st then if tk=="number" then if hexify then handle(format("%s [0x%X]={ %s },",depth,k,concat(st,", "))) + elseif accurate then + handle(format("%s [%q]={ %s },",depth,k,concat(st,", "))) else handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) end @@ -2669,6 +2699,8 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]=%s,",depth,k,v and "true" or "false")) + elseif accurate then + handle(format("%s [%q]=%s,",depth,k,v and "true" or "false")) else handle(format("%s [%s]=%s,",depth,k,v and "true" or "false")) end @@ -2688,6 +2720,8 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]=load(%q),",depth,k,f)) + elseif accurate then + handle(format("%s [%q]=load(%q),",depth,k,f)) else handle(format("%s [%s]=load(%q),",depth,k,f)) end @@ -2705,6 +2739,8 @@ local function do_serialize(root,name,depth,level,indexed) if tk=="number" then if hexify then handle(format("%s [0x%X]=%q,",depth,k,tostring(v))) + elseif accurate then + handle(format("%s [%q]=%q,",depth,k,tostring(v))) else handle(format("%s [%s]=%q,",depth,k,tostring(v))) end @@ -2728,6 +2764,7 @@ local function serialize(_handle,root,name,specification) if type(specification)=="table" then noquotes=specification.noquotes hexify=specification.hexify + accurate=specification.accurate handle=_handle or specification.handle or print functions=specification.functions compact=specification.compact @@ -3043,7 +3080,7 @@ end local function sequenced(t,sep,simple) if not t then return "" - elseif type(t)=="string" then + elseif type(t)~="table" then return t end local n=#t @@ -3082,7 +3119,11 @@ local function sequenced(t,sep,simple) end end end - return concat(s,sep or " | ") + if sep==true then + return "{ "..concat(s,", ").." }" + else + return concat(s,sep or " | ") + end end table.sequenced=sequenced function table.print(t,...) @@ -3213,7 +3254,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-io"] = package.loaded["l-io"] or true --- original size: 11823, stripped down to: 6325 +-- original size: 11829, stripped down to: 6331 if not modules then modules={} end modules ['l-io']={ version=1.001, @@ -3227,7 +3268,7 @@ local open,flush,write,read=io.open,io.flush,io.write,io.read local byte,find,gsub,format=string.byte,string.find,string.gsub,string.format local concat=table.concat local type=type -if string.find(os.getenv("PATH"),";",1,true) then +if string.find(os.getenv("PATH") or "",";",1,true) then io.fileseparator,io.pathseparator="\\",";" else io.fileseparator,io.pathseparator="/",":" @@ -3765,7 +3806,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-os"] = package.loaded["l-os"] or true --- original size: 18916, stripped down to: 10126 +-- original size: 19102, stripped down to: 10192 if not modules then modules={} end modules ['l-os']={ version=1.001, @@ -3779,7 +3820,7 @@ local date,time=os.date,os.time local find,format,gsub,upper,gmatch=string.find,string.format,string.gsub,string.upper,string.gmatch local concat=table.concat local random,ceil,randomseed=math.random,math.ceil,math.randomseed -local rawget,rawset,type,getmetatable,setmetatable,tonumber,tostring=rawget,rawset,type,getmetatable,setmetatable,tonumber,tostring +local type,setmetatable,tonumber,tostring=type,setmetatable,tonumber,tostring do local selfdir=os.selfdir if selfdir=="" then @@ -3930,7 +3971,8 @@ local launchers={ unix="xdg-open %s &> /dev/null &", } function os.launch(str) - execute(format(launchers[os.name] or launchers.unix,str)) + local command=format(launchers[os.name] or launchers.unix,str) + execute(command) end local gettimeofday=os.gettimeofday or os.clock os.gettimeofday=gettimeofday @@ -4185,6 +4227,12 @@ function os.validdate(year,month,day) end return year,month,day end +function os.date(fmt,...) + if not fmt then + fmt="%Y-%m-%d %H:%M" + end + return date(fmt,...) +end local osexit=os.exit local exitcode=nil function os.setexitcode(code) @@ -4207,7 +4255,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-file"] = package.loaded["l-file"] or true --- original size: 21984, stripped down to: 10148 +-- original size: 22175, stripped down to: 10302 if not modules then modules={} end modules ['l-file']={ version=1.001, @@ -4229,15 +4277,24 @@ local checkedsplit=string.checkedsplit local P,R,S,C,Cs,Cp,Cc,Ct=lpeg.P,lpeg.R,lpeg.S,lpeg.C,lpeg.Cs,lpeg.Cp,lpeg.Cc,lpeg.Ct local attributes=lfs.attributes function lfs.isdir(name) - return attributes(name,"mode")=="directory" + if name then + return attributes(name,"mode")=="directory" + end end function lfs.isfile(name) - local a=attributes(name,"mode") - return a=="file" or a=="link" or nil + if name then + local a=attributes(name,"mode") + return a=="file" or a=="link" or nil + end end function lfs.isfound(name) - local a=attributes(name,"mode") - return (a=="file" or a=="link") and name or nil + if name then + local a=attributes(name,"mode") + return (a=="file" or a=="link") and name or nil + end +end +function lfs.modification(name) + return name and attributes(name,"modification") or nil end if sandbox then sandbox.redefine(lfs.isfile,"lfs.isfile") @@ -4616,7 +4673,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-gzip"] = package.loaded["l-gzip"] or true --- original size: 1211, stripped down to: 951 +-- original size: 5115, stripped down to: 1699 if not modules then modules={} end modules ['l-gzip']={ version=1.001, @@ -4624,43 +4681,75 @@ if not modules then modules={} end modules ['l-gzip']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -if not gzip then - return -end -local suffix,suffixes=file.suffix,file.suffixes -function gzip.load(filename) - local f=io.open(filename,"rb") - if not f then - elseif suffix(filename)=="gz" then - f:close() - local g=gzip.open(filename,"rb") - if g then - local str=g:read("*all") - g:close() - return str +gzip=gzip or {} +if not zlib then + zlib=xzip +elseif not xzip then + xzip=zlib +end +if zlib then + local suffix=file.suffix + local suffixes=file.suffixes + local find=string.find + local openfile=io.open + local gzipwindow=15+16 + local gziplevel=3 + local identifier="^\x1F\x8B\x08" + local compress=zlib.compress + local decompress=zlib.decompress + function gzip.load(filename) + local f=openfile(filename,"rb") + if not f then + else + local data=f:read("*all") + f:close() + if data and data~="" then + if suffix(filename)=="gz" then + data=decompress(data,gzipwindow) + end + return data + end end - else - local str=f:read("*all") - f:close() - return str end -end -function gzip.save(filename,data) - if suffix(filename)~="gz" then - filename=filename..".gz" + function gzip.save(filename,data,level) + if suffix(filename)~="gz" then + filename=filename..".gz" + end + local f=openfile(filename,"wb") + if f then + data=compress(data or "",level or gziplevel,nil,gzipwindow) + f:write(data) + f:close() + return #data + end end - local f=io.open(filename,"wb") - if f then - local s=zlib.compress(data or "",9,nil,15+16) - f:write(s) - f:close() - return #s + function gzip.suffix(filename) + local suffix,extra=suffixes(filename) + local gzipped=extra=="gz" + return suffix,gzipped + end + function gzip.compressed(s) + return s and find(s,identifier) + end + function gzip.compress(s,level) + if s and not find(s,identifier) then + if not level then + level=gziplevel + elseif level<=0 then + return s + elseif level>9 then + level=9 + end + return compress(s,level or gziplevel,nil,gzipwindow) or s + end + end + function gzip.decompress(s) + if s and find(s,identifier) then + return decompress(s,gzipwindow) + else + return s + end end -end -function gzip.suffix(filename) - local suffix,extra=suffixes(filename) - local gzipped=extra=="gz" - return suffix,gzipped end @@ -4670,7 +4759,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-md5"] = package.loaded["l-md5"] or true --- original size: 3309, stripped down to: 2218 +-- original size: 3414, stripped down to: 2307 if not modules then modules={} end modules ['l-md5']={ version=1.001, @@ -4689,6 +4778,8 @@ if not md5 then end local md5,file=md5,file local gsub=string.gsub +local modification,isfile,touch=lfs.modification,lfs.isfile,lfs.touch +local loaddata,savedata=io.loaddata,io.savedata do local patterns=lpeg and lpeg.patterns if patterns then @@ -4704,10 +4795,11 @@ do md5.sumHEXA=md5.HEX end end +local md5HEX=md5.HEX function file.needsupdating(oldname,newname,threshold) - local oldtime=lfs.attributes(oldname,"modification") + local oldtime=modification(oldname) if oldtime then - local newtime=lfs.attributes(newname,"modification") + local newtime=modification(newname) if not newtime then return true elseif newtime>=oldtime then @@ -4723,31 +4815,32 @@ function file.needsupdating(oldname,newname,threshold) end file.needs_updating=file.needsupdating function file.syncmtimes(oldname,newname) - local oldtime=lfs.attributes(oldname,"modification") - if oldtime and lfs.isfile(newname) then - lfs.touch(newname,oldtime,oldtime) + local oldtime=modification(oldname) + if oldtime and isfile(newname) then + touch(newname,oldtime,oldtime) end end -function file.checksum(name) +local function checksum(name) if md5 then - local data=io.loaddata(name) + local data=loaddata(name) if data then - return md5.HEX(data) + return md5HEX(data) end end return nil end +file.checksum=checksum function file.loadchecksum(name) if md5 then - local data=io.loaddata(name..".md5") + local data=loaddata(name..".md5") return data and (gsub(data,"%s","")) end return nil end function file.savechecksum(name,checksum) - if not checksum then checksum=file.checksum(name) end + if not checksum then checksum=checksum(name) end if checksum then - io.savedata(name..".md5",checksum) + savedata(name..".md5",checksum) return checksum end return nil @@ -5636,7 +5729,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-unicode"] = package.loaded["l-unicode"] or true --- original size: 41047, stripped down to: 17171 +-- original size: 41281, stripped down to: 17261 if not modules then modules={} end modules ['l-unicode']={ version=1.001, @@ -6151,49 +6244,52 @@ end function utf.utf32_to_utf8_t(t,endian) return endian and utf32_to_utf8_be_t(t) or utf32_to_utf8_le_t(t) or t end -local function little(b) - if b<0x10000 then - return char(b%256,rshift(b,8)) - else - b=b-0x10000 - local b1=rshift(b,10)+0xD800 - local b2=b%1024+0xDC00 - return char(b1%256,rshift(b1,8),b2%256,rshift(b2,8)) +if bit32 then + local rshift=bit32.rshift + local function little(b) + if b<0x10000 then + return char(b%256,rshift(b,8)) + else + b=b-0x10000 + local b1=rshift(b,10)+0xD800 + local b2=b%1024+0xDC00 + return char(b1%256,rshift(b1,8),b2%256,rshift(b2,8)) + end end -end -local function big(b) - if b<0x10000 then - return char(rshift(b,8),b%256) - else - b=b-0x10000 - local b1=rshift(b,10)+0xD800 - local b2=b%1024+0xDC00 - return char(rshift(b1,8),b1%256,rshift(b2,8),b2%256) + local function big(b) + if b<0x10000 then + return char(rshift(b,8),b%256) + else + b=b-0x10000 + local b1=rshift(b,10)+0xD800 + local b2=b%1024+0xDC00 + return char(rshift(b1,8),b1%256,rshift(b2,8),b2%256) + end end -end -local l_remap=Cs((p_utf8byte/little+P(1)/"")^0) -local b_remap=Cs((p_utf8byte/big+P(1)/"")^0) -local function utf8_to_utf16_be(str,nobom) - if nobom then - return lpegmatch(b_remap,str) - else - return char(254,255)..lpegmatch(b_remap,str) + local l_remap=Cs((p_utf8byte/little+P(1)/"")^0) + local b_remap=Cs((p_utf8byte/big+P(1)/"")^0) + local function utf8_to_utf16_be(str,nobom) + if nobom then + return lpegmatch(b_remap,str) + else + return char(254,255)..lpegmatch(b_remap,str) + end end -end -local function utf8_to_utf16_le(str,nobom) - if nobom then - return lpegmatch(l_remap,str) - else - return char(255,254)..lpegmatch(l_remap,str) + local function utf8_to_utf16_le(str,nobom) + if nobom then + return lpegmatch(l_remap,str) + else + return char(255,254)..lpegmatch(l_remap,str) + end end -end -utf.utf8_to_utf16_be=utf8_to_utf16_be -utf.utf8_to_utf16_le=utf8_to_utf16_le -function utf.utf8_to_utf16(str,littleendian,nobom) - if littleendian then - return utf8_to_utf16_le(str,nobom) - else - return utf8_to_utf16_be(str,nobom) + utf.utf8_to_utf16_be=utf8_to_utf16_be + utf.utf8_to_utf16_le=utf8_to_utf16_le + function utf.utf8_to_utf16(str,littleendian,nobom) + if littleendian then + return utf8_to_utf16_le(str,nobom) + else + return utf8_to_utf16_be(str,nobom) + end end end local pattern=Cs ( @@ -6467,7 +6563,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-str"] = package.loaded["util-str"] or true --- original size: 43488, stripped down to: 21595 +-- original size: 45188, stripped down to: 22734 if not modules then modules={} end modules ['util-str']={ version=1.001, @@ -6797,6 +6893,13 @@ local template=[[ %s return function(%s) return %s end ]] +local pattern=Cs(Cc('"')*( + (1-S('"\\\n\r'))^1+P('"')/'\\"'+P('\\')/'\\\\'+P('\n')/'\\n'+P('\r')/'\\r' +)^0*Cc('"')) +patterns.escapedquotes=pattern +function string.escapedquotes(s) + return lpegmatch(pattern,s) +end local preamble="" local environment={ global=global or _G, @@ -6821,9 +6924,10 @@ local environment={ formattednumber=number.formatted, sparseexponent=number.sparseexponent, formattedfloat=number.formattedfloat, - stripzero=lpeg.patterns.stripzero, - stripzeros=lpeg.patterns.stripzeros, - FORMAT=string.f9, + stripzero=patterns.stripzero, + stripzeros=patterns.stripzeros, + escapedquotes=string.escapedquotes, + FORMAT=string.f6, } local arguments={ "a1" } setmetatable(arguments,{ __index=function(t,k) @@ -6874,13 +6978,16 @@ local format_left=function(f) return format("a%s..utfpadding(a%s,%i)",n,n,-f) end end -local format_q=function() +local format_q=JITSUPPORTED and function() n=n+1 return format("(a%s ~= nil and format('%%q',tostring(a%s)) or '')",n,n) +end or function() + n=n+1 + return format("(a%s ~= nil and format('%%q',a%s) or '')",n,n) end local format_Q=function() n=n+1 - return format("format('%%q',tostring(a%s))",n) + return format("escapedquotes(tostring(a%s))",n) end local format_i=function(f) n=n+1 @@ -7031,12 +7138,25 @@ local format_n=function() n=n+1 return format("((a%s %% 1 == 0) and format('%%i',a%s) or tostring(a%s))",n,n,n) end -local format_N=function(f) - n=n+1 - if not f or f=="" then - f=".9" - end - return format("(((a%s %% 1 == 0) and format('%%i',a%s)) or lpegmatch(stripzero,format('%%%sf',a%s)))",n,n,f,n) +local format_N if environment.FORMAT then + format_N=function(f) + n=n+1 + if not f or f=="" then + return format("FORMAT(a%s,'%%.9f')",n) + elseif f==".6" or f=="0.6" then + return format("FORMAT(a%s)",n) + else + return format("FORMAT(a%s,'%%%sf')",n,f) + end + end +else + format_N=function(f) + n=n+1 + if not f or f=="" then + f=".9" + end + return format("(((a%s %% 1 == 0) and format('%%i',a%s)) or lpegmatch(stripzero,format('%%%sf',a%s)))",n,n,f,n) + end end local format_a=function(f) n=n+1 @@ -7265,9 +7385,9 @@ patterns.xmlescape=Cs((P("<")/"<"+P(">")/">"+P("&")/"&"+P('"')/"" patterns.texescape=Cs((C(S("#$%\\{}"))/"\\%1"+anything)^0) patterns.luaescape=Cs(((1-S('"\n'))^1+P('"')/'\\"'+P('\n')/'\\n"')^0) patterns.luaquoted=Cs(Cc('"')*((1-S('"\n'))^1+P('"')/'\\"'+P('\n')/'\\n"')^0*Cc('"')) -add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],{ xmlescape=lpeg.patterns.xmlescape }) -add(formatters,"tex",[[lpegmatch(texescape,%s)]],{ texescape=lpeg.patterns.texescape }) -add(formatters,"lua",[[lpegmatch(luaescape,%s)]],{ luaescape=lpeg.patterns.luaescape }) +add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],{ xmlescape=patterns.xmlescape }) +add(formatters,"tex",[[lpegmatch(texescape,%s)]],{ texescape=patterns.texescape }) +add(formatters,"lua",[[lpegmatch(luaescape,%s)]],{ luaescape=patterns.luaescape }) local dquote=patterns.dquote local equote=patterns.escaped+dquote/'\\"'+1 local cquote=Cc('"') @@ -7299,6 +7419,27 @@ local f_16_16=formatters["%0.5N"] function number.to16dot16(n) return f_16_16(n/65536.0) end +if not string.explode then + local tsplitat=lpeg.tsplitat + local p_utf=patterns.utf8character + local p_check=C(p_utf)*(P("+")*Cc(true))^0 + local p_split=Ct(C(p_utf)^0) + local p_space=Ct((C(1-P(" ")^1)+P(" ")^1)^0) + function string.explode(str,symbol) + if symbol=="" then + return lpegmatch(p_split,str) + elseif symbol then + local a,b=lpegmatch(p_check,symbol) + if b then + return lpegmatch(tsplitat(P(a)^1),str) + else + return lpegmatch(tsplitat(a),str) + end + else + return lpegmatch(p_space,str) + end + end +end end -- of closure @@ -7307,7 +7448,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tab"] = package.loaded["util-tab"] or true --- original size: 28866, stripped down to: 16134 +-- original size: 32649, stripped down to: 18257 if not modules then modules={} end modules ['util-tab']={ version=1.001, @@ -7544,78 +7685,160 @@ function tables.encapsulate(core,capsule,protect) } ) end end -local f_hashed_string=formatters["[%q]=%q,"] -local f_hashed_number=formatters["[%q]=%s,"] -local f_hashed_boolean=formatters["[%q]=%l,"] -local f_hashed_table=formatters["[%q]="] -local f_indexed_string=formatters["[%s]=%q,"] -local f_indexed_number=formatters["[%s]=%s,"] -local f_indexed_boolean=formatters["[%s]=%l,"] -local f_indexed_table=formatters["[%s]="] -local f_ordered_string=formatters["%q,"] -local f_ordered_number=formatters["%s,"] -local f_ordered_boolean=formatters["%l,"] -function table.fastserialize(t,prefix) - local r={ type(prefix)=="string" and prefix or "return" } - local m=1 - local function fastserialize(t,outer) - local n=#t - m=m+1 - r[m]="{" - if n>0 then - for i=0,n do - local v=t[i] - local tv=type(v) - if tv=="string" then - m=m+1 r[m]=f_ordered_string(v) - elseif tv=="number" then - m=m+1 r[m]=f_ordered_number(v) - elseif tv=="table" then - fastserialize(v) - elseif tv=="boolean" then - m=m+1 r[m]=f_ordered_boolean(v) +if JITSUPPORTED then + local f_hashed_string=formatters["[%Q]=%Q,"] + local f_hashed_number=formatters["[%Q]=%s,"] + local f_hashed_boolean=formatters["[%Q]=%l,"] + local f_hashed_table=formatters["[%Q]="] + local f_indexed_string=formatters["[%s]=%Q,"] + local f_indexed_number=formatters["[%s]=%s,"] + local f_indexed_boolean=formatters["[%s]=%l,"] + local f_indexed_table=formatters["[%s]="] + local f_ordered_string=formatters["%Q,"] + local f_ordered_number=formatters["%s,"] + local f_ordered_boolean=formatters["%l,"] + function table.fastserialize(t,prefix) + local r={ type(prefix)=="string" and prefix or "return" } + local m=1 + local function fastserialize(t,outer) + local n=#t + m=m+1 + r[m]="{" + if n>0 then + local v=t[0] + if v then + local tv=type(v) + if tv=="string" then + m=m+1 r[m]=f_indexed_string(0,v) + elseif tv=="number" then + m=m+1 r[m]=f_indexed_number(0,v) + elseif tv=="table" then + m=m+1 r[m]=f_indexed_table(0) + fastserialize(v) + m=m+1 r[m]=f_indexed_table(0) + elseif tv=="boolean" then + m=m+1 r[m]=f_indexed_boolean(0,v) + end + end + for i=1,n do + local v=t[i] + local tv=type(v) + if tv=="string" then + m=m+1 r[m]=f_ordered_string(v) + elseif tv=="number" then + m=m+1 r[m]=f_ordered_number(v) + elseif tv=="table" then + fastserialize(v) + elseif tv=="boolean" then + m=m+1 r[m]=f_ordered_boolean(v) + end end end - end - for k,v in next,t do - local tk=type(k) - if tk=="number" then - if k>n or k<0 then + for k,v in next,t do + local tk=type(k) + if tk=="number" then + if k>n or k<0 then + local tv=type(v) + if tv=="string" then + m=m+1 r[m]=f_indexed_string(k,v) + elseif tv=="number" then + m=m+1 r[m]=f_indexed_number(k,v) + elseif tv=="table" then + m=m+1 r[m]=f_indexed_table(k) + fastserialize(v) + elseif tv=="boolean" then + m=m+1 r[m]=f_indexed_boolean(k,v) + end + end + else local tv=type(v) if tv=="string" then - m=m+1 r[m]=f_indexed_string(k,v) + m=m+1 r[m]=f_hashed_string(k,v) elseif tv=="number" then - m=m+1 r[m]=f_indexed_number(k,v) + m=m+1 r[m]=f_hashed_number(k,v) elseif tv=="table" then - m=m+1 r[m]=f_indexed_table(k) + m=m+1 r[m]=f_hashed_table(k) fastserialize(v) elseif tv=="boolean" then - m=m+1 r[m]=f_indexed_boolean(k,v) + m=m+1 r[m]=f_hashed_boolean(k,v) end end + end + m=m+1 + if outer then + r[m]="}" else - local tv=type(v) - if tv=="string" then - m=m+1 r[m]=f_hashed_string(k,v) - elseif tv=="number" then - m=m+1 r[m]=f_hashed_number(k,v) - elseif tv=="table" then - m=m+1 r[m]=f_hashed_table(k) - fastserialize(v) - elseif tv=="boolean" then - m=m+1 r[m]=f_hashed_boolean(k,v) - end + r[m]="}," end + return r end - m=m+1 - if outer then - r[m]="}" - else - r[m]="}," + return concat(fastserialize(t,true)) + end +else + local f_v=formatters["[%q]=%q,"] + local f_t=formatters["[%q]="] + local f_q=formatters["%q,"] + function table.fastserialize(t,prefix) + local r={ type(prefix)=="string" and prefix or "return" } + local m=1 + local function fastserialize(t,outer) + local n=#t + m=m+1 + r[m]="{" + if n>0 then + local v=t[0] + if v then + m=m+1 + r[m]="[0]='" + if type(v)=="table" then + fastserialize(v) + else + r[m]=format("%q,",v) + end + end + for i=1,n do + local v=t[i] + m=m+1 + if type(v)=="table" then + r[m]=format("[%i]=",i) + fastserialize(v) + else + r[m]=format("[%i]=%q,",i,v) + end + end + end + for k,v in next,t do + local tk=type(k) + if tk=="number" then + if k>n or k<0 then + m=m+1 + if type(v)=="table" then + r[m]=format("[%i]=",k) + fastserialize(v) + else + r[m]=format("[%i]=%q,",k,v) + end + end + else + m=m+1 + if type(v)=="table" then + r[m]=format("[%q]=",k) + fastserialize(v) + else + r[m]=format("[%q]=%q,",k,v) + end + end + end + m=m+1 + if outer then + r[m]="}" + else + r[m]="}," + end + return r end - return r + return concat(fastserialize(t,true)) end - return concat(fastserialize(t,true)) end function table.deserialize(str) if not str or str=="" then @@ -7709,28 +7932,28 @@ function table.twowaymapper(t) return t end local f_start_key_idx=formatters["%w{"] -local f_start_key_num=formatters["%w[%s]={"] +local f_start_key_num=JITSUPPORTED and formatters["%w[%s]={"] or formatters["%w[%q]={"] local f_start_key_str=formatters["%w[%q]={"] local f_start_key_boo=formatters["%w[%l]={"] local f_start_key_nop=formatters["%w{"] local f_stop=formatters["%w},"] -local f_key_num_value_num=formatters["%w[%s]=%s,"] -local f_key_str_value_num=formatters["%w[%q]=%s,"] -local f_key_boo_value_num=formatters["%w[%l]=%s,"] -local f_key_num_value_str=formatters["%w[%s]=%q,"] -local f_key_str_value_str=formatters["%w[%q]=%q,"] -local f_key_boo_value_str=formatters["%w[%l]=%q,"] -local f_key_num_value_boo=formatters["%w[%s]=%l,"] -local f_key_str_value_boo=formatters["%w[%q]=%l,"] +local f_key_num_value_num=JITSUPPORTED and formatters["%w[%s]=%s,"] or formatters["%w[%s]=%q,"] +local f_key_str_value_num=JITSUPPORTED and formatters["%w[%Q]=%s,"] or formatters["%w[%Q]=%q,"] +local f_key_boo_value_num=JITSUPPORTED and formatters["%w[%l]=%s,"] or formatters["%w[%l]=%q,"] +local f_key_num_value_str=JITSUPPORTED and formatters["%w[%s]=%Q,"] or formatters["%w[%q]=%Q,"] +local f_key_str_value_str=formatters["%w[%Q]=%Q,"] +local f_key_boo_value_str=formatters["%w[%l]=%Q,"] +local f_key_num_value_boo=JITSUPPORTED and formatters["%w[%s]=%l,"] or formatters["%w[%q]=%l,"] +local f_key_str_value_boo=formatters["%w[%Q]=%l,"] local f_key_boo_value_boo=formatters["%w[%l]=%l,"] -local f_key_num_value_not=formatters["%w[%s]={},"] -local f_key_str_value_not=formatters["%w[%q]={},"] +local f_key_num_value_not=JITSUPPORTED and formatters["%w[%s]={},"] or formatters["%w[%q]={},"] +local f_key_str_value_not=formatters["%w[%Q]={},"] local f_key_boo_value_not=formatters["%w[%l]={},"] -local f_key_num_value_seq=formatters["%w[%s]={ %, t },"] -local f_key_str_value_seq=formatters["%w[%q]={ %, t },"] +local f_key_num_value_seq=JITSUPPORTED and formatters["%w[%s]={ %, t },"] or formatters["%w[%q]={ %, t },"] +local f_key_str_value_seq=formatters["%w[%Q]={ %, t },"] local f_key_boo_value_seq=formatters["%w[%l]={ %, t },"] -local f_val_num=formatters["%w%s,"] -local f_val_str=formatters["%w%q,"] +local f_val_num=JITSUPPORTED and formatters["%w%s,"] or formatters["%w%q,"] +local f_val_str=formatters["%w%Q,"] local f_val_boo=formatters["%w%l,"] local f_val_not=formatters["%w{},"] local f_val_seq=formatters["%w{ %, t },"] @@ -7738,7 +7961,7 @@ local f_fin_seq=formatters[" %, t }"] local f_table_return=formatters["return {"] local f_table_name=formatters["%s={"] local f_table_direct=formatters["{"] -local f_table_entry=formatters["[%q]={"] +local f_table_entry=formatters["[%Q]={"] local f_table_finish=formatters["}"] local spaces=utilities.strings.newrepeater(" ") local original_serialize=table.serialize @@ -8339,7 +8562,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sac"] = package.loaded["util-sac"] or true --- original size: 11332, stripped down to: 8420 +-- original size: 12946, stripped down to: 9507 if not modules then modules={} end modules ['util-sac']={ version=1.001, @@ -8374,6 +8597,7 @@ end function streams.size(f) return f and f[3] or 0 end +streams.getsize=streams.size function streams.setposition(f,i) if f[4] then if i<=0 then @@ -8566,9 +8790,9 @@ function streams.readfixed2(f) f[2]=j+1 local a,b=byte(f[1],i,j) if a>=0x80 then - tonumber((a-0x100).."."..b) + return tonumber((a-0x100).."."..b) or 0 else - tonumber((a ).."."..b) + return tonumber((a ).."."..b) or 0 end end function streams.readfixed4(f) @@ -8577,9 +8801,9 @@ function streams.readfixed4(f) f[2]=j+1 local a,b,c,d=byte(f[1],i,j) if a>=0x80 then - tonumber((0x100*a+b-0x10000).."."..(0x100*c+d)) + return tonumber((0x100*a+b-0x10000).."."..(0x100*c+d)) or 0 else - tonumber((0x100*a+b ).."."..(0x100*c+d)) + return tonumber((0x100*a+b ).."."..(0x100*c+d)) or 0 end end if bit32 then @@ -8659,6 +8883,16 @@ if sio and sio.readcardinal2 then f[2]=i+4 return readinteger4(f[1],i) end + function streams.readfixed2(f) + local i=f[2] + f[2]=i+2 + return readfixed2(f[1],i) + end + function streams.readfixed4(f) + local i=f[2] + f[2]=i+4 + return readfixed4(f[1],i) + end function streams.read2dot4(f) local i=f[2] f[2]=i+2 @@ -8758,6 +8992,50 @@ else return t end end +do + local files=utilities.files + if files then + local openfile=files.open + local openstream=streams.open + local openstring=streams.openstring + local setmetatable=setmetatable + function io.newreader(str,method) + local f,m + if method=="string" then + f=openstring(str) + m=streams + elseif method=="stream" then + f=openstream(str) + m=streams + else + f=openfile(str,"rb") + m=files + end + if f then + local t={} + setmetatable(t,{ + __index=function(t,k) + local r=m[k] + if k=="close" then + if f then + m.close(f) + f=nil + end + return function() end + elseif r then + local v=function(_,a,b) return r(f,a,b) end + t[k]=v + return v + else + print("unknown key",k) + end + end + } ) + return t + end + end + end +end end -- of closure @@ -9801,7 +10079,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-soc-imp-copas"] = package.loaded["util-soc-imp-copas"] or true --- original size: 25844, stripped down to: 14821 +-- original size: 25959, stripped down to: 14893 local socket=socket or require("socket") @@ -9838,6 +10116,7 @@ local copas={ autoclose=true, running=false, report=report, + trace=false, } local function statushandler(status,...) if status then @@ -9847,7 +10126,9 @@ local function statushandler(status,...) if type(err)=="table" then err=err[1] end - report("error: %s",tostring(err)) + if copas.trace then + report("error: %s",tostring(err)) + end return nil,err end function socket.protect(func) @@ -9861,7 +10142,9 @@ function socket.newtry(finalizer) if not status then local detail=select(2,...) pcall(finalizer,detail) - report("error: %s",tostring(detail)) + if copas.trace then + report("error: %s",tostring(detail)) + end return end return... @@ -12429,7 +12712,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 13340, stripped down to: 8826 +-- original size: 13394, stripped down to: 8882 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -12439,8 +12722,9 @@ if not modules then modules={} end modules ['trac-set']={ license="see context related readme files" } local type,next,tostring,tonumber=type,next,tostring,tonumber +local print=print local concat,sortedhash=table.concat,table.sortedhash -local format,find,lower,gsub,topattern=string.format,string.find,string.lower,string.gsub,string.topattern +local formatters,find,lower,gsub,topattern=string.formatters,string.find,string.lower,string.gsub,string.topattern local is_boolean=string.is_boolean local settings_to_hash=utilities.parsers.settings_to_hash local allocate=utilities.storage.allocate @@ -12450,10 +12734,10 @@ local setters=utilities.setters or {} utilities.setters=setters local data={} local trace_initialize=false +local frozen=true function setters.initialize(filename,name,values) local setter=data[name] if setter then - frozen=true local data=setter.data if data then for key,newvalue in sortedhash(values) do @@ -12647,8 +12931,8 @@ function setters.show(t) end end local enable,disable,register,list,show=setters.enable,setters.disable,setters.register,setters.list,setters.show -function setters.report(setter,...) - print(format("%-15s : %s\n",setter.name,format(...))) +function setters.report(setter,fmt,...) + print(formatters["%-15s : %s\n"](setter.name,formatters[fmt](...))) end local function default(setter,name) local d=setter.data[name] @@ -12668,7 +12952,7 @@ function setters.new(name) disable=function(...) disable (setter,...) end, reset=function(...) reset (setter,...) end, register=function(...) register(setter,...) end, - list=function(...) list (setter,...) end, + list=function(...) return list (setter,...) end, show=function(...) show (setter,...) end, default=function(...) return default (setter,...) end, value=function(...) return value (setter,...) end, @@ -12771,7 +13055,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 32618, stripped down to: 20935 +-- original size: 33003, stripped down to: 21667 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -12816,14 +13100,30 @@ local function ignore() end setmetatableindex(logs,function(t,k) t[k]=ignore;return ignore end) local report,subreport,status,settarget,setformats,settranslations local direct,subdirect,writer,pushtarget,poptarget,setlogfile,settimedlog,setprocessor,setformatters,newline +local function ansisupported(specification) + if specification~="ansi" and specification~="ansilog" then + return false + elseif os and os.enableansi then + return os.enableansi() + else + return false + end +end if runningtex and texio then if texio.setescape then texio.setescape(0) end - if arg then + if arg and ansisupported then for k,v in next,arg do if v=="--ansi" or v=="--c:ansi" then - variant="ansi" + if ansisupported("ansi") then + variant="ansi" + end + break + elseif v=="--ansilog" or v=="--c:ansilog" then + if ansisupported("ansilog") then + variant="ansilog" + end break end end @@ -12928,6 +13228,10 @@ if runningtex and texio then }, } } + variants.ansilog={ + formats=variants.ansi.formats, + targets=variants.default.targets, + } logs.flush=io.flush writer=function(...) write_nl(target,...) @@ -13034,6 +13338,9 @@ if runningtex and texio then t=specification.targets f=specification.formats or specification else + if not ansisupported(specification) then + specification="default" + end local v=variants[specification] if v then t=v.targets @@ -13060,7 +13367,7 @@ if runningtex and texio then subdirect_nop=f.subdirect_nop status_yes=f.status_yes status_nop=f.status_nop - if variant=="ansi" then + if variant=="ansi" or variant=="ansilog" then useluawrites() end settarget(whereto) @@ -13153,6 +13460,9 @@ else if type(specification)=="table" then f=specification.formats or specification else + if not ansisupported(specification) then + specification="default" + end local v=variants[specification] if v then f=v.formats @@ -13408,12 +13718,6 @@ end local nesting=0 local verbose=false local hasscheme=url.hasscheme -function logs.show_open(name) -end -function logs.show_close(name) -end -function logs.show_load(name) -end local simple=logs.reporter("comment") logs.simple=simple logs.simpleline=simple @@ -13480,6 +13784,13 @@ local exporters={ logs.reporters=reporters logs.exporters=exporters function logs.application(t) + local arguments=environment and environment.arguments + if arguments then + local ansi=arguments.ansi or arguments.ansilog + if ansi then + logs.setformatters(arguments.ansi and "ansi" or "ansilog") + end + end t.name=t.name or "unknown" t.banner=t.banner t.moreinfo=moreinfo @@ -13553,8 +13864,6 @@ else print(format(...)) end end -io.stdout:setvbuf('no') -io.stderr:setvbuf('no') if package.helpers.report then package.helpers.report=logs.reporter("package loader") end @@ -13652,7 +13961,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-inf"] = package.loaded["trac-inf"] or true --- original size: 8966, stripped down to: 5972 +-- original size: 9973, stripped down to: 7492 if not modules then modules={} end modules ['trac-inf']={ version=1.001, @@ -13674,7 +13983,7 @@ statistics.enable=true statistics.threshold=0.01 local statusinfo,n,registered,timers={},0,{},{} setmetatableindex(timers,function(t,k) - local v={ timing=0,loadtime=0 } + local v={ timing=0,loadtime=0,offset=0 } t[k]=v return v end) @@ -13682,10 +13991,40 @@ local function hastiming(instance) return instance and timers[instance] end local function resettiming(instance) - timers[instance or "notimer"]={ timing=0,loadtime=0 } + timers[instance or "notimer"]={ timing=0,loadtime=0,offset=0 } end local ticks=clock local seconds=function(n) return n or 0 end +if lua.getpreciseticks then + ticks=lua.getpreciseticks + seconds=lua.getpreciseseconds +elseif FFISUPPORTED and ffi and os.type=="windows" then + local okay,kernel=pcall(ffi.load,"kernel32") + if kernel then + local tonumber=ffi.number or tonumber + ffi.cdef[[ + int QueryPerformanceFrequency(int64_t *lpFrequency); + int QueryPerformanceCounter(int64_t *lpPerformanceCount); + ]] + local target=ffi.new("__int64[1]") + ticks=function() + if kernel.QueryPerformanceCounter(target)==1 then + return tonumber(target[0]) + else + return 0 + end + end + local target=ffi.new("__int64[1]") + seconds=function(ticks) + if kernel.QueryPerformanceFrequency(target)==1 then + return ticks/tonumber(target[0]) + else + return 0 + end + end + end +else +end local function starttiming(instance,reset) local timer=timers[instance or "notimer"] local it=timer.timing @@ -13720,12 +14059,26 @@ local function stoptiming(instance) end return 0 end +local function benchmarktimer(instance) + local timer=timers[instance or "notimer"] + local it=timer.timing + if it>1 then + timer.timing=it-1 + else + local starttime=timer.starttime + if starttime and starttime>0 then + timer.offset=ticks()-starttime + else + timer.offset=0 + end + end +end local function elapsed(instance) if type(instance)=="number" then return instance else local timer=timers[instance or "notimer"] - return timer and seconds(timer.loadtime) or 0 + return timer and seconds(timer.loadtime-2*(timer.offset or 0)) or 0 end end local function currenttime(instance) @@ -13738,7 +14091,7 @@ local function currenttime(instance) else local starttime=timer.starttime if starttime and starttime>0 then - return seconds(timer.loadtime+ticks()-starttime) + return seconds(timer.loadtime+ticks()-starttime-2*(timer.offset or 0)) end end return 0 @@ -13764,6 +14117,7 @@ statistics.elapsed=elapsed statistics.elapsedtime=elapsedtime statistics.elapsedindeed=elapsedindeed statistics.elapsedseconds=elapsedseconds +statistics.benchmarktimer=benchmarktimer function statistics.register(tag,fnc) if statistics.enable and type(fnc)=="function" then local rt=registered[tag] or (#statusinfo+1) @@ -13780,10 +14134,17 @@ function statistics.show() return format("%s, type: %s, binary subtree: %s", os.platform or "unknown",os.type or "unknown",environment.texos or "unknown") end) - register("used engine",function() - return format("%s version %s with functionality level %s, banner: %s", - LUATEXENGINE,LUATEXVERSION,LUATEXFUNCTIONALITY,lower(status.banner)) - end) + if LUATEXENGINE=="luametatex" then + register("used engine",function() + return format("%s version %s, functionality level %s, format id %s", + LUATEXENGINE,LUATEXVERSION,LUATEXFUNCTIONALITY,LUATEXFORMATID) + end) + else + register("used engine",function() + return format("%s version %s with functionality level %s, banner: %s", + LUATEXENGINE,LUATEXVERSION,LUATEXFUNCTIONALITY,lower(status.banner)) + end) + end register("control sequences",function() return format("%s of %s + %s",status.cs_count,status.hash_size,status.hash_extra) end) @@ -13822,7 +14183,11 @@ function statistics.show() end function statistics.memused() local round=math.round or math.floor - return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000),round(status.luastate_bytes/1000000)) + return format("%s MB, ctx: %s MB, max: %s MB)", + round(collectgarbage("count")/1000), + round(status.luastate_bytes/1000000), + status.luastate_bytes_max and round(status.luastate_bytes_max/1000000) or "unknown" + ) end starttiming(statistics) function statistics.formatruntime(runtime) @@ -14013,7 +14378,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lua"] = package.loaded["util-lua"] or true --- original size: 6664, stripped down to: 4589 +-- original size: 7149, stripped down to: 4997 if not modules then modules={} end modules ['util-lua']={ version=1.001, @@ -14038,16 +14403,21 @@ luautilities.nofstrippedchunks=0 luautilities.nofstrippedbytes=0 local strippedchunks={} luautilities.strippedchunks=strippedchunks +if not LUATEXENGINE then + LUATEXENGINE=status.luatex_engine and string.lower(status.luatex_engine) + JITSUPPORTED=LUATEXENGINE=="luajittex" or jit + CONTEXTLMTXMODE=CONTEXTLMTXMODE or (LUATEXENGINE=="luametatex" and 1) or 0 +end luautilities.suffixes={ tma="tma", - tmc=jit and "tmb" or "tmc", + tmc=(CONTEXTLMTXMODE and CONTEXTLMTXMODE>0 and "tmd") or (jit and "tmb") or "tmc", lua="lua", - luc=jit and "lub" or "luc", + luc=(CONTEXTLMTXMODE and CONTEXTLMTXMODE>0 and "lud") or (jit and "lub") or "luc", lui="lui", luv="luv", luj="luj", tua="tua", - tuc="tuc", + tuc=(CONTEXTLMTXMODE and CONTEXTLMTXMODE>0 and "tud") or (jit and "tub") or "tuc", } local function register(name) if tracestripping then @@ -14186,7 +14556,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-deb"] = package.loaded["util-deb"] or true --- original size: 9955, stripped down to: 6693 +-- original size: 10136, stripped down to: 6832 if not modules then modules={} end modules ['util-deb']={ version=1.001, @@ -14210,7 +14580,13 @@ local dummycalls=10*1000 local nesting=0 local names={} local initialize=false -if not (FFISUPPORTED and ffi) then +if lua.getpreciseticks then + initialize=function() + ticks=lua.getpreciseticks + seconds=lua.getpreciseseconds + initialize=false + end +elseif not (FFISUPPORTED and ffi) then elseif os.type=="windows" then initialize=function() local kernel=ffilib("kernel32","system") @@ -14482,7 +14858,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tpl"] = package.loaded["util-tpl"] or true --- original size: 7112, stripped down to: 3887 +-- original size: 7722, stripped down to: 4212 if not modules then modules={} end modules ['util-tpl']={ version=1.001, @@ -14498,6 +14874,7 @@ local report_template=logs.reporter("template") local tostring,next=tostring,next local format,sub,byte=string.format,string.sub,string.byte local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns +local formatters=string.formatters local replacer local function replacekey(k,t,how,recursive) local v=t[k] @@ -14566,6 +14943,10 @@ local function replaceoptional(l,m,r,t,how,recurse) local v=t[l] return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" end +local function replaceformatted(l,m,r,t,how,recurse) + local v=t[r] + return v and formatters[l](v) +end local single=P("%") local double=P("%%") local lquoted=P("%[") @@ -14579,16 +14960,19 @@ local nolquoted=lquoted/'' local norquoted=rquoted/'' local nolquotedq=lquotedq/'' local norquotedq=rquotedq/'' +local nolformatted=P(":")/"%%" +local norformatted=P(":")/"" local noloptional=P("%?")/'' local noroptional=P("?%")/'' local nomoptional=P(":")/'' local args=Carg(1)*Carg(2)*Carg(3) -local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle -local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq -local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted +local key=nosingle*((C((1-nosingle)^1)*args)/replacekey)*nosingle +local quoted=nolquotedq*((C((1-norquotedq)^1)*args)/replacekeyquoted)*norquotedq +local unquoted=nolquoted*((C((1-norquoted)^1)*args)/replacekeyunquoted)*norquoted local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional +local formatted=nosingle*((Cs(nolformatted*(1-norformatted )^1)*norformatted*C((1-nosingle)^1)*args)/replaceformatted)*nosingle local any=P(1) - replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) + replacer=Cs((unquoted+quoted+formatted+escape+optional+key+any)^0) local function replace(str,mapping,how,recurse) if mapping and str then return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str @@ -14627,7 +15011,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sbx"] = package.loaded["util-sbx"] or true --- original size: 20393, stripped down to: 13121 +-- original size: 21084, stripped down to: 13214 if not modules then modules={} end modules ['util-sbx']={ version=1.001, @@ -14873,37 +15257,50 @@ local iopopen=sandbox.original(io.popen) local reported={} local function validcommand(name,program,template,checkers,defaults,variables,reporter,strict) if validbinaries~=false and (validbinaries==true or validbinaries[program]) then + local binpath=nil if variables then for variable,value in next,variables do - local checker=validators[checkers[variable]] - if checker then - value=checker(unquoted(value),strict) - if value then - variables[variable]=optionalquoted(value) + local chktype=checkers[variable] + if chktype=="verbose" then + else + local checker=validators[chktype] + if checker then + value=checker(unquoted(value),strict) + if value then + variables[variable]=optionalquoted(value) + else + report("variable %a with value %a fails the check",variable,value) + return + end else - report("variable %a with value %a fails the check",variable,value) + report("variable %a has no checker",variable) return end - else - report("variable %a has no checker",variable) - return end end for variable,default in next,defaults do local value=variables[variable] if not value or value=="" then - local checker=validators[checkers[variable]] - if checker then - default=checker(unquoted(default),strict) - if default then - variables[variable]=optionalquoted(default) - else - report("variable %a with default %a fails the check",variable,default) - return + local chktype=checkers[variable] + if chktype=="verbose" then + else + local checker=validators[chktype] + if checker then + default=checker(unquoted(default),strict) + if default then + variables[variable]=optionalquoted(default) + else + report("variable %a with default %a fails the check",variable,default) + return + end end end end end + binpath=variables.binarypath + end + if type(binpath)=="string" and binpath~="" then + program=binpath.."/"..program end local command=program.." "..replace(template,variables) if reporter then @@ -14938,7 +15335,8 @@ local runners={ if trace then report("execute: %s",command) end - return osexecute(command) + local okay=osexecute(command) + return okay end end, pipeto=function(...) @@ -14972,7 +15370,7 @@ function sandbox.registerrunner(specification) return end if validrunners[name] then - report("invalid name, runner %a already defined") + report("invalid name, runner %a already defined",name) return end local program=specification.program @@ -15090,8 +15488,8 @@ if io then end if os then overload(os.execute,binaryrunner,"os.execute") - overload(os.spawn,dummyrunner,"os.spawn") - overload(os.exec,dummyrunner,"os.exec") + overload(os.spawn,dummyrunner,"os.spawn") + overload(os.exec,dummyrunner,"os.exec") overload(os.resultof,binaryrunner,"os.resultof") overload(os.pipeto,binaryrunner,"os.pipeto") overload(os.rename,filehandlertwo,"os.rename") @@ -15116,13 +15514,6 @@ end if zip then zip.open=register(zip.open,filehandlerone,"zip.open") end -if fontloader then - fontloader.open=register(fontloader.open,filehandlerone,"fontloader.open") - fontloader.info=register(fontloader.info,filehandlerone,"fontloader.info") -end -if epdf then - epdf.open=register(epdf.open,filehandlerone,"epdf.open") -end sandbox.registerroot=registerroot sandbox.registerbinary=registerbinary sandbox.registerlibrary=registerlibrary @@ -15528,7 +15919,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-env"] = package.loaded["luat-env"] or true --- original size: 6134, stripped down to: 4118 +-- original size: 6551, stripped down to: 4315 if not modules then modules={} end modules ['luat-env']={ version=1.001, @@ -15537,7 +15928,7 @@ package.loaded["luat-env"] = package.loaded["luat-env"] or true copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local rawset,rawget,loadfile=rawset,rawget,loadfile +local rawset,loadfile=rawset,loadfile local gsub=string.gsub local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_lua=logs.reporter("resolvers","lua") @@ -15584,6 +15975,12 @@ function environment.texfile(filename) return resolvers.findfile(filename,'tex') end function environment.luafile(filename) + if CONTEXTLMTXMODE and CONTEXTLMTXMODE>0 and file.suffix(filename)=="lua" then + local resolved=resolvers.findfile(file.replacesuffix(filename,"lmt")) or "" + if resolved~="" then + return resolved + end + end local resolved=resolvers.findfile(filename,'tex') or "" if resolved~="" then return resolved @@ -15695,7 +16092,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-zip"] = package.loaded["util-zip"] or true --- original size: 18645, stripped down to: 11291 +-- original size: 19496, stripped down to: 10858 if not modules then modules={} end modules ['util-zip']={ version=1.001, @@ -15706,7 +16103,7 @@ if not modules then modules={} end modules ['util-zip']={ local type,tostring,tonumber=type,tostring,tonumber local sort=table.sort local find,format,sub,gsub=string.find,string.format,string.sub,string.gsub -local osdate,ostime=os.date,os.time +local osdate,ostime,osclock=os.date,os.time,os.clock local ioopen=io.open local loaddata,savedata=io.loaddata,io.savedata local filejoin,isdir,dirname,mkdirs=file.join,lfs.isdir,file.dirname,dir.mkdirs @@ -15721,25 +16118,19 @@ local getposition=files.getposition local band=bit32.band local rshift=bit32.rshift local lshift=bit32.lshift -local decompress,calculatecrc -if flate then - decompress=flate.flate_decompress - calculatecrc=flate.update_crc32 -else +local decompress,expandsize,calculatecrc local zlibdecompress=zlib.decompress + local zlibexpandsize=zlib.expandsize local zlibchecksum=zlib.crc32 - decompress=function(source,targetsize) - local target=zlibdecompress(source,-15) - if target then - return target - else - return false,1 - end + decompress=function(source) + return zlibdecompress(source,-15) end + expandsize=zlibexpandsize and function(source,targetsize) + return zlibexpandsize(source,targetsize,-15) + end or decompress calculatecrc=function(buffer,initial) return zlibchecksum(initial or 0,buffer) end -end local zipfiles={} utilities.zipfiles=zipfiles local openzipfile,closezipfile,unzipfile,foundzipfile,getziphash,getziplist do @@ -15864,7 +16255,11 @@ local openzipfile,closezipfile,unzipfile,foundzipfile,getziphash,getziplist do setposition(handle,position) local result=readstring(handle,compressed) if data.method==8 then - result=decompress(result,data.uncompressed) + if expandsize then + result=expandsize(result,data.uncompressed) + else + result=decompress(result) + end end if check and data.crc32~=calculatecrc(result) then print("checksum mismatch") @@ -15883,14 +16278,14 @@ local openzipfile,closezipfile,unzipfile,foundzipfile,getziphash,getziplist do zipfiles.list=getziplist zipfiles.found=foundzipfile end -if flate then do +if xzip then local writecardinal1=files.writebyte local writecardinal2=files.writecardinal2le local writecardinal4=files.writecardinal4le local logwriter=logs.writer local globpattern=dir.globpattern - local compress=flate.flate_compress - local checksum=flate.update_crc32 + local compress=xzip.compress + local checksum=xzip.crc32 local function fromdostime(dostime,dosdate) return ostime { year=rshift(dosdate,9)+1980, @@ -16087,27 +16482,33 @@ if flate then do local count=#list local step=number.idiv(count,10) local done=0 + local steps=verbose=="steps" + local time=steps and osclock() for i=1,count do local l=list[i] local n=l.filename local d=unzipfile(z,n) - local p=filejoin(path,n) - if mkdirs(dirname(p)) then - if verbose=="steps" then - total=total+#d - done=done+1 - if done>=step then - done=0 - logwriter(format("%4i files of %4i done, %10i bytes",i,count,total)) + if d then + local p=filejoin(path,n) + if mkdirs(dirname(p)) then + if steps then + total=total+#d + done=done+1 + if done>=step then + done=0 + logwriter(format("%4i files of %4i done, %10i bytes, %0.3f seconds",i,count,total,osclock()-time)) + end + elseif verbose then + logwriter(n) end - elseif verbose then - logwriter(n) + savedata(p,d) end - savedata(p,d) + else + logwriter(format("problem with file %s",n)) end end - if verbose=="steps" then - logwriter(format("%4i files of %4i done, %10i bytes",count,count,total)) + if steps then + logwriter(format("%4i files of %4i done, %10i bytes, %0.3f seconds",count,count,total,osclock()-time)) end closezipfile(z) return true @@ -16118,37 +16519,8 @@ if flate then do end zipfiles.zipdir=zipdir zipfiles.unzipdir=unzipdir -end end -if flate then - local streams=utilities.streams - local openfile=streams.open - local closestream=streams.close - local setposition=streams.setposition - local getsize=streams.size - local readcardinal4=streams.readcardinal4le - local getstring=streams.getstring - local decompress=flate.gz_decompress - function zipfiles.gunzipfile(filename) - local strm=openfile(filename) - if strm then - setposition(strm,getsize(strm)-4+1) - local size=readcardinal4(strm) - local data=decompress(getstring(strm),size) - closestream(strm) - return data - end - end -elseif gzip then - local openfile=gzip.open - function zipfiles.gunzipfile(filename) - local g=openfile(filename,"rb") - if g then - local d=g:read("*a") - d:close() - return d - end - end end +zipfiles.gunzipfile=gzip.load end -- of closure @@ -16157,7 +16529,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-tab"] = package.loaded["lxml-tab"] or true --- original size: 60383, stripped down to: 35698 +-- original size: 61191, stripped down to: 35864 if not modules then modules={} end modules ['lxml-tab']={ version=1.001, @@ -16881,7 +17253,10 @@ local slash=P('/') local colon=P(':') local semicolon=P(';') local ampersand=P('&') -local valid=R('az','AZ','09')+S('_-.') +local valid_0=R("\128\255") +local valid_1=R('az','AZ')+S('_')+valid_0 +local valid_2=valid_1+R('09')+S('-.') +local valid=valid_1*valid_2^0 local name_yes=C(valid^1)*colon*C(valid^1) local name_nop=C(P(true))*C(valid^1) local name=name_yes+name_nop @@ -16917,8 +17292,9 @@ end local function entityfile(pattern,k,v,n) if n then local okay,data - if resolvers then - okay,data=resolvers.loadbinfile(n) + local loadbinfile=resolvers and resolvers.loadbinfile + if loadbinfile then + okay,data=loadbinfile(n) else data=io.loaddata(n) okay=data and data~="" @@ -17027,12 +17403,14 @@ publicentityfile+publicdoctype+systemdoctype+definitiondoctype+simpledoctype)*op } return grammar_parsed_text_one,grammar_parsed_text_two,grammar_unparsed_text end -grammar_parsed_text_one_nop, -grammar_parsed_text_two_nop, -grammar_unparsed_text_nop=install(space,spacing,anything) -grammar_parsed_text_one_yes, -grammar_parsed_text_two_yes, -grammar_unparsed_text_yes=install(space_nl,spacing_nl,anything_nl) +local + grammar_parsed_text_one_nop, + grammar_parsed_text_two_nop, + grammar_unparsed_text_nop=install(space,spacing,anything) +local + grammar_parsed_text_one_yes, + grammar_parsed_text_two_yes, + grammar_unparsed_text_yes=install(space_nl,spacing_nl,anything_nl) local function _xmlconvert_(data,settings,detail) settings=settings or {} preparexmlstate(settings) @@ -17613,7 +17991,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-lpt"] = package.loaded["lxml-lpt"] or true --- original size: 55145, stripped down to: 30992 +-- original size: 54626, stripped down to: 31255 if not modules then modules={} end modules ['lxml-lpt']={ version=1.001, @@ -18095,6 +18473,8 @@ local builtin={ lastindex="(#ll.__p__.dt or 1)", lastelement="(ll.__p__.en or 1)", last="#list", + list="list", + self="ll", rootposition="order", order="order", element="(ll.ei or 1)", @@ -18203,7 +18583,8 @@ local function register_selector(specification) end local function register_expression(expression) local converted=lpegmatch(converter,expression) - local runner=load(format(template_e,converted)) + local wrapped=format(template_e,converted) + local runner=load(wrapped) runner=(runner and runner()) or function() errorrunner_e(expression,converted) end return { kind="expression",expression=expression,converted=converted,evaluator=runner } end @@ -18575,6 +18956,20 @@ expressions.count=function(e,pattern) local collected=applylpath(e,pattern) return pattern and (collected and #collected) or 0 end +expressions.attribute=function(e,name,value) + if type(e)=="table" and name then + local a=e.at + if a then + local v=a[name] + if value then + return v==value + else + return v + end + end + end + return nil +end expressions.oneof=function(s,...) for i=1,select("#",...) do if s==select(i,...) then @@ -18621,7 +19016,7 @@ function expressions.contains(str,pattern) end return false end -function xml.expressions.idstring(str) +function expressions.idstring(str) return type(str)=="string" and gsub(str,"^#","") or "" end local function traverse(root,pattern,handle) @@ -20394,7 +20789,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 11099, stripped down to: 7152 +-- original size: 11019, stripped down to: 7086 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -20407,9 +20802,9 @@ local next,type,getmetatable,rawset=next,type,getmetatable,rawset local gsub,find,gmatch,char=string.gsub,string.find,string.gmatch,string.char local filedirname,filebasename,filejoin=file.dirname,file.basename,file.join local ostype,osname,osuname,ossetenv,osgetenv=os.type,os.name,os.uname,os.setenv,os.getenv +local sortedpairs=table.sortedpairs local P,S,R,C,Cs,Cc,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) -local trace_detail=false trackers.register("resolvers.details",function(v) trace_detail=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) local report_initialization=logs.reporter("resolvers","initialization") resolvers=resolvers or {} @@ -20618,7 +21013,7 @@ if ostype=="unix" then rawset(t,k,v) end local colon=P(":") - for k,v in table.sortedpairs(prefixes) do + for k,v in sortedpairs(prefixes) do if p then p=P(k)+p else @@ -20645,7 +21040,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 18154, stripped down to: 10416 +-- original size: 18179, stripped down to: 10432 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -20929,7 +21324,7 @@ local function scan(files,remap,spec,path,n,m,r,onlyone,tolerant) scancache[sub(full,1,-2)]=files return files,remap,n,m,r end -function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) +local function scanfiles(path,branch,usecache,onlyonce,tolerant) local realpath=resolveprefix(path) if usecache then local content=fullcache[realpath] @@ -20984,8 +21379,9 @@ function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) statistics.stoptiming(timer) return content end +resolvers.scanfiles=scanfiles function resolvers.simplescanfiles(path,branch,usecache) - return resolvers.scanfiles(path,branch,usecache,true,true) + return scanfiles(path,branch,usecache,true,true) end function resolvers.scandata() table.sort(scanned) @@ -21052,7 +21448,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 9360, stripped down to: 6312 +-- original size: 9400, stripped down to: 6347 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -21062,7 +21458,7 @@ if not modules then modules={} end modules ['data-env']={ license="see context related readme files", } local lower,gsub=string.lower,string.gsub -local next=next +local next,rawget=next,rawget local resolvers=resolvers local allocate=utilities.storage.allocate local setmetatableindex=table.setmetatableindex @@ -21143,13 +21539,13 @@ local relations=allocate { mp={ names={ "mp" }, variable='MPINPUTS', - suffixes={ 'mp','mpvi','mpiv','mpii' }, + suffixes={ 'mp','mpvi','mpiv','mpxl','mpii' }, usertype=true, }, tex={ names={ "tex" }, variable='TEXINPUTS', - suffixes={ "tex","mkvi","mkiv","mkii","cld","lfg","xml" }, + suffixes={ "tex","mkiv","mkvi","mkxl","mklx","mkii","cld","lfg","xml" }, usertype=true, }, icc={ @@ -21337,7 +21733,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 16284, stripped down to: 10938 +-- original size: 16099, stripped down to: 11379 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -21346,48 +21742,83 @@ if not modules then modules={} end modules ['data-tmp']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local format,lower,gsub,concat=string.format,string.lower,string.gsub,table.concat -local concat=table.concat -local mkdirs,isdir,isfile=dir.mkdirs,lfs.isdir,lfs.isfile -local addsuffix,is_writable,is_readable=file.addsuffix,file.is_writable,file.is_readable -local formatters=string.formatters local next,type=next,type +local pcall,loadfile,collectgarbage=pcall,loadfile,collectgarbage +local format,lower,gsub=string.format,string.lower,string.gsub +local concat,serialize,fastserialize,serializetofile=table.concat,table.serialize,table.fastserialize,table.tofile +local mkdirs,expanddirname,isdir,isfile=dir.mkdirs,dir.expandname,lfs.isdir,lfs.isfile +local is_writable,is_readable=file.is_writable,file.is_readable +local collapsepath,joinfile,addsuffix,dirname=file.collapsepath,file.join,file.addsuffix,file.dirname +local savedata=file.savedata +local formatters=string.formatters +local osexit,osdate,osuuid=os.exit,os.date,os.uuid +local removefile=os.remove +local md5hex=md5.hex local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) local report_caches=logs.reporter("resolvers","caches") local report_resolvers=logs.reporter("resolvers","caching") local resolvers=resolvers local cleanpath=resolvers.cleanpath -local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) -local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) -local compile=utilities.lua.compile -function utilities.lua.compile(luafile,lucfile,cleanup,strip) - if cleanup==nil then cleanup=directive_cleanup end - if strip==nil then strip=directive_strip end - return compile(luafile,lucfile,cleanup,strip) +local resolvepath=resolvers.resolve +local luautilities=utilities.lua +do + local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) + local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) + local compilelua=luautilities.compile + function luautilities.compile(luafile,lucfile,cleanup,strip) + if cleanup==nil then cleanup=directive_cleanup end + if strip==nil then strip=directive_strip end + return compilelua(luafile,lucfile,cleanup,strip) + end end caches=caches or {} local caches=caches -local luasuffixes=utilities.lua.suffixes -caches.base=caches.base or "luatex-cache" -caches.more=caches.more or "context" -caches.direct=false -caches.tree=false -caches.force=true -caches.ask=false -caches.relocate=false +local writable=nil +local readables={} +local usedreadables={} +local compilelua=luautilities.compile +local luasuffixes=luautilities.suffixes +caches.base=caches.base or "luatex-cache" +caches.more=caches.more or "context" caches.defaults={ "TMPDIR","TEMPDIR","TMP","TEMP","HOME","HOMEPATH" } -directives.register("system.caches.fast",function(v) caches.fast=true end) -local writable,readables,usedreadables=nil,{},{} +local direct_cache=false +local fast_cache=false +local cache_tree=false +directives.register("system.caches.direct",function(v) direct_cache=true end) +directives.register("system.caches.fast",function(v) fast_cache=true end) +local function configfiles() + return concat(resolvers.configurationfiles(),";") +end +local function hashed(tree) + tree=gsub(tree,"[\\/]+$","") + tree=lower(tree) + local hash=md5hex(tree) + if trace_cache or trace_locating then + report_caches("hashing tree %a, hash %a",tree,hash) + end + return hash +end +local function treehash() + local tree=configfiles() + if not tree or tree=="" then + return false + else + return hashed(tree) + end +end +caches.hashed=hashed +caches.treehash=treehash +caches.configfiles=configfiles local function identify() local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") if texmfcaches then for k=1,#texmfcaches do local cachepath=texmfcaches[k] if cachepath~="" then - cachepath=resolvers.resolve(cachepath) - cachepath=resolvers.cleanpath(cachepath) - cachepath=file.collapsepath(cachepath) + cachepath=resolvepath(cachepath) + cachepath=cleanpath(cachepath) + cachepath=collapsepath(cachepath) local valid=isdir(cachepath) if valid then if is_readable(cachepath) then @@ -21396,16 +21827,14 @@ local function identify() writable=cachepath end end - elseif not writable and caches.force then - local cacheparent=file.dirname(cachepath) - if is_writable(cacheparent) and true then - if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath),"no",{ "yes","no" })=="yes" then - mkdirs(cachepath) - if isdir(cachepath) and is_writable(cachepath) then - report_caches("path %a created",cachepath) - writable=cachepath - readables[#readables+1]=cachepath - end + elseif not writable then + local cacheparent=dirname(cachepath) + if is_writable(cacheparent) then + mkdirs(cachepath) + if isdir(cachepath) and is_writable(cachepath) then + report_caches("path %a created",cachepath) + writable=cachepath + readables[#readables+1]=cachepath end end end @@ -21418,8 +21847,8 @@ local function identify() local cachepath=texmfcaches[k] cachepath=resolvers.expansion(cachepath) if cachepath~="" then - cachepath=resolvers.resolve(cachepath) - cachepath=resolvers.cleanpath(cachepath) + cachepath=resolvepath(cachepath) + cachepath=cleanpath(cachepath) local valid=isdir(cachepath) if valid and is_readable(cachepath) then if not writable and is_writable(cachepath) then @@ -21433,23 +21862,25 @@ local function identify() end if not writable then report_caches("fatal error: there is no valid writable cache path defined") - os.exit() + osexit() elseif #readables==0 then report_caches("fatal error: there is no valid readable cache path defined") - os.exit() + osexit() end - writable=dir.expandname(resolvers.cleanpath(writable)) - local base,more,tree=caches.base,caches.more,caches.tree or caches.treehash() + writable=expanddirname(cleanpath(writable)) + local base=caches.base + local more=caches.more + local tree=cache_tree or treehash() if tree then - caches.tree=tree + cache_tree=tree writable=mkdirs(writable,base,more,tree) for i=1,#readables do - readables[i]=file.join(readables[i],base,more,tree) + readables[i]=joinfile(readables[i],base,more,tree) end else writable=mkdirs(writable,base,more) for i=1,#readables do - readables[i]=file.join(readables[i],base,more) + readables[i]=joinfile(readables[i],base,more) end end if trace_cache then @@ -21486,27 +21917,8 @@ function caches.usedpaths(separator) return writable or "?" end end -function caches.configfiles() - return concat(resolvers.configurationfiles(),";") -end -function caches.hashed(tree) - tree=gsub(tree,"[\\/]+$","") - tree=lower(tree) - local hash=md5.hex(tree) - if trace_cache or trace_locating then - report_caches("hashing tree %a, hash %a",tree,hash) - end - return hash -end -function caches.treehash() - local tree=caches.configfiles() - if not tree or tree=="" then - return false - else - return caches.hashed(tree) - end -end -local r_cache,w_cache={},{} +local r_cache={} +local w_cache={} local function getreadablepaths(...) local tags={... } local hash=concat(tags,"/") @@ -21516,7 +21928,7 @@ local function getreadablepaths(...) if #tags>0 then done={} for i=1,#readables do - done[i]=file.join(readables[i],...) + done[i]=joinfile(readables[i],...) end else done=readables @@ -21540,17 +21952,25 @@ local function getwritablepath(...) end return done end -caches.getreadablepaths=getreadablepaths -caches.getwritablepath=getwritablepath -function caches.getfirstreadablefile(filename,...) - local fullname,path=caches.setfirstwritablefile(filename,...) +local function setfirstwritablefile(filename,...) + local wr=getwritablepath(...) + local fullname=joinfile(wr,filename) + return fullname,wr +end +local function setluanames(path,name) + return + format("%s/%s.%s",path,name,luasuffixes.tma), + format("%s/%s.%s",path,name,luasuffixes.tmc) +end +local function getfirstreadablefile(filename,...) + local fullname,path=setfirstwritablefile(filename,...) if is_readable(fullname) then return fullname,path end local rd=getreadablepaths(...) for i=1,#rd do local path=rd[i] - local fullname=file.join(path,filename) + local fullname=joinfile(path,filename) if is_readable(fullname) then usedreadables[i]=true return fullname,path @@ -21558,19 +21978,11 @@ function caches.getfirstreadablefile(filename,...) end return fullname,path end -function caches.setfirstwritablefile(filename,...) - local wr=getwritablepath(...) - local fullname=file.join(wr,filename) - return fullname,wr -end -function caches.define(category,subcategory) - return function() - return getwritablepath(category,subcategory) - end -end -function caches.setluanames(path,name) - return format("%s/%s.%s",path,name,luasuffixes.tma),format("%s/%s.%s",path,name,luasuffixes.tmc) -end +caches.getreadablepaths=getreadablepaths +caches.getwritablepath=getwritablepath +caches.setfirstwritablefile=setfirstwritablefile +caches.getfirstreadablefile=getfirstreadablefile +caches.setluanames=setluanames function caches.loaddata(readables,name,writable) if type(readables)=="string" then readables={ readables } @@ -21578,21 +21990,22 @@ function caches.loaddata(readables,name,writable) for i=1,#readables do local path=readables[i] local loader=false - local tmaname,tmcname=caches.setluanames(path,name) + local state=false + local tmaname,tmcname=setluanames(path,name) if isfile(tmcname) then - loader=loadfile(tmcname) + state,loader=pcall(loadfile,tmcname) end if not loader and isfile(tmaname) then - local tmacrap,tmcname=caches.setluanames(writable,name) + local tmacrap,tmcname=setluanames(writable,name) if isfile(tmcname) then - loader=loadfile(tmcname) + state,loader=pcall(loadfile,tmcname) end - utilities.lua.compile(tmaname,tmcname) + compilelua(tmaname,tmcname) if isfile(tmcname) then - loader=loadfile(tmcname) + state,loader=pcall(loadfile,tmcname) end if not loader then - loader=loadfile(tmaname) + state,loader=pcall(loadfile,tmaname) end end if loader then @@ -21604,21 +22017,21 @@ function caches.loaddata(readables,name,writable) return false end function caches.is_writable(filepath,filename) - local tmaname,tmcname=caches.setluanames(filepath,filename) + local tmaname,tmcname=setluanames(filepath,filename) return is_writable(tmaname) end -local saveoptions={ compact=true } -function caches.savedata(filepath,filename,data,raw) - local tmaname,tmcname=caches.setluanames(filepath,filename) - data.cache_uuid=os.uuid() - if caches.fast then - file.savedata(tmaname,table.fastserialize(data,true)) - elseif caches.direct then - file.savedata(tmaname,table.serialize(data,true,saveoptions)) +local saveoptions={ compact=true,accurate=not JITSUPPORTED } +function caches.savedata(filepath,filename,data,fast) + local tmaname,tmcname=setluanames(filepath,filename) + data.cache_uuid=osuuid() + if fast or fast_cache then + savedata(tmaname,fastserialize(data,true)) + elseif direct_cache then + savedata(tmaname,serialize(data,true,saveoptions)) else - table.tofile(tmaname,data,true,saveoptions) + serializetofile(tmaname,data,true,saveoptions) end - utilities.lua.compile(tmaname,tmcname) + compilelua(tmaname,tmcname) end local content_state={} function caches.contentstate() @@ -21626,11 +22039,14 @@ function caches.contentstate() end function caches.loadcontent(cachename,dataname,filename) if not filename then - local name=caches.hashed(cachename) - local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") - filename=file.join(path,name) + local name=hashed(cachename) + local full,path=getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=joinfile(path,name) + end + local state,blob=pcall(loadfile,addsuffix(filename,luasuffixes.luc)) + if not blob then + state,blob=pcall(loadfile,addsuffix(filename,luasuffixes.lua)) end - local blob=loadfile(addsuffix(filename,luasuffixes.luc)) or loadfile(addsuffix(filename,luasuffixes.lua)) if blob then local data=blob() if data and data.content then @@ -21663,9 +22079,9 @@ function caches.collapsecontent(content) end function caches.savecontent(cachename,dataname,content,filename) if not filename then - local name=caches.hashed(cachename) - local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") - filename=file.join(path,name) + local name=hashed(cachename) + local full,path=setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=joinfile(path,name) end local luaname=addsuffix(filename,luasuffixes.lua) local lucname=addsuffix(filename,luasuffixes.luc) @@ -21676,17 +22092,17 @@ function caches.savecontent(cachename,dataname,content,filename) type=dataname, root=cachename, version=resolvers.cacheversion, - date=os.date("%Y-%m-%d"), - time=os.date("%H:%M:%S"), + date=osdate("%Y-%m-%d"), + time=osdate("%H:%M:%S"), content=content, - uuid=os.uuid(), + uuid=osuuid(), } - local ok=io.savedata(luaname,table.serialize(data,true)) + local ok=savedata(luaname,serialize(data,true)) if ok then if trace_locating then report_resolvers("category %a, cachename %a saved in %a",dataname,cachename,luaname) end - if utilities.lua.compile(luaname,lucname) then + if compilelua(luaname,lucname) then if trace_locating then report_resolvers("%a compiled to %a",dataname,lucname) end @@ -21695,7 +22111,7 @@ function caches.savecontent(cachename,dataname,content,filename) if trace_locating then report_resolvers("compiling failed for %a, deleting file %a",dataname,lucname) end - os.remove(lucname) + removefile(lucname) end elseif trace_locating then report_resolvers("unable to save %a in %a (access error)",dataname,luaname) @@ -21709,7 +22125,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5310, stripped down to: 3784 +-- original size: 5518, stripped down to: 3854 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -21718,31 +22134,45 @@ if not modules then modules={} end modules ['data-met']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local find,format=string.find,string.format -local sequenced=table.sequenced +local type=type +local find=string.find local addurlscheme,urlhashed=url.addscheme,url.hashed +local collapsepath,joinfile=file.collapsepath,file.join +local report_methods=logs.reporter("resolvers","methods") local trace_locating=false local trace_methods=false trackers.register("resolvers.locating",function(v) trace_methods=v end) trackers.register("resolvers.methods",function(v) trace_methods=v end) -local report_methods=logs.reporter("resolvers","methods") local allocate=utilities.storage.allocate local resolvers=resolvers local registered={} local function splitmethod(filename) if not filename then - return { scheme="unknown",original=filename } + return { + scheme="unknown", + original=filename, + } end if type(filename)=="table" then return filename end - filename=file.collapsepath(filename,".") + filename=collapsepath(filename,".") if not find(filename,"://",1,true) then - return { scheme="file",path=filename,original=filename,filename=filename } + return { + scheme="file", + path=filename, + original=filename, + filename=filename, + } end - local specification=url.hashed(filename) + local specification=urlhashed(filename) if not specification.scheme or specification.scheme=="" then - return { scheme="file",path=filename,original=filename,filename=filename } + return { + scheme="file", + path=filename, + original=filename, + filename=filename, + } else return specification end @@ -21751,7 +22181,8 @@ resolvers.splitmethod=splitmethod local function methodhandler(what,first,...) local method=registered[what] if method then - local how,namespace=method.how,method.namespace + local how=method.how + local namespace=method.namespace if how=="uri" or how=="url" then local specification=splitmethod(first) local scheme=specification.scheme @@ -21797,7 +22228,10 @@ local function methodhandler(what,first,...) end resolvers.methodhandler=methodhandler function resolvers.registermethod(name,namespace,how) - registered[name]={ how=how or "tag",namespace=namespace } + registered[name]={ + how=how or "tag", + namespace=namespace + } namespace["byscheme"]=function(scheme,filename,...) if scheme=="file" then return methodhandler(name,filename,...) @@ -21806,7 +22240,7 @@ function resolvers.registermethod(name,namespace,how) end end end -local concatinators=allocate { notfound=file.join } +local concatinators=allocate { notfound=joinfile } local locators=allocate { notfound=function() end } local hashers=allocate { notfound=function() end } local generators=allocate { notfound=function() end } @@ -21827,7 +22261,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 68195, stripped down to: 43680 +-- original size: 69576, stripped down to: 44470 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -21838,7 +22272,8 @@ if not modules then modules={} end modules ['data-res']={ } local gsub,find,lower,upper,match,gmatch=string.gsub,string.find,string.lower,string.upper,string.match,string.gmatch local concat,insert,remove=table.concat,table.insert,table.remove -local next,type,rawget=next,type,rawget +local next,type,rawget,loadfile=next,type,rawget,loadfile +local mergedtable=table.merged local os=os local P,S,R,C,Cc,Cs,Ct,Carg=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Carg local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns @@ -21854,13 +22289,14 @@ local joinpath=file.joinpath local is_qualified_path=file.is_qualified_path local allocate=utilities.storage.allocate local settings_to_array=utilities.parsers.settings_to_array +local urlhasscheme=url.hasscheme local getcurrentdir=lfs.currentdir local isfile=lfs.isfile local isdir=lfs.isdir local setmetatableindex=table.setmetatableindex local luasuffixes=utilities.lua.suffixes local trace_locating=false trackers .register("resolvers.locating",function(v) trace_locating=v end) -local trace_detail=false trackers .register("resolvers.details",function(v) trace_detail=v end) +local trace_details=false trackers .register("resolvers.details",function(v) trace_details=v end) local trace_expansions=false trackers .register("resolvers.expansions",function(v) trace_expansions=v end) local trace_paths=false trackers .register("resolvers.paths",function(v) trace_paths=v end) local resolve_otherwise=true directives.register("resolvers.otherwise",function(v) resolve_otherwise=v end) @@ -21879,10 +22315,17 @@ local ostype,osname,osenv,ossetenv,osgetenv=os.type,os.name,os.env,os.setenv,os. resolvers.cacheversion="1.100" resolvers.configbanner="" resolvers.homedir=environment.homedir -resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } resolvers.luacnfname="texmfcnf.lua" resolvers.luacnffallback="contextcnf.lua" resolvers.luacnfstate="unknown" +local criticalvars={ + "SELFAUTOLOC", + "SELFAUTODIR", + "SELFAUTOPARENT", + "TEXMFCNF", + "TEXMF", + "TEXOS", +} if environment.default_texmfcnf then resolvers.luacnfspec="home:texmf/web2c;"..environment.default_texmfcnf else @@ -21902,13 +22345,20 @@ local dangerous=resolvers.dangerous local suffixmap=resolvers.suffixmap resolvers.defaultsuffixes={ "tex" } local instance=nil -function resolvers.setenv(key,value,raw) +local variable +local expansion +local setenv +local getenv +local formatofsuffix=resolvers.formatofsuffix +local splitpath=resolvers.splitpath +local splitmethod=resolvers.splitmethod +setenv=function(key,value,raw) if instance then instance.environment[key]=value ossetenv(key,raw and value or resolveprefix(value)) end end -local function getenv(key) +getenv=function(key) local value=rawget(instance.environment,key) if value and value~="" then return value @@ -21918,94 +22368,97 @@ local function getenv(key) end end resolvers.getenv=getenv -resolvers.env=getenv -local function resolvevariable(k) - return instance.expansions[k] -end +resolvers.setenv=setenv local dollarstripper=lpeg.stripper("$") local inhibitstripper=P("!")^0*Cs(P(1)^0) -local somevariable=P("$")/"" -local somekey=C(R("az","AZ","09","__","--")^1) -local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) -local variableexpander=Cs((somevariable*(somekey/resolvevariable)+somethingelse)^1 ) -local cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" -local variablecleaner=Cs((cleaner+P(1))^0) -local somevariable=R("az","AZ","09","__","--")^1/resolvevariable -local variable=(P("$")/"")*(somevariable+(P("{")/"")*somevariable*(P("}")/"")) -local variableresolver=Cs((variable+P(1))^0) -local function expandedvariable(var) - return lpegmatch(variableexpander,var) or var -end -function resolvers.reset() - if trace_locating then - report_resolving("creating instance") - end - local environment={} - local variables={} - local expansions={} - local order={} - instance={ - environment=environment, - variables=variables, - expansions=expansions, - order=order, - files={}, - setups={}, - found={}, - foundintrees={}, - hashes={}, - hashed={}, - pathlists=false, - specification={}, - lists={}, - data={}, - fakepaths={}, - remember=true, - diskcache=true, - renewcache=false, - renewtree=false, - loaderror=false, - savelists=true, - pattern=nil, - force_suffixes=true, - pathstack={}, - } - setmetatableindex(variables,function(t,k) - local v - for i=1,#order do - v=order[i][k] +local expandedvariable,resolvedvariable do + local function resolveinstancevariable(k) + return instance.expansions[k] + end + local p_variable=P("$")/"" + local p_key=C(R("az","AZ","09","__","--")^1) + local p_whatever=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) + local variableexpander=Cs((p_variable*(p_key/resolveinstancevariable)+p_whatever)^1 ) + local p_cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" + local variablecleaner=Cs((p_cleaner+P(1))^0) + local p_variable=R("az","AZ","09","__","--")^1/resolveinstancevariable + local p_variable=(P("$")/"")*(p_variable+(P("{")/"")*p_variable*(P("}")/"")) + local variableresolver=Cs((p_variable+P(1))^0) + expandedvariable=function(var) + return lpegmatch(variableexpander,var) or var + end + function resolvers.reset() + if trace_locating then + report_resolving("creating instance") + end + local environment={} + local variables={} + local expansions={} + local order={} + instance={ + environment=environment, + variables=variables, + expansions=expansions, + order=order, + files={}, + setups={}, + found={}, + foundintrees={}, + hashes={}, + hashed={}, + pathlists=false, + specification={}, + lists={}, + data={}, + fakepaths={}, + remember=true, + diskcache=true, + renewcache=false, + renewtree=false, + loaderror=false, + savelists=true, + pattern=nil, + force_suffixes=true, + pathstack={}, + } + setmetatableindex(variables,function(t,k) + local v + for i=1,#order do + v=order[i][k] + if v~=nil then + t[k]=v + return v + end + end + if v==nil then + v="" + end + t[k]=v + return v + end) + local repath=resolvers.repath + setmetatableindex(environment,function(t,k) + local v=osgetenv(k) + if v==nil then + v=variables[k] + end if v~=nil then - t[k]=v - return v + v=checkedvariable(v) or "" end - end - if v==nil then - v="" - end - t[k]=v - return v - end) - setmetatableindex(environment,function(t,k) - local v=osgetenv(k) - if v==nil then - v=variables[k] - end - if v~=nil then - v=checkedvariable(v) or "" - end - v=resolvers.repath(v) - t[k]=v - return v - end) - setmetatableindex(expansions,function(t,k) - local v=environment[k] - if type(v)=="string" then - v=lpegmatch(variableresolver,v) - v=lpegmatch(variablecleaner,v) - end - t[k]=v - return v - end) + v=repath(v) + t[k]=v + return v + end) + setmetatableindex(expansions,function(t,k) + local v=environment[k] + if type(v)=="string" then + v=lpegmatch(variableresolver,v) + v=lpegmatch(variablecleaner,v) + end + t[k]=v + return v + end) + end end function resolvers.initialized() return instance~=nil @@ -22019,31 +22472,33 @@ local function reset_caches() instance.lists={} instance.pathlists=false end -local slash=P("/") -local pathexpressionpattern=Cs ( - Cc("^")*( - Cc("%")*S(".-")+slash^2*P(-1)/"/.*" +local makepathexpression do + local slash=P("/") + local pathexpressionpattern=Cs ( + Cc("^")*( + Cc("%")*S(".-")+slash^2*P(-1)/"/.*" +slash^2/"/"+(1-slash)*P(-1)*Cc("/")+P(1) - )^1*Cc("$") -) -local cache={} -local function makepathexpression(str) - if str=="." then - return "^%./$" - else - local c=cache[str] - if not c then - c=lpegmatch(pathexpressionpattern,str) - cache[str]=c + )^1*Cc("$") + ) + local cache={} + makepathexpression=function(str) + if str=="." then + return "^%./$" + else + local c=cache[str] + if not c then + c=lpegmatch(pathexpressionpattern,str) + cache[str]=c + end + return c end - return c end end local function reportcriticalvariables(cnfspec) if trace_locating then - for i=1,#resolvers.criticalvars do - local k=resolvers.criticalvars[i] - local v=resolvers.getenv(k) or "unknown" + for i=1,#criticalvars do + local k=criticalvars[i] + local v=getenv(k) or "unknown" report_resolving("variable %a set to %a",k,v) end report_resolving() @@ -22065,7 +22520,7 @@ local function identify_configuration_files() resolvers.luacnfstate="environment" end reportcriticalvariables(cnfspec) - local cnfpaths=expandedpathfromlist(resolvers.splitpath(cnfspec)) + local cnfpaths=expandedpathfromlist(splitpath(cnfspec)) local function locatecnf(luacnfname,kind) for i=1,#cnfpaths do local filepath=cnfpaths[i] @@ -22098,6 +22553,8 @@ local function identify_configuration_files() end local function load_configuration_files() local specification=instance.specification + local setups=instance.setups + local order=instance.order if #specification>0 then local luacnfname=resolvers.luacnfname for i=1,#specification do @@ -22107,7 +22564,6 @@ local function load_configuration_files() local realname=resolveprefix(filename) local blob=loadfile(realname) if blob then - local setups=instance.setups local data=blob() local parent=data and data.parent if parent then @@ -22118,7 +22574,7 @@ local function load_configuration_files() local parentdata=blob() if parentdata then report_resolving("loading configuration file %a",filename) - data=table.merged(parentdata,data) + data=mergedtable(parentdata,data) end end end @@ -22150,7 +22606,7 @@ local function load_configuration_files() if trace_locating then report_resolving("reloading configuration due to TEXMF redefinition") end - resolvers.setenv("TEXMFCNF",cnfspec) + setenv("TEXMFCNF",cnfspec) instance.specification={} identify_configuration_files() load_configuration_files() @@ -22168,7 +22624,7 @@ local function load_configuration_files() elseif trace_locating then report_resolving("skipping configuration file %a (no valid format)",filename) end - instance.order[#instance.order+1]=instance.setups[pathname] + order[#order+1]=setups[pathname] if instance.loaderror then break end @@ -22177,6 +22633,8 @@ local function load_configuration_files() report_resolving("warning: no lua configuration files found") end end +local expandedpathlist +local unexpandedpathlist function resolvers.configurationfiles() return instance.specification or {} end @@ -22193,7 +22651,7 @@ local function load_file_databases() end end local function locate_file_databases() - local texmfpaths=resolvers.expandedpathlist("TEXMF") + local texmfpaths=expandedpathlist("TEXMF") if #texmfpaths>0 then for i=1,#texmfpaths do local path=collapsepath(texmfpaths[i]) @@ -22202,7 +22660,7 @@ local function locate_file_databases() if stripped~="" then local runtime=stripped==path path=cleanpath(path) - local spec=resolvers.splitmethod(stripped) + local spec=splitmethod(stripped) if runtime and (spec.noscheme or spec.scheme=="file") then stripped="tree:///"..stripped elseif spec.scheme=="cache" or spec.scheme=="file" then @@ -22236,11 +22694,13 @@ local function generate_file_databases() end end local function save_file_databases() - for i=1,#instance.hashes do - local hash=instance.hashes[i] + local hashes=instance.hashes + local files=instance.files + for i=1,#hashes do + local hash=hashes[i] local cachename=hash.name if hash.cache then - local content=instance.files[cachename] + local content=files[cachename] caches.collapsecontent(content) if trace_locating then report_resolving("saving tree %a",cachename) @@ -22252,8 +22712,9 @@ local function save_file_databases() end end function resolvers.renew(hashname) + local files=instance.files if hashname and hashname~="" then - local expanded=resolvers.expansion(hashname) or "" + local expanded=expansion(hashname) or "" if expanded~="" then if trace_locating then report_resolving("identifying tree %a from %a",expanded,hashname) @@ -22270,7 +22731,7 @@ function resolvers.renew(hashname) report_resolving("using path %a",realpath) end methodhandler('generators',hashname) - local content=instance.files[hashname] + local content=files[hashname] caches.collapsecontent(content) if trace_locating then report_resolving("saving tree %a",hashname) @@ -22297,38 +22758,46 @@ local function load_databases() end end function resolvers.appendhash(type,name,cache) - if not instance.hashed[name] then + local hashed=instance.hashed + local hashes=instance.hashes + if hashed[name] then + else if trace_locating then report_resolving("hash %a appended",name) end - insert(instance.hashes,{ type=type,name=name,cache=cache } ) - instance.hashed[name]=cache + insert(hashes,{ type=type,name=name,cache=cache } ) + hashed[name]=cache end end function resolvers.prependhash(type,name,cache) - if not instance.hashed[name] then + local hashed=instance.hashed + local hashes=instance.hashes + if hashed[name] then + else if trace_locating then report_resolving("hash %a prepended",name) end - insert(instance.hashes,1,{ type=type,name=name,cache=cache } ) - instance.hashed[name]=cache + insert(hashes,1,{ type=type,name=name,cache=cache } ) + hashed[name]=cache end end function resolvers.extendtexmfvariable(specification) - local t=resolvers.splitpath(getenv("TEXMF")) - insert(t,1,specification) - local newspec=concat(t,",") - if instance.environment["TEXMF"] then - instance.environment["TEXMF"]=newspec - elseif instance.variables["TEXMF"] then - instance.variables["TEXMF"]=newspec + local environment=instance.environment + local variables=instance.variables + local texmftrees=splitpath(getenv("TEXMF")) + insert(texmftrees,1,specification) + texmftrees=concat(texmftrees,",") + if environment["TEXMF"] then + environment["TEXMF"]=texmftrees + elseif variables["TEXMF"] then + variables["TEXMF"]=texmftrees else end reset_hashes() end function resolvers.splitexpansions() - local ie=instance.expansions - for k,v in next,ie do + local expansions=instance.expansions + for k,v in next,expansions do local t,tn,h,p={},0,{},splitconfigurationpath(v) for kk=1,#p do local vv=p[kk] @@ -22338,33 +22807,37 @@ function resolvers.splitexpansions() h[vv]=true end end - if #t>1 then - ie[k]=t + if tn>1 then + expansions[k]=t else - ie[k]=t[1] + expansions[k]=t[1] end end end function resolvers.datastate() return caches.contentstate() end -function resolvers.variable(name) +variable=function(name) + local variables=instance.variables local name=name and lpegmatch(dollarstripper,name) - local result=name and instance.variables[name] + local result=name and variables[name] return result~=nil and result or "" end -function resolvers.expansion(name) +expansion=function(name) + local expansions=instance.expansions local name=name and lpegmatch(dollarstripper,name) - local result=name and instance.expansions[name] + local result=name and expansions[name] return result~=nil and result or "" end -function resolvers.unexpandedpathlist(str) - local pth=resolvers.variable(str) - local lst=resolvers.splitpath(pth) +resolvers.variable=variable +resolvers.expansion=expansion +unexpandedpathlist=function(str) + local pth=variable(str) + local lst=splitpath(pth) return expandedpathfromlist(lst) end function resolvers.unexpandedpath(str) - return joinpath(resolvers.unexpandedpathlist(str)) + return joinpath(unexpandedpathlist(str)) end function resolvers.pushpath(name) local pathstack=instance.pathstack @@ -22394,8 +22867,8 @@ function resolvers.stackpath() end local done={} function resolvers.resetextrapaths() - local ep=instance.extra_paths - if not ep then + local extra_paths=instance.extra_paths + if not extra_paths then done={} instance.extra_paths={} elseif #ep>0 then @@ -22416,8 +22889,8 @@ function resolvers.registerextrapath(paths,subpaths) end local paths=settings_to_array(paths) local subpaths=settings_to_array(subpaths) - local ep=instance.extra_paths or {} - local oldn=#ep + local extra_paths=instance.extra_paths or {} + local oldn=#extra_paths local newn=oldn local nofpaths=#paths local nofsubpaths=#subpaths @@ -22430,7 +22903,7 @@ function resolvers.registerextrapath(paths,subpaths) local ps=p.."/"..s if not done[ps] then newn=newn+1 - ep[newn]=cleanpath(ps) + extra_paths[newn]=cleanpath(ps) done[ps]=true end end @@ -22440,7 +22913,7 @@ function resolvers.registerextrapath(paths,subpaths) local p=paths[i] if not done[p] then newn=newn+1 - ep[newn]=cleanpath(p) + extra_paths[newn]=cleanpath(p) done[p]=true end end @@ -22449,17 +22922,17 @@ function resolvers.registerextrapath(paths,subpaths) for i=1,oldn do for j=1,nofsubpaths do local s=subpaths[j] - local ps=ep[i].."/"..s + local ps=extra_paths[i].."/"..s if not done[ps] then newn=newn+1 - ep[newn]=cleanpath(ps) + extra_paths[newn]=cleanpath(ps) done[ps]=true end end end end if newn>0 then - instance.extra_paths=ep + instance.extra_paths=extra_paths end if newn~=oldn then reset_caches() @@ -22467,17 +22940,19 @@ function resolvers.registerextrapath(paths,subpaths) end function resolvers.pushextrapath(path) local paths=settings_to_array(path) - if instance.extra_stack then - insert(instance.extra_stack,1,paths) + local extra_stack=instance.extra_stack + if extra_stack then + insert(extra_stack,1,paths) else instance.extra_stack={ paths } end reset_caches() end function resolvers.popextrapath() - if instance.extra_stack then + local extra_stack=instance.extra_stack + if extra_stack then reset_caches() - return remove(instance.extra_stack,1) + return remove(extra_stack,1) end end local function made_list(instance,list,extra_too) @@ -22506,33 +22981,21 @@ local function made_list(instance,list,extra_too) end end if extra_too then - local es=instance.extra_stack - if es and #es>0 then - for k=1,#es do - add(es[k]) + local extra_stack=instance.extra_stack + local extra_paths=instance.extra_paths + if extra_stack and #extra_stack>0 then + for k=1,#extra_stack do + add(extra_stack[k]) end end - local ep=instance.extra_paths - if ep and #ep>0 then - add(ep) + if extra_paths and #extra_paths>0 then + add(extra_paths) end end add(list) return new end -function resolvers.cleanpathlist(str) - local t=resolvers.expandedpathlist(str) - if t then - for i=1,#t do - t[i]=collapsepath(cleanpath(t[i])) - end - end - return t -end -function resolvers.expandpath(str) - return joinpath(resolvers.expandedpathlist(str)) -end -function resolvers.expandedpathlist(str,extra_too) +expandedpathlist=function(str,extra_too) if not str then return {} elseif instance.savelists then @@ -22540,40 +23003,56 @@ function resolvers.expandedpathlist(str,extra_too) local lists=instance.lists local lst=lists[str] if not lst then - local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str)),extra_too) + local l=made_list(instance,splitpath(expansion(str)),extra_too) lst=expandedpathfromlist(l) lists[str]=lst end return lst else - local lst=resolvers.splitpath(resolvers.expansion(str)) + local lst=splitpath(expansion(str)) return made_list(instance,expandedpathfromlist(lst),extra_too) end end -function resolvers.expandedpathlistfromvariable(str) +resolvers.expandedpathlist=expandedpathlist +resolvers.unexpandedpathlist=unexpandedpathlist +function resolvers.cleanpathlist(str) + local t=expandedpathlist(str) + if t then + for i=1,#t do + t[i]=collapsepath(cleanpath(t[i])) + end + end + return t +end +function resolvers.expandpath(str) + return joinpath(expandedpathlist(str)) +end +local function expandedpathlistfromvariable(str) str=lpegmatch(dollarstripper,str) local tmp=resolvers.variableofformatorsuffix(str) - return resolvers.expandedpathlist(tmp~="" and tmp or str) + return expandedpathlist(tmp~="" and tmp or str) end function resolvers.expandpathfromvariable(str) - return joinpath(resolvers.expandedpathlistfromvariable(str)) + return joinpath(expandedpathlistfromvariable(str)) end +resolvers.expandedpathlistfromvariable=expandedpathlistfromvariable function resolvers.cleanedpathlist(v) - local t=resolvers.expandedpathlist(v) + local t=expandedpathlist(v) for i=1,#t do - t[i]=resolvers.resolve(resolvers.cleanpath(t[i])) + t[i]=resolveprefix(cleanpath(t[i])) end return t end function resolvers.expandbraces(str) - local pth=expandedpathfromlist(resolvers.splitpath(str)) + local pth=expandedpathfromlist(splitpath(str)) return joinpath(pth) end function resolvers.registerfilehash(name,content,someerror) + local files=instance.files if content then - instance.files[name]=content + files[name]=content else - instance.files[name]={} + files[name]={} if somerror==true then instance.loaderror=someerror end @@ -22592,7 +23071,7 @@ function resolvers.renewcache() end local function isreadable(name) local readable=isfile(name) - if trace_detail then + if trace_details then if readable then report_resolving("file %a is readable",name) else @@ -22609,7 +23088,7 @@ local function collect_files(names) local variant=hash.type local search=filejoin(root,path,name) local result=methodhandler('concatinators',variant,root,path,name) - if trace_detail then + if trace_details then report_resolving("match: variant %a, search %a, result %a",variant,search,result) end noffiles=noffiles+1 @@ -22618,7 +23097,7 @@ local function collect_files(names) end for k=1,#names do local filename=names[k] - if trace_detail then + if trace_details then report_resolving("checking name %a",filename) end local basename=filebasename(filename) @@ -22630,12 +23109,13 @@ local function collect_files(names) pathname="/"..pathname.."$" end local hashes=instance.hashes + local files=instance.files for h=1,#hashes do local hash=hashes[h] local hashname=hash.name - local content=hashname and instance.files[hashname] + local content=hashname and files[hashname] if content then - if trace_detail then + if trace_details then report_resolving("deep checking %a, base %a, pattern %a",hashname,basename,pathname) end local path,name=lookup(content,basename) @@ -22703,7 +23183,6 @@ local function find_analyze(filename,askedformat,allresults) if askedformat=="" then if filesuffix=="" or not suffixmap[filesuffix] then local defaultsuffixes=resolvers.defaultsuffixes - local formatofsuffix=resolvers.formatofsuffix for i=1,#defaultsuffixes do local forcedname=filename..'.'..defaultsuffixes[i] wantedfiles[#wantedfiles+1]=forcedname @@ -22713,7 +23192,7 @@ local function find_analyze(filename,askedformat,allresults) end end else - filetype=resolvers.formatofsuffix(filename) + filetype=formatofsuffix(filename) if trace_locating then report_resolving("using suffix based filetype %a",filetype) end @@ -22736,7 +23215,7 @@ local function find_analyze(filename,askedformat,allresults) end local function find_direct(filename,allresults) if not dangerous[askedformat] and isreadable(filename) then - if trace_detail then + if trace_details then report_resolving("file %a found directly",filename) end return "direct",{ filename } @@ -22761,12 +23240,12 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) report_resolving("checking qualified name %a",filename) end if isreadable(filename) then - if trace_detail then + if trace_details then report_resolving("qualified file %a found",filename) end return "qualified",{ filename } end - if trace_detail then + if trace_details then report_resolving("locating qualified file %a",filename) end local forcedname,suffix="",suffixonly(filename) @@ -22774,11 +23253,11 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) local format_suffixes=askedformat=="" and resolvers.defaultsuffixes or suffixes[askedformat] if format_suffixes then for i=1,#format_suffixes do - local s=format_suffixes[i] - forcedname=filename.."."..s + local suffix=format_suffixes[i] + forcedname=filename.."."..suffix if isreadable(forcedname) then if trace_locating then - report_resolving("no suffix, forcing format filetype %a",s) + report_resolving("no suffix, forcing format filetype %a",suffix) end return "qualified",{ forcedname } end @@ -22791,7 +23270,7 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) local savedformat=askedformat local format=savedformat or "" if format=="" then - askedformat=resolvers.formatofsuffix(suffix) + askedformat=formatofsuffix(suffix) end if not format then askedformat="othertextfiles" @@ -22822,7 +23301,7 @@ local function find_qualified(filename,allresults,askedformat,alsostripped) end local function check_subpath(fname) if isreadable(fname) then - if trace_detail then + if trace_details then report_resolving("found %a by deep scanning",fname) end return fname @@ -22830,7 +23309,7 @@ local function check_subpath(fname) end local function makepathlist(list,filetype) local typespec=resolvers.variableofformat(filetype) - local pathlist=resolvers.expandedpathlist(typespec,filetype and usertypes[filetype]) + local pathlist=expandedpathlist(typespec,filetype and usertypes[filetype]) local entry={} if pathlist and #pathlist>0 then for k=1,#pathlist do @@ -22841,7 +23320,7 @@ local function makepathlist(list,filetype) local expression=makepathexpression(pathname) local barename=gsub(pathname,"/+$","") barename=resolveprefix(barename) - local scheme=url.hasscheme(barename) + local scheme=urlhasscheme(barename) local schemename=gsub(barename,"%.%*$",'') entry[k]={ path=path, @@ -22878,7 +23357,7 @@ local function find_intree(filename,filetype,wantedfiles,allresults) dirlist[i]=filedirname(filelist[i][3]).."/" end end - if trace_detail then + if trace_details then report_resolving("checking filename %a in tree",filename) end for k=1,#pathlist do @@ -22888,7 +23367,7 @@ local function find_intree(filename,filetype,wantedfiles,allresults) local done=false if filelist then local expression=entry.expression - if trace_detail then + if trace_details then report_resolving("using pattern %a for path %a",expression,pathname) end for k=1,#filelist do @@ -22899,16 +23378,16 @@ local function find_intree(filename,filetype,wantedfiles,allresults) result[#result+1]=resolveprefix(fl[3]) done=true if allresults then - if trace_detail then + if trace_details then report_resolving("match to %a in hash for file %a and path %a, continue scanning",expression,f,d) end else - if trace_detail then + if trace_details then report_resolving("match to %a in hash for file %a and path %a, quit scanning",expression,f,d) end break end - elseif trace_detail then + elseif trace_details then report_resolving("no match to %a in hash for file %a and path %a",expression,f,d) end end @@ -22923,7 +23402,7 @@ local function find_intree(filename,filetype,wantedfiles,allresults) if not find(pname,"*",1,true) then if can_be_dir(pname) then if not done and not entry.prescanned then - if trace_detail then + if trace_details then report_resolving("quick root scan for %a",pname) end for k=1,#wantedfiles do @@ -22938,7 +23417,7 @@ local function find_intree(filename,filetype,wantedfiles,allresults) end end if not done and entry.recursive then - if trace_detail then + if trace_details then report_resolving("scanning filesystem for %a",pname) end local files=resolvers.simplescanfiles(pname,false,true) @@ -23004,7 +23483,7 @@ local function find_intree(filename,filetype,wantedfiles,allresults) end end local function find_onpath(filename,filetype,wantedfiles,allresults) - if trace_detail then + if trace_details then report_resolving("checking filename %a, filetype %a, wanted files %a",filename,filetype,concat(wantedfiles," | ")) end local result={} @@ -23046,7 +23525,9 @@ collect_instance_files=function(filename,askedformat,allresults) { find_onpath (filename,filetype,wantedfiles,true) }, { find_otherwise(filename,filetype,wantedfiles,true) }, } - local result,status,done={},{},{} + local result={} + local status={} + local done={} for k,r in next,results do local method,list=r[1],r[2] if method and list then @@ -23060,7 +23541,7 @@ collect_instance_files=function(filename,askedformat,allresults) end end end - if trace_detail then + if trace_details then report_resolving("lookup status: %s",table.serialize(status,filename)) end return result,status @@ -23117,6 +23598,9 @@ local function findfiles(filename,filetype,allresults) if not filename or filename=="" then return {} end + if allresults==nil then + allresults=true + end local result,status=collect_instance_files(filename,filetype or "",allresults) if not result or #result==0 then local lowered=lower(filename) @@ -23126,27 +23610,25 @@ local function findfiles(filename,filetype,allresults) end return result or {},status end -function resolvers.findfiles(filename,filetype) - if not filename or filename=="" then - return "" - else - return findfiles(filename,filetype,true) - end -end -function resolvers.findfile(filename,filetype) +local function findfile(filename,filetype) if not filename or filename=="" then return "" else return findfiles(filename,filetype,false)[1] or "" end end +resolvers.findfiles=findfiles +resolvers.findfile=findfile +resolvers.find_file=findfile +resolvers.find_files=findfiles function resolvers.findpath(filename,filetype) return filedirname(findfiles(filename,filetype,false)[1] or "") end local function findgivenfiles(filename,allresults) + local hashes=instance.hashes + local files=instance.files local base=filebasename(filename) local result={} - local hashes=instance.hashes local function okay(hash,path,name) local found=methodhandler('concatinators',hash.type,hash.name,path,name) if found and found~="" then @@ -23156,7 +23638,7 @@ local function findgivenfiles(filename,allresults) end for k=1,#hashes do local hash=hashes[k] - local content=instance.files[hash.name] + local content=files[hash.name] if content then local path,name=lookup(content,base) if not path then @@ -23188,14 +23670,14 @@ function resolvers.wildcardpattern(pattern) return lpegmatch(makewildcard,pattern) or pattern end local function findwildcardfiles(filename,allresults,result) + local files=instance.files + local hashes=instance.hashes local result=result or {} local base=filebasename(filename) local dirn=filedirname(filename) local path=lower(lpegmatch(makewildcard,dirn) or dirn) local name=lower(lpegmatch(makewildcard,base) or base) - local files=instance.files if find(name,"*",1,true) then - local hashes=instance.hashes local function okay(found,path,base,hashname,hashtype) if find(found,path) then local full=methodhandler('concatinators',hashtype,hashname,found,base) @@ -23235,7 +23717,6 @@ local function findwildcardfiles(filename,allresults,result) end end end - local hashes=instance.hashes for k=1,#hashes do local hash=hashes[k] local hashname=hash.name @@ -23265,13 +23746,21 @@ end function resolvers.findwildcardfile(filename) return findwildcardfiles(filename,false)[1] or "" end -function resolvers.automount() -end -function resolvers.starttiming() - statistics.starttiming(instance) +do + local starttiming=statistics.starttiming + local stoptiming=statistics.stoptiming + local elapsedtime=statistics.elapsedtime + function resolvers.starttiming() + starttiming(instance) + end + function resolvers.stoptiming() + stoptiming(instance) + end + function resolvers.loadtime() + return elapsedtime(instance) + end end -function resolvers.stoptiming() - statistics.stoptiming(instance) +function resolvers.automount() end function resolvers.load(option) resolvers.starttiming() @@ -23285,9 +23774,6 @@ function resolvers.load(option) local files=instance.files return files and next(files) and true end -function resolvers.loadtime() - return statistics.elapsedtime(instance) -end local function report(str) if trace_locating then report_resolving(str) @@ -23317,7 +23803,7 @@ function resolvers.dowithfilesandreport(command,files,...) end end function resolvers.showpath(str) - return joinpath(resolvers.expandedpathlist(resolvers.formatofvariable(str))) + return joinpath(expandedpathlist(resolvers.formatofvariable(str))) end function resolvers.registerfile(files,name,path) if files[name] then @@ -23331,7 +23817,7 @@ function resolvers.registerfile(files,name,path) end end function resolvers.dowithpath(name,func) - local pathlist=resolvers.expandedpathlist(name) + local pathlist=expandedpathlist(name) for i=1,#pathlist do func("^"..cleanpath(pathlist[i])) end @@ -23341,11 +23827,11 @@ function resolvers.dowithvariable(name,func) end function resolvers.locateformat(name) local engine=environment.ownmain or "luatex" - local barename=removesuffix(name) + local barename=removesuffix(file.basename(name)) local fullname=addsuffix(barename,"fmt") local fmtname=caches.getfirstreadablefile(fullname,"formats",engine) or "" if fmtname=="" then - fmtname=resolvers.findfile(fullname) + fmtname=findfile(fullname) fmtname=cleanpath(fmtname) end if fmtname~="" then @@ -23354,17 +23840,17 @@ function resolvers.locateformat(name) local lucname=addsuffix(barename,luasuffixes.luc) local luiname=addsuffix(barename,luasuffixes.lui) if isfile(luiname) then - return barename,luiname + return fmtname,luiname elseif isfile(lucname) then - return barename,lucname + return fmtname,lucname elseif isfile(luaname) then - return barename,luaname + return fmtname,luaname end end return nil,nil end function resolvers.booleanvariable(str,default) - local b=resolvers.expansion(str) + local b=expansion(str) if b=="" then return default else @@ -23374,6 +23860,7 @@ function resolvers.booleanvariable(str,default) end function resolvers.dowithfilesintree(pattern,handle,before,after) local hashes=instance.hashes + local files=instance.files for i=1,#hashes do local hash=hashes[i] local blobtype=hash.type @@ -23385,7 +23872,7 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) if before then before(blobtype,blobpath,pattern) end - for path,name in filtered(instance.files[blobpath],pattern) do + for path,name in filtered(files[blobpath],pattern) do if type(path)=="string" then checked=checked+1 if handle(blobtype,blobpath,path,name) then @@ -23406,10 +23893,6 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) end end end -local obsolete=resolvers.obsolete or {} -resolvers.obsolete=obsolete -resolvers.find_file=resolvers.findfile obsolete.find_file=resolvers.findfile -resolvers.find_files=resolvers.findfiles obsolete.find_files=resolvers.findfiles function resolvers.knownvariables(pattern) if instance then local environment=instance.environment @@ -23443,7 +23926,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 4854, stripped down to: 2889 +-- original size: 5088, stripped down to: 3144 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -23452,6 +23935,7 @@ if not modules then modules={} end modules ['data-pre']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local insert,remove=table.insert,table.remove local resolvers=resolvers local prefixes=resolvers.prefixes local cleanpath=resolvers.cleanpath @@ -23529,8 +24013,9 @@ prefixes.kpse=prefixes.locate prefixes.full=prefixes.locate prefixes.file=prefixes.filename prefixes.path=prefixes.pathname +local inputstack={} +local stackpath=resolvers.stackpath local function toppath() - local inputstack=resolvers.inputstack if not inputstack then return "." end @@ -23542,15 +24027,23 @@ local function toppath() end end local function jobpath() - local path=resolvers.stackpath() + local path=stackpath() if not path or path=="" then return "." else return path end end +local function pushinputname(name) + insert(inputstack,name) +end +local function popinputname(name) + return remove(inputstack) +end resolvers.toppath=toppath resolvers.jobpath=jobpath +resolvers.pushinputname=pushinputname +resolvers.popinputname=popinputname prefixes.toppath=function(str) return cleanpath(joinpath(toppath(),str)) end prefixes.jobpath=function(str) return cleanpath(joinpath(jobpath(),str)) end resolvers.setdynamic("toppath") @@ -23593,7 +24086,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-out"] = package.loaded["data-out"] or true --- original size: 530, stripped down to: 470 +-- original size: 551, stripped down to: 470 if not modules then modules={} end modules ['data-out']={ version=1.001, @@ -23616,7 +24109,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3863, stripped down to: 3170 +-- original size: 4365, stripped down to: 3588 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -23625,39 +24118,46 @@ if not modules then modules={} end modules ['data-fil']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local ioopen=io.open +local isdir=lfs.isdir local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_files=logs.reporter("resolvers","files") local resolvers=resolvers local resolveprefix=resolvers.resolve -local finders,openers,loaders,savers=resolvers.finders,resolvers.openers,resolvers.loaders,resolvers.savers -local locators,hashers,generators,concatinators=resolvers.locators,resolvers.hashers,resolvers.generators,resolvers.concatinators +local findfile=resolvers.findfile +local scanfiles=resolvers.scanfiles +local registerfilehash=resolvers.registerfilehash +local appendhash=resolvers.appendhash +local loadcachecontent=caches.loadcontent local checkgarbage=utilities.garbagecollector and utilities.garbagecollector.check -function locators.file(specification) +function resolvers.locators.file(specification) local filename=specification.filename local realname=resolveprefix(filename) - if realname and realname~='' and lfs.isdir(realname) then + if realname and realname~='' and isdir(realname) then if trace_locating then report_files("file locator %a found as %a",filename,realname) end - resolvers.appendhash('file',filename,true) + appendhash('file',filename,true) elseif trace_locating then report_files("file locator %a not found",filename) end end -function hashers.file(specification) +function resolvers.hashers.file(specification) local pathname=specification.filename - local content=caches.loadcontent(pathname,'files') - resolvers.registerfilehash(pathname,content,content==nil) + local content=loadcachecontent(pathname,'files') + registerfilehash(pathname,content,content==nil) end -function generators.file(specification) +function resolvers.generators.file(specification) local pathname=specification.filename - local content=resolvers.scanfiles(pathname,false,true) - resolvers.registerfilehash(pathname,content,true) + local content=scanfiles(pathname,false,true) + registerfilehash(pathname,content,true) end -concatinators.file=file.join +resolvers.concatinators.file=file.join +local finders=resolvers.finders +local notfound=finders.notfound function finders.file(specification,filetype) local filename=specification.filename - local foundname=resolvers.findfile(filename,filetype) + local foundname=findfile(filename,filetype) if foundname and foundname~="" then if trace_locating then report_files("file finder: %a found",filename) @@ -23667,37 +24167,55 @@ function finders.file(specification,filetype) if trace_locating then report_files("file finder: %a not found",filename) end - return finders.notfound() + return notfound() end end -function openers.helpers.textopener(tag,filename,f) +local openers=resolvers.openers +local notfound=openers.notfound +local overloaded=false +local function textopener(tag,filename,f) return { - reader=function() return f:read () end, - close=function() logs.show_close(filename) return f:close() end, + reader=function() return f:read () end, + close=function() return f:close() end, } end +function openers.helpers.textopener(...) + return textopener(...) +end +function openers.helpers.settextopener(opener) + if overloaded then + report_files("file opener: %s overloaded","already") + else + if trace_locating then + report_files("file opener: %s overloaded","once") + end + overloaded=true + textopener=opener + end +end function openers.file(specification,filetype) local filename=specification.filename if filename and filename~="" then - local f=io.open(filename,"r") + local f=ioopen(filename,"r") if f then if trace_locating then report_files("file opener: %a opened",filename) end - return openers.helpers.textopener("file",filename,f) + return textopener("file",filename,f) end end if trace_locating then report_files("file opener: %a not found",filename) end - return openers.notfound() + return notfound() end +local loaders=resolvers.loaders +local notfound=loaders.notfound function loaders.file(specification,filetype) local filename=specification.filename if filename and filename~="" then - local f=io.open(filename,"rb") + local f=ioopen(filename,"rb") if f then - logs.show_load(filename) if trace_locating then report_files("file loader: %a loaded",filename) end @@ -23714,7 +24232,7 @@ function loaders.file(specification,filetype) if trace_locating then report_files("file loader: %a not found",filename) end - return loaders.notfound() + return notfound() end @@ -23724,7 +24242,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-con"] = package.loaded["data-con"] or true --- original size: 5029, stripped down to: 3432 +-- original size: 5388, stripped down to: 3685 if not modules then modules={} end modules ['data-con']={ version=1.100, @@ -23733,6 +24251,7 @@ if not modules then modules={} end modules ['data-con']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local setmetatable=setmetatable local format,lower,gsub=string.format,string.lower,string.gsub local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) local trace_containers=false trackers.register("resolvers.containers",function(v) trace_containers=v end) @@ -23740,16 +24259,21 @@ local trace_storage=false trackers.register("resolvers.storage",function(v) tra containers=containers or {} local containers=containers containers.usecache=true +local getwritablepath=caches.getwritablepath +local getreadablepaths=caches.getreadablepaths +local cacheiswritable=caches.is_writable +local loaddatafromcache=caches.loaddata +local savedataincache=caches.savedata local report_containers=logs.reporter("resolvers","containers") local allocated={} local mt={ __index=function(t,k) if k=="writable" then - local writable=caches.getwritablepath(t.category,t.subcategory) or { "." } + local writable=getwritablepath(t.category,t.subcategory) or { "." } t.writable=writable return writable elseif k=="readables" then - local readables=caches.getreadablepaths(t.category,t.subcategory) or { "." } + local readables=getreadablepaths(t.category,t.subcategory) or { "." } t.readables=readables return readables end @@ -23780,7 +24304,7 @@ function containers.define(category,subcategory,version,enabled) end end function containers.is_usable(container,name) - return container.enabled and caches and caches.is_writable(container.writable,name) + return container.enabled and caches and cacheiswritable(container.writable,name) end function containers.is_valid(container,name) if name and name~="" then @@ -23794,7 +24318,7 @@ function containers.read(container,name) local storage=container.storage local stored=storage[name] if not stored and container.enabled and caches and containers.usecache then - stored=caches.loaddata(container.readables,name,container.writable) + stored=loaddatafromcache(container.readables,name,container.writable) if stored and stored.cache_version==container.version then if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","load",container.subcategory,name) @@ -23810,17 +24334,20 @@ function containers.read(container,name) end return stored end -function containers.write(container,name,data) +function containers.write(container,name,data,fast) if data then data.cache_version=container.version if container.enabled and caches then - local unique,shared=data.unique,data.shared - data.unique,data.shared=nil,nil - caches.savedata(container.writable,name,data) + local unique=data.unique + local shared=data.shared + data.unique=nil + data.shared=nil + savedataincache(container.writable,name,data,fast) if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","save",container.subcategory,name) end - data.unique,data.shared=unique,shared + data.unique=unique + data.shared=shared end if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","store",container.subcategory,name) @@ -23843,7 +24370,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-use"] = package.loaded["data-use"] or true --- original size: 4434, stripped down to: 3180 +-- original size: 5790, stripped down to: 2910 if not modules then modules={} end modules ['data-use']={ version=1.001, @@ -23852,40 +24379,11 @@ if not modules then modules={} end modules ['data-use']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local format,lower,gsub,find=string.format,string.lower,string.gsub,string.find +local format=string.format local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_mounts=logs.reporter("resolvers","mounts") local resolvers=resolvers -resolvers.automounted=resolvers.automounted or {} -function resolvers.automount(usecache) - local mountpaths=resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) - if (not mountpaths or #mountpaths==0) and usecache then - mountpaths=caches.getreadablepaths("mount") - end - if mountpaths and #mountpaths>0 then - resolvers.starttiming() - for k=1,#mountpaths do - local root=mountpaths[k] - local f=io.open(root.."/url.tmi") - if f then - for line in f:lines() do - if line then - if find(line,"^[%%#%-]") then - elseif find(line,"^zip://") then - if trace_locating then - report_mounts("mounting %a",line) - end - table.insert(resolvers.automounted,line) - resolvers.usezipfile(line) - end - end - end - f:close() - end - end - resolvers.stoptiming() - end -end +local findfile=resolvers.findfile statistics.register("used config file",function() return caches.configfiles() end) statistics.register("used cache path",function() return caches.usedpaths() end) function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) @@ -23895,9 +24393,11 @@ function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) local luvdata={ enginebanner=enginebanner, formatbanner=formatbanner, - sourcehash=md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), + sourcehash=md5.hex(io.loaddata(findfile(sourcefile)) or "unknown"), sourcefile=sourcefile, luaversion=LUAVERSION, + formatid=LUATEXFORMATID, + functionality=LUATEXFUNCTIONALITY, } io.savedata(luvname,table.serialize(luvdata,true)) lua.registerfinalizer(function() @@ -23917,7 +24417,7 @@ function statistics.checkfmtstatus(texname) if lfs.isfile(luvname) then local luv=dofile(luvname) if luv and luv.sourcefile then - local sourcehash=md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") + local sourcehash=md5.hex(io.loaddata(findfile(luv.sourcefile)) or "unknown") local luvbanner=luv.enginebanner or "?" if luvbanner~=enginebanner then return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) @@ -23927,8 +24427,19 @@ function statistics.checkfmtstatus(texname) return format("source mismatch (luv: %s <> bin: %s)",luvhash,sourcehash) end local luvluaversion=luv.luaversion or 0 - if luvluaversion~=LUAVERSION then - return format("lua mismatch (luv: %s <> bin: %s)",luvluaversion,LUAVERSION) + local engluaversion=LUAVERSION or 0 + if luvluaversion~=engluaversion then + return format("lua mismatch (luv: %s <> bin: %s)",luvluaversion,engluaversion) + end + local luvfunctionality=luv.functionality or 0 + local engfunctionality=status.development_id or 0 + if luvfunctionality~=engfunctionality then + return format("functionality mismatch (luv: %s <> bin: %s)",luvfunctionality,engfunctionality) + end + local luvformatid=luv.formatid or 0 + local engformatid=status.format_id or 0 + if luvformatid~=engformatid then + return format("formatid mismatch (luv: %s <> bin: %s)",luvformatid,engformatid) end else return "invalid status file" @@ -23947,7 +24458,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 10263, stripped down to: 7556 +-- original size: 10725, stripped down to: 7949 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -23960,6 +24471,14 @@ local format,find,match=string.format,string.find,string.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_zip=logs.reporter("resolvers","zip") local resolvers=resolvers +local findfile=resolvers.findfile +local registerfile=resolvers.registerfile +local splitmethod=resolvers.splitmethod +local prependhash=resolvers.prependhash +local starttiming=resolvers.starttiming +local extendtexmf=resolvers.extendtexmfvariable +local stoptiming=resolvers.stoptiming +local urlquery=url.query zip=zip or {} local zip=zip local archives=zip.archives or {} @@ -23974,8 +24493,9 @@ if zipfiles then closezip=zipfiles.close validfile=zipfiles.found wholefile=zipfiles.unzip + local listzip=zipfiles.list traversezip=function(zfile) - return ipairs(zipfiles.list(zfile)) + return ipairs(listzip(zfile)) end local streams=utilities.streams local openstream=streams.open @@ -24032,28 +24552,30 @@ local function validzip(str) return str end end -function zip.openarchive(name) +local function openarchive(name) if not name or name=="" then return nil else local arch=archives[name] if not arch then - local full=resolvers.findfile(name) or "" + local full=findfile(name) or "" arch=full~="" and openzip(full) or false archives[name]=arch end return arch end end -function zip.closearchive(name) +local function closearchive(name) if not name or (name=="" and archives[name]) then closezip(archives[name]) archives[name]=nil end end +zip.openarchive=openarchive +zip.closearchive=closearchive function resolvers.locators.zip(specification) local archive=specification.filename - local zipfile=archive and archive~="" and zip.openarchive(archive) + local zipfile=archive and archive~="" and openarchive(archive) if trace_locating then if zipfile then report_zip("locator: archive %a found",archive) @@ -24062,13 +24584,6 @@ function resolvers.locators.zip(specification) end end end -function resolvers.hashers.zip(specification) - local archive=specification.filename - if trace_locating then - report_zip("loading file %a",archive) - end - resolvers.usezipfile(specification.original) -end function resolvers.concatinators.zip(zipfile,path,name) if not path or path=="" then return format('%s?name=%s',zipfile,name) @@ -24076,14 +24591,16 @@ function resolvers.concatinators.zip(zipfile,path,name) return format('%s?name=%s/%s',zipfile,path,name) end end -function resolvers.finders.zip(specification) +local finders=resolvers.finders +local notfound=finders.notfound +function finders.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("finder: archive %a found",archive) @@ -24104,16 +24621,19 @@ function resolvers.finders.zip(specification) if trace_locating then report_zip("finder: %a not found",original) end - return resolvers.finders.notfound() + return notfound() end -function resolvers.openers.zip(specification) +local openers=resolvers.openers +local notfound=openers.notfound +local textopener=openers.helpers.textopener +function openers.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("opener; archive %a opened",archive) @@ -24123,7 +24643,7 @@ function resolvers.openers.zip(specification) if trace_locating then report_zip("opener: file %a found",queryname) end - return resolvers.openers.helpers.textopener('zip',original,handle) + return textopener('zip',original,handle) elseif trace_locating then report_zip("opener: file %a not found",queryname) end @@ -24135,23 +24655,24 @@ function resolvers.openers.zip(specification) if trace_locating then report_zip("opener: %a not found",original) end - return resolvers.openers.notfound() + return notfound() end -function resolvers.loaders.zip(specification) +local loaders=resolvers.loaders +local notfound=loaders.notfound +function loaders.zip(specification) local original=specification.original local archive=specification.filename if archive then - local query=url.query(specification.query) + local query=urlquery(specification.query) local queryname=query.name if queryname then - local zfile=zip.openarchive(archive) + local zfile=openarchive(archive) if zfile then if trace_locating then report_zip("loader: archive %a opened",archive) end local data=wholefile(zfile,queryname) if data then - logs.show_load(original) if trace_locating then report_zip("loader; file %a loaded",original) end @@ -24167,47 +24688,24 @@ function resolvers.loaders.zip(specification) if trace_locating then report_zip("loader: %a not found",original) end - return resolvers.openers.notfound() -end -function resolvers.usezipfile(archive) - local specification=resolvers.splitmethod(archive) - local archive=specification.filename - if archive and not registeredfiles[archive] then - local z=zip.openarchive(archive) - if z then - local tree=url.query(specification.query).tree or "" - if trace_locating then - report_zip("registering: archive %a",archive) - end - resolvers.starttiming() - resolvers.prependhash('zip',archive) - resolvers.extendtexmfvariable(archive) - registeredfiles[archive]=z - resolvers.registerfilehash(archive,resolvers.registerzipfile(z,tree)) - resolvers.stoptiming() - elseif trace_locating then - report_zip("registering: unknown archive %a",archive) - end - elseif trace_locating then - report_zip("registering: archive %a not found",archive) - end + return notfound() end -function resolvers.registerzipfile(z,tree) +local function registerzipfile(z,tree) local names={} local files={} local remap={} local n=0 local filter=tree=="" and "^(.+)/(.-)$" or format("^%s/(.+)/(.-)$",tree) - local register=resolvers.registerfile if trace_locating then report_zip("registering: using filter %a",filter) end + starttiming() for i in traversezip(z) do local filename=i.filename local path,name=match(filename,filter) if not path then n=n+1 - register(names,filename,"") + registerfile(names,filename,"") local usedname=lower(filename) files[usedname]="" if usedname~=filename then @@ -24224,12 +24722,43 @@ function resolvers.registerzipfile(z,tree) else end end + stoptiming() report_zip("registering: %s files registered",n) return { files=files, remap=remap, } end +local function usezipfile(archive) + local specification=splitmethod(archive) + local archive=specification.filename + if archive and not registeredfiles[archive] then + local z=openarchive(archive) + if z then + local tree=urlquery(specification.query).tree or "" + if trace_locating then + report_zip("registering: archive %a",archive) + end + prependhash('zip',archive) + extendtexmf(archive) + registeredfiles[archive]=z + registerfilehash(archive,registerzipfile(z,tree)) + elseif trace_locating then + report_zip("registering: unknown archive %a",archive) + end + elseif trace_locating then + report_zip("registering: archive %a not found",archive) + end +end +resolvers.usezipfile=usezipfile +resolvers.registerzipfile=registerzipfile +function resolvers.hashers.zip(specification) + local archive=specification.filename + if trace_locating then + report_zip("loading file %a",archive) + end + usezipfile(specification.original) +end end -- of closure @@ -24238,7 +24767,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 8478, stripped down to: 5223 +-- original size: 10802, stripped down to: 6619 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -24247,180 +24776,255 @@ if not modules then modules={} end modules ['data-tre']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local type=type local find,gsub,lower=string.find,string.gsub,string.lower -local basename,dirname,joinname=file.basename,file.dirname,file .join +local basename,dirname,joinname=file.basename,file.dirname,file.join local globdir,isdir,isfile=dir.glob,lfs.isdir,lfs.isfile local P,lpegmatch=lpeg.P,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_trees=logs.reporter("resolvers","trees") local resolvers=resolvers -local resolveprefix=resolvers.resolve -local notfound=resolvers.finders.notfound -local lookup=resolvers.get_from_content -local collectors={} -local found={} -function resolvers.finders.tree(specification) - local spec=specification.filename - local okay=found[spec] - if okay==nil then - if spec~="" then - local path=dirname(spec) - local name=basename(spec) - if path=="" then - path="." - end - local names=collectors[path] - if not names then - local pattern=find(path,"/%*+$") and path or (path.."/*") - names=globdir(pattern) - collectors[path]=names - end - local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" - for i=1,#names do - local fullname=names[i] - if find(fullname,pattern) then - found[spec]=fullname - return fullname - end - end - local pattern=lower(pattern) - for i=1,#names do - local fullname=lower(names[i]) - if find(fullname,pattern) then - if isfile(fullname) then +local finders=resolvers.finders +local openers=resolvers.openers +local loaders=resolvers.loaders +local locators=resolvers.locators +local hashers=resolvers.hashers +local generators=resolvers.generators +do + local collectors={} + local found={} + local notfound=finders.notfound + function finders.tree(specification) + local spec=specification.filename + local okay=found[spec] + if okay==nil then + if spec~="" then + local path=dirname(spec) + local name=basename(spec) + if path=="" then + path="." + end + local names=collectors[path] + if not names then + local pattern=find(path,"/%*+$") and path or (path.."/*") + names=globdir(pattern) + collectors[path]=names + end + local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" + for i=1,#names do + local fullname=names[i] + if find(fullname,pattern) then found[spec]=fullname return fullname - else - break + end + end + local pattern=lower(pattern) + for i=1,#names do + local fullname=lower(names[i]) + if find(fullname,pattern) then + if isfile(fullname) then + found[spec]=fullname + return fullname + else + break + end end end end + okay=notfound() + found[spec]=okay end - okay=notfound() - found[spec]=okay + return okay end - return okay end -function resolvers.locators.tree(specification) - local name=specification.filename - local realname=resolveprefix(name) - if realname and realname~='' and isdir(realname) then - if trace_locating then - report_trees("locator %a found",realname) +do + local resolveprefix=resolvers.resolve + local appendhash=resolvers.appendhash + local function dolocate(specification) + local name=specification.filename + local realname=resolveprefix(name) + if realname and realname~='' and isdir(realname) then + if trace_locating then + report_trees("locator %a found",realname) + end + appendhash('tree',name,false) + elseif trace_locating then + report_trees("locator %a not found",name) end - resolvers.appendhash('tree',name,false) - elseif trace_locating then - report_trees("locator %a not found",name) end + locators.tree=dolocate + locators.dirlist=dolocate + locators.dirfile=dolocate end -function resolvers.hashers.tree(specification) - local name=specification.filename - if trace_locating then - report_trees("analyzing %a",name) - end - resolvers.methodhandler("hashers",name) - resolvers.generators.file(specification) -end -local collectors={} -local splitter=lpeg.splitat("/**/") -local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } -table.setmetatableindex(collectors,function(t,k) - local rootname=lpegmatch(stripper,k) - local dataname=joinname(rootname,"dirlist") - local content=caches.loadcontent(dataname,"files",dataname) - if not content then - content=resolvers.scanfiles(rootname,nil,nil,false,true) - caches.savecontent(dataname,"files",content,dataname) - end - t[k]=content - return content -end) -local function checked(root,p,n) - if p then - if type(p)=="table" then - for i=1,#p do - local fullname=joinname(root,p[i],n) +do + local filegenerator=generators.file + generators.dirlist=filegenerator + generators.dirfile=filegenerator +end +do + local filegenerator=generators.file + local methodhandler=resolvers.methodhandler + local function dohash(specification) + local name=specification.filename + if trace_locating then + report_trees("analyzing %a",name) + end + methodhandler("hashers",name) + filegenerator(specification) + end + hashers.tree=dohash + hashers.dirlist=dohash + hashers.dirfile=dohash +end +local resolve do + local collectors={} + local splitter=lpeg.splitat("/**/") + local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } + local loadcontent=caches.loadcontent + local savecontent=caches.savecontent + local notfound=finders.notfound + local scanfiles=resolvers.scanfiles + local lookup=resolvers.get_from_content + table.setmetatableindex(collectors,function(t,k) + local rootname=lpegmatch(stripper,k) + local dataname=joinname(rootname,"dirlist") + local content=loadcontent(dataname,"files",dataname) + if not content then + content=scanfiles(rootname,nil,nil,false,true) + savecontent(dataname,"files",content,dataname) + end + t[k]=content + return content + end) + local function checked(root,p,n) + if p then + if type(p)=="table" then + for i=1,#p do + local fullname=joinname(root,p[i],n) + if isfile(fullname) then + return fullname + end + end + else + local fullname=joinname(root,p,n) if isfile(fullname) then return fullname end end - else - local fullname=joinname(root,p,n) - if isfile(fullname) then - return fullname - end end + return notfound() end - return notfound() -end -local function resolve(specification) - local filename=specification.filename - if filename~="" then - local root,rest=lpegmatch(splitter,filename) - if root and rest then - local path,name=dirname(rest),basename(rest) - if name~=rest then - local content=collectors[root] - local p,n=lookup(content,name) - if not p then - return notfound() - end - local pattern=".*/"..path.."$" - local istable=type(p)=="table" - if istable then - for i=1,#p do - local pi=p[i] - if pi==path or find(pi,pattern) then - local fullname=joinname(root,pi,n) - if isfile(fullname) then - return fullname + resolve=function(specification) + local filename=specification.filename + if filename~="" then + local root,rest=lpegmatch(splitter,filename) + if root and rest then + local path,name=dirname(rest),basename(rest) + if name~=rest then + local content=collectors[root] + local p,n=lookup(content,name) + if not p then + return notfound() + end + local pattern=".*/"..path.."$" + local istable=type(p)=="table" + if istable then + for i=1,#p do + local pi=p[i] + if pi==path or find(pi,pattern) then + local fullname=joinname(root,pi,n) + if isfile(fullname) then + return fullname + end end end + elseif p==path or find(p,pattern) then + local fullname=joinname(root,p,n) + if isfile(fullname) then + return fullname + end end - elseif p==path or find(p,pattern) then - local fullname=joinname(root,p,n) - if isfile(fullname) then - return fullname + local queries=specification.queries + if queries and queries.option=="fileonly" then + return checked(root,p,n) + else + return notfound() end end - local queries=specification.queries - if queries and queries.option=="fileonly" then - return checked(root,p,n) - else - return notfound() - end + end + local path=dirname(filename) + local name=basename(filename) + local root=lpegmatch(stripper,path) + local content=collectors[path] + local p,n=lookup(content,name) + if p then + return checked(root,p,n) end end - local path,name=dirname(filename),basename(filename) - local root=lpegmatch(stripper,path) - local content=collectors[path] - local p,n=lookup(content,name) - if p then - return checked(root,p,n) + return notfound() + end + finders.dirlist=resolve + function finders.dirfile(specification) + local queries=specification.queries + if queries then + queries.option="fileonly" + else + specification.queries={ option="fileonly" } end + return resolve(specification) end - return notfound() end -resolvers.finders .dirlist=resolve -resolvers.locators .dirlist=resolvers.locators .tree -resolvers.hashers .dirlist=resolvers.hashers .tree -resolvers.generators.dirlist=resolvers.generators.file -resolvers.openers .dirlist=resolvers.openers .file -resolvers.loaders .dirlist=resolvers.loaders .file -function resolvers.finders.dirfile(specification) - local queries=specification.queries - if queries then - queries.option="fileonly" - else - specification.queries={ option="fileonly" } +do + local fileopener=openers.file + local fileloader=loaders.file + openers.dirlist=fileopener + loaders.dirlist=fileloader + openers.dirfile=fileopener + loaders.dirfile=fileloader +end +do + local hashfile="dirhash.lua" + local kind="HASH256" + local version=1.0 + local loadtable=table.load + local savetable=table.save + local loaddata=io.loaddata + function resolvers.dirstatus(patterns) + local t=type(patterns) + if t=="string" then + patterns={ patterns } + elseif t~="table" then + return false + end + local status=loadtable(hashfile) + if not status or status.version~=version or status.kind~=kind then + status={ + version=1.0, + kind=kind, + hashes={}, + } + end + local hashes=status.hashes + local changed={} + local action=sha2[kind] + local update={} + for i=1,#patterns do + local pattern=patterns[i] + local files=globdir(pattern) + for i=1,#files do + local name=files[i] + local hash=action(loaddata(name)) + if hashes[name]~=hash then + changed[#changed+1]=name + end + update[name]=hash + end + end + status.hashes=update + savetable(hashfile,status) + return #changed>0 and changed or false end - return resolve(specification) end -resolvers.locators .dirfile=resolvers.locators .dirlist -resolvers.hashers .dirfile=resolvers.hashers .dirlist -resolvers.generators.dirfile=resolvers.generators.dirlist -resolvers.openers .dirfile=resolvers.openers .dirlist -resolvers.loaders .dirfile=resolvers.loaders .dirlist end -- of closure @@ -24429,7 +25033,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6753, stripped down to: 5268 +-- original size: 6945, stripped down to: 5408 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -24439,8 +25043,11 @@ if not modules then modules={} end modules ['data-sch']={ license="see context related readme files" } local load,tonumber=load,tonumber -local gsub,concat,format=string.gsub,table.concat,string.format +local gsub,format=string.gsub,string.format +local sortedhash,concat=table.sortedhash,table.concat local finders,openers,loaders=resolvers.finders,resolvers.openers,resolvers.loaders +local addsuffix,suffix,splitbase=file.addsuffix,file.suffix,file.splitbase +local md5hex=md5.hex local trace_schemes=false trackers.register("resolvers.schemes",function(v) trace_schemes=v end) local report_schemes=logs.reporter("resolvers","schemes") local http=require("socket.http") @@ -24457,7 +25064,7 @@ function cleaners.none(specification) return specification.original end function cleaners.strip(specification) - local path,name=file.splitbase(specification.original) + local path,name=splitbase(specification.original) if path=="" then return (gsub(name,"[^%a%d%.]+","-")) else @@ -24465,7 +25072,7 @@ function cleaners.strip(specification) end end function cleaners.md5(specification) - return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) + return addsuffix(md5hex(specification.original),suffix(specification.path)) end local cleaner=cleaners.strip directives.register("schemes.cleanmethod",function(v) cleaner=cleaners[v] or cleaners.strip end) @@ -24485,7 +25092,7 @@ local runner=sandbox.registerrunner { name="curl resolver", method="execute", program="curl", - template="--silent --insecure --create-dirs --output %cachename% %original%", + template='--silent --insecure --create-dirs --output "%cachename%" "%original%"', checkers={ cachename="cache", original="url", @@ -24556,10 +25163,10 @@ end schemes.install=install local function http_handler(specification,cachename) local tempname=cachename..".tmp" - local f=io.open(tempname,"wb") + local handle=io.open(tempname,"wb") local status,message=http.request { url=specification.original, - sink=ltn12.sink.file(f) + sink=ltn12.sink.file(handle) } if not status then os.remove(tempname) @@ -24574,13 +25181,13 @@ install('https') install('ftp') statistics.register("scheme handling time",function() local l,r,nl,nr={},{},0,0 - for k,v in table.sortedhash(loaded) do + for k,v in sortedhash(loaded) do if v>0 then nl=nl+1 l[nl]=k..":"..v end end - for k,v in table.sortedhash(reused) do + for k,v in sortedhash(reused) do if v>0 then nr=nr+1 r[nr]=k..":"..v @@ -24588,10 +25195,10 @@ statistics.register("scheme handling time",function() end local n=nl+nr if n>0 then - l=nl>0 and concat(l) or "none" - r=nr>0 and concat(r) or "none" + if nl==0 then l={ "none" } end + if nr==0 then r={ "none" } end return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s", - statistics.elapsedtime(schemes),n,threshold,l,r) + statistics.elapsedtime(schemes),n,threshold,concat(l," "),concat(l," ")) else return nil end @@ -24624,7 +25231,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4207, stripped down to: 3041 +-- original size: 4227, stripped down to: 3049 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -24634,8 +25241,7 @@ if not modules then modules={} end modules ['data-lua']={ license="see context related readme files" } local package,lpeg=package,lpeg -local gsub=string.gsub -local concat=table.concat +local loadfile=loadfile local addsuffix=file.addsuffix local P,S,Cs,lpegmatch=lpeg.P,lpeg.S,lpeg.Cs,lpeg.match local luasuffixes={ 'tex','lua' } @@ -24646,6 +25252,8 @@ local helpers=package.helpers or {} local methods=helpers.methods or {} local resolvers=resolvers local resolveprefix=resolvers.resolve +local expandedpaths=resolvers.expandedpathlistfromvariable +local findfile=resolvers.findfile helpers.report=logs.reporter("resolvers","libraries") trackers.register("resolvers.libraries",function(v) helpers.trace=v end) trackers.register("resolvers.locating",function(v) helpers.trace=v end) @@ -24674,7 +25282,7 @@ local function getluaformatpaths() if not luaformatpaths then luaformatpaths={} for i=1,#luaformats do - registerpath("lua format","lua",luaformatpaths,resolvers.expandedpathlistfromvariable(luaformats[i])) + registerpath("lua format","lua",luaformatpaths,expandedpaths(luaformats[i])) end end return luaformatpaths @@ -24683,7 +25291,7 @@ local function getlibformatpaths() if not libformatpaths then libformatpaths={} for i=1,#libformats do - registerpath("lib format","lib",libformatpaths,resolvers.expandedpathlistfromvariable(libformats[i])) + registerpath("lib format","lib",libformatpaths,expandedpaths(libformats[i])) end end return libformatpaths @@ -24693,7 +25301,7 @@ local function loadedbyformat(name,rawname,suffixes,islib,what) local report=helpers.report for i=1,#suffixes do local format=suffixes[i] - local resolved=resolvers.findfile(name,format) or "" + local resolved=findfile(name,format) or "" if trace then report("%s format, identifying %a using format %a",what,name,format) end @@ -24731,7 +25339,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-aux"] = package.loaded["data-aux"] or true --- original size: 2452, stripped down to: 1877 +-- original size: 2610, stripped down to: 2019 if not modules then modules={} end modules ['data-aux']={ version=1.001, @@ -24742,24 +25350,28 @@ if not modules then modules={} end modules ['data-aux']={ } local find=string.find local type,next=type,next +local addsuffix,removesuffix=file.addsuffix,file.removesuffix +local loaddata,savedata=io.loaddata,io.savedata local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local resolvers=resolvers +local cleanpath=resolvers.cleanpath +local findfiles=resolvers.findfiles local report_scripts=logs.reporter("resolvers","scripts") function resolvers.updatescript(oldname,newname) local scriptpath="context/lua" - newname=file.addsuffix(newname,"lua") - local oldscript=resolvers.cleanpath(oldname) + local oldscript=cleanpath(oldname) + local newname=addsuffix(newname,"lua") + local newscripts=findfiles(newname) or {} if trace_locating then report_scripts("to be replaced old script %a",oldscript) end - local newscripts=resolvers.findfiles(newname) or {} if #newscripts==0 then if trace_locating then report_scripts("unable to locate new script") end else for i=1,#newscripts do - local newscript=resolvers.cleanpath(newscripts[i]) + local newscript=cleanpath(newscripts[i]) if trace_locating then report_scripts("checking new script %a",newscript) end @@ -24771,17 +25383,17 @@ function resolvers.updatescript(oldname,newname) if trace_locating then report_scripts("new script should come from %a",scriptpath) end - elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then + elseif not (find(oldscript,removesuffix(newname).."$") or find(oldscript,newname.."$")) then if trace_locating then report_scripts("invalid new script name") end else - local newdata=io.loaddata(newscript) + local newdata=loaddata(newscript) if newdata then if trace_locating then report_scripts("old script content replaced by new content: %s",oldscript) end - io.savedata(oldscript,newdata) + savedata(oldscript,newdata) break elseif trace_locating then report_scripts("unable to load new script") @@ -24854,7 +25466,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 1823, stripped down to: 1542 +-- original size: 2038, stripped down to: 1696 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -24864,15 +25476,22 @@ if not modules then modules={} end modules ['data-lst']={ license="see context related readme files" } local type=type -local concat,sortedhash=table.concat,table.sortedhash +local sortedhash=table.sortedhash +local isdir=lfs.isdir local resolvers=resolvers local listers=resolvers.listers or {} resolvers.listers=listers local resolveprefix=resolvers.resolve +local configurationfiles=resolvers.configurationfiles +local expandedpathfromlist=resolvers.expandedpathfromlist +local splitpath=resolvers.splitpath +local knownvariables=resolvers.knownvariables local report_lists=logs.reporter("resolvers","lists") local report_resolved=logs.reporter("system","resolved") local function tabstr(str) - if type(str)=='table' then + if not str then + return "unset" + elseif type(str)=='table' then return concat(str," | ") else return str @@ -24882,22 +25501,22 @@ function listers.variables(pattern) local result=resolvers.knownvariables(pattern) for key,value in sortedhash(result) do report_lists(key) - report_lists(" env: %s",tabstr(value.environment or "unset")) - report_lists(" var: %s",tabstr(value.variable or "unset")) - report_lists(" exp: %s",tabstr(value.expansion or "unset")) - report_lists(" res: %s",tabstr(value.resolved or "unset")) + report_lists(" env: %s",tabstr(value.environment)) + report_lists(" var: %s",tabstr(value.variable)) + report_lists(" exp: %s",tabstr(value.expansion)) + report_lists(" res: %s",tabstr(value.resolved)) end end function listers.configurations() - local configurations=resolvers.configurationfiles() + local configurations=configurationfiles() for i=1,#configurations do report_resolved("file : %s",resolveprefix(configurations[i])) end report_resolved("") - local list=resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) + local list=expandedpathfromlist(splitpath(resolvers.luacnfspec)) for i=1,#list do local li=resolveprefix(list[i]) - if lfs.isdir(li) then + if isdir(li) then report_resolved("path - %s",li) else report_resolved("path + %s",li) @@ -24910,318 +25529,147 @@ end -- of closure do -- create closure to overcome 200 locals limit -package.loaded["util-lib"] = package.loaded["util-lib"] or true +package.loaded["libs-ini"] = package.loaded["libs-ini"] or true --- original size: 16094, stripped down to: 8443 +-- original size: 5822, stripped down to: 3629 -if not modules then modules={} end modules ['util-lib']={ +if not modules then modules={} end modules ['libs-ini']={ version=1.001, comment="companion to luat-lib.mkiv", author="Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright="PRAGMA ADE / ConTeXt Development Team", - license="see context related readme files", + license="see context related readme files" } +local type,unpack=type,unpack local type=type -local next=next -local pcall=pcall -local gsub=string.gsub -local find=string.find -local sort=table.sort -local pathpart=file.pathpart local nameonly=file.nameonly local joinfile=file.join -local removesuffix=file.removesuffix local addsuffix=file.addsuffix -local findfile=resolvers.findfile -local findfiles=resolvers.findfiles -local expandpaths=resolvers.expandedpathlistfromvariable local qualifiedpath=file.is_qualified_path local isfile=lfs.isfile -local done=false -local function locate(required,version,trace,report,action) - if type(required)~="string" then - report("provide a proper library name") - return - end - if trace then - report("requiring library %a with version %a",required,version or "any") - end - local found_library=nil - local required_full=gsub(required,"%.","/") - local required_path=pathpart(required_full) - local required_base=nameonly(required_full) - if qualifiedpath(required) then - if isfile(addsuffix(required,os.libsuffix)) then - if trace then - report("qualified name %a found",required) +local findfile=resolvers.findfile +local expandpaths=resolvers.expandedpathlistfromvariable +local report=logs.reporter("resolvers","libraries") +local trace=false +trackers.register("resolvers.lib",function(v) trace=v end) +local function findlib(required) + local suffix=os.libsuffix or "so" + if not qualifiedpath(required) then + local list=directives.value("system.librarynames" ) + local only=nameonly(required) + if type(list)=="table" then + list=list[only] + if type(list)~="table" then + list={ only } end - found_library=required else - if trace then - report("qualified name %a not found",required) - end + list={ only } end - else - local required_name=required_base.."."..os.libsuffix - local version=type(version)=="string" and version~="" and version or false - local engine="luatex" - if trace and not done then - local list=expandpaths("lib") - for i=1,#list do - report("tds path %i: %s",i,list[i]) - end - end - local function found(locate,asked_library,how,...) - if trace then - report("checking %s: %a",how,asked_library) - end - return locate(asked_library,...) + if trace then + report("using lookup list for library %a: % | t",only,list) end - local function check(locate,...) - local found=nil - if version then - local asked_library=joinfile(required_path,version,required_name) - if trace then - report("checking %s: %a","with version",asked_library) - end - found=locate(asked_library,...) + for i=1,#list do + local name=list[i] + local found=findfile(name,"lib") + if not found then + found=findfile(addsuffix(name,suffix),"lib") end - if not found or found=="" then - local asked_library=joinfile(required_path,required_name) + if found then if trace then - report("checking %s: %a","with version",asked_library) + report("library %a resolved via %a path to %a",name,"tds lib",found) end - found=locate(asked_library,...) - end - return found and found~="" and found or false - end - local function attempt(checkpattern) - if trace then - report("checking tds lib paths strictly") - end - local found=findfile and check(findfile,"lib") - if found and (not checkpattern or find(found,checkpattern)) then return found end - if trace then - report("checking tds lib paths with wildcard") - end - local asked_library=joinfile(required_path,".*",required_name) - if trace then - report("checking %s: %a","latest version",asked_library) - end - local list=findfiles(asked_library,"lib",true) - if list and #list>0 then - sort(list) - local found=list[#list] - if found and (not checkpattern or find(found,checkpattern)) then - return found - end - end - if trace then - report("checking lib paths") - end - package.extralibpath(environment.ownpath) - local paths=package.libpaths() - local pattern="/[^/]+%."..os.libsuffix.."$" - for i=1,#paths do - required_path=gsub(paths[i],pattern,"") - local found=check(lfs.isfound) - if type(found)=="string" and (not checkpattern or find(found,checkpattern)) then - return found - end - end - return false end - if engine then - if trace then - report("attemp 1, engine %a",engine) - end - found_library=attempt("/"..engine.."/") - if not found_library then - if trace then - report("attemp 2, no engine",asked_library) + if expandpaths then + local list=expandpaths("PATH") + local base=addsuffix(only,suffix) + for i=1,#list do + local full=joinfile(list[i],base) + local found=isfile(full) and full + if found then + if trace then + report("library %a resolved via %a path to %a",name,"system",found) + end + return found end - found_library=attempt() end - else - found_library=attempt() end - end - if not found_library then + elseif isfile(addsuffix(required,suffix)) then if trace then - report("not found: %a",required) + report("library with qualified name %a %sfound",required,"") end - library=false + return required else if trace then - report("found: %a",found_library) - end - local result,message=action(found_library,required_base) - if result then - library=result - else - library=false - report("load error: message %a, library %a",tostring(message or "unknown"),found_library or "no library") + report("library with qualified name %a %sfound",required,"not ") end end - if trace then - if not library then - report("unknown library: %a",required) - else - report("stored library: %a",required) - end - end - return library or nil + return false end -do - local report_swiglib=logs.reporter("swiglib") - local trace_swiglib=false - local savedrequire=require - local loadedlibs={} - local loadlib=package.loadlib - local pushdir=dir.push - local popdir=dir.pop - trackers.register("resolvers.swiglib",function(v) trace_swiglib=v end) - function requireswiglib(required,version) - local library=loadedlibs[library] - if library==nil then - local trace_swiglib=trace_swiglib or package.helpers.trace - library=locate(required,version,trace_swiglib,report_swiglib,function(name,base) - pushdir(pathpart(name)) - local opener="luaopen_"..base - if trace_swiglib then - report_swiglib("opening: %a with %a",name,opener) - end - local library,message=loadlib(name,opener) - local libtype=type(library) - if libtype=="function" then - library=library() +local foundlibraries=table.setmetatableindex(function(t,k) + local v=findlib(k) + t[k]=v + return v +end) +function resolvers.findlib(required) + return foundlibraries[required] +end +local libraries={} +resolvers.libraries=libraries +local report=logs.reporter("optional") +if optional then optional.loaded={} end +function libraries.validoptional(name) + local thelib=optional and optional[name] + if not thelib then + elseif thelib.initialize then + return thelib + else + report("invalid optional library %a",libname) + end +end +function libraries.optionalloaded(name,libnames) + local thelib=optional and optional[name] + if not thelib then + report("no optional %a library found",name) + else + local thelib_initialize=thelib.initialize + if not thelib_initialize then + report("invalid optional library %a",name) + else + if type(libnames)=="string" then + libnames={ libnames } + end + if type(libnames)=="table" then + for i=1,#libnames do + local libname=libnames[i] + local filename=foundlibraries[libname] + if filename then + libnames[i]=filename + else + report("unable to locate library %a",libname) + return + end + end + local initialized=thelib_initialize(unpack(libnames)) + if initialized then + report("using library '% + t'",libnames) else - report_swiglib("load error: %a returns %a, message %a, library %a",opener,libtype,(string.gsub(message or "no message","[%s]+$","")),found_library or "no library") - library=false + report("unable to initialize library '% + t'",libnames) end - popdir() - return library - end) - loadedlibs[required]=library or false - end - return library - end - function require(name,version) - if find(name,"^swiglib%.") then - return requireswiglib(name,version) - else - return savedrequire(name) - end - end - local swiglibs={} - local initializer="core" - function swiglib(name,version) - local library=swiglibs[name] - if not library then - statistics.starttiming(swiglibs) - if trace_swiglib then - report_swiglib("loading %a",name) - end - if not find(name,"%."..initializer.."$") then - fullname="swiglib."..name.."."..initializer - else - fullname="swiglib."..name + return initialized end - library=requireswiglib(fullname,version) - swiglibs[name]=library - statistics.stoptiming(swiglibs) end - return library end - statistics.register("used swiglibs",function() - if next(swiglibs) then - return string.format("%s, initial load time %s seconds",table.concat(table.sortedkeys(swiglibs)," "),statistics.elapsedtime(swiglibs)) - end - end) end if FFISUPPORTED and ffi and ffi.load then - local report_ffilib=logs.reporter("ffilib") - local trace_ffilib=false - local savedffiload=ffi.load - trackers.register("resolvers.ffilib",function(v) trace_ffilib=v end) - local loaded={} - local function locateindeed(name) - name=removesuffix(name) - local l=loaded[name] - if l==nil then - local state,library=pcall(savedffiload,name) - if type(library)=="userdata" then - l=library - elseif type(state)=="userdata" then - l=state - else - l=false - end - loaded[name]=l - elseif trace_ffilib then - report_ffilib("reusing already loaded %a",name) - end - return l - end - local function getlist(required) - local list=directives.value("system.librarynames" ) - if type(list)=="table" then - list=list[required] - if type(list)=="table" then - if trace then - report("using lookup list for library %a: % | t",required,list) - end - return list - end - end - return { required } - end - function ffilib(name,version) - name=removesuffix(name) - local l=loaded[name] - if l~=nil then - if trace_ffilib then - report_ffilib("reusing already loaded %a",name) - end - return l - end - local list=getlist(name) - if version=="system" then - for i=1,#list do - local library=locateindeed(list[i]) - if type(library)=="userdata" then - return library - end - end - else - for i=1,#list do - local library=locate(list[i],version,trace_ffilib,report_ffilib,locateindeed) - if type(library)=="userdata" then - return library - end - end - end - end + local ffiload=ffi.load function ffi.load(name) - local list=getlist(name) - for i=1,#list do - local library=ffilib(list[i]) - if type(library)=="userdata" then - return library - end - end - if trace_ffilib then - report_ffilib("trying to load %a using normal loader",name) - end - for i=1,#list do - local state,library=pcall(savedffiload,list[i]) - if type(library)=="userdata" then - return library - elseif type(state)=="userdata" then - return library - end + local full=name and foundlibraries[name] + if full then + return ffiload(full) + else + return ffiload(name) end end end @@ -25336,7 +25784,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 9637, stripped down to: 7253 +-- original size: 13964, stripped down to: 10026 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -25350,16 +25798,14 @@ local concat=table.concat local quoted=string.quoted local luasuffixes=utilities.lua.suffixes local report_format=logs.reporter("resolvers","formats") -local function primaryflags() - local arguments=environment.arguments +local function primaryflags(arguments) local flags={} if arguments.silent then flags[#flags+1]="--interaction=batchmode" end return concat(flags," ") end -local function secondaryflags() - local arguments=environment.arguments +local function secondaryflags(arguments) local trackers=arguments.trackers local directives=arguments.directives local flags={} @@ -25381,6 +25827,9 @@ local function secondaryflags() if arguments.ansi then flags[#flags+1]="--c:ansi" end + if arguments.ansilog then + flags[#flags+1]="--c:ansilog" + end if arguments.strip then flags[#flags+1]="--c:strip" end @@ -25391,12 +25840,13 @@ local function secondaryflags() end local template=[[--ini %primaryflags% --lua=%luafile% %texfile% %secondaryflags% %dump% %redirect%]] local checkers={ - primaryflags="string", - secondaryflags="string", + primaryflags="verbose", + secondaryflags="verbose", luafile="readable", texfile="readable", redirect="string", dump="string", + binarypath="string", } local runners={ luatex=sandbox.registerrunner { @@ -25421,46 +25871,88 @@ local runners={ reporter=report_format, }, } -function environment.make_format(name,arguments) +local function validbinarypath() + if not environment.arguments.nobinarypath then + local path=environment.ownpath or file.dirname(environment.ownname) + if path and path~="" then + path=dir.expandname(path) + if path~="" and lfs.isdir(path) then + return path + end + end + end +end +function environment.make_format(formatname) + local arguments=environment.arguments local engine=environment.ownmain or "luatex" - local silent=environment.arguments.silent - local errors=environment.arguments.errors - local olddir=dir.current() - local path=caches.getwritablepath("formats",engine) or "" - if path~="" then - lfs.chdir(path) - end - report_format("using format path %a",dir.current()) - local texsourcename=file.addsuffix(name,"mkiv") - local fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + local silent=arguments.silent + local errors=arguments.errors + local texsourcename="" + local texsourcepath="" + local fulltexsourcename="" + if engine=="luametatex" then + texsourcename=file.addsuffix(formatname,"mkxl") + fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + end + if fulltexsourcename=="" then + texsourcename=file.addsuffix(formatname,"mkiv") + fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + end if fulltexsourcename=="" then - texsourcename=file.addsuffix(name,"tex") + texsourcename=file.addsuffix(formatname,"tex") fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" end if fulltexsourcename=="" then - report_format("no tex source file with name %a (mkiv or tex)",name) - lfs.chdir(olddir) + report_format("no tex source file with name %a (mkiv or tex)",formatname) return - else - report_format("using tex source file %a",fulltexsourcename) end - local texsourcepath=dir.expandname(file.dirname(fulltexsourcename)) - local specificationname=file.replacesuffix(fulltexsourcename,"lus") - local fullspecificationname=resolvers.findfile(specificationname,"tex") or "" - if fullspecificationname=="" then - specificationname=file.join(texsourcepath,"context.lus") - fullspecificationname=resolvers.findfile(specificationname,"tex") or "" + report_format("using tex source file %a",fulltexsourcename) + fulltexsourcename=dir.expandname(fulltexsourcename) + texsourcepath=file.dirname(fulltexsourcename) + if not lfs.isfile(fulltexsourcename) then + report_format("no accessible tex source file with name %a",fulltexsourcename) + return end + local specificationname="context.lus" + local specificationpath="" + local fullspecificationname=resolvers.findfile(specificationname) or "" if fullspecificationname=="" then - report_format("unknown stub specification %a",specificationname) - lfs.chdir(olddir) + report_format("unable to locate specification file %a",specificationname) + return + end + report_format("using specification file %a",fullspecificationname) + fullspecificationname=dir.expandname(fullspecificationname) + specificationpath=file.dirname(fullspecificationname) + if texsourcepath~=specificationpath then + report_format("tex source file and specification file are on different paths") + return + end + if not lfs.isfile(fulltexsourcename) then + report_format("no accessible tex source file with name %a",fulltexsourcename) + return + end + if not lfs.isfile(fullspecificationname) then + report_format("no accessible specification file with name %a",fulltexsourcename) + return + end + report_format("using tex source path %a",texsourcepath) + local validformatpath=caches.getwritablepath("formats",engine) or "" + local startupdir=dir.current() + if validformatpath=="" then + report_format("invalid format path, insufficient write access") + return + end + local binarypath=validbinarypath() + report_format("changing to format path %a",validformatpath) + lfs.chdir(validformatpath) + if dir.current()~=validformatpath then + report_format("unable to change to format path %a",validformatpath) return end - local specificationpath=file.dirname(fullspecificationname) local usedluastub=nil local usedlualibs=dofile(fullspecificationname) if type(usedlualibs)=="string" then - usedluastub=file.join(file.dirname(fullspecificationname),usedlualibs) + usedluastub=file.join(specificationpath,usedlualibs) elseif type(usedlualibs)=="table" then report_format("using stub specification %a",fullspecificationname) local texbasename=file.basename(name) @@ -25477,48 +25969,57 @@ function environment.make_format(name,arguments) end else report_format("invalid stub specification %a",fullspecificationname) - lfs.chdir(olddir) + lfs.chdir(startupdir) return end + local runner=runners[engine] + if not runner then + report_format("the format %a cannot be generated, no runner available for engine %a",name,engine) + lfs.chdir(startupdir) + return + end + local primaryflags=primaryflags(arguments) + local secondaryflags=secondaryflags(arguments) local specification={ - primaryflags=primaryflags(), - secondaryflags=secondaryflags(), + binarypath=binarypath, + primaryflags=primaryflags, + secondaryflags=secondaryflags, luafile=quoted(usedluastub), texfile=quoted(fulltexsourcename), dump=os.platform=="unix" and "\\\\dump" or "\\dump", } - local runner=runners[engine] - if not runner then - report_format("format %a cannot be generated, no runner available for engine %a",name,engine) - elseif silent then - statistics.starttiming() + if silent then specification.redirect="> temp.log" - local result=runner(specification) - local runtime=statistics.stoptiming() - if result~=0 then - print(format("%s silent make > fatal error when making format %q",engine,name)) - else - print(format("%s silent make > format %q made in %.3f seconds",engine,name,runtime)) - end + end + statistics.starttiming() + local result=runner(specification) + local runtime=statistics.stoptiming() + if silent then os.remove("temp.log") - else - runner(specification) end - local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" - local mp=dir.glob(pattern) - if mp then - for i=1,#mp do - local name=mp[i] - report_format("removing related mplib format %a",file.basename(name)) - os.remove(name) + report_format() + if binarypath and binarypath~="" then + report_format("binary path : %s",binarypath or "?") end - end - lfs.chdir(olddir) + report_format("format path : %s",validformatpath) + report_format("luatex engine : %s",engine) + report_format("lua startup file : %s",usedluastub) + if primaryflags~="" then + report_format("primary flags : %s",primaryflags) + end + if secondaryflags~="" then + report_format("secondary flags : %s",secondaryflags) + end + report_format("context file : %s",fulltexsourcename) + report_format("run time : %.3f seconds",runtime) + report_format("return value : %s",result==0 and "okay" or "error") + report_format() + lfs.chdir(startupdir) end -local template=[[%flags% --fmt=%fmtfile% --lua=%luafile% %texfile% %more%]] +local template=[[%primaryflags% --fmt=%fmtfile% --lua=%luafile% %texfile% %secondaryflags%]] local checkers={ - flags="string", - more="string", + primaryflags="verbose", + secondaryflags="verbose", fmtfile="readable", luafile="readable", texfile="readable", @@ -25531,6 +26032,13 @@ local runners={ checkers=checkers, reporter=report_format, }, + luametatex=sandbox.registerrunner { + name="run luametatex format", + program="luametatex", + template=template, + checkers=checkers, + reporter=report_format, + }, luajittex=sandbox.registerrunner { name="run luajittex format", program="luajittex", @@ -25539,51 +26047,78 @@ local runners={ reporter=report_format, }, } -function environment.run_format(name,data,more) - if name and name~="" then - local engine=environment.ownmain or "luatex" - local barename=file.removesuffix(name) - local fmtname=caches.getfirstreadablefile(file.addsuffix(barename,"fmt"),"formats",engine) - if fmtname=="" then - fmtname=resolvers.findfile(file.addsuffix(barename,"fmt")) or "" - end - fmtname=resolvers.cleanpath(fmtname) - if fmtname=="" then - report_format("no format with name %a",name) - else - local barename=file.removesuffix(name) - local luaname=file.addsuffix(barename,"luc") - if not lfs.isfile(luaname) then - luaname=file.addsuffix(barename,"lua") - end - if not lfs.isfile(luaname) then - report_format("using format name %a",fmtname) - report_format("no luc/lua file with name %a",barename) - else - local runner=runners[engine] - if not runner then - report_format("format %a cannot be run, no runner available for engine %a",name,engine) - else - runner { - flags=primaryflags(), - fmtfile=quoted(barename), - luafile=quoted(luaname), - texfile=quoted(data), - more=more, - } - end - end - end +function environment.run_format(formatname,scriptname,filename,primaryflags,secondaryflags,verbose) + local engine=environment.ownmain or "luatex" + if not formatname or formatname=="" then + report_format("missing format name") + return + end + if not scriptname or scriptname=="" then + report_format("missing script name") + return + end + if not lfs.isfile(formatname) or not lfs.isfile(scriptname) then + formatname,scriptname=resolvers.locateformat(formatname) end + if not formatname or formatname=="" then + report_format("invalid format name") + return + end + if not scriptname or scriptname=="" then + report_format("invalid script name") + return + end + local runner=runners[engine] + if not runner then + report_format("format %a cannot be run, no runner available for engine %a",file.nameonly(name),engine) + return + end + if not filename then + filename "" + end + local binarypath=validbinarypath() + local specification={ + binarypath=binarypath, + primaryflags=primaryflags or "", + secondaryflags=secondaryflags or "", + fmtfile=quoted(formatname), + luafile=quoted(scriptname), + texfile=filename~="" and quoted(filename) or "", + } + statistics.starttiming() + local result=runner(specification) + local runtime=statistics.stoptiming() + if verbose then + report_format() + if binarypath and binarypath~="" then + report_format("binary path : %s",binarypath) + end + report_format("luatex engine : %s",engine) + report_format("lua startup file : %s",scriptname) + report_format("tex format file : %s",formatname) + if filename~="" then + report_format("tex input file : %s",filename) + end + if primaryflags~="" then + report_format("primary flags : %s",primaryflags) + end + if secondaryflags~="" then + report_format("secondary flags : %s",secondaryflags) + end + report_format("run time : %.3f seconds",runtime) + report_format("return value : %s",result==0 and "okay" or "error") + report_format() + end + return result end end -- of closure --- used libraries : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua +-- used libraries : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua libs-ini.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 1019480 --- stripped bytes : 403728 +-- original bytes : 1038245 +-- stripped bytes : 409841 -- end library merge @@ -25697,7 +26232,7 @@ local ownlibs = { -- order can be made better 'data-tmf.lua', 'data-lst.lua', - 'util-lib.lua', -- swiglib + 'libs-ini.lua', 'luat-sta.lua', 'luat-fmt.lua', @@ -25846,7 +26381,7 @@ local helpinfo = [[ <category name="basic"> <subcategory> <flag name="script"><short>run an mtx script (lua prefered method) (<ref name="noquotes"/>), no script gives list</short></flag> - <flag name="evaluate"><short>run code passed on the commandline (between quotes)</short></flag> + <flag name="evaluate"><short>run code passed on the commandline (between quotes) (=loop) (exit|quit aborts)</short></flag> <flag name="execute"><short>run a script or program (texmfstart method) (<ref name="noquotes"/>)</short></flag> <flag name="resolve"><short>resolve prefixed arguments</short></flag> <flag name="ctxlua"><short>run internally (using preloaded libs)</short></flag> @@ -25865,6 +26400,7 @@ local helpinfo = [[ <flag name="stubpath" value="binpath"><short>paths where stubs wil be written</short></flag> <flag name="windows"><short>create windows (mswin) stubs</short></flag> <flag name="unix"><short>create unix (linux) stubs</short></flag> + <flag name="addbinarypath"><short>prepend the (found) binarypath to runners</short></flag> </subcategory> <subcategory> <flag name="verbose"><short>give a bit more info</short></flag> @@ -26086,8 +26622,7 @@ end report() io.flush() end - -- no os.exec because otherwise we get the wrong return value - local code = os.execute(command) -- maybe spawn + local code = os.execute(command) if code == 0 then return true else @@ -26130,7 +26665,7 @@ function runners.execute_program(fullname) report() report() io.flush() - local code = os.exec(command) -- (fullname,unpack(after)) does not work / maybe spawn + local code = os.execute(command) return code == 0 end end @@ -26517,16 +27052,22 @@ function runners.associate(filename) end function runners.evaluate(code,filename) -- for Luigi + local environment = table.setmetatableindex(_G) if code == "loop" then while true do - io.write("> ") + io.write("lua > ") local code = io.read() - if code ~= "" then + if code == "quit" or code == "exit" then + break + elseif code ~= "" then local temp = string.match(code,"^= (.*)$") if temp then - code = "print("..temp..")" + code = "inspect("..temp..")" + end + local compiled, message = load(code,"console","t",environment) + if type(compiled) ~= "function" then + compiled = load("inspect("..code..")","console","t",environment) end - local compiled, message = loadstring(code) if type(compiled) ~= "function" then io.write("! " .. (message or code).."\n") else @@ -26539,7 +27080,10 @@ function runners.evaluate(code,filename) -- for Luigi code = filename end if code ~= "" then - local compiled, message = loadstring(code) + local compiled, message = load(code,"console","t",environment) + if type(compiled) ~= "function" then + compiled = load("inspect("..code..")","console","t",environment) + end if type(compiled) ~= "function" then io.write("invalid lua code: " .. (message or code)) return @@ -26716,17 +27260,17 @@ do end -if e_argument("ansi") then +-- if e_argument("ansi") or e_argument("ansilog") then - logs.setformatters("ansi") +-- logs.setformatters(e_argument("ansi") and "ansi" or "ansilog") - local script = e_argument("script") or e_argument("scripts") +-- -- local script = e_argument("script") or e_argument("scripts") +-- -- +-- -- if type(script) == "string" then +-- -- logs.writer("]0;"..script.."") -- for Alan to test +-- -- end - if type(script) == "string" then - logs.writer("]0;"..script.."") -- for Alan to test - end - -end +-- end if e_argument("script") or e_argument("scripts") then @@ -26975,11 +27519,11 @@ elseif e_argument("format-path") then resolvers.load() report(caches.getwritablepath("format")) -elseif e_argument("pattern") then - - -- luatools - - runners.execute_ctx_script("mtx-base","--pattern='" .. e_argument("pattern") .. "'",filename) +-- elseif e_argument("pattern") then +-- +-- -- luatools +-- +-- runners.execute_ctx_script("mtx-base","--pattern='" .. e_argument("pattern") .. "'",filename) elseif e_argument("generate") then |